Merge pull request #32 from opencybersecurityalliance/develop

Try to improve markroot

firepit/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """IBM Security"""
 __email__ = 'pcoccoli@us.ibm.com'
-__version__ = '1.2.0'
+__version__ = '1.2.1'
 
 
 from importlib import import_module

firepit/raft.py

@@ -7,6 +7,7 @@
 
 import re
 from collections import OrderedDict
+from collections import defaultdict
 
 import ijson
 import orjson
@@ -275,7 +276,11 @@ def markroot(obj, viewname='observed-data'):
         return [obj]
     objs = obj['objects']
     reffed = set()
+
+    # Keep track of the preference order of each reffed object, by type
+    prefs = defaultdict(list)
     for idx, sco in objs.items():
+        prefs[sco['type']].append(idx)
         for attr, val in json_normalize(sco, flat_lists=False).items():
             if attr.endswith('_ref'):
                 if val not in objs or val == idx:
@@ -284,11 +289,13 @@ def markroot(obj, viewname='observed-data'):
                 # only mark the root (think process:parent_ref)
                 if objs[idx]['type'] == objs[val]['type']:
                     _mark_tree(objs, val, reffed)
-                elif (objs[idx]['type'] == 'network-traffic' and
-                      objs[val]['type'].endswith('-addr') and
-                      'dst_' in attr):
-                    # For src/dst pairs, consider the src as the root (so add dst to reffed)
-                    reffed.add(val)
+                elif (objs[val]['type'].endswith('-addr')):
+                    if 'dst_' in attr:
+                        # For src/dst pairs, consider the src as the root (so add dst to reffed)
+                        reffed.add(val)
+                    elif attr.endswith('src_ref'):
+                        # Save ref as the "preferred" object for this type
+                        prefs[objs[val]['type']].insert(0, val)
                 elif val in reffed:
                     reffed.add(idx)
             elif attr.endswith('_refs'):
@@ -299,7 +306,9 @@ def markroot(obj, viewname='observed-data'):
                         reffed.add(ref)
     for k, v in objs.items():
         if k not in reffed:
-            objs[k]['x_root'] = 1
+            # Check if there's a more preferred object
+            if v['type'] not in prefs or prefs[v['type']][0] == k:
+                objs[k]['x_root'] = 1
     return [obj]
 
 

firepit/splint.py

@@ -35,11 +35,11 @@ def _timefmt(ts):
 
 def _start_bundle():
     bundle_id = 'bundle--' + str(uuid.uuid4())
-    sys.stdout.write('{"type": "bundle",'
+    sys.stdout.write('{"type":"bundle",'
                      '"id": "')
     sys.stdout.write(bundle_id + '",')
-    sys.stdout.write('"spec_version": "2.0",'
-                     '"objects": [')
+    sys.stdout.write('"spec_version":"2.0",'
+                     '"objects":[')
 
 
 def _end_bundle():
@@ -116,5 +116,25 @@ def dedup_ids(
     _end_bundle()
 
 
+@app.command()
+def limit(
+    n: int = typer.Argument(..., help="Max number of observations"),
+    filename: str = typer.Argument(..., help="STIX bundle file"),
+):
+    """Truncate STIX bundle"""
+    _start_bundle()
+    count = 0
+    for obj in raft.get_objects(filename):
+        if count > n:
+            break
+        blob = json.dumps(obj, separators=(',', ':'))
+        if count:
+            sys.stdout.write(f',{blob}')
+        else:
+            sys.stdout.write(f'{blob}')
+        count += 1
+    _end_bundle()
+
+
 if __name__ == "__main__":
     app()

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.2.0
+current_version = 1.2.1
 commit = True
 tag = True
 

setup.py

@@ -57,6 +57,6 @@
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/opencybersecurityalliance/firepit',
-    version='1.2.0',
+    version='1.2.1',
     zip_safe=False,
 )

tests/conftest.py

@@ -35,3 +35,9 @@ def fake_bundle_list():
         os.path.join(cwd, 'conn_a.json'),
         os.path.join(cwd, 'conn_b.json')
     ]
+
+
+@pytest.fixture
+def one_event_bundle():
+    cwd = os.path.dirname(os.path.abspath(__file__))
+    return os.path.join(cwd, 'one_event.json')

tests/one_event.json

@@ -0,0 +1,152 @@
+{
+  "type": "bundle",
+  "id": "bundle--1198c768-9949-4052-8d49-018e99534c86",
+  "spec_version": "2.0",
+  "objects": [
+    {
+      "type": "identity",
+      "id": "identity--4216159b-35ba-4b08-8dd5-896a03c5368e",
+      "identity_class": "system",
+      "name": "qradar",
+      "created": "2021-08-23T14:07:19.596Z",
+      "modified": "2021-08-23T14:07:19.596Z"
+    },
+    {
+      "id": "observed-data--b0ced57e-970b-4767-b3df-ffddb168e0bc",
+      "type": "observed-data",
+      "created_by_ref": "identity--4216159b-35ba-4b08-8dd5-896a03c5368e",
+      "created": "2021-08-23T14:07:20.112Z",
+      "modified": "2021-08-23T14:07:20.112Z",
+      "objects": {
+        "0": {
+          "type": "x-oca-event",
+          "action": "Traffic Start",
+          "outcome": "Firewall Session Opened",
+          "category": [
+            "Access"
+          ],
+          "provider": "Palo Alto PA Series",
+          "agent": "PaSeries @ firewall.time4demo.com",
+          "created": "2021-08-23T14:02:09.000Z",
+          "network_ref": "4",
+          "host_ref": "5",
+          "original_ref": "10"
+        },
+        "1": {
+          "type": "x-qradar",
+          "qid": 53512405,
+          "category_id": 4007,
+          "high_level_category_id": 4000,
+          "log_source_id": 114,
+          "device_type": 206,
+          "direction": "L2L",
+          "credibility": 10,
+          "relevance": 10,
+          "cre_event_list": [
+            "100013",
+            "100033",
+            "100489",
+            "100103",
+            "100227",
+            "100225",
+            "100258",
+            "100257",
+            "122989",
+            "100221",
+            "100297",
+            "100272",
+            "100219",
+            "100218",
+            "100215"
+          ],
+          "domain_id": 0,
+          "domain_name": "Default Domain",
+          "has_offense": "false"
+        },
+        "2": {
+          "type": "x-ibm-finding",
+          "start": "2021-08-23T14:02:11.113Z",
+          "end": "2021-08-23T14:02:11.113Z",
+          "src_ip_ref": "3",
+          "dst_ip_ref": "7",
+          "event_count": 1,
+          "finding_type": "event",
+          "magnitude": 6,
+          "severity": 0,
+          "src_geolocation": "NorthAmerica.UnitedStates",
+          "dst_geolocation": "NorthAmerica.UnitedStates",
+          "rule_names": [
+            "BB:PortDefinition: DNS Ports",
+            "BB:ProtocolDefinition: Windows Protocols",
+            "BB:UBA : Common Log Source Filters",
+            "BB:CategoryDefinition: Firewall or ACL Accept",
+            "Source Asset Weight is Low",
+            "Source Asset Exists",
+            "BB:DeviceDefinition: IDS / IPS",
+            "BB:DeviceDefinition: FW / Router / Switch",
+            "BB:CategoryDefinition: Firewall or ACL Accept Event for a FW/Router/Switch Device",
+            "Destination Asset Weight is Low",
+            "BB:PortDefinition: Authorized L2R Ports",
+            "Load Basic Building Blocks",
+            "Destination Asset Port is Open",
+            "Destination Asset Exists",
+            "Context is Local to Local"
+          ]
+        },
+        "3": {
+          "type": "ipv4-addr",
+          "value": "10.95.79.130",
+          "resolves_to_refs": [
+            "6"
+          ]
+        },
+        "4": {
+          "type": "network-traffic",
+          "src_ref": "3",
+          "src_port": 48589,
+          "dst_ref": "7",
+          "dst_port": 53,
+          "protocols": [
+            "udp"
+          ]
+        },
+        "5": {
+          "type": "x-oca-asset",
+          "ip_refs": [
+            "3",
+            "9"
+          ],
+          "mac_refs": [
+            "6"
+          ]
+        },
+        "6": {
+          "type": "mac-addr",
+          "value": "00:00:00:00:00:00"
+        },
+        "7": {
+          "type": "ipv4-addr",
+          "value": "10.5.146.130",
+          "resolves_to_refs": [
+            "8"
+          ]
+        },
+        "8": {
+          "type": "mac-addr",
+          "value": "00:00:00:00:00:00"
+        },
+        "9": {
+          "type": "ipv4-addr",
+          "value": "0.0.0.0"
+        },
+        "10": {
+          "type": "artifact",
+          "payload_bin": "PDE0PkF1ZyAyMyAxMDowMjoxMCBmaXJld2FsbC50aW1lNGRlbW8uY29tIExFRUY6Mi4wfFBhbG8gQWx0byBOZXR3b3Jrc3xQQU4tT1MgU3lzbG9nIEludGVncmF0aW9ufDEwLjAuMHxhbGxvd3x4N0N8Y2F0PVRSQUZGSUN8UmVjZWl2ZVRpbWU9MjAyMS8wOC8yMyAxMDowMjowOXxTZXJpYWxOdW1iZXI9fFR5cGU9VFJBRkZJQ3xTdWJ0eXBlPXN0YXJ0fGRldlRpbWU9QXVnIDIzIDIwMjEgMTQ6MDI6MDkgR01UfHNyYz0xMC45NS43OS4xMzB8ZHN0PTEwLjUuMTQ2LjEzMHxzcmNQb3N0TkFUPTAuMC4wLjB8ZHN0UG9zdE5BVD0wLjAuMC4wfFJ1bGVOYW1lPW1nbXQtYWxsb3ctYWxsfHVzck5hbWU9fFNvdXJjZVVzZXI9fERlc3RpbmF0aW9uVXNlcj18QXBwbGljYXRpb249ZG5zfFZpcnR1YWxTeXN0ZW09dnN5czF8U291cmNlWm9uZT1TRUNORVRfTUdUfERlc3RpbmF0aW9uWm9uZT1TRUNORVRfTUdUfEluZ3Jlc3NJbnRlcmZhY2U9ZXRoZXJuZXQxLzF8RWdyZXNzSW50ZXJmYWNlPWV0aGVybmV0MS8xfExvZ0ZvcndhcmRpbmdQcm9maWxlPXFyYWRhci1mb3J3YXJkLXByb2ZpbGV8U2Vzc2lvbklEPTg5MXxSZXBlYXRDb3VudD0xfHNyY1BvcnQ9NDg1ODl8ZHN0UG9ydD01M3xzcmNQb3N0TkFUUG9ydD0wfGRzdFBvc3ROQVRQb3J0PTB8RmxhZ3M9MHg4MDAwfHByb3RvPXVkcHxhY3Rpb249YWxsb3d8dG90YWxCeXRlcz04Nnxkc3RCeXRlcz0wfHNyY0J5dGVzPTg2fHRvdGFsUGFja2V0cz0xfFN0YXJ0VGltZT0yMDIxLzA4LzIzIDEwOjAyOjA3fEVsYXBzZWRUaW1lPTB8VVJMQ2F0ZWdvcnk9YW55fHNlcXVlbmNlPTE1NTk5MTczfEFjdGlvbkZsYWdzPTB4MHxTb3VyY2VMb2NhdGlvbj0xMC4wLjAuMC0xMC4yNTUuMjU1LjI1NXxEZXN0aW5hdGlvbkxvY2F0aW9uPTEwLjAuMC4wLTEwLjI1NS4yNTUuMjU1fGRzdFBhY2tldHM9MHxzcmNQYWNrZXRzPTF8U2Vzc2lvbkVuZFJlYXNvbj1uL2F8RGV2aWNlR3JvdXBIaWVyYXJjaHlMMT0wfERldmljZUdyb3VwSGllcmFyY2h5TDI9MHxEZXZpY2VHcm91cEhpZXJhcmNoeUwzPTB8RGV2aWNlR3JvdXBIaWVyYXJjaHlMND0wfHZTcmNOYW1lPXxEZXZpY2VOYW1lPWZpcmV3YWxsfEFjdGlvblNvdXJjZT1mcm9tLXBvbGljeXxTcmNVVUlEPXxEc3RVVUlEPXxUdW5uZWxJRD0wfE1vbml0b3JUYWc9fFBhcmVudFNlc3Npb25JRD0wfFBhcmVudFN0YXJ0VGltZT18VHVubmVsVHlwZT1OL0EK"
+        }
+      },
+      "first_observed": "2021-08-23T14:02:11.113Z",
+      "last_observed": "2021-08-23T14:02:11.113Z",
+      "number_observed": 1
+    }
+  ]
+}

tests/test_storage.py

@@ -544,3 +544,29 @@ def test_clobber_viewname(fake_bundle_file_2, tmpdir):
     # conns2 should be no more:
     with pytest.raises(UnknownViewname):
         store.lookup('conns2')
+
+
+def test_three_ips(one_event_bundle, tmpdir):
+    """A single Observation SDO can contain any arbitrary number and type
+    of SCOs.  In the case that one type appears multiple times,
+    firepit will attempt to mark one as the "primary", or most
+    significant, instance by setting an "x_root" attribute to 1 (note
+    that this name may change in the future).
+
+    A common case is `ipv4-addr`: if you have a `network-traffic`
+    object, then you usally have 2 `ipv4-addr` (or `ipv6-addr`)
+    objects.  In that case, firepit will (arbitrarily) pick the object
+    referenced as the `src_ref` to be the "primary".
+
+    This test case involves 1 Observation with 3 IP addresses.  One is
+    `src_ref`, one is `dst_ref`, and third is...well, ask the QRadar
+    people, I guess.
+
+    """
+    store = tmp_storage(tmpdir)
+    store.cache('q1', [one_event_bundle])
+
+    results = store._query('SELECT value FROM "ipv4-addr" WHERE "x_root" IS NOT NULL')
+    rows = results.fetchall()
+    assert len(rows) == 1  # There can be only 1!!!
+    assert rows[0]['value'] == '10.95.79.130'