From 92d1e969ebc6e94f3e0de40bee0a0babc38995ab Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Mon, 23 Aug 2021 16:44:05 -0400 Subject: [PATCH 1/2] Try to improve markroot transform --- firepit/raft.py | 21 ++++-- firepit/splint.py | 26 +++++++- tests/conftest.py | 6 ++ tests/one_event.json | 152 ++++++++++++++++++++++++++++++++++++++++++ tests/test_storage.py | 26 ++++++++ 5 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 tests/one_event.json diff --git a/firepit/raft.py b/firepit/raft.py index d071eb5..b4f6656 100644 --- a/firepit/raft.py +++ b/firepit/raft.py @@ -7,6 +7,7 @@ import re from collections import OrderedDict +from collections import defaultdict import ijson import orjson @@ -275,7 +276,11 @@ def markroot(obj, viewname='observed-data'): return [obj] objs = obj['objects'] reffed = set() + + # Keep track of the preference order of each reffed object, by type + prefs = defaultdict(list) for idx, sco in objs.items(): + prefs[sco['type']].append(idx) for attr, val in json_normalize(sco, flat_lists=False).items(): if attr.endswith('_ref'): if val not in objs or val == idx: @@ -284,11 +289,13 @@ def markroot(obj, viewname='observed-data'): # only mark the root (think process:parent_ref) if objs[idx]['type'] == objs[val]['type']: _mark_tree(objs, val, reffed) - elif (objs[idx]['type'] == 'network-traffic' and - objs[val]['type'].endswith('-addr') and - 'dst_' in attr): - # For src/dst pairs, consider the src as the root (so add dst to reffed) - reffed.add(val) + elif (objs[val]['type'].endswith('-addr')): + if 'dst_' in attr: + # For src/dst pairs, consider the src as the root (so add dst to reffed) + reffed.add(val) + elif attr.endswith('src_ref'): + # Save ref as the "preferred" object for this type + prefs[objs[val]['type']].insert(0, val) elif val in reffed: reffed.add(idx) elif attr.endswith('_refs'): @@ -299,7 +306,9 @@ def markroot(obj, viewname='observed-data'): reffed.add(ref) for k, v in objs.items(): if k not in reffed: - objs[k]['x_root'] = 1 + # Check if there's a more preferred object + if v['type'] not in prefs or prefs[v['type']][0] == k: + objs[k]['x_root'] = 1 return [obj] diff --git a/firepit/splint.py b/firepit/splint.py index d35e6a0..9f56771 100644 --- a/firepit/splint.py +++ b/firepit/splint.py @@ -35,11 +35,11 @@ def _timefmt(ts): def _start_bundle(): bundle_id = 'bundle--' + str(uuid.uuid4()) - sys.stdout.write('{"type": "bundle",' + sys.stdout.write('{"type":"bundle",' '"id": "') sys.stdout.write(bundle_id + '",') - sys.stdout.write('"spec_version": "2.0",' - '"objects": [') + sys.stdout.write('"spec_version":"2.0",' + '"objects":[') def _end_bundle(): @@ -116,5 +116,25 @@ def dedup_ids( _end_bundle() +@app.command() +def limit( + n: int = typer.Argument(..., help="Max number of observations"), + filename: str = typer.Argument(..., help="STIX bundle file"), +): + """Truncate STIX bundle""" + _start_bundle() + count = 0 + for obj in raft.get_objects(filename): + if count > n: + break + blob = json.dumps(obj, separators=(',', ':')) + if count: + sys.stdout.write(f',{blob}') + else: + sys.stdout.write(f'{blob}') + count += 1 + _end_bundle() + + if __name__ == "__main__": app() diff --git a/tests/conftest.py b/tests/conftest.py index 1a85d7d..f177ac8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,3 +35,9 @@ def fake_bundle_list(): os.path.join(cwd, 'conn_a.json'), os.path.join(cwd, 'conn_b.json') ] + + +@pytest.fixture +def one_event_bundle(): + cwd = os.path.dirname(os.path.abspath(__file__)) + return os.path.join(cwd, 'one_event.json') diff --git a/tests/one_event.json b/tests/one_event.json new file mode 100644 index 0000000..aa03e22 --- /dev/null +++ b/tests/one_event.json @@ -0,0 +1,152 @@ +{ + "type": "bundle", + "id": "bundle--1198c768-9949-4052-8d49-018e99534c86", + "spec_version": "2.0", + "objects": [ + { + "type": "identity", + "id": "identity--4216159b-35ba-4b08-8dd5-896a03c5368e", + "identity_class": "system", + "name": "qradar", + "created": "2021-08-23T14:07:19.596Z", + "modified": "2021-08-23T14:07:19.596Z" + }, + { + "id": "observed-data--b0ced57e-970b-4767-b3df-ffddb168e0bc", + "type": "observed-data", + "created_by_ref": "identity--4216159b-35ba-4b08-8dd5-896a03c5368e", + "created": "2021-08-23T14:07:20.112Z", + "modified": "2021-08-23T14:07:20.112Z", + "objects": { + "0": { + "type": "x-oca-event", + "action": "Traffic Start", + "outcome": "Firewall Session Opened", + "category": [ + "Access" + ], + "provider": "Palo Alto PA Series", + "agent": "PaSeries @ firewall.time4demo.com", + "created": "2021-08-23T14:02:09.000Z", + "network_ref": "4", + "host_ref": "5", + "original_ref": "10" + }, + "1": { + "type": "x-qradar", + "qid": 53512405, + "category_id": 4007, + "high_level_category_id": 4000, + "log_source_id": 114, + "device_type": 206, + "direction": "L2L", + "credibility": 10, + "relevance": 10, + "cre_event_list": [ + "100013", + "100033", + "100489", + "100103", + "100227", + "100225", + "100258", + "100257", + "122989", + "100221", + "100297", + "100272", + "100219", + "100218", + "100215" + ], + "domain_id": 0, + "domain_name": "Default Domain", + "has_offense": "false" + }, + "2": { + "type": "x-ibm-finding", + "start": "2021-08-23T14:02:11.113Z", + "end": "2021-08-23T14:02:11.113Z", + "src_ip_ref": "3", + "dst_ip_ref": "7", + "event_count": 1, + "finding_type": "event", + "magnitude": 6, + "severity": 0, + "src_geolocation": "NorthAmerica.UnitedStates", + "dst_geolocation": "NorthAmerica.UnitedStates", + "rule_names": [ + "BB:PortDefinition: DNS Ports", + "BB:ProtocolDefinition: Windows Protocols", + "BB:UBA : Common Log Source Filters", + "BB:CategoryDefinition: Firewall or ACL Accept", + "Source Asset Weight is Low", + "Source Asset Exists", + "BB:DeviceDefinition: IDS / IPS", + "BB:DeviceDefinition: FW / Router / Switch", + "BB:CategoryDefinition: Firewall or ACL Accept Event for a FW/Router/Switch Device", + "Destination Asset Weight is Low", + "BB:PortDefinition: Authorized L2R Ports", + "Load Basic Building Blocks", + "Destination Asset Port is Open", + "Destination Asset Exists", + "Context is Local to Local" + ] + }, + "3": { + "type": "ipv4-addr", + "value": "10.95.79.130", + "resolves_to_refs": [ + "6" + ] + }, + "4": { + "type": "network-traffic", + "src_ref": "3", + "src_port": 48589, + "dst_ref": "7", + "dst_port": 53, + "protocols": [ + "udp" + ] + }, + "5": { + "type": "x-oca-asset", + "ip_refs": [ + "3", + "9" + ], + "mac_refs": [ + "6" + ] + }, + "6": { + "type": "mac-addr", + "value": "00:00:00:00:00:00" + }, + "7": { + "type": "ipv4-addr", + "value": "10.5.146.130", + "resolves_to_refs": [ + "8" + ] + }, + "8": { + "type": "mac-addr", + "value": "00:00:00:00:00:00" + }, + "9": { + "type": "ipv4-addr", + "value": "0.0.0.0" + }, + "10": { + "type": "artifact", + "payload_bin": "PDE0PkF1ZyAyMyAxMDowMjoxMCBmaXJld2FsbC50aW1lNGRlbW8uY29tIExFRUY6Mi4wfFBhbG8gQWx0byBOZXR3b3Jrc3xQQU4tT1MgU3lzbG9nIEludGVncmF0aW9ufDEwLjAuMHxhbGxvd3x4N0N8Y2F0PVRSQUZGSUN8UmVjZWl2ZVRpbWU9MjAyMS8wOC8yMyAxMDowMjowOXxTZXJpYWxOdW1iZXI9fFR5cGU9VFJBRkZJQ3xTdWJ0eXBlPXN0YXJ0fGRldlRpbWU9QXVnIDIzIDIwMjEgMTQ6MDI6MDkgR01UfHNyYz0xMC45NS43OS4xMzB8ZHN0PTEwLjUuMTQ2LjEzMHxzcmNQb3N0TkFUPTAuMC4wLjB8ZHN0UG9zdE5BVD0wLjAuMC4wfFJ1bGVOYW1lPW1nbXQtYWxsb3ctYWxsfHVzck5hbWU9fFNvdXJjZVVzZXI9fERlc3RpbmF0aW9uVXNlcj18QXBwbGljYXRpb249ZG5zfFZpcnR1YWxTeXN0ZW09dnN5czF8U291cmNlWm9uZT1TRUNORVRfTUdUfERlc3RpbmF0aW9uWm9uZT1TRUNORVRfTUdUfEluZ3Jlc3NJbnRlcmZhY2U9ZXRoZXJuZXQxLzF8RWdyZXNzSW50ZXJmYWNlPWV0aGVybmV0MS8xfExvZ0ZvcndhcmRpbmdQcm9maWxlPXFyYWRhci1mb3J3YXJkLXByb2ZpbGV8U2Vzc2lvbklEPTg5MXxSZXBlYXRDb3VudD0xfHNyY1BvcnQ9NDg1ODl8ZHN0UG9ydD01M3xzcmNQb3N0TkFUUG9ydD0wfGRzdFBvc3ROQVRQb3J0PTB8RmxhZ3M9MHg4MDAwfHByb3RvPXVkcHxhY3Rpb249YWxsb3d8dG90YWxCeXRlcz04Nnxkc3RCeXRlcz0wfHNyY0J5dGVzPTg2fHRvdGFsUGFja2V0cz0xfFN0YXJ0VGltZT0yMDIxLzA4LzIzIDEwOjAyOjA3fEVsYXBzZWRUaW1lPTB8VVJMQ2F0ZWdvcnk9YW55fHNlcXVlbmNlPTE1NTk5MTczfEFjdGlvbkZsYWdzPTB4MHxTb3VyY2VMb2NhdGlvbj0xMC4wLjAuMC0xMC4yNTUuMjU1LjI1NXxEZXN0aW5hdGlvbkxvY2F0aW9uPTEwLjAuMC4wLTEwLjI1NS4yNTUuMjU1fGRzdFBhY2tldHM9MHxzcmNQYWNrZXRzPTF8U2Vzc2lvbkVuZFJlYXNvbj1uL2F8RGV2aWNlR3JvdXBIaWVyYXJjaHlMMT0wfERldmljZUdyb3VwSGllcmFyY2h5TDI9MHxEZXZpY2VHcm91cEhpZXJhcmNoeUwzPTB8RGV2aWNlR3JvdXBIaWVyYXJjaHlMND0wfHZTcmNOYW1lPXxEZXZpY2VOYW1lPWZpcmV3YWxsfEFjdGlvblNvdXJjZT1mcm9tLXBvbGljeXxTcmNVVUlEPXxEc3RVVUlEPXxUdW5uZWxJRD0wfE1vbml0b3JUYWc9fFBhcmVudFNlc3Npb25JRD0wfFBhcmVudFN0YXJ0VGltZT18VHVubmVsVHlwZT1OL0EK" + } + }, + "first_observed": "2021-08-23T14:02:11.113Z", + "last_observed": "2021-08-23T14:02:11.113Z", + "number_observed": 1 + } + ] +} diff --git a/tests/test_storage.py b/tests/test_storage.py index a313dd6..2b8b954 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -544,3 +544,29 @@ def test_clobber_viewname(fake_bundle_file_2, tmpdir): # conns2 should be no more: with pytest.raises(UnknownViewname): store.lookup('conns2') + + +def test_three_ips(one_event_bundle, tmpdir): + """A single Observation SDO can contain any arbitrary number and type + of SCOs. In the case that one type appears multiple times, + firepit will attempt to mark one as the "primary", or most + significant, instance by setting an "x_root" attribute to 1 (note + that this name may change in the future). + + A common case is `ipv4-addr`: if you have a `network-traffic` + object, then you usally have 2 `ipv4-addr` (or `ipv6-addr`) + objects. In that case, firepit will (arbitrarily) pick the object + referenced as the `src_ref` to be the "primary". + + This test case involves 1 Observation with 3 IP addresses. One is + `src_ref`, one is `dst_ref`, and third is...well, ask the QRadar + people, I guess. + + """ + store = tmp_storage(tmpdir) + store.cache('q1', [one_event_bundle]) + + results = store._query('SELECT value FROM "ipv4-addr" WHERE "x_root" IS NOT NULL') + rows = results.fetchall() + assert len(rows) == 1 # There can be only 1!!! + assert rows[0]['value'] == '10.95.79.130' From 3436eda3b114534cc36a52095116125f3f3e676d Mon Sep 17 00:00:00 2001 From: Paul Coccoli Date: Mon, 23 Aug 2021 16:51:12 -0400 Subject: [PATCH 2/2] =?UTF-8?q?Bump=20version:=201.2.0=20=E2=86=92=201.2.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- firepit/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/firepit/__init__.py b/firepit/__init__.py index 80f9b58..85a9109 100644 --- a/firepit/__init__.py +++ b/firepit/__init__.py @@ -2,7 +2,7 @@ __author__ = """IBM Security""" __email__ = 'pcoccoli@us.ibm.com' -__version__ = '1.2.0' +__version__ = '1.2.1' from importlib import import_module diff --git a/setup.cfg b/setup.cfg index d1ff5f9..7b9ceca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.0 +current_version = 1.2.1 commit = True tag = True diff --git a/setup.py b/setup.py index 4e5bb3a..e5b80e6 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/opencybersecurityalliance/firepit', - version='1.2.0', + version='1.2.1', zip_safe=False, )