From 1b7840afd0023ecef2c492a4132409f03f5f600c Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Wed, 27 Apr 2022 11:27:30 +0200 Subject: [PATCH 1/6] Raise the default vast query job limit to 500 We also manually force the threadpool worker count to 1 to avoid hitting ulimit or cgroup limits. --- apps/vast/vast_threatbus/vast_threatbus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/vast/vast_threatbus/vast_threatbus.py b/apps/vast/vast_threatbus/vast_threatbus.py index 86eeb4d0..f7624c76 100755 --- a/apps/vast/vast_threatbus/vast_threatbus.py +++ b/apps/vast/vast_threatbus/vast_threatbus.py @@ -98,7 +98,7 @@ def validate_config(config: Settings): Validator("retro_match", is_type_of=bool, default=True), Validator("snapshot", is_type_of=int, default=30), Validator("retro_match_max_events", is_type_of=int, default=0), - Validator("max_background_tasks", is_type_of=int, default=100), + Validator("max_background_tasks", is_type_of=int, default=500), Validator("retro_match_timeout", is_type_of=float, default=5.0), Validator("transform_context", "sink", default=None), ] @@ -362,6 +362,8 @@ async def retro_match_vast( start = time.time() vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger) kwargs = {} + # Don't allocate unnecessary resources. + kwargs["caf.scheduler.max_threads"] = 1 if low_priority_support: kwargs["low_priority"] = True if retro_match_max_events > 0: From 55ba5da51464dc3ad33fb9c79f6d20895e4ee01a Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 16 May 2022 11:44:54 +0200 Subject: [PATCH 2/6] Increase the default retro-match timeout to one hour --- apps/vast/vast_threatbus/vast_threatbus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/vast/vast_threatbus/vast_threatbus.py b/apps/vast/vast_threatbus/vast_threatbus.py index f7624c76..0daef969 100755 --- a/apps/vast/vast_threatbus/vast_threatbus.py +++ b/apps/vast/vast_threatbus/vast_threatbus.py @@ -99,7 +99,7 @@ def validate_config(config: Settings): Validator("snapshot", is_type_of=int, default=30), Validator("retro_match_max_events", is_type_of=int, default=0), Validator("max_background_tasks", is_type_of=int, default=500), - Validator("retro_match_timeout", is_type_of=float, default=5.0), + Validator("retro_match_timeout", is_type_of=float, default=3600.0), Validator("transform_context", "sink", default=None), ] From c6472e3f0e449e0c34cbcc4acd19114e5b9d6762 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 16 May 2022 11:55:12 +0200 Subject: [PATCH 3/6] Add a changelog entry for new defaults --- apps/vast/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/vast/CHANGELOG.md b/apps/vast/CHANGELOG.md index 4d9a93be..3880c035 100644 --- a/apps/vast/CHANGELOG.md +++ b/apps/vast/CHANGELOG.md @@ -10,6 +10,12 @@ Every entry has a category for which we use the following visual abbreviations: - ⚡️ Breaking Changes - 🐞 Bug Fixes +## Unreleased + +- ⚠️ To make use of VASTs new query query scheduler `vast-threatbus` now runs up + to 500 queries in parallel by default. It also waits for one hour before + aborting a query. + ## [2022.01.27] No user-facing changes. From 6ee1ae33fbeefd580b491a59c4bd8f31220afcf3 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 16 May 2022 12:02:18 +0200 Subject: [PATCH 4/6] Prepare the changelog for v2022.05.16 --- CHANGELOG.md | 7 +++++++ apps/stix-shifter/CHANGELOG.md | 4 ++++ apps/suricata/CHANGELOG.md | 4 ++++ apps/vast/CHANGELOG.md | 3 ++- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83aacc7c..e831d039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ Every entry has a category for which we use the following visual abbreviations: - 🧬 Experimental Features - 🐞 Bug Fixes +## [2022.05.16] + +- ⚠️ To make use of VASTs new query query scheduler `vast-threatbus` now runs up + to 500 queries in parallel by default. It also waits for one hour before + aborting a query. + [#190](https://github.com/tenzir/threatbus/pull/190) + ## [2022.01.27] No user-facing changes. diff --git a/apps/stix-shifter/CHANGELOG.md b/apps/stix-shifter/CHANGELOG.md index 803a218e..7cad8d9a 100644 --- a/apps/stix-shifter/CHANGELOG.md +++ b/apps/stix-shifter/CHANGELOG.md @@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations: - ⚡️ Breaking Changes - 🐞 Bug Fixes +## [2022.05.16] + +No user-facing changes. + ## [2022.01.27] No user-facing changes. diff --git a/apps/suricata/CHANGELOG.md b/apps/suricata/CHANGELOG.md index effd4c86..ebd3d400 100644 --- a/apps/suricata/CHANGELOG.md +++ b/apps/suricata/CHANGELOG.md @@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations: - ⚡️ Breaking Changes - 🐞 Bug Fixes +## [2022.05.16] + +No user-facing changes. + ## [2022.01.27] No user-facing changes. diff --git a/apps/vast/CHANGELOG.md b/apps/vast/CHANGELOG.md index 3880c035..6e5e833a 100644 --- a/apps/vast/CHANGELOG.md +++ b/apps/vast/CHANGELOG.md @@ -10,11 +10,12 @@ Every entry has a category for which we use the following visual abbreviations: - ⚡️ Breaking Changes - 🐞 Bug Fixes -## Unreleased +## [2022.05.16] - ⚠️ To make use of VASTs new query query scheduler `vast-threatbus` now runs up to 500 queries in parallel by default. It also waits for one hour before aborting a query. + [#190](https://github.com/tenzir/threatbus/pull/190) ## [2022.01.27] From 460aafbfaa49fd4409b15e6932f0e6ff3b3b04f0 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 16 May 2022 12:09:18 +0200 Subject: [PATCH 5/6] Fix grammar --- CHANGELOG.md | 4 ++-- apps/vast/CHANGELOG.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e831d039..b46c7790 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ Every entry has a category for which we use the following visual abbreviations: ## [2022.05.16] -- ⚠️ To make use of VASTs new query query scheduler `vast-threatbus` now runs up - to 500 queries in parallel by default. It also waits for one hour before +- ⚠️ To make use of VAST's new query query scheduler `vast-threatbus` now runs + up to 500 queries in parallel by default. It also waits for one hour before aborting a query. [#190](https://github.com/tenzir/threatbus/pull/190) diff --git a/apps/vast/CHANGELOG.md b/apps/vast/CHANGELOG.md index 6e5e833a..896e581a 100644 --- a/apps/vast/CHANGELOG.md +++ b/apps/vast/CHANGELOG.md @@ -12,8 +12,8 @@ Every entry has a category for which we use the following visual abbreviations: ## [2022.05.16] -- ⚠️ To make use of VASTs new query query scheduler `vast-threatbus` now runs up - to 500 queries in parallel by default. It also waits for one hour before +- ⚠️ To make use of VAST's new query query scheduler `vast-threatbus` now runs + up to 500 queries in parallel by default. It also waits for one hour before aborting a query. [#190](https://github.com/tenzir/threatbus/pull/190) From feb9408d1b86328360b6c675368c76e9a04ae410 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Mon, 16 May 2022 12:12:32 +0200 Subject: [PATCH 6/6] Update version to 2022.5.16 --- apps/stix-shifter/setup.py | 4 ++-- apps/suricata/setup.py | 4 ++-- apps/vast/setup.py | 4 ++-- apps/zmq-app-template/setup.py | 4 ++-- plugins/apps/threatbus_cif3/setup.py | 4 ++-- plugins/apps/threatbus_misp/setup.py | 4 ++-- plugins/apps/threatbus_zeek/setup.py | 4 ++-- plugins/apps/threatbus_zmq/setup.py | 4 ++-- plugins/backbones/file_benchmark/setup.py | 4 ++-- plugins/backbones/threatbus_inmem/setup.py | 4 ++-- plugins/backbones/threatbus_rabbitmq/setup.py | 4 ++-- setup.py | 2 +- 12 files changed, 23 insertions(+), 23 deletions(-) diff --git a/apps/stix-shifter/setup.py b/apps/stix-shifter/setup.py index ce648893..7b2f66ae 100644 --- a/apps/stix-shifter/setup.py +++ b/apps/stix-shifter/setup.py @@ -36,7 +36,7 @@ "stix2 >= 3.0", "stix-shifter >= 3.4.2", "stix-shifter-utils >= 3.4.2", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "open source", @@ -54,5 +54,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/apps/suricata/setup.py b/apps/suricata/setup.py index 4f0f171e..3d138233 100644 --- a/apps/suricata/setup.py +++ b/apps/suricata/setup.py @@ -32,7 +32,7 @@ "pyzmq >= 19", "parsuricata", "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "open source", @@ -53,5 +53,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/apps/vast/setup.py b/apps/vast/setup.py index c62aaa40..b0713897 100644 --- a/apps/vast/setup.py +++ b/apps/vast/setup.py @@ -33,7 +33,7 @@ "pyzmq >= 19", "pyvast >= 1.0.0", "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "open source", @@ -52,5 +52,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/apps/zmq-app-template/setup.py b/apps/zmq-app-template/setup.py index 21a54efd..4251ccd8 100644 --- a/apps/zmq-app-template/setup.py +++ b/apps/zmq-app-template/setup.py @@ -31,7 +31,7 @@ "dynaconf>=3.1.4,!=3.1.8", "pyzmq >= 19", "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "open source", @@ -49,5 +49,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/apps/threatbus_cif3/setup.py b/plugins/apps/threatbus_cif3/setup.py index 25c5fff1..bd98f79e 100644 --- a/plugins/apps/threatbus_cif3/setup.py +++ b/plugins/apps/threatbus_cif3/setup.py @@ -27,7 +27,7 @@ entry_points={"threatbus.app": ["cif3 = threatbus_cif3.plugin"]}, install_requires=[ "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", "cifsdk > 3.0.0rc4, < 4.0", ], keywords=[ @@ -49,5 +49,5 @@ packages=["threatbus_cif3"], python_requires=">=3.6", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/apps/threatbus_misp/setup.py b/plugins/apps/threatbus_misp/setup.py index fc05c5e5..d2c6d553 100644 --- a/plugins/apps/threatbus_misp/setup.py +++ b/plugins/apps/threatbus_misp/setup.py @@ -28,7 +28,7 @@ install_requires=[ "pymisp >= 2.4.120", "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], extras_require={"kafka": ["confluent-kafka>=1.3.0"], "zmq": ["pyzmq>=18.1.1"]}, keywords=[ @@ -49,5 +49,5 @@ packages=["threatbus_misp"], python_requires=">=3.7", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/apps/threatbus_zeek/setup.py b/plugins/apps/threatbus_zeek/setup.py index 555f76c6..52787a32 100644 --- a/plugins/apps/threatbus_zeek/setup.py +++ b/plugins/apps/threatbus_zeek/setup.py @@ -27,7 +27,7 @@ entry_points={"threatbus.app": ["zeek = threatbus_zeek.plugin"]}, install_requires=[ "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "Zeek", @@ -50,5 +50,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/apps/threatbus_zmq/setup.py b/plugins/apps/threatbus_zmq/setup.py index 66f1c8a3..938c0417 100644 --- a/plugins/apps/threatbus_zmq/setup.py +++ b/plugins/apps/threatbus_zmq/setup.py @@ -29,7 +29,7 @@ "pyzmq>=19", "python-dateutil>=2.8.1", "stix2>=3.0", - "threatbus>=2022.1.27", + "threatbus>=2022.5.16", ], keywords=[ "zeromq", @@ -47,5 +47,5 @@ packages=["threatbus_zmq"], python_requires=">=3.7", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/backbones/file_benchmark/setup.py b/plugins/backbones/file_benchmark/setup.py index c44a26e5..2ea28042 100644 --- a/plugins/backbones/file_benchmark/setup.py +++ b/plugins/backbones/file_benchmark/setup.py @@ -24,7 +24,7 @@ entry_points={"threatbus.backbone": ["file_benchmark = file_benchmark.plugin"]}, install_requires=[ "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=["threatbus", "plugin"], license="BSD 3-clause", @@ -34,5 +34,5 @@ packages=["file_benchmark"], python_requires=">=3.7", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/backbones/threatbus_inmem/setup.py b/plugins/backbones/threatbus_inmem/setup.py index 0feaac85..e18f057e 100644 --- a/plugins/backbones/threatbus_inmem/setup.py +++ b/plugins/backbones/threatbus_inmem/setup.py @@ -24,7 +24,7 @@ entry_points={"threatbus.backbone": ["inmem = threatbus_inmem.plugin"]}, install_requires=[ "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "message broker", @@ -41,5 +41,5 @@ packages=["threatbus_inmem"], python_requires=">=3.7", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/plugins/backbones/threatbus_rabbitmq/setup.py b/plugins/backbones/threatbus_rabbitmq/setup.py index b51dac2d..d6d84224 100644 --- a/plugins/backbones/threatbus_rabbitmq/setup.py +++ b/plugins/backbones/threatbus_rabbitmq/setup.py @@ -26,7 +26,7 @@ "pika >= 1.1.0", "retry", "stix2 >= 3.0", - "threatbus >= 2022.1.27", + "threatbus >= 2022.5.16", ], keywords=[ "message broker", @@ -46,5 +46,5 @@ packages=["threatbus_rabbitmq"], python_requires=">=3.7", url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", ) diff --git a/setup.py b/setup.py index de9e3ad3..64e1acf5 100644 --- a/setup.py +++ b/setup.py @@ -49,5 +49,5 @@ python_requires=">=3.7", setup_requires=["setuptools", "wheel"], url="https://github.com/tenzir/threatbus", - version="2022.1.27", + version="2022.5.16", )