Merge pull request #190

Raise the default vast query job limit to 500

CHANGELOG.md

@@ -10,6 +10,13 @@ Every entry has a category for which we use the following visual abbreviations:
 - 🧬 Experimental Features
 - 🐞 Bug Fixes
 
+## [2022.05.16]
+
+- ⚠️ To make use of VAST's new query query scheduler `vast-threatbus` now runs
+  up to 500 queries in parallel by default. It also waits for one hour before
+  aborting a query.
+  [#190](https://github.com/tenzir/threatbus/pull/190)
+
 ## [2022.01.27]
 
 No user-facing changes.

apps/stix-shifter/CHANGELOG.md

@@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations:
 - ⚡️ Breaking Changes
 - 🐞 Bug Fixes
 
+## [2022.05.16]
+
+No user-facing changes.
+
 ## [2022.01.27]
 
 No user-facing changes.

apps/stix-shifter/setup.py

@@ -36,7 +36,7 @@
         "stix2 >= 3.0",
         "stix-shifter >= 3.4.2",
         "stix-shifter-utils >= 3.4.2",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "open source",
@@ -54,5 +54,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

apps/suricata/CHANGELOG.md

@@ -11,6 +11,10 @@ Every entry has a category for which we use the following visual abbreviations:
 - ⚡️ Breaking Changes
 - 🐞 Bug Fixes
 
+## [2022.05.16]
+
+No user-facing changes.
+
 ## [2022.01.27]
 
 No user-facing changes.

apps/suricata/setup.py

@@ -32,7 +32,7 @@
         "pyzmq >= 19",
         "parsuricata",
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "open source",
@@ -53,5 +53,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

apps/vast/CHANGELOG.md

@@ -10,6 +10,13 @@ Every entry has a category for which we use the following visual abbreviations:
 - ⚡️ Breaking Changes
 - 🐞 Bug Fixes
 
+## [2022.05.16]
+
+- ⚠️ To make use of VAST's new query query scheduler `vast-threatbus` now runs
+  up to 500 queries in parallel by default. It also waits for one hour before
+  aborting a query.
+  [#190](https://github.com/tenzir/threatbus/pull/190)
+
 ## [2022.01.27]
 
 No user-facing changes.

apps/vast/setup.py

@@ -33,7 +33,7 @@
         "pyzmq >= 19",
         "pyvast >= 1.0.0",
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "open source",
@@ -52,5 +52,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

apps/vast/vast_threatbus/vast_threatbus.py

@@ -98,8 +98,8 @@ def validate_config(config: Settings):
         Validator("retro_match", is_type_of=bool, default=True),
         Validator("snapshot", is_type_of=int, default=30),
         Validator("retro_match_max_events", is_type_of=int, default=0),
-        Validator("max_background_tasks", is_type_of=int, default=100),
-        Validator("retro_match_timeout", is_type_of=float, default=5.0),
+        Validator("max_background_tasks", is_type_of=int, default=500),
+        Validator("retro_match_timeout", is_type_of=float, default=3600.0),
         Validator("transform_context", "sink", default=None),
     ]
 
@@ -362,6 +362,8 @@ async def retro_match_vast(
         start = time.time()
         vast = VAST(binary=vast_binary, endpoint=vast_endpoint, logger=logger)
         kwargs = {}
+        # Don't allocate unnecessary resources.
+        kwargs["caf.scheduler.max_threads"] = 1
         if low_priority_support:
             kwargs["low_priority"] = True
         if retro_match_max_events > 0:

apps/zmq-app-template/setup.py

@@ -31,7 +31,7 @@
         "dynaconf>=3.1.4,!=3.1.8",
         "pyzmq >= 19",
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "open source",
@@ -49,5 +49,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/apps/threatbus_cif3/setup.py

@@ -27,7 +27,7 @@
     entry_points={"threatbus.app": ["cif3 = threatbus_cif3.plugin"]},
     install_requires=[
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
         "cifsdk > 3.0.0rc4, < 4.0",
     ],
     keywords=[
@@ -49,5 +49,5 @@
     packages=["threatbus_cif3"],
     python_requires=">=3.6",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/apps/threatbus_misp/setup.py

@@ -28,7 +28,7 @@
     install_requires=[
         "pymisp >= 2.4.120",
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     extras_require={"kafka": ["confluent-kafka>=1.3.0"], "zmq": ["pyzmq>=18.1.1"]},
     keywords=[
@@ -49,5 +49,5 @@
     packages=["threatbus_misp"],
     python_requires=">=3.7",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/apps/threatbus_zeek/setup.py

@@ -27,7 +27,7 @@
     entry_points={"threatbus.app": ["zeek = threatbus_zeek.plugin"]},
     install_requires=[
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "Zeek",
@@ -50,5 +50,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/apps/threatbus_zmq/setup.py

@@ -29,7 +29,7 @@
         "pyzmq>=19",
         "python-dateutil>=2.8.1",
         "stix2>=3.0",
-        "threatbus>=2022.1.27",
+        "threatbus>=2022.5.16",
     ],
     keywords=[
         "zeromq",
@@ -47,5 +47,5 @@
     packages=["threatbus_zmq"],
     python_requires=">=3.7",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/backbones/file_benchmark/setup.py

@@ -24,7 +24,7 @@
     entry_points={"threatbus.backbone": ["file_benchmark = file_benchmark.plugin"]},
     install_requires=[
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=["threatbus", "plugin"],
     license="BSD 3-clause",
@@ -34,5 +34,5 @@
     packages=["file_benchmark"],
     python_requires=">=3.7",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/backbones/threatbus_inmem/setup.py

@@ -24,7 +24,7 @@
     entry_points={"threatbus.backbone": ["inmem = threatbus_inmem.plugin"]},
     install_requires=[
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "message broker",
@@ -41,5 +41,5 @@
     packages=["threatbus_inmem"],
     python_requires=">=3.7",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

plugins/backbones/threatbus_rabbitmq/setup.py

@@ -26,7 +26,7 @@
         "pika >= 1.1.0",
         "retry",
         "stix2 >= 3.0",
-        "threatbus >= 2022.1.27",
+        "threatbus >= 2022.5.16",
     ],
     keywords=[
         "message broker",
@@ -46,5 +46,5 @@
     packages=["threatbus_rabbitmq"],
     python_requires=">=3.7",
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )

setup.py

@@ -49,5 +49,5 @@
     python_requires=">=3.7",
     setup_requires=["setuptools", "wheel"],
     url="https://github.com/tenzir/threatbus",
-    version="2022.1.27",
+    version="2022.5.16",
 )