Changed the file filters functionality (new args : select & avoid)

Updated the readme with details on file filters

Readme.md

@@ -47,6 +47,40 @@ python3 zircolite.py --evtx ../Logs --ruleset rules/rules_windows_sysmon.json
 
 ## Advanced use
 
+### File filters
+
+Some EVTX files are not used by SIGMA rules but can become quite large (Microsoft-Windows-SystemDataArchiver%4Diagnostic.evtx etc.), if you use `Zircolite` with directory as input argument, all EVTX files will be converted, saved and matched against the SIGMA Rules. 
+
+To speed up the detection process, you may want to use Zircolite on files matching or not matching a specific pattern. For that you can use **filters** provided by the two command line arguments :
+- `-s` or `--select` : select files partly matching the provided a string (case insensitive)
+- `-a` or `--avoid` : exclude files partly matching the provided a string (case insensitive)
+
+```shell
+# Only use EVTX files that contains "sysmon" in their names
+python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json --select sysmon
+
+# Exclude "Microsoft-Windows-SystemDataArchiver%4Diagnostic.evtx" 
+python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json --avoid systemdataarchiver
+
+# Only use EVTX files with "operational" in their names but exclude "defender" related logs
+python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json \
+--select operational --avoid defender
+
+```
+
+For example, the **Sysmon** ruleset available in the `rules` directory only use the following channels (names have been shortened) : Sysmon, Security, System, Powershell, Defender, AppLocker, DriverFrameworks, Application, NTLM, DNS, MSexchange, WMI-activity, TaskScheduler. So if you use the sysmon ruleset with the following rules, you should speed up `Zircolite`execution : 
+
+```shell
+python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json \
+--select sysmon --select security.evtx --select system.evtx \
+--select application.evtx --select Windows-NTLM --select DNS \
+--select powershell --select defender --select applocker \
+--select driverframeworks --select "msexchange management" \
+--select TaskScheduler --select WMI-activity
+```
+
+:information_source: the "select" argument is always applied first and then the "avoid" argument is applied. So, it is possible to exclude files from included files but not the opposite.
+
 ### Templating
 
 Zircolite provides a templating system based on Jinja 2. It allows you to change the output format to suits your needs (Splunk or ELK integration, Grep-able output...). To use the template system, use these arguments :
@@ -182,7 +216,7 @@ Since the Docker image mirrors Zircolite's repository, all options are also avai
 ## "Battle-tested" ?
 
 Zircolite has been used to perform cold-analysis (in Lab) on EVTX in multiple "real-life" situations. 
-However, even if Zircolite has been used many times to perform analysis directly on an Microsoft Windows endpoint there is not yet a pipeline to thoroughly test every release.
+However, even if Zircolite has been used many times to perform analysis directly on an Microsoft Windows endpoint, there is not yet a pipeline to thoroughly test every release.
 
 ## License
 

zircolite.py

@@ -274,6 +274,16 @@ def sendLogsHTTP(host, payload = ""):
         logging.debug(f"{Fore.RED}   [-] {e}")
         return False
 
+def selectFiles(pathList, selectFilesList):
+    if selectFilesList is not None:
+        return [evtx for evtx in [str(element) for element in list(pathList)] if any(fileFilters[0].lower() in evtx.lower() for fileFilters in selectFilesList)]
+    return pathList
+
+def avoidFiles(pathList, avoidFilesList):
+    if avoidFilesList is not None:
+        return [evtx for evtx in [str(element) for element in list(pathList)] if all(fileFilters[0].lower() not in evtx.lower() for fileFilters in avoidFilesList)]
+    return pathList
+
 ################################################################
 # MAIN()
 ################################################################
@@ -291,7 +301,8 @@ def sendLogsHTTP(host, payload = ""):
     tmpDir = "tmp-" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(8))
     parser = argparse.ArgumentParser()
     parser.add_argument("-e", "--evtx", help="EVTX log file or directory where EVTX log files are stored in JSON or EVTX format", type=str, required=True)
-    parser.add_argument("--filter", help="Filter on these filenames. Only EVTX files containing the provided string will be used", action='append', nargs='+')
+    parser.add_argument("-s", "--select", help="Only EVTX files containing the provided string will be used. If there is/are exclusion(s) (--avoid) they will be handled after selection", action='append', nargs='+')
+    parser.add_argument("-a", "--avoid", help="EVTX files containing the provided string will NOT be used", action='append', nargs='+')
     parser.add_argument("-r", "--ruleset", help="JSON File containing SIGMA rules", type=str, required=True)
     parser.add_argument("-c", "--config", help="JSON File containing field mappings and exclusions", type=str, default="config/fieldMappings.json")
     parser.add_argument("-o", "--outfile", help="JSON file that will contains all detected events", type=str, default="detected_events.json")
@@ -366,18 +377,14 @@ def sendLogsHTTP(host, payload = ""):
             EVTXList = [EVTXPath]
         else:
             quitOnError(f"{Fore.RED}   [-] Unable to extract EVTX from submitted path")
-        if len(EVTXList) > 0:
-            for evtx in tqdm(EVTXList, colour="yellow"):
-                # If there are file filter(s) : check if current file is in file filters
-                if (args.filter is not None): 
-                    if any(fileFilters[0].lower() in str(evtx).lower() for fileFilters in args.filter): 
-                        extractEvtx(evtx, args.tmpdir, evtx_dumpBinary)
-                else:
-                    extractEvtx(evtx, args.tmpdir, evtx_dumpBinary)
+        FileList = avoidFiles(selectFiles(EVTXList, args.select), args.avoid) # Apply file filters in ths order "select" than "avoid"
+        if len(FileList) > 0:
+            for evtx in tqdm(FileList, colour="yellow"):
+                extractEvtx(evtx, args.tmpdir, evtx_dumpBinary)
             # Set the path for the next step
             EVTXJSONList = list(Path(args.tmpdir).rglob("*.json"))
         else:
-            quitOnError(f"{Fore.RED}   [-] No EVTX files found. Please verify the directory or the extension with '--fileext'")
+            quitOnError(f"{Fore.RED}   [-] No EVTX files found. Please verify filters, the directory or the extension with '--fileext'")
     else:
         EVTXJSONList = list(Path(args.evtx).rglob("*.json"))