From 7fc8fd4c8321a05dc4d9f359222b3d53fc31722e Mon Sep 17 00:00:00 2001 From: wagga40 <6437862+wagga40@users.noreply.github.com> Date: Wed, 19 May 2021 07:22:52 +0200 Subject: [PATCH] Changed the file filters functionality (new args : select & avoid) Updated the readme with details on file filters --- Readme.md | 36 +++++++++++++++++++++++++++++++++++- zircolite.py | 27 +++++++++++++++++---------- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/Readme.md b/Readme.md index 12e266a..7b0b142 100644 --- a/Readme.md +++ b/Readme.md @@ -47,6 +47,40 @@ python3 zircolite.py --evtx ../Logs --ruleset rules/rules_windows_sysmon.json ## Advanced use +### File filters + +Some EVTX files are not used by SIGMA rules but can become quite large (Microsoft-Windows-SystemDataArchiver%4Diagnostic.evtx etc.), if you use `Zircolite` with directory as input argument, all EVTX files will be converted, saved and matched against the SIGMA Rules. + +To speed up the detection process, you may want to use Zircolite on files matching or not matching a specific pattern. For that you can use **filters** provided by the two command line arguments : +- `-s` or `--select` : select files partly matching the provided a string (case insensitive) +- `-a` or `--avoid` : exclude files partly matching the provided a string (case insensitive) + +```shell +# Only use EVTX files that contains "sysmon" in their names +python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json --select sysmon + +# Exclude "Microsoft-Windows-SystemDataArchiver%4Diagnostic.evtx" +python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json --avoid systemdataarchiver + +# Only use EVTX files with "operational" in their names but exclude "defender" related logs +python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json \ +--select operational --avoid defender + +``` + +For example, the **Sysmon** ruleset available in the `rules` directory only use the following channels (names have been shortened) : Sysmon, Security, System, Powershell, Defender, AppLocker, DriverFrameworks, Application, NTLM, DNS, MSexchange, WMI-activity, TaskScheduler. So if you use the sysmon ruleset with the following rules, you should speed up `Zircolite`execution : + +```shell +python3 zircolite.py --evtx logs/ --ruleset rules/rules_windows_sysmon.json \ +--select sysmon --select security.evtx --select system.evtx \ +--select application.evtx --select Windows-NTLM --select DNS \ +--select powershell --select defender --select applocker \ +--select driverframeworks --select "msexchange management" \ +--select TaskScheduler --select WMI-activity +``` + +:information_source: the "select" argument is always applied first and then the "avoid" argument is applied. So, it is possible to exclude files from included files but not the opposite. + ### Templating Zircolite provides a templating system based on Jinja 2. It allows you to change the output format to suits your needs (Splunk or ELK integration, Grep-able output...). To use the template system, use these arguments : @@ -182,7 +216,7 @@ Since the Docker image mirrors Zircolite's repository, all options are also avai ## "Battle-tested" ? Zircolite has been used to perform cold-analysis (in Lab) on EVTX in multiple "real-life" situations. -However, even if Zircolite has been used many times to perform analysis directly on an Microsoft Windows endpoint there is not yet a pipeline to thoroughly test every release. +However, even if Zircolite has been used many times to perform analysis directly on an Microsoft Windows endpoint, there is not yet a pipeline to thoroughly test every release. ## License diff --git a/zircolite.py b/zircolite.py index daf0b37..cd3b8ad 100755 --- a/zircolite.py +++ b/zircolite.py @@ -274,6 +274,16 @@ def sendLogsHTTP(host, payload = ""): logging.debug(f"{Fore.RED} [-] {e}") return False +def selectFiles(pathList, selectFilesList): + if selectFilesList is not None: + return [evtx for evtx in [str(element) for element in list(pathList)] if any(fileFilters[0].lower() in evtx.lower() for fileFilters in selectFilesList)] + return pathList + +def avoidFiles(pathList, avoidFilesList): + if avoidFilesList is not None: + return [evtx for evtx in [str(element) for element in list(pathList)] if all(fileFilters[0].lower() not in evtx.lower() for fileFilters in avoidFilesList)] + return pathList + ################################################################ # MAIN() ################################################################ @@ -291,7 +301,8 @@ def sendLogsHTTP(host, payload = ""): tmpDir = "tmp-" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(8)) parser = argparse.ArgumentParser() parser.add_argument("-e", "--evtx", help="EVTX log file or directory where EVTX log files are stored in JSON or EVTX format", type=str, required=True) - parser.add_argument("--filter", help="Filter on these filenames. Only EVTX files containing the provided string will be used", action='append', nargs='+') + parser.add_argument("-s", "--select", help="Only EVTX files containing the provided string will be used. If there is/are exclusion(s) (--avoid) they will be handled after selection", action='append', nargs='+') + parser.add_argument("-a", "--avoid", help="EVTX files containing the provided string will NOT be used", action='append', nargs='+') parser.add_argument("-r", "--ruleset", help="JSON File containing SIGMA rules", type=str, required=True) parser.add_argument("-c", "--config", help="JSON File containing field mappings and exclusions", type=str, default="config/fieldMappings.json") parser.add_argument("-o", "--outfile", help="JSON file that will contains all detected events", type=str, default="detected_events.json") @@ -366,18 +377,14 @@ def sendLogsHTTP(host, payload = ""): EVTXList = [EVTXPath] else: quitOnError(f"{Fore.RED} [-] Unable to extract EVTX from submitted path") - if len(EVTXList) > 0: - for evtx in tqdm(EVTXList, colour="yellow"): - # If there are file filter(s) : check if current file is in file filters - if (args.filter is not None): - if any(fileFilters[0].lower() in str(evtx).lower() for fileFilters in args.filter): - extractEvtx(evtx, args.tmpdir, evtx_dumpBinary) - else: - extractEvtx(evtx, args.tmpdir, evtx_dumpBinary) + FileList = avoidFiles(selectFiles(EVTXList, args.select), args.avoid) # Apply file filters in ths order "select" than "avoid" + if len(FileList) > 0: + for evtx in tqdm(FileList, colour="yellow"): + extractEvtx(evtx, args.tmpdir, evtx_dumpBinary) # Set the path for the next step EVTXJSONList = list(Path(args.tmpdir).rglob("*.json")) else: - quitOnError(f"{Fore.RED} [-] No EVTX files found. Please verify the directory or the extension with '--fileext'") + quitOnError(f"{Fore.RED} [-] No EVTX files found. Please verify filters, the directory or the extension with '--fileext'") else: EVTXJSONList = list(Path(args.evtx).rglob("*.json"))