diff --git a/docs/Advanced.md b/docs/Advanced.md index 7972b48..884b4b9 100644 --- a/docs/Advanced.md +++ b/docs/Advanced.md @@ -266,4 +266,80 @@ Then you just have to open `index.html` in your favorite browser and click on a * After Python 3.8 install, you will need Nuitka : `pip3 install nuitka` * In the root folder of Zircolite type : `python3 -m nuitka --onefile zircolite.py` -:warning: When packaging with PyInstaller some AV may not like your package. \ No newline at end of file +:warning: When packaging with PyInstaller some AV may not like your package. + +--- + +### Using With DFIR Orc + +**DFIR Orc** is a Forensics artefact collection tool for systems running Microsoft Windows (pretty much like [Kape](https://www.kroll.com/en/services/cyber-risk/incident-response-litigation-support/kroll-artifact-parser-extractor-kape) or [artifactcollector](https://forensicanalysis.github.io/documentation/manual/usage/acquisition/)). For more detailed explaination, please check here : [https://dfir-orc.github.io](https://dfir-orc.github.io). + +[ZikyHD](https://github.com/ZikyHD) has done a pretty good job at integrating **Zircolite** with **DFIR Orc** in this repository : [https://github.com/Zircocorp/dfir-orc-config](https://github.com/Zircocorp/dfir-orc-config). + +Basically, if you want to integrate Zircolite with **DFIR Orc** : + +- Clone the [DFIR Orc Config repository](https://github.com/DFIR-ORC/dfir-orc-config) : `git clone https://github.com/Zircocorp/dfir-orc-config.git` +- Create a `DFIR-ORC_config.xml` (or add to an existing one) in the `config` directory containing : + +```xml + + + DFIR-ORC_{SystemType}_{FullComputerName}_{TimeStamp}.log + DFIR-ORC_{SystemType}_{FullComputerName}_{TimeStamp}.json + + + + + + + + + + --cores 1 --noexternal -e C:\windows\System32\winevt\Logs + + + + + + + +``` + +:information_source: Please note that if you add this configuration to an existing one, you only need to keep the part between `` and `` blocks. + +- Put your custom or default mapping file `zircolite_win10_nuitka.exe ` (the default one is in the Zircolite repository `config` directory) `rules_windows_generic.json` (the default one is in the Zircolite repository `rules` directory) in the the `config` directory. + +- Put **Zircolite** [binary](https://github.com/wagga40/Zircolite/releases) (in this example `zircolite_win10_nuitka.exe`) and **DFIR Orc** [binaries](https://github.com/DFIR-ORC/dfir-orc/releases) (x86 and x64) in the the `tools` directory. + +- Create a `DFIR-ORC_Embed.xml` (or add to an existing one) in the `config` directory containing : + +```xml + + + .\tools\DFIR-Orc_x86.exe + .\output\%ORC_OUTPUT% + + 7z:#Tools|DFIR-Orc_x64.exe + self:# + + + + + + + + + + + + + + + + + +``` +:information_source: Please note that if you add this configuration to an existing one, you only need to keep the part between `` and `` blocks. + +- Now you need to generate the **DFIR Orc** binary by executing `.\configure.ps1` at the root of the repository +- The final output will be in the `output` directory \ No newline at end of file diff --git a/docs/Readme.md b/docs/Readme.md index a897bab..ae953d6 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -24,6 +24,7 @@ * [Templating and Formatting](Advanced.md#templating-and-formatting) * [Mini GUI](Advanced.md#mini-gui) * [Packaging Zircolite](Advanced.md#packaging-zircolite) +* [Using With DFIR Orc](Advanced.md#using-with-dfir-orc) ## Internals diff --git a/docs/Usage.md b/docs/Usage.md index 391892c..795deca 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -71,6 +71,16 @@ python3 zircolite.py --evtx --ruleset --db If you need to re-execute Zircolite, you can do it directly using the SQLite database as the EVTX source (with `--evtx ` and `--dbonly`) and avoid to convert the EVTX, post-process the EVTX and insert data to database. **Using this technique can save a lot of time...** +#### Sysmon for Linux XML log files + +Sysmon for linux has been released in October 2021. It outputs XML in text format with one event per-line. As of version 2.6.0, **Zircolite** has an *initial* support of Sysmon for Linux log files. To test it, just add `-S` to you command line : + +```shell +python3 zircolite.py --evtx --ruleset -S +``` + +:information_source: Since the logs come from Linux, the default file extension when using `-S` case is `.log` + --- ### Generate your own rulesets diff --git a/requirements.txt b/requirements.txt index ebdf439..dd20ea7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ tqdm>=4.58.0 requests>=2.24.0 evtx>=0.7 aiohttp[speedups]~=3.7 +lxml~=4.6 diff --git a/tools/genEmbed/genEmbed.py b/tools/genEmbed/genEmbed.py index 1528680..c5755b9 100644 --- a/tools/genEmbed/genEmbed.py +++ b/tools/genEmbed/genEmbed.py @@ -126,6 +126,7 @@ def render(self): rulesIf=self.rulesIf, rulesCheck=self.rulesCheck, noPackage = "args.package = False", + binPathVar = "binPath = None", executeRuleSetFromVar='zircoliteCore.loadRulesetFromVar(ruleset=ruleset, ruleFilters=args.rulefilter)', fieldMappingsLines=self.fieldMappingsLines )) diff --git a/zircolite.py b/zircolite.py index 02e2800..6da5431 100755 --- a/zircolite.py +++ b/zircolite.py @@ -2,36 +2,38 @@ # -*- coding: utf-8 -*- # Standard libs +import argparse +import base64 +import csv import json -import sqlite3 import logging -from sqlite3 import Error +import multiprocessing as mp import os +from pathlib import Path +import random +import shutil +import signal import socket +import sqlite3 +from sqlite3 import Error +import string import subprocess -import argparse -import sys import time -import random -import string -import signal -import base64 -from pathlib import Path -import shutil +import sys from sys import platform as _platform import zlib -import csv # External libs +import aiohttp +import asyncio +from colorama import Fore +from evtx import PyEvtxParser +from lxml import etree +import socket from tqdm import tqdm import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -from colorama import Fore from jinja2 import Template -from evtx import PyEvtxParser -import aiohttp -import asyncio -import socket def signal_handler(sig, frame): consoleLogger.info("[-] Execution interrupted !") @@ -228,7 +230,7 @@ def flatten(x, name=''): JSONLine[key] = value # Creating the CREATE TABLE SQL statement if key.lower() not in self.keyDict: - self.keyDict[key.lower()] = "" + self.keyDict[key.lower()] = key if type(value) is int: fieldStmt += f"'{key}' INTEGER,\n" else: @@ -449,20 +451,23 @@ def executeRuleset(self, outFile, writeMode='w', forwarder=None, showAll=False, self.logger.error(f"{Fore.RED} [-] Error saving some results : {e}") if not self.noOutput and not self.csvMode: fileHandle.write('{}]') - def run(self, EVTXJSONList): + def run(self, EVTXJSONList, Insert2Db=True): self.logger.info("[+] Processing EVTX") flattener = JSONFlattener(configFile=self.config, timeAfter=self.timeAfter, timeBefore=self.timeBefore) flattener.runAll(EVTXJSONList) - self.logger.info("[+] Creating model") - self.createDb(flattener.fieldStmt) - self.logger.info("[+] Inserting data") - self.insertFlattenedJSON2Db(flattener.valuesStmt) - self.logger.info("[+] Cleaning unused objects") + if Insert2Db: + self.logger.info("[+] Creating model") + self.createDb(flattener.fieldStmt) + self.logger.info("[+] Inserting data") + self.insertFlattenedJSON2Db(flattener.valuesStmt) + self.logger.info("[+] Cleaning unused objects") + else: + return flattener.keyDict del flattener class evtxExtractor: - def __init__(self, logger=None, providedTmpDir=None, coreCount=None, useExternalBinaries=True): + def __init__(self, logger=None, providedTmpDir=None, coreCount=None, useExternalBinaries=True, binPath = None, xmlLogs=False): self.logger = logger or logging.getLogger(__name__) if Path(str(providedTmpDir)).is_dir(): self.tmpDir = f"tmp-{self.randString()}" @@ -472,8 +477,9 @@ def __init__(self, logger=None, providedTmpDir=None, coreCount=None, useExternal os.mkdir(self.tmpDir) self.cores = coreCount or os.cpu_count() self.useExternalBinaries = useExternalBinaries + self.xmlLogs = xmlLogs #{% if not embeddedMode %} - self.evtxDumpCmd = self.getOSExternalTools() + self.evtxDumpCmd = self.getOSExternalTools(binPath) #{% else %} #{{ evtxDumpCmdEmbed }} #{% endif %} @@ -493,14 +499,17 @@ def getOSExternalToolsEmbed(self): self.makeExecutable("{{ externalTool }}") return "{{ externalTool }}" #{% else %} - def getOSExternalTools(self): + def getOSExternalTools(self, binPath): """ Determine which binaries to run depending on host OS : 32Bits is NOT supported for now since evtx_dump is 64bits only""" - if _platform == "linux" or _platform == "linux2": - return "bin/evtx_dump_lin" - elif _platform == "darwin": - return "bin/evtx_dump_mac" - elif _platform == "win32": - return "bin\\evtx_dump_win.exe" + if binPath is None: + if _platform == "linux" or _platform == "linux2": + return "bin/evtx_dump_lin" + elif _platform == "darwin": + return "bin/evtx_dump_mac" + elif _platform == "win32": + return "bin\\evtx_dump_win.exe" + else: + return binPath #{% endif %} def runUsingBindings(self, file): @@ -512,12 +521,66 @@ def runUsingBindings(self, file): filepath = Path(file) filename = filepath.name parser = PyEvtxParser(str(filepath)) - with open(f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "w") as f: + with open(f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "w", encoding="utf-8") as f: for record in parser.records_json(): f.write(f'{json.dumps(json.loads(record["data"]))}\n') except Exception as e: self.logger.error(f"{Fore.RED} [-] {e}") + def SysmonXMLLine2JSON(self, xmlLine): + """ + Remove syslog header and convert xml data to json : code from ZikyHD (https://github.com/ZikyHD) + """ + def cleanTag(tag, ns): + if ns in tag: + return tag[len(ns):] + return tag + + if not 'Event' in xmlLine: + return None + xmlLine = "" + xmlLine.split("")[1] + root = etree.fromstring(xmlLine) + ns = u'http://schemas.microsoft.com/win/2004/08/events/event' + child = {"#attributes": {"xmlns": ns}} + for appt in root.getchildren(): + nodename = cleanTag(appt.tag,ns) + nodevalue = {} + for elem in appt.getchildren(): + if not elem.text: + text = "" + else: + try: + text = int(elem.text) + except: + text = elem.text + if elem.tag == 'Data': + childnode = elem.get("Name") + else: + childnode = cleanTag(elem.tag,ns) + if elem.attrib: + text = {"#attributes": dict(elem.attrib)} + obj={childnode:text} + nodevalue = {**nodevalue, **obj} + node = {nodename: nodevalue} + child = {**child, **node} + event = { "Event": child } + return event + + def SysmonXMLLogs2JSON(self, file, outfile): + """ + Use multiprocessing to convert Sysmon for Linux XML logs to JSON + """ + with open(file, "r", encoding="ISO-8859-1") as fp: + data = fp.readlines() + pool = mp.Pool(self.cores) + result = pool.map(self.SysmonXMLLine2JSON, data) + pool.close() + pool.join() + with open(outfile, "w", encoding="UTF-8") as fp: + for element in result: + if element is not None: + fp.write(json.dumps(element) + '\n') + def run(self, file): """ Convert EVTX to JSON using evtx_dump : https://github.com/omerbenamram/evtx. @@ -525,18 +588,25 @@ def run(self, file): """ self.logger.debug(f"EXTRACTING : {file}") - if not self.useExternalBinaries or not Path(self.evtxDumpCmd).is_file(): - self.logger.debug(f"No external binaries args or evtx_dump is missing") - self.runUsingBindings(file) - else: + if self.xmlLogs: try: - filepath = Path(file) - filename = filepath.name - cmd = [self.evtxDumpCmd, "--no-confirm-overwrite", "-o", "jsonl", str(file), "-f", f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "-t", str(self.cores)] - subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + filename = Path(file).name + self.SysmonXMLLogs2JSON(str(file), f"{self.tmpDir}/{str(filename)}-{self.randString()}.json") except Exception as e: self.logger.error(f"{Fore.RED} [-] {e}") - + else: + if not self.useExternalBinaries or not Path(self.evtxDumpCmd).is_file(): + self.logger.debug(f" [-] No external binaries args or evtx_dump is missing") + self.runUsingBindings(file) + else: + try: + filepath = Path(file) + filename = filepath.name + cmd = [self.evtxDumpCmd, "--no-confirm-overwrite", "-o", "jsonl", str(file), "-f", f"{self.tmpDir}/{str(filename)}-{self.randString()}.json", "-t", str(self.cores)] + subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + except Exception as e: + self.logger.error(f"{Fore.RED} [-] {e}") + def cleanup(self): shutil.rmtree(self.tmpDir) #{% if embeddedMode %} @@ -628,9 +698,11 @@ def avoidFiles(pathList, avoidFilesList): parser.add_argument("-a", "--avoid", help="EVTX files containing the provided string will NOT be used", action='append', nargs='+') #{% if not embeddedMode %} parser.add_argument("-r", "--ruleset", help="JSON File containing SIGMA rules", type=str, required=True) + parser.add_argument("--fieldlist", help="Get all EVTX fields", action='store_true') parser.add_argument("-sg", "--sigma", help="Tell Zircolite to directly use SIGMA rules (slower) instead of the converted ones, you must provide SIGMA config file path", type=str) parser.add_argument("-sc", "--sigmac", help="Sigmac path (version >= 0.20), this arguments is mandatary only if you use '--sigma'", type=str) parser.add_argument("-se", "--sigmaerrors", help="Show rules conversion error (i.e not supported by the SIGMA SQLite backend)", action='store_true') + parser.add_argument("--evtx_dump", help="Tell Zircolite to use this binary for EVTX conversion, on Linux and MacOS the path must launch the binary (eg. './evtx_dump' and not 'evtx_dump')", type=str, default=None) #{% else %} #{% for rule in rules %} #{{ rule -}} @@ -649,6 +721,7 @@ def avoidFiles(pathList, avoidFilesList): parser.add_argument("-n", "--nolog", help="Don't create a log file or a result file (useful when forwarding)", action='store_true') parser.add_argument("-j", "--jsononly", help="If logs files are already in JSON lines format ('jsonl' in evtx_dump) ", action='store_true') parser.add_argument("-D", "--dbonly", help="Directly use a previously saved database file, timerange filters will not work", action='store_true') + parser.add_argument("-S", "--sysmon4linux", help="Use this option if your log file is a Sysmon for linux log file, default file extension is '.log'", action='store_true') parser.add_argument("-A", "--after", help="Limit to events that happened after the provided timestamp (UTC). Format : 1970-01-01T00:00:00", type=str, default="1970-01-01T00:00:00") parser.add_argument("-B", "--before", help="Limit to events that happened before the provided timestamp (UTC). Format : 1970-01-01T00:00:00", type=str, default="9999-12-12T23:59:59") parser.add_argument("--remote", help="Forward results to a HTTP/Splunk, please provide the full address e.g [http://]address:port[/uri]", type=str) @@ -716,7 +789,9 @@ def avoidFiles(pathList, avoidFilesList): #{% if embeddedMode %} readyForTemplating = True + #{{ binPathVar }} #{% else %} + binPath = args.evtx_dump # Check Sigma config file & Sigmac path if args.sigma and args.sigmac : checkIfExists(args.sigma, f"{Fore.RED} [-] Cannot find SIGMA config file : {args.sigma}") @@ -750,11 +825,10 @@ def avoidFiles(pathList, avoidFilesList): # If we are not working directly with the db if not args.dbonly: - # Init EVTX extractor object - extractor = evtxExtractor(logger=consoleLogger, providedTmpDir=args.tmpdir, coreCount=args.cores, useExternalBinaries=(not args.noexternal)) # If we are working with json we change the file extension if it is not user-provided if args.jsononly and args.fileext == "evtx": args.fileext = "json" - if not args.jsononly: consoleLogger.info(f"[+] Extracting EVTX Using '{extractor.tmpDir}' directory ") + if args.sysmon4linux and args.fileext == "evtx": args.fileext = "log" + EVTXPath = Path(args.evtx) if EVTXPath.is_dir(): # EVTX recursive search in given directory with given file extension @@ -762,7 +836,7 @@ def avoidFiles(pathList, avoidFilesList): elif EVTXPath.is_file(): EVTXList = [EVTXPath] else: - quitOnError(f"{Fore.RED} [-] Unable to extract EVTX from submitted path") + quitOnError(f"{Fore.RED} [-] Unable to find EVTX from submitted path") # Applying file filters in this order : "select" than "avoid" FileList = avoidFiles(selectFiles(EVTXList, args.select), args.avoid) @@ -770,6 +844,9 @@ def avoidFiles(pathList, avoidFilesList): quitOnError(f"{Fore.RED} [-] No file found. Please verify filters, the directory or the extension with '--fileext'") if not args.jsononly: + # Init EVTX extractor object + extractor = evtxExtractor(logger=consoleLogger, providedTmpDir=args.tmpdir, coreCount=args.cores, useExternalBinaries=(not args.noexternal), binPath=binPath, xmlLogs=args.sysmon4linux) + consoleLogger.info(f"[+] Extracting EVTX Using '{extractor.tmpDir}' directory ") for evtx in tqdm(FileList, colour="yellow"): extractor.run(evtx) # Set the path for the next step @@ -782,7 +859,17 @@ def avoidFiles(pathList, avoidFilesList): #{% endif %} if EVTXJSONList == []: quitOnError(f"{Fore.RED} [-] No JSON files found.") - + + #{% if not embeddedMode -%} + # Print field list and exit + if args.fieldlist: + fields = zircoliteCore.run(EVTXJSONList, False) + zircoliteCore.close() + if not args.jsononly and not args.keeptmp: extractor.cleanup() + [print(sortedField) for sortedField in sorted([field for field in fields.values()])] + sys.exit(0) + #{% endif %} + # Flatten and insert to Db zircoliteCore.run(EVTXJSONList) # Unload In memory DB to disk. Done here to allow debug in case of ruleset execution error @@ -812,7 +899,6 @@ def avoidFiles(pathList, avoidFilesList): consoleLogger.info(f"[+] These rules were not converted (not supported by backend) : ") for error in convertedRules["errors"]: consoleLogger.info(f'{Fore.LIGHTYELLOW_EX} [-] "{error}"{Fore.RESET}') - #exportList = [rule["rule"] for rule in generatedRules if not rule["notsupported"]] zircoliteCore.loadRulesetFromVar(ruleset=convertedRules["ruleset"], ruleFilters=args.rulefilter) else: zircoliteCore.loadRulesetFromFile(filename=args.ruleset, ruleFilters=args.rulefilter)