From 1c4aabfac5092076d965d128a6e5aebafa69527b Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 18 Aug 2020 12:22:25 +0530 Subject: [PATCH 1/7] add exclude arg --- cve_bin_tool/cli.py | 21 ++++++++++++++++++--- cve_bin_tool/version_scanner.py | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cve_bin_tool/cli.py b/cve_bin_tool/cli.py index 9f5e179d60..de58adbf25 100755 --- a/cve_bin_tool/cli.py +++ b/cve_bin_tool/cli.py @@ -77,6 +77,13 @@ def main(argv=None): input_group.add_argument( "directory", help="directory to scan", nargs="?", default=None ) + input_group.add_argument( + "-e", + "--exclude", + action=StringToListAction, + help="Comma separated Exclude directory path", + default=None, + ) input_group.add_argument( "-i", "--input-file", action="store", default="", help="provide input filename", @@ -164,6 +171,7 @@ def main(argv=None): ) defaults = { "directory": "", + "exclude": [], "input_file": "", "log_level": "info", "format": "console", @@ -293,9 +301,16 @@ def main(argv=None): LOGGER.warning(f"{product_info}, {triage_data}") cve_scanner.get_cves(product_info, triage_data) if args["directory"]: - version_scanner = VersionScanner( - should_extract=args["extract"], error_mode=error_mode - ) + if args["exclude"]: + version_scanner = VersionScanner( + should_extract=args["extract"], + exclude_folders=args["exclude"], + error_mode=error_mode, + ) + else: + version_scanner = VersionScanner( + should_extract=args["extract"], error_mode=error_mode + ) version_scanner.remove_skiplist(skips) LOGGER.info(version_scanner.print_checkers()) for scan_info in version_scanner.recursive_scan(args["directory"]): diff --git a/cve_bin_tool/version_scanner.py b/cve_bin_tool/version_scanner.py index bd7e9a1e16..d011e023e9 100644 --- a/cve_bin_tool/version_scanner.py +++ b/cve_bin_tool/version_scanner.py @@ -25,6 +25,7 @@ class VersionScanner: def __init__( self, should_extract=False, + exclude_folders=[], checkers=None, logger=None, error_mode=ErrorMode.TruncTrace, @@ -40,7 +41,7 @@ def __init__( self.checkers = checkers or self.load_checkers() self.score = score self.total_scanned_files = 0 - exclude_folders = [".git"] + self.exclude_folders = exclude_folders self.walker = DirWalk(folder_exclude_pattern=";".join(exclude_folders)).walk self.should_extract = should_extract self.file_stack = [] From e9ccde4b692f7ac0fd3ad6a4906526ac2a739091 Mon Sep 17 00:00:00 2001 From: Sahil Date: Tue, 18 Aug 2020 12:23:59 +0530 Subject: [PATCH 2/7] exclude for compressed files in sub directory --- cve_bin_tool/util.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cve_bin_tool/util.py b/cve_bin_tool/util.py index 44a392ac15..3949c95664 100644 --- a/cve_bin_tool/util.py +++ b/cve_bin_tool/util.py @@ -150,6 +150,9 @@ def walk(self, roots=None): for i in filenames if self.pattern_match(i, self.pattern) and not self.pattern_match(i, self.file_exclude_pattern) + and not self.directory_pattern( + os.path.join(dirpath, i), self.folder_exclude_pattern + ) ] dirnames[:] = [ i @@ -174,3 +177,13 @@ def pattern_match(text, patterns): if fnmatch.fnmatch(text, pattern): return True return False + + @staticmethod + def directory_pattern(text, patterns): + """ Match dirname patterns """ + if not patterns: + return False + for pattern in patterns.split(";"): + if text.startswith(pattern): + return True + return False From 3bd6370fb553d1e75dbc45169f58fb99b2cef239 Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 20 Aug 2020 22:33:51 +0530 Subject: [PATCH 3/7] exclude_path works with fnmatch --- cve_bin_tool/util.py | 12 +----------- cve_bin_tool/version_scanner.py | 6 +++++- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cve_bin_tool/util.py b/cve_bin_tool/util.py index 3949c95664..82516db570 100644 --- a/cve_bin_tool/util.py +++ b/cve_bin_tool/util.py @@ -150,7 +150,7 @@ def walk(self, roots=None): for i in filenames if self.pattern_match(i, self.pattern) and not self.pattern_match(i, self.file_exclude_pattern) - and not self.directory_pattern( + and not self.pattern_match( os.path.join(dirpath, i), self.folder_exclude_pattern ) ] @@ -177,13 +177,3 @@ def pattern_match(text, patterns): if fnmatch.fnmatch(text, pattern): return True return False - - @staticmethod - def directory_pattern(text, patterns): - """ Match dirname patterns """ - if not patterns: - return False - for pattern in patterns.split(";"): - if text.startswith(pattern): - return True - return False diff --git a/cve_bin_tool/version_scanner.py b/cve_bin_tool/version_scanner.py index d011e023e9..b5199b21fd 100644 --- a/cve_bin_tool/version_scanner.py +++ b/cve_bin_tool/version_scanner.py @@ -42,7 +42,11 @@ def __init__( self.score = score self.total_scanned_files = 0 self.exclude_folders = exclude_folders - self.walker = DirWalk(folder_exclude_pattern=";".join(exclude_folders)).walk + self.walker = DirWalk( + folder_exclude_pattern="".join( + [exclude + "*;" for exclude in exclude_folders] + ) + ).walk self.should_extract = should_extract self.file_stack = [] self.error_mode = error_mode From 116ce952206cf694c4a07a944ac9d42d3528c1ae Mon Sep 17 00:00:00 2001 From: Sahil Date: Fri, 21 Aug 2020 00:53:03 +0530 Subject: [PATCH 4/7] add tests for --exclude --- test/test_cli.py | 23 +++++++++++++++++++++++ test/test_config.py | 1 + 2 files changed, 24 insertions(+) diff --git a/test/test_cli.py b/test/test_cli.py index f34963d00a..f7c8aea854 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -56,6 +56,15 @@ def test_no_extraction(self): """ Test scanner against curl-7.20.0 rpm with extraction turned off """ assert main(["cve-bin-tool", os.path.join(self.tempdir, CURL_7_20_0_RPM)]) == 0 + def test_exclude(self, caplog): + """ Test that the exclude paths are not scanned """ + test_path = os.path.abspath(os.path.dirname(__file__)) + exclude_path = os.path.join(test_path, "assets/") + checkers = ["curl", "libcurl", "kerberos", "kerberos_5"] + with caplog.at_level(logging.INFO): + main(["cve-bin-tool", test_path, "-e", ",".join(exclude_path)]) + self.check_exclude_log(caplog, exclude_path, checkers) + def test_usage(self): """ Test that the usage returns 0 """ with pytest.raises(SystemExit) as e: @@ -101,6 +110,20 @@ def test_update_flags(self): main(["cve-bin-tool", "-u", "whatever", self.tempdir]) assert e.value.args[0] == -2 + @staticmethod + def check_exclude_log(caplog, exclude_path, checkers): + # The final log has all the checkers detected + final_log = [ + record for record in caplog.records if "NewFound CVEs" in record.message + ] + assert len(final_log) == 0, "CVEs from excluded path scanned!!" + if final_log: + final_log = final_log[0].message + for checker in checkers: + assert checker in final_log, f"found a CVE {checker} in {exclude_path}" + + caplog.clear() + @staticmethod def check_checkers_log(caplog, skip_checkers, include_checkers): # The final log has all the checkers detected diff --git a/test/test_config.py b/test/test_config.py index 3e9cd15459..2c31dee1b2 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -12,6 +12,7 @@ class TestConfig: "cvss": 0, "directory": "test/assets", "disable_version_check": False, + "exclude": [], "extract": False, "format": "console", "input_file": "test/csv/triage.csv", From 3df7f87d9a7c0ad30597e01a8eb7a30cad6bd238 Mon Sep 17 00:00:00 2001 From: Sahil Date: Fri, 21 Aug 2020 23:57:36 +0530 Subject: [PATCH 5/7] exclude path [.git] appended --- test/test_cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_cli.py b/test/test_cli.py index f7c8aea854..343fca520b 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -12,6 +12,7 @@ from cve_bin_tool.cli import main from cve_bin_tool.cvedb import DISK_LOCATION_DEFAULT from cve_bin_tool.extractor import Extractor +from cve_bin_tool.version_scanner import VersionScanner from .utils import ( TempDirTest, download_file, @@ -60,7 +61,7 @@ def test_exclude(self, caplog): """ Test that the exclude paths are not scanned """ test_path = os.path.abspath(os.path.dirname(__file__)) exclude_path = os.path.join(test_path, "assets/") - checkers = ["curl", "libcurl", "kerberos", "kerberos_5"] + checkers = list(VersionScanner().checkers.keys()) with caplog.at_level(logging.INFO): main(["cve-bin-tool", test_path, "-e", ",".join(exclude_path)]) self.check_exclude_log(caplog, exclude_path, checkers) @@ -116,7 +117,7 @@ def check_exclude_log(caplog, exclude_path, checkers): final_log = [ record for record in caplog.records if "NewFound CVEs" in record.message ] - assert len(final_log) == 0, "CVEs from excluded path scanned!!" + assert len(final_log) == 0, "Checkers from excluded path scanned!!" if final_log: final_log = final_log[0].message for checker in checkers: From ca31ba3f3fbc5edda62be34138393aa80e3cf30a Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 24 Aug 2020 12:31:09 +0530 Subject: [PATCH 6/7] changes in --exclude --- cve_bin_tool/cli.py | 15 +++++---------- cve_bin_tool/util.py | 20 +++++++++++--------- cve_bin_tool/version_scanner.py | 7 +++++-- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/cve_bin_tool/cli.py b/cve_bin_tool/cli.py index de58adbf25..266e0e9d11 100755 --- a/cve_bin_tool/cli.py +++ b/cve_bin_tool/cli.py @@ -301,16 +301,11 @@ def main(argv=None): LOGGER.warning(f"{product_info}, {triage_data}") cve_scanner.get_cves(product_info, triage_data) if args["directory"]: - if args["exclude"]: - version_scanner = VersionScanner( - should_extract=args["extract"], - exclude_folders=args["exclude"], - error_mode=error_mode, - ) - else: - version_scanner = VersionScanner( - should_extract=args["extract"], error_mode=error_mode - ) + version_scanner = VersionScanner( + should_extract=args["extract"], + exclude_folders=args["exclude"], + error_mode=error_mode, + ) version_scanner.remove_skiplist(skips) LOGGER.info(version_scanner.print_checkers()) for scan_info in version_scanner.recursive_scan(args["directory"]): diff --git a/cve_bin_tool/util.py b/cve_bin_tool/util.py index 82516db570..eb9cd4400a 100644 --- a/cve_bin_tool/util.py +++ b/cve_bin_tool/util.py @@ -146,19 +146,21 @@ def walk(self, roots=None): for dirpath, dirnames, filenames in os.walk(root): # Filters filenames[:] = [ - i - for i in filenames - if self.pattern_match(i, self.pattern) - and not self.pattern_match(i, self.file_exclude_pattern) + filename + for filename in filenames + if self.pattern_match(os.path.join(dirpath, filename), self.pattern) and not self.pattern_match( - os.path.join(dirpath, i), self.folder_exclude_pattern + os.path.join(dirpath, filename), self.file_exclude_pattern + ) + and not self.pattern_match( + os.path.join(dirpath, filename), self.folder_exclude_pattern ) ] dirnames[:] = [ - i - for i in dirnames - if self.pattern_match(i, self.folder_include_pattern) - and not self.pattern_match(i, self.folder_exclude_pattern) + dirname + for dirname in dirnames + if self.pattern_match(dirname, self.folder_include_pattern) + and not self.pattern_match(dirname, self.folder_exclude_pattern) ] # Yields if self.yield_files: diff --git a/cve_bin_tool/version_scanner.py b/cve_bin_tool/version_scanner.py index b5199b21fd..710fced33d 100644 --- a/cve_bin_tool/version_scanner.py +++ b/cve_bin_tool/version_scanner.py @@ -42,9 +42,12 @@ def __init__( self.score = score self.total_scanned_files = 0 self.exclude_folders = exclude_folders + self.exclude_folders.append(".git") + self.walker = DirWalk( - folder_exclude_pattern="".join( - [exclude + "*;" for exclude in exclude_folders] + folder_exclude_pattern=";".join( + exclude if exclude.endswith("*") else exclude + "*" + for exclude in exclude_folders ) ).walk self.should_extract = should_extract From b028714edbd1b26b3143ccb511a68f073076b156 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 24 Aug 2020 16:13:35 +0530 Subject: [PATCH 7/7] join path in dirnames --- cve_bin_tool/util.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cve_bin_tool/util.py b/cve_bin_tool/util.py index eb9cd4400a..9ab5efa074 100644 --- a/cve_bin_tool/util.py +++ b/cve_bin_tool/util.py @@ -159,8 +159,12 @@ def walk(self, roots=None): dirnames[:] = [ dirname for dirname in dirnames - if self.pattern_match(dirname, self.folder_include_pattern) - and not self.pattern_match(dirname, self.folder_exclude_pattern) + if self.pattern_match( + os.path.join(dirpath, dirname), self.folder_include_pattern + ) + and not self.pattern_match( + os.path.join(dirpath, dirname), self.folder_exclude_pattern + ) ] # Yields if self.yield_files: