From cb15dc146415f1e7398b1ee2d674b05541910753 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Thu, 5 Dec 2019 16:14:24 +0100 Subject: [PATCH 1/5] Create data manager for the vts dictionary. A new process is spawn, and it will load the vts in a dictionary. This process run and keep the collection. The main process it is lighter now. Therefore, the new fork()'ed scan processes inherite less memory. --- CHANGELOG.md | 1 + ospd/ospd.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f51c1dc7..53d474fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Set loglevel to debug for some message. [#159](https://github.com/greenbone/ospd/pull/159) - Improve error handling when stop a scan. [#163](https://github.com/greenbone/ospd/pull/163) - Check the existence and status of an scan_id. [#179](https://github.com/greenbone/ospd/pull/179) +- Create data manager and spawn new process to keep the vts dictionary. [#191](https://github.com/greenbone/ospd/pull/191) ### Fixed - Fix set permission in unix socket. [#157](https://github.com/greenbone/ospd/pull/157) diff --git a/ospd/ospd.py b/ospd/ospd.py index 5ab5bf0b..937533b3 100644 --- a/ospd/ospd.py +++ b/ospd/ospd.py @@ -262,6 +262,8 @@ def add_vt( ): """ Add a vulnerability test information. """ + if self.vts is None: + self.vts = multiprocessing.Manager().dict() if not vt_id: raise OspdError('Invalid vt_id {}'.format(vt_id)) From af6cfdee1ea3598f61989dca7f891b33072d49aa Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Fri, 6 Dec 2019 10:00:39 +0100 Subject: [PATCH 2/5] Init self.vts as None. Also check for None before iterating. --- ospd/ospd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ospd/ospd.py b/ospd/ospd.py index 937533b3..fb8b07a9 100644 --- a/ospd/ospd.py +++ b/ospd/ospd.py @@ -205,7 +205,7 @@ def __init__( for name, param in BASE_SCANNER_PARAMS.items(): self.add_scanner_param(name, param) - self.vts = dict() + self.vts = None self.vt_id_pattern = re.compile("[0-9a-zA-Z_\\-:.]{1,80}") self.vts_version = None @@ -1584,6 +1584,9 @@ def get_vts_xml(self, vt_id=None, filtered_vts=None): vts_xml = Element('vts') + if not self.vts: + return vts_xml + if filtered_vts is not None and len(filtered_vts) == 0: return vts_xml From fa3367a43bac676ece2c2fdbd89fc54eae09861e Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Fri, 6 Dec 2019 13:58:04 +0100 Subject: [PATCH 3/5] Iterate over DictProxy with iter(). DictProxy in Python 3.5 does not support iteration as over a regular dict. --- ospd/ospd.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ospd/ospd.py b/ospd/ospd.py index fb8b07a9..0c2f53c4 100644 --- a/ospd/ospd.py +++ b/ospd/ospd.py @@ -1596,7 +1596,10 @@ def get_vts_xml(self, vt_id=None, filtered_vts=None): elif vt_id: vts_xml.append(self.get_vt_xml(vt_id)) else: - for vt_id in self.vts: + # TODO: Because DictProxy for python3.5 doesn't support + # iterkeys(), itervalues(), or iteritems() either, the iteration + # must be done as follow. + for vt_id in iter(self.vts.keys()): vts_xml.append(self.get_vt_xml(vt_id)) return vts_xml From dd7b4a008dc19beb15b715a4e4d960f6b3aac216 Mon Sep 17 00:00:00 2001 From: Juan Jose Nicola Date: Mon, 9 Dec 2019 12:31:07 +0100 Subject: [PATCH 4/5] Improve docstring --- ospd/ospd.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ospd/ospd.py b/ospd/ospd.py index 0c2f53c4..18976381 100644 --- a/ospd/ospd.py +++ b/ospd/ospd.py @@ -261,6 +261,13 @@ def add_vt( severities=None, ): """ Add a vulnerability test information. + + IMPORTANT: The VT's Data Manager will store the vts collection. + If the collection is considerably big and it will be consultated + intensible during a routine, consider to do a deepcopy(), since + accessing the shared memory in the data manager is very expencive. + At the end of the routine, the temporal copy must be set to None + and deleted. """ if self.vts is None: self.vts = multiprocessing.Manager().dict() From fc6ac7bade82715dbd79cc6d78e55da7be47b5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Nicola?= Date: Mon, 9 Dec 2019 13:35:02 +0100 Subject: [PATCH 5/5] Update ospd/ospd.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Björn Ricks --- ospd/ospd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ospd/ospd.py b/ospd/ospd.py index 18976381..afe1d8ce 100644 --- a/ospd/ospd.py +++ b/ospd/ospd.py @@ -265,7 +265,7 @@ def add_vt( IMPORTANT: The VT's Data Manager will store the vts collection. If the collection is considerably big and it will be consultated intensible during a routine, consider to do a deepcopy(), since - accessing the shared memory in the data manager is very expencive. + accessing the shared memory in the data manager is very expensive. At the end of the routine, the temporal copy must be set to None and deleted. """