From 1eddad6f80d8ab01851f916c0a290a2108f305f6 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 17 Oct 2022 20:40:03 +0000 Subject: [PATCH 1/8] first implementation of --try-dependencies and --try-builddependencies --- easybuild/framework/easyconfig/tweak.py | 41 +++++++++++++++++++++++-- easybuild/tools/options.py | 6 ++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c3fdfa0904..6c063f22ea 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -252,7 +252,6 @@ def tweak_one(orig_ec, tweaked_ec, tweaks, targetdir=None): :param targetdir: target directory for tweaked easyconfig file, defaults to temporary directory (only used if tweaked_ec is None) """ - # read easyconfig file ectxt = read_file(orig_ec) @@ -299,7 +298,39 @@ def __repr__(self): for key in list(tweaks): val = tweaks[key] - if isinstance(val, list): + if key in ['dependencies', 'builddependencies']: + import ast + new_or_updated_deps = [] + for dep in tweaks[key]: + new_or_updated_deps += [tuple(ast.literal_eval(dep))] + + # use non-greedy matching for list value using '*?' to avoid including other parameters in match, + # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % key, re.M) + res = regexp.search(ectxt) + if res: + current_deps = ast.literal_eval(str(res.group('val'))) + + # loop through new or updated deps, if match is found, replace it, else append new + newval = current_deps + for new_dep in new_or_updated_deps: + if new_dep in newval: + continue + + newval = [new_dep if new_dep[0] == curr_dep[0] else curr_dep for curr_dep in newval] + if new_dep not in newval: + _log.debug("Adding dependency %s to %s" % (str(new_dep), key)) + newval += [new_dep] + else: + _log.debug("Updated %s dependency in %s to %s" % (new_dep[0], key, str(new_dep))) + + ectxt = regexp.sub("%s = %s" % (res.group('key'), str(newval)), ectxt) + _log.info("Tweaked %s list to '%s'" % (key, str(newval))) + else: + ectxt += "%s = %s" % (key, str(new_or_updated_deps)) + _log.info("Tweaked %s list to '%s'" % (key, str(new_or_updated_deps))) + + elif isinstance(val, list): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % key, re.M) @@ -329,6 +360,12 @@ def __repr__(self): tweaks.pop(key) + # these are already handled + if 'dependencies' in list(tweaks): + tweaks.pop('dependencies') + if 'builddependencies' in list(tweaks): + tweaks.pop('builddependencies') + # add parameters or replace existing ones special_values = { # if the value is True/False/None then take that diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6128c3e650..79d28e5cea 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -300,6 +300,10 @@ def software_options(self): 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'builddependencies': (("Specify dependencies to replace or add."), + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'dependencies': (("Specify dependencies to replace or add."), + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software': ("Search and build software with given name and version", 'strlist', 'extend', None, {'metavar': 'NAME,VERSION'}), 'software-name': ("Search and build software with given name", @@ -1661,6 +1665,8 @@ def process_software_build_specs(options): 'toolchain_version': options.try_toolchain_version, 'update_deps': options.try_update_deps, 'ignore_versionsuffixes': options.try_ignore_versionsuffixes, + 'dependencies': options.try_dependencies, + 'builddependencies': options.try_builddependencies, } # process easy options From 06a4ddfa0d92c34c13e89ba6a67aa8637b37651f Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Mon, 17 Oct 2022 20:50:24 +0000 Subject: [PATCH 2/8] appeasing hound --- easybuild/tools/options.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 79d28e5cea..e8fd370a45 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -300,10 +300,10 @@ def software_options(self): 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), - 'builddependencies': (("Specify dependencies to replace or add."), - None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), - 'dependencies': (("Specify dependencies to replace or add."), - None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'builddependencies': ("Specify builddependencies to replace or add.", + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'dependencies': ("Specify dependencies to replace or add.", + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software': ("Search and build software with given name and version", 'strlist', 'extend', None, {'metavar': 'NAME,VERSION'}), 'software-name': ("Search and build software with given name", From 02b9397cb58e79772e3de6465cafe60af953f582 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 18 Oct 2022 14:07:35 +0000 Subject: [PATCH 3/8] renamed dependencies to dependency in --try-[build]dependencies to avoid clash with other code --- easybuild/framework/easyconfig/tweak.py | 25 +++++++++++++------------ easybuild/tools/options.py | 8 ++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 6c063f22ea..9da68e2ba3 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -298,7 +298,7 @@ def __repr__(self): for key in list(tweaks): val = tweaks[key] - if key in ['dependencies', 'builddependencies']: + if key in ['dependency', 'builddependency']: import ast new_or_updated_deps = [] for dep in tweaks[key]: @@ -306,7 +306,8 @@ def __repr__(self): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line - regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % key, re.M) + param = key.replace('y','ies') + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % param, re.M) res = regexp.search(ectxt) if res: current_deps = ast.literal_eval(str(res.group('val'))) @@ -319,16 +320,16 @@ def __repr__(self): newval = [new_dep if new_dep[0] == curr_dep[0] else curr_dep for curr_dep in newval] if new_dep not in newval: - _log.debug("Adding dependency %s to %s" % (str(new_dep), key)) + _log.debug("Adding dependency %s to %s" % (str(new_dep), param)) newval += [new_dep] else: - _log.debug("Updated %s dependency in %s to %s" % (new_dep[0], key, str(new_dep))) + _log.debug("Updated %s dependency in %s to %s" % (new_dep[0], param, str(new_dep))) - ectxt = regexp.sub("%s = %s" % (res.group('key'), str(newval)), ectxt) - _log.info("Tweaked %s list to '%s'" % (key, str(newval))) + ectxt = regexp.sub("%s = %s" % (res.group('param'), str(newval)), ectxt) + _log.info("Tweaked %s list to '%s'" % (param, str(newval))) else: - ectxt += "%s = %s" % (key, str(new_or_updated_deps)) - _log.info("Tweaked %s list to '%s'" % (key, str(new_or_updated_deps))) + ectxt += "%s = %s" % (param, str(new_or_updated_deps)) + _log.info("Tweaked %s list to '%s'" % (param, str(new_or_updated_deps))) elif isinstance(val, list): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, @@ -361,10 +362,10 @@ def __repr__(self): tweaks.pop(key) # these are already handled - if 'dependencies' in list(tweaks): - tweaks.pop('dependencies') - if 'builddependencies' in list(tweaks): - tweaks.pop('builddependencies') + if 'dependency' in list(tweaks): + tweaks.pop('dependency') + if 'builddependency' in list(tweaks): + tweaks.pop('builddependency') # add parameters or replace existing ones special_values = { diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index e8fd370a45..081cdb0f54 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -300,9 +300,9 @@ def software_options(self): 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), - 'builddependencies': ("Specify builddependencies to replace or add.", + 'builddependency': ("Specify builddependency to replace or add.", None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), - 'dependencies': ("Specify dependencies to replace or add.", + 'dependency': ("Specify dependency to replace or add.", None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software': ("Search and build software with given name and version", 'strlist', 'extend', None, {'metavar': 'NAME,VERSION'}), @@ -1665,8 +1665,8 @@ def process_software_build_specs(options): 'toolchain_version': options.try_toolchain_version, 'update_deps': options.try_update_deps, 'ignore_versionsuffixes': options.try_ignore_versionsuffixes, - 'dependencies': options.try_dependencies, - 'builddependencies': options.try_builddependencies, + 'dependency': options.try_dependency, + 'builddependency': options.try_builddependency, } # process easy options From ea2fc011fd77a96f821e4a0d2c4e46bdbb2b81d2 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 18 Oct 2022 14:08:52 +0000 Subject: [PATCH 4/8] appeasing hound --- easybuild/framework/easyconfig/tweak.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 9da68e2ba3..11bab40ce4 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -307,7 +307,8 @@ def __repr__(self): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line param = key.replace('y','ies') - regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % param, re.M) + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % param, + re.M) res = regexp.search(ectxt) if res: current_deps = ast.literal_eval(str(res.group('val'))) From 1b751dbac55526d603ff3260b343c33b174421b4 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 18 Oct 2022 14:09:21 +0000 Subject: [PATCH 5/8] appeasing hound --- easybuild/framework/easyconfig/tweak.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index 11bab40ce4..0229b4e61e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -306,7 +306,7 @@ def __repr__(self): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line - param = key.replace('y','ies') + param = key.replace('y', 'ies') regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % param, re.M) res = regexp.search(ectxt) From 79e8f277997e777362f83968ec2c29f1c522211c Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 18 Oct 2022 14:10:09 +0000 Subject: [PATCH 6/8] appeasing hound --- easybuild/tools/options.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 081cdb0f54..6b120dcd9d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -301,9 +301,9 @@ def software_options(self): "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'builddependency': ("Specify builddependency to replace or add.", - None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'dependency': ("Specify dependency to replace or add.", - None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software': ("Search and build software with given name and version", 'strlist', 'extend', None, {'metavar': 'NAME,VERSION'}), 'software-name': ("Search and build software with given name", From 75b103d656882671a39aff9bb68cf0eda62fac72 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 19 Oct 2022 15:43:22 +0000 Subject: [PATCH 7/8] added test for try-dependency and try-builddependency --- test/framework/options.py | 83 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index 126c52931b..13b8da5c6a 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1636,6 +1636,89 @@ def test_try_update_deps(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + def test_try_dependency(self): + """Test for --try-dependency and --try-builddependency.""" + + # first, construct a toy easyconfig that is well suited for testing (multiple deps) + test_ectxt = '\n'.join([ + "easyblock = 'ConfigureMake'", + '', + "name = 'test'", + "version = '1.2.3'", + '' + "homepage = 'https://test.org'", + "description = 'this is just a test'", + '', + "toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}", + '', + "dependencies = [('hwloc', '1.6.2')]", + ]) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ectxt) + + args = [ + test_ec, + '--try-toolchain-version=6.4.0-2.28', + "--try-dependency=('hwloc', '1.11.8')", + "--try-dependency=('FFTW', '3.3.7', '-serial')", + "--disable-map-toolchains", + '-D', + ] + + outtxt = self.eb_main(args, raise_error=True, do_build=True) + patterns = [ + # toolchain got updated + r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", + # hwloc was updated to 1.11.8, thanks to available easyconfig + r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", + # FFTW was added, thanks to available easyconfig + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb \(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + # also generated easyconfig for test/1.2.3 with expected toolchain + r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + + # construct another toy easyconfig that is well suited for testing builddependency + test_ectxt = '\n'.join([ + "easyblock = 'ConfigureMake'", + '', + "name = 'test'", + "version = '1.2.3'", + '' + "homepage = 'https://test.org'", + "description = 'this is just a test'", + '', + "toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}", + '', + "builddependencies = [('hwloc', '1.6.2')]", + ]) + write_file(test_ec, test_ectxt) + args = [ + test_ec, + '--try-toolchain-version=6.4.0-2.28', + "--try-builddependency=('hwloc', '1.11.8')", + "--try-builddependency=('FFTW', '3.3.7', '-serial')", + "--disable-map-toolchains", + '-D', + ] + outtxt = self.eb_main(args, raise_error=True, do_build=True) + + patterns = [ + # toolchain got updated + r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", + # hwloc was updated to 1.11.8, thanks to available easyconfig + r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", + # FFTW was added, thanks to available easyconfig + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb \(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + # also generated easyconfig for test/1.2.3 with expected toolchain + r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + def test_dry_run_hierarchical(self): """Test dry run using a hierarchical module naming scheme.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') From 198ac9ede747ca6a8d3ce8bbd3f84bed52c135a6 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 19 Oct 2022 15:48:16 +0000 Subject: [PATCH 8/8] appeasing hound --- test/framework/options.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 13b8da5c6a..db60b309a8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1672,7 +1672,8 @@ def test_try_dependency(self): # hwloc was updated to 1.11.8, thanks to available easyconfig r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", # FFTW was added, thanks to available easyconfig - r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb \(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb " + + r"\(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", # also generated easyconfig for test/1.2.3 with expected toolchain r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", ] @@ -1711,7 +1712,8 @@ def test_try_dependency(self): # hwloc was updated to 1.11.8, thanks to available easyconfig r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", # FFTW was added, thanks to available easyconfig - r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb \(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb " + + r"\(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", # also generated easyconfig for test/1.2.3 with expected toolchain r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", ]