diff --git a/.github/workflows/container_tests.yml b/.github/workflows/container_tests.yml index ef310a3816..41e6820dd8 100644 --- a/.github/workflows/container_tests.yml +++ b/.github/workflows/container_tests.yml @@ -74,7 +74,7 @@ jobs: ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz - name: run test run: | @@ -95,7 +95,7 @@ jobs: echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; # see https://docs.easybuild.io/en/latest/Containers.html - curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64' eb bzip2-1.0.8.eb --containerize --experimental --container-build-image diff --git a/.github/workflows/container_tests_apptainer.yml b/.github/workflows/container_tests_apptainer.yml index 35c26c26c9..1f46f2f5f6 100644 --- a/.github/workflows/container_tests_apptainer.yml +++ b/.github/workflows/container_tests_apptainer.yml @@ -66,7 +66,7 @@ jobs: ls dist export PREFIX=/tmp/$USER/$GITHUB_SHA pip install --prefix $PREFIX dist/easybuild-framework*tar.gz - pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/develop.tar.gz + pip install --prefix $PREFIX https://github.com/easybuilders/easybuild-easyblocks/archive/5.0.x.tar.gz - name: run test run: | @@ -87,7 +87,7 @@ jobs: echo '%_dbpath %{_var}/lib/rpm' >> $HOME/.rpmmacros # build CentOS 7 container image for bzip2 1.0.8 using EasyBuild; # see https://docs.easybuild.io/en/latest/Containers.html - curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb + curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/5.0.x/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb export EASYBUILD_CONTAINERPATH=$PWD export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64' export EASYBUILD_CONTAINER_TYPE='apptainer' diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml new file mode 100644 index 0000000000..fc4936286a --- /dev/null +++ b/.github/workflows/end2end.yml @@ -0,0 +1,60 @@ +name: End-to-end test of EasyBuild in different distros +on: [push, pull_request] +jobs: + build_publish: + name: End-to-end test + runs-on: ubuntu-latest + strategy: + matrix: + container: + - centos-8.5 + - fedora-36 + - opensuse-15.4 + - rockylinux-8.8 + - rockylinux-9.2 + - ubuntu-20.04 + - ubuntu-22.04 + fail-fast: false + container: + image: ghcr.io/easybuilders/${{ matrix.container }}-amd64 + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: download and unpack easyblocks and easyconfigs repositories + run: | + cd $HOME + for pkg in easyblocks easyconfigs; do + curl -OL https://github.com/easybuilders/easybuild-${pkg}/archive/5.0.x.tar.gz + tar xfz 5.0.x.tar.gz + rm -f 5.0.x.tar.gz + done + + - name: Set up environment + shell: bash + run: | + # collect environment variables to be set in subsequent steps in script that can be sourced + echo "export PATH=$PWD:$PATH" > /tmp/eb_env + echo "export PYTHONPATH=$PWD:$HOME/easybuild-easyblocks-5.0.x:$HOME/easybuild-easyconfigs-5.0.x" >> /tmp/eb_env + + - name: Run commands to check test environment + shell: bash + run: | + cmds=( + "whoami" + "pwd" + "env | sort" + "eb --version" + "eb --show-system-info" + "eb --check-eb-deps" + "eb --show-config" + ) + for cmd in "${cmds[@]}"; do + echo ">>> $cmd" + sudo -u easybuild bash -l -c "source /tmp/eb_env; $cmd" + done + + - name: End-to-end test of installing bzip2 with EasyBuild + shell: bash + run: | + sudo -u easybuild bash -l -c "source /tmp/eb_env; eb bzip2-1.0.8.eb --trace --robot" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3b88c58436..324b64630b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -13,10 +13,7 @@ jobs: setup: runs-on: ubuntu-20.04 outputs: - lmod7: Lmod-7.8.22 lmod8: Lmod-8.7.6 - modulesTcl: modules-tcl-1.147 - modules3: modules-3.2.10 modules4: modules-4.1.4 steps: - run: "true" @@ -29,49 +26,25 @@ jobs: modules_tool: # use variables defined by 'setup' job above, see also # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context - - ${{needs.setup.outputs.lmod7}} - ${{needs.setup.outputs.lmod8}} - - ${{needs.setup.outputs.modulesTcl}} - - ${{needs.setup.outputs.modules3}} - ${{needs.setup.outputs.modules4}} - module_syntax: [Lua, Tcl] lc_all: [""] - # don't test with Lua module syntax (only supported in Lmod) - exclude: - - modules_tool: ${{needs.setup.outputs.modulesTcl}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules3}} - module_syntax: Lua - - modules_tool: ${{needs.setup.outputs.modules4}} - module_syntax: Lua include: # Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax) - python: 3.7 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: 3.8 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: 3.8 - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl - python: 3.9 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.10' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - - python: '3.11' - modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua - python: '3.11' modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Tcl # There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set # Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7) - python: 3.6 modules_tool: ${{needs.setup.outputs.lmod8}} - module_syntax: Lua lc_all: C fail-fast: false steps: @@ -122,16 +95,11 @@ jobs: # and are only run after the PR gets merged GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}} run: | - # don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, - # and only when testing with Lua as module syntax, - # to avoid hitting GitHub rate limit; + # only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9, to avoid hitting GitHub rate limit; # tests that require a GitHub token are skipped automatically when no GitHub token is available - if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then + if [[ "${{matrix.modules_tool}}" =~ 'Lmod-8' ]] && [[ "${{matrix.python}}" =~ 3.[69] ]]; then if [ ! -z $GITHUB_TOKEN ]; then - if [ "x${{matrix.python}}" == 'x2.6' ]; - then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())"; - else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; - fi; + SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())"; python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')"; fi echo "GitHub token installed!" @@ -169,8 +137,6 @@ jobs: - name: run test suite env: EB_VERBOSE: 1 - EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} - TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}} LC_ALL: ${{matrix.lc_all}} run: | # run tests *outside* of checked out easybuild-framework directory, @@ -186,28 +152,37 @@ jobs: export PYTHONPATH=$PREFIX/lib/python${{matrix.python}}/site-packages:$PYTHONPATH eb --version # tell EasyBuild which modules tool is available - if [[ ${{matrix.modules_tool}} =~ ^modules-tcl- ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesTcl - elif [[ ${{matrix.modules_tool}} =~ ^modules-3 ]]; then - export EASYBUILD_MODULES_TOOL=EnvironmentModulesC - elif [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then + if [[ ${{matrix.modules_tool}} =~ ^modules-4 ]]; then export EASYBUILD_MODULES_TOOL=EnvironmentModules else export EASYBUILD_MODULES_TOOL=Lmod fi - export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL - eb --show-config - # gather some useful info on test system - eb --show-system-info - # check GitHub configuration - eb --check-github --github-user=easybuild_test - # create file owned by root but writable by anyone (used by test_copy_file) - sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt - # run test suite - python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log - # try and make sure output of running tests is clean (no printed messages/warnings) - IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" - # '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches) - PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) - test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL} + + # Run tests with LUA and Tcl module syntax (where supported) + for module_syntax in Lua Tcl; do + # Only Lmod supports Lua + if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then + echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool" + continue + fi + printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n' + export EASYBUILD_MODULE_SYNTAX="${module_syntax}" + export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}" + + eb --show-config + # gather some useful info on test system + eb --show-system-info + # check GitHub configuration + eb --check-github --github-user=easybuild_test + # create file owned by root but writable by anyone (used by test_copy_file) + sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt + # run test suite + python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log + # try and make sure output of running tests is clean (no printed messages/warnings) + IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test" + # '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches) + PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true) + test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1) + done diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8f5ec0f1e0..94b66db903 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,6 +4,81 @@ For more detailed information, please see the git log. These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html. +v4.9.0 (30 December 2023) +------------------------- + +feature release + +- various enhancements, including: + - allow tweaking of easyconfigs from different toolchains (#3669) + - add support for `--list-software --output-format=json` (#4152) + - add `--module-cache-suffix` configuration setting to allow multiple (Lmod) caches (#4403) +- various bug fixes, including: + - deduplicate warnings & errors found in logs and add initial newline + tab in output (#4361) + - fix support for Environment Modules as modules tool to pass unit tests with v4.2+ (#4369) + - adapt module function check for Environment Modules v4+ (#4371) + - only install GitHub token when testing with Lmod 8.x + Python 3.6 or 3.9 (#4375) + - use `-qopenmp` instead of `-fiopenmp` for OpenMP in Intel compilers (#4377) + - fix `LIBBLAS_MT` for FlexiBLAS, ensure `-lpthread` is included (#4379) + - relax major version match regex in `find_related_easyconfigs` using for `--review-pr` (#4385) + - eliminate duplicate multideps from generated module files (#4386) + - resolve templated values in extension names in `_make_extension_list` (#4392) + - use source toolchain version when passing only `--try-toolchain` (#4395) + - fix writing spider cache for Lmod >= 8.7.12 (#4402) + - fix `--inject-checksums` when extension specifies patch file in tuple format (#4405) + - fix `LooseVersion` when running with Python 2.7 (#4408) + - use more recent easyblocks PR in `test_github_merge_pr` (#4414) +- other changes: + - extend test that checks build environment to recent `foss/2023a` toolchain (#4391) + + +v4.8.2 (29 October 2023) +------------------------ + +update/bugfix release + +- various enhancements, including: + - add support for `%(sysroot)s` template value (#4359) + - add `dependency_names` method to `EasyConfig` class to get set of names of (direct) dependencies (#4360) +- various bug fixes, including: + - add CI workflow to run unit tests with Python 2 (again) (#4333) + - fix typo in help message for `--silence-hook-trigger` (#4343) + - include major version (`*majver`) templates in auto-generated documentation (#4347) + - reset `tempfile.tempdir` to `None` to avoid that tmpdir path gets progressively deeper with each easystack item (#4350) + - fix `findPythonDeps.py` script when called with an (absolute or relative) path to an easyconfig instead of a filename (#4365) + - fix broken test for `reasons_for_closing`, which fails because commit status of easyconfigs PR is no longer available (#4366) +- other changes: + - reduce number of CI jobs by testing for Lua and Tcl module syntax in a single CI job (#4192) + + +v4.8.1 (11 September 2023) +-------------------------- + +update/bugfix release + +- various enhancements, including: + - add end-to-end test for running EasyBuild in different Linux distros using containers (#3968) + - suggest default title in `--review-pr` (#4287) + - add `build_and_install_loop` hooks to run before and after the install loop for individual easyconfigs (#4304) + - implement support for `cancel_hook` and `fail_hook` (#4315, #4325) + - add postiter hook to the list of steps so the corresponding hook can be used (#4316) + - add `run_shell_cmd` hook (#4323) + - add `build_info_msg` easyconfig parameter to print message during installation of an easyconfig (#4324) + - add `--silence-hook-trigger` configuration option to supress printing of debug message every time a hook is triggered (#4329) + - add support for using fine grained Github tokens (#4332) + - add definitions for ifbf and iofbf toolchain (#4337) + - add support for submodule filtering and specifying extra Git configuration in `git_config` (#4338, #4339) +- various bug fixes, including: + - improve error when checksum dict has no entry for a file (#4150) + - avoid error being logged when `checksums.json` is not found (#4261) + - don't fail in `mkdir` if path gets created while processing it (#4300, #4328) + - ignore request for external module (meta)data when no modules tool is active (#4308) + - use sys.executable to obtain path to `python` command in tests, rather than assuming that `python` command is available in `$PATH` (#4309) + - fix `test_add_and_remove_module_path` by replacing string comparison of paths by checking whether they point to the same path (since symlinks may cause trouble) (#4312) + - enhance `Toolchain.get_flag` to handle lists (#4319) + - only add extensions in module file if there are extensions (#4331) + + v4.8.0 (7 July 2023) -------------------- diff --git a/easybuild/base/fancylogger.py b/easybuild/base/fancylogger.py index df7baa0e0e..96ec7c1f8b 100644 --- a/easybuild/base/fancylogger.py +++ b/easybuild/base/fancylogger.py @@ -285,7 +285,7 @@ def makeRecord(self, name, level, pathname, lineno, msg, args, excinfo, func=Non overwrite make record to use a fancy record (with more options) """ logrecordcls = logging.LogRecord - if hasattr(self, 'fancyrecord') and self.fancyrecord: + if getattr(self, 'fancyrecord', None): logrecordcls = FancyLogRecord try: new_msg = str(msg) diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index 3e9788ac44..4b99eab61d 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -93,7 +93,7 @@ def set_columns(cols=None): pass if cols is not None: - os.environ['COLUMNS'] = "%s" % cols + os.environ['COLUMNS'] = str(cols) def what_str_list_tuple(name): @@ -825,8 +825,8 @@ def get_env_options(self): self.environment_arguments.append("%s=%s" % (lo, val)) else: # interpretation of values: 0/no/false means: don't set it - if ("%s" % val).lower() not in ("0", "no", "false",): - self.environment_arguments.append("%s" % lo) + if str(val).lower() not in ("0", "no", "false",): + self.environment_arguments.append(str(lo)) else: self.log.debug("Environment variable %s is not set" % env_opt_name) @@ -1034,7 +1034,7 @@ def main_options(self): # make_init is deprecated if hasattr(self, 'make_init'): self.log.debug('main_options: make_init is deprecated. Rename function to main_options.') - getattr(self, 'make_init')() + self.make_init() else: # function names which end with _options and do not start with main or _ reg_main_options = re.compile("^(?!_|main).*_options$") @@ -1192,7 +1192,7 @@ def add_group_parser(self, opt_dict, description, prefix=None, otherdefaults=Non for extra_detail in details[4:]: if isinstance(extra_detail, (list, tuple,)): # choices - nameds['choices'] = ["%s" % x for x in extra_detail] # force to strings + nameds['choices'] = [str(x) for x in extra_detail] # force to strings hlp += ' (choices: %s)' % ', '.join(nameds['choices']) elif isinstance(extra_detail, str) and len(extra_detail) == 1: args.insert(0, "-%s" % extra_detail) @@ -1711,7 +1711,7 @@ class SimpleOption(GeneralOption): PARSER = SimpleOptionParser SETROOTLOGGER = True - def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): + def __init__(self, go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None): """Initialisation :param go_dict: General Option option dict :param short_groupdescr: short description of main options @@ -1740,18 +1740,13 @@ def __init__(self, go_dict=None, descr=None, short_groupdescr=None, long_groupde super(SimpleOption, self).__init__(**kwargs) - if descr is not None: - # TODO: as there is no easy/clean way to access the version of the vsc-base package, - # this is equivalent to a warning - self.log.deprecated('SimpleOption descr argument', '2.5.0', '3.0.0') - def main_options(self): if self.go_dict is not None: prefix = None self.add_group_parser(self.go_dict, self.descr, prefix=prefix) -def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdescr=None, config_files=None): +def simple_option(go_dict=None, short_groupdescr=None, long_groupdescr=None, config_files=None): """A function that returns a single level GeneralOption option parser :param go_dict: General Option option dict @@ -1765,5 +1760,5 @@ def simple_option(go_dict=None, descr=None, short_groupdescr=None, long_groupdes the generated help will include the docstring """ - return SimpleOption(go_dict=go_dict, descr=descr, short_groupdescr=short_groupdescr, - long_groupdescr=long_groupdescr, config_files=config_files) + return SimpleOption(go_dict=go_dict, short_groupdescr=short_groupdescr, long_groupdescr=long_groupdescr, + config_files=config_files) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index eaf4dc630c..7a46c49921 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -512,14 +512,15 @@ def autocomplete(parser, arg_completer=None, opt_completer=None, subcmd_complete if option: if option.nargs > 0: optarg = True - if hasattr(option, 'completer'): + try: completer = option.completer - elif option.choices: - completer = ListCompleter(option.choices) - elif option.type in ('string',): - completer = opt_completer - else: - completer = NoneCompleter() + except AttributeError: + if option.choices: + completer = ListCompleter(option.choices) + elif option.type in ('string',): + completer = opt_completer + else: + completer = NoneCompleter() # Warn user at least, it could help him figure out the problem. elif hasattr(option, 'completer'): msg = "Error: optparse option with a completer does not take arguments: %s" % (option) @@ -615,11 +616,9 @@ class CmdComplete(object): def autocomplete(self, completer=None): parser = OPTIONPARSER_CLASS(self.__doc__.strip()) if hasattr(self, 'addopts'): - fnc = getattr(self, 'addopts') - fnc(parser) + self.addopts(parser) - if hasattr(self, 'completer'): - completer = getattr(self, 'completer') + completer = getattr(self, 'completer', completer) return autocomplete(parser, completer) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5b1aecad80..4482b0a71a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -41,7 +41,7 @@ * Davide Vanzo (Vanderbilt University) * Caspar van Leeuwen (SURF) """ - +import concurrent import copy import glob import inspect @@ -52,7 +52,9 @@ import tempfile import time import traceback +from concurrent.futures import ThreadPoolExecutor from datetime import datetime +from textwrap import indent import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain @@ -87,7 +89,7 @@ from easybuild.tools.hooks import MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP from easybuild.tools.hooks import POSTPROC_STEP, PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP from easybuild.tools.hooks import SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook -from easybuild.tools.run import check_async_cmd, run_cmd +from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -394,13 +396,13 @@ def get_checksums_from_json(self, always_read=False): :param always_read: always read the checksums.json file, even if it has been read before """ if always_read or self.json_checksums is None: - try: - path = self.obtain_file("checksums.json", no_download=True) + path = self.obtain_file("checksums.json", no_download=True, warning_only=True) + if path is not None: self.log.info("Loading checksums from file %s", path) json_txt = read_file(path) self.json_checksums = json.loads(json_txt) - # if the file can't be found, return an empty dict - except EasyBuildError: + else: + # if the file can't be found, return an empty dict self.json_checksums = {} return self.json_checksums @@ -727,7 +729,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True): return exts_sources def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False, - git_config=None, no_download=False, download_instructions=None, alt_location=None): + git_config=None, no_download=False, download_instructions=None, alt_location=None, + warning_only=False): """ Locate the file with the given name - searches in different subdirectories of source path @@ -780,7 +783,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No return fullpath except IOError as err: - raise EasyBuildError("Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err) + if not warning_only: + raise EasyBuildError("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + else: + self.log.warning("Downloading file %s " + "from url %s to %s failed: %s", filename, url, fullpath, err) + return None else: # try and find file in various locations @@ -857,8 +866,13 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No self.dry_run_msg(" * %s (MISSING)", filename) return filename else: - raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + if not warning_only: + raise EasyBuildError("Couldn't find file %s anywhere, and downloading it is disabled... " + "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + else: + self.log.warning("Couldn't find file %s anywhere, and downloading it is disabled... " "Paths attempted (in order): %s ", filename, ', '.join(failedpaths)) + return None elif git_config: return get_source_tarball_from_git(filename, targetdir, git_config) else: @@ -940,7 +954,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No if download_instructions is None: download_instructions = self.cfg['download_instructions'] if download_instructions is not None and download_instructions != "": - msg = "\nDownload instructions:\n\n" + download_instructions + '\n' + msg = "\nDownload instructions:\n\n" + indent(download_instructions, ' ') + '\n\n' + msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths()) print_msg(msg, prefix=False, stderr=True) error_msg += "please follow the download instructions above, and make the file available " error_msg += "in the active source path (%s)" % ':'.join(source_paths()) @@ -950,7 +965,11 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No error_msg += "and downloading it didn't work either... " error_msg += "Paths attempted (in order): %s " % failedpaths_msg - raise EasyBuildError(error_msg, filename) + if not warning_only: + raise EasyBuildError(error_msg, filename) + else: + self.log.warning(error_msg, filename) + return None # # GETTER/SETTER UTILITY FUNCTIONS @@ -1396,6 +1415,14 @@ def make_module_extra(self, altroot=None, altversion=None): value, type(value)) lines.append(self.module_generator.prepend_paths(key, value, allow_abs=self.cfg['allow_prepend_abs_path'])) + for (key, value) in self.cfg['modextrapaths_append'].items(): + if isinstance(value, str): + value = [value] + elif not isinstance(value, (tuple, list)): + raise EasyBuildError("modextrapaths_append dict value %s (type: %s) is not a list or tuple", + value, type(value)) + lines.append(self.module_generator.append_paths(key, value, allow_abs=self.cfg['allow_append_abs_path'])) + modloadmsg = self.cfg['modloadmsg'] if modloadmsg: # add trailing newline to prevent that shell prompt is 'glued' to module load message @@ -1714,9 +1741,15 @@ def _make_extension_list(self): Each entry should be a (name, version) tuple or just (name, ) if no version exists """ - # We need only name and version, so don't resolve templates # Each extension in exts_list is either a string or a list/tuple with name, version as first entries - return [(ext, ) if isinstance(ext, str) else ext[:2] for ext in self.cfg.get_ref('exts_list')] + # As name can be a templated value we must resolve templates + exts_list = [] + for ext in self.cfg.get_ref('exts_list'): + if isinstance(ext, str): + exts_list.append((resolve_template(ext, self.cfg.template_values), )) + else: + exts_list.append((resolve_template(ext[0], self.cfg.template_values), ext[1])) + return exts_list def make_extension_string(self, name_version_sep='-', ext_sep=', ', sort=True): """ @@ -1764,21 +1797,20 @@ def skip_extensions_sequential(self, exts_filter): exts_cnt = len(self.ext_instances) - res = [] + exts = [] for idx, ext_inst in enumerate(self.ext_instances): cmd, stdin = resolve_exts_filter_template(exts_filter, ext_inst) - (out, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False) - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_inst.name, ec, out) - if ec == 0: - print_msg("skipping extension %s" % ext_inst.name, silent=self.silent, log=self.log) + res = run_shell_cmd(cmd, stdin=stdin, fail_on_error=False, hidden=True) + self.log.info(f"exts_filter result for {ext_inst.name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_inst.name}", silent=self.silent, log=self.log) else: - self.log.info("Not skipping %s", ext_inst.name) - res.append(ext_inst) + self.log.info(f"Not skipping {ext_inst.name}") + exts.append(ext_inst) - self.update_exts_progress_bar("skipping installed extensions (%d/%d checked)" % (idx + 1, exts_cnt)) + self.update_exts_progress_bar(f"skipping installed extensions ({idx + 1}/{exts_cnt} checked)") - self.ext_instances = res + self.ext_instances = exts self.update_exts_progress_bar("already installed extensions filtered out", total=len(self.ext_instances)) def skip_extensions_parallel(self, exts_filter): @@ -1789,41 +1821,31 @@ def skip_extensions_parallel(self, exts_filter): self.log.experimental("Skipping installed extensions in parallel") print_msg("skipping installed extensions (in parallel)", log=self.log) - async_cmd_info_cache = {} - running_checks_ids = [] installed_exts_ids = [] - exts_queue = list(enumerate(self.ext_instances[:])) checked_exts_cnt = 0 exts_cnt = len(self.ext_instances) + cmds = [resolve_exts_filter_template(exts_filter, ext) for ext in self.ext_instances] + + with ThreadPoolExecutor(max_workers=self.cfg['parallel']) as thread_pool: - # asynchronously run checks to see whether extensions are already installed - while exts_queue or running_checks_ids: + # list of command to run asynchronously + async_cmds = [thread_pool.submit(run_shell_cmd, cmd, stdin=stdin, hidden=True, fail_on_error=False, + asynchronous=True, task_id=idx) for (idx, (cmd, stdin)) in enumerate(cmds)] - # first handle completed checks - for idx in running_checks_ids[:]: + # process result of commands as they have completed running + for done_task in concurrent.futures.as_completed(async_cmds): + res = done_task.result() + idx = res.task_id ext_name = self.ext_instances[idx].name - # don't read any output, just check whether command completed - async_cmd_info = check_async_cmd(*async_cmd_info_cache[idx], output_read_size=0, fail_on_error=False) - if async_cmd_info['done']: - out, ec = async_cmd_info['output'], async_cmd_info['exit_code'] - self.log.info("exts_filter result for %s: exit code %s; output: %s", ext_name, ec, out) - running_checks_ids.remove(idx) - if ec == 0: - print_msg("skipping extension %s" % ext_name, log=self.log) - installed_exts_ids.append(idx) - - checked_exts_cnt += 1 - exts_pbar_label = "skipping installed extensions " - exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) - self.update_exts_progress_bar(exts_pbar_label) - - # start additional checks asynchronously - while exts_queue and len(running_checks_ids) < self.cfg['parallel']: - idx, ext = exts_queue.pop(0) - cmd, stdin = resolve_exts_filter_template(exts_filter, ext) - async_cmd_info_cache[idx] = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, - regexp=False, trace=False, asynchronous=True) - running_checks_ids.append(idx) + self.log.info(f"exts_filter result for {ext_name}: exit code {res.exit_code}; output: {res.output}") + if res.exit_code == 0: + print_msg(f"skipping extension {ext_name}", log=self.log) + installed_exts_ids.append(idx) + + checked_exts_cnt += 1 + exts_pbar_label = "skipping installed extensions " + exts_pbar_label += "(%d/%d checked)" % (checked_exts_cnt, exts_cnt) + self.update_exts_progress_bar(exts_pbar_label) # compose new list of extensions, skip over the ones that are already installed; # note: original order in extensions list should be preserved! @@ -1928,6 +1950,8 @@ def install_extensions_parallel(self, install=True): """ self.log.info("Installing extensions in parallel...") + thread_pool = ThreadPoolExecutor(max_workers=self.cfg['parallel']) + running_exts = [] installed_ext_names = [] @@ -1964,16 +1988,23 @@ def update_exts_progress_bar_helper(running_exts, progress_size): # check for extension installations that have completed if running_exts: - self.log.info("Checking for completed extension installations (%d running)...", len(running_exts)) + self.log.info(f"Checking for completed extension installations ({len(running_exts)} running)...") for ext in running_exts[:]: - if self.dry_run or ext.async_cmd_check(): - self.log.info("Installation of %s completed!", ext.name) - ext.postrun() - running_exts.remove(ext) - installed_ext_names.append(ext.name) - update_exts_progress_bar_helper(running_exts, 1) + if self.dry_run or ext.async_cmd_task.done(): + res = ext.async_cmd_task.result() + if res.exit_code == 0: + self.log.info(f"Installation of extension {ext.name} completed!") + # run post-install method for extension from same working dir as installation of extension + cwd = change_dir(res.work_dir) + ext.postrun() + change_dir(cwd) + running_exts.remove(ext) + installed_ext_names.append(ext.name) + update_exts_progress_bar_helper(running_exts, 1) + else: + raise_run_shell_cmd_error(res) else: - self.log.debug("Installation of %s is still running...", ext.name) + self.log.debug(f"Installation of extension {ext.name} is still running...") # try to start as many extension installations as we can, taking into account number of available cores, # but only consider first 100 extensions still in the queue @@ -2040,9 +2071,9 @@ def update_exts_progress_bar_helper(running_exts, progress_size): rpath_filter_dirs=self.rpath_filter_dirs) if install: ext.prerun() - ext.run_async() + ext.async_cmd_task = ext.run_async(thread_pool) running_exts.append(ext) - self.log.info("Started installation of extension %s in the background...", ext.name) + self.log.info(f"Started installation of extension {ext.name} in the background...") update_exts_progress_bar_helper(running_exts, 0) # print progress info after every iteration (unless that info is already shown via progress bar) @@ -2055,6 +2086,8 @@ def update_exts_progress_bar_helper(running_exts, progress_size): running_ext_names = ', '.join(x.name for x in running_exts[:3]) + ", ..." print_msg(msg % (installed_cnt, exts_cnt, queued_cnt, running_cnt, running_ext_names), log=self.log) + thread_pool.shutdown() + # # MISCELLANEOUS UTILITY FUNCTIONS # @@ -2698,18 +2731,19 @@ def test_step(self): """Run unit tests provided by software (if any).""" unit_test_cmd = self.cfg['runtest'] if unit_test_cmd: - - self.log.debug("Trying to execute %s as a command for running unit tests...", unit_test_cmd) - (out, _) = run_cmd(unit_test_cmd, log_all=True, simple=False) - - return out + self.log.debug(f"Trying to execute {unit_test_cmd} as a command for running unit tests...") + res = run_shell_cmd(unit_test_cmd) + return res.output def _test_step(self): """Run the test_step and handles failures""" try: self.test_step() - except EasyBuildError as err: - self.report_test_failure(err) + except RunShellCmdError as err: + err.print() + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in test step for {ec_path}" + self.report_test_failure(error_msg) def stage_install_step(self): """ @@ -2961,17 +2995,17 @@ def run_post_install_commands(self, commands=None): commands = self.cfg['postinstallcmds'] if commands: - self.log.debug("Specified post install commands: %s", commands) + self.log.debug(f"Specified post install commands: {commands}") # make sure we have a list of commands if not isinstance(commands, (list, tuple)): - error_msg = "Invalid value for 'postinstallcmds', should be list or tuple of strings: %s" - raise EasyBuildError(error_msg, commands) + error_msg = f"Invalid value for 'postinstallcmds', should be list or tuple of strings: {commands}" + raise EasyBuildError(error_msg) for cmd in commands: if not isinstance(cmd, str): - raise EasyBuildError("Invalid element in 'postinstallcmds', not a string: %s", cmd) - run_cmd(cmd, simple=True, log_ok=True, log_all=True) + raise EasyBuildError(f"Invalid element in 'postinstallcmds', not a string: {cmd}") + run_shell_cmd(cmd) def apply_post_install_patches(self, patches=None): """ @@ -3000,12 +3034,6 @@ def post_install_step(self): - run post install commands if any were specified """ - self.run_post_install_commands() - self.apply_post_install_patches() - self.print_post_install_messages() - - self.fix_shebang() - lib_dir = os.path.join(self.installdir, 'lib') lib64_dir = os.path.join(self.installdir, 'lib64') @@ -3026,6 +3054,12 @@ def post_install_step(self): # create *relative* 'lib' symlink to 'lib64'; symlink('lib64', lib_dir, use_abspath_source=False) + self.run_post_install_commands() + self.apply_post_install_patches() + self.print_post_install_messages() + + self.fix_shebang() + def sanity_check_step(self, *args, **kwargs): """ Do a sanity check on the installation @@ -3088,8 +3122,10 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # hard reset $LD_LIBRARY_PATH before running RPATH sanity check orig_env = env.unset_env_vars(['LD_LIBRARY_PATH']) - self.log.debug("$LD_LIBRARY_PATH during RPATH sanity check: %s", os.getenv('LD_LIBRARY_PATH', '(empty)')) - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + ld_library_path = os.getenv('LD_LIBRARY_PATH', '(empty)') + self.log.debug(f"$LD_LIBRARY_PATH during RPATH sanity check: {ld_library_path}") + modules_list = self.modules_tool.list() + self.log.debug(f"List of loaded modules: {modules_list}") not_found_regex = re.compile(r'(\S+)\s*\=\>\s*not found') readelf_rpath_regex = re.compile('(RPATH)', re.M) @@ -3098,33 +3134,31 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # For example, libcuda.so.1 should never be RPATH-ed by design, # see https://github.com/easybuilders/easybuild-framework/issues/4095 filter_rpath_sanity_libs = build_option('filter_rpath_sanity_libs') - msg = "Ignoring the following libraries if they are not found by RPATH sanity check: %s" - self.log.info(msg, filter_rpath_sanity_libs) + msg = "Ignoring the following libraries if they are not found by RPATH sanity check: {filter_rpath_sanity_libs}" + self.log.info(msg) if rpath_dirs is None: rpath_dirs = self.cfg['bin_lib_subdirs'] or self.bin_lib_subdirs() if not rpath_dirs: rpath_dirs = DEFAULT_BIN_LIB_SUBDIRS - self.log.info("Using default subdirectories for binaries/libraries to verify RPATH linking: %s", - rpath_dirs) + self.log.info(f"Using default subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}") else: - self.log.info("Using specified subdirectories for binaries/libraries to verify RPATH linking: %s", - rpath_dirs) + self.log.info(f"Using specified subdirs for binaries/libraries to verify RPATH linking: {rpath_dirs}") for dirpath in [os.path.join(self.installdir, d) for d in rpath_dirs]: if os.path.exists(dirpath): - self.log.debug("Sanity checking RPATH for files in %s", dirpath) + self.log.debug(f"Sanity checking RPATH for files in {dirpath}") for path in [os.path.join(dirpath, x) for x in os.listdir(dirpath)]: - self.log.debug("Sanity checking RPATH for %s", path) + self.log.debug(f"Sanity checking RPATH for {path}") out = get_linked_libs_raw(path) if out is None: - msg = "Failed to determine dynamically linked libraries for %s, " + msg = "Failed to determine dynamically linked libraries for {path}, " msg += "so skipping it in RPATH sanity check" - self.log.debug(msg, path) + self.log.debug(msg) else: # check whether all required libraries are found via 'ldd' matches = re.findall(not_found_regex, out) @@ -3132,34 +3166,34 @@ def sanity_check_rpath(self, rpath_dirs=None, check_readelf_rpath=True): # For each match, check if the library is in the exception list for match in matches: if match in filter_rpath_sanity_libs: - msg = "Library %s not found for %s, but ignored " - msg += "since it is on the rpath exception list: %s" - self.log.info(msg, match, path, filter_rpath_sanity_libs) + msg = f"Library {match} not found for {path}, but ignored " + msg += f"since it is on the rpath exception list: {filter_rpath_sanity_libs}" + self.log.info(msg) else: - fail_msg = "Library %s not found for %s" % (match, path) + fail_msg = f"Library {match} not found for {path}" self.log.warning(fail_msg) fails.append(fail_msg) else: - self.log.debug("Output of 'ldd %s' checked, looks OK", path) + self.log.debug(f"Output of 'ldd {path}' checked, looks OK") # check whether RPATH section in 'readelf -d' output is there if check_readelf_rpath: fail_msg = None - out, ec = run_cmd("readelf -d %s" % path, simple=False, trace=False) - if ec: - fail_msg = "Failed to run 'readelf %s': %s" % (path, out) - elif not readelf_rpath_regex.search(out): - fail_msg = "No '(RPATH)' found in 'readelf -d' output for %s: %s" % (path, out) + res = run_shell_cmd(f"readelf -d {path}", fail_on_error=False) + if res.exit_code: + fail_msg = f"Failed to run 'readelf -d {path}': {res.output}" + elif not readelf_rpath_regex.search(res.output): + fail_msg = f"No '(RPATH)' found in 'readelf -d' output for {path}: {out}" if fail_msg: self.log.warning(fail_msg) fails.append(fail_msg) else: - self.log.debug("Output of 'readelf -d %s' checked, looks OK", path) + self.log.debug(f"Output of 'readelf -d {path}' checked, looks OK") else: self.log.debug("Skipping the RPATH section check with 'readelf -d', as requested") else: - self.log.debug("Not sanity checking files in non-existing directory %s", dirpath) + self.log.debug(f"Not sanity checking files in non-existing directory {dirpath}") env.restore_env_vars(orig_env) @@ -3290,6 +3324,19 @@ def regex_for_lib(lib): return fail_msg + def sanity_check_mod_files(self): + """ + Check installation for Fortran .mod files + """ + self.log.debug(f"Checking for .mod files in install directory {self.installdir}...") + mod_files = glob.glob(os.path.join(self.installdir, '**', '*.mod'), recursive=True) + + fail_msg = None + if mod_files: + fail_msg = f"One or more .mod files found in {self.installdir}: " + ', '.join(mod_files) + + return fail_msg + def _sanity_check_step_common(self, custom_paths, custom_commands): """ Determine sanity check paths and commands to use. @@ -3588,19 +3635,20 @@ def xs2str(xs): change_dir(self.installdir) # run sanity check commands - for command in commands: + for cmd in commands: - trace_msg("running command '%s' ..." % command) + trace_msg(f"running command '{cmd}' ...") - out, ec = run_cmd(command, simple=False, log_ok=False, log_all=False, trace=False) - if ec != 0: - fail_msg = "sanity check command %s exited with code %s (output: %s)" % (command, ec, out) + res = run_shell_cmd(cmd, fail_on_error=False, hidden=True) + if res.exit_code != 0: + fail_msg = f"sanity check command {cmd} exited with code {res.exit_code} (output: {res.output})" self.sanity_check_fail_msgs.append(fail_msg) - self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) + self.log.warning(f"Sanity check: {fail_msg}") else: - self.log.info("sanity check command %s ran successfully! (output: %s)" % (command, out)) + self.log.info(f"sanity check command {cmd} ran successfully! (output: {res.output})") - trace_msg("result for command '%s': %s" % (command, ('FAILED', 'OK')[ec == 0])) + cmd_result_str = ('FAILED', 'OK')[res.exit_code == 0] + trace_msg(f"result for command '{cmd}': {cmd_result_str}") # also run sanity check for extensions (unless we are an extension ourselves) if not extension: @@ -3611,6 +3659,16 @@ def xs2str(xs): self.log.warning("Check for required/banned linked shared libraries failed!") self.sanity_check_fail_msgs.append(linked_shared_lib_fails) + # software installed with GCCcore toolchain should not have Fortran module files (.mod), + # unless that's explicitly allowed + if self.toolchain.name in ('GCCcore',) and not self.cfg['skip_mod_files_sanity_check']: + mod_files_found_msg = self.sanity_check_mod_files() + if mod_files_found_msg: + if build_option('fail_on_mod_files_gcccore'): + self.sanity_check_fail_msgs.append(mod_files_found_msg) + else: + print_warning(mod_files_found_msg) + # cleanup if self.fake_mod_data: self.clean_up_fake_module(self.fake_mod_data) @@ -3627,7 +3685,7 @@ def xs2str(xs): # pass or fail if self.sanity_check_fail_msgs: - raise EasyBuildError("Sanity check failed: %s", '\n'.join(self.sanity_check_fail_msgs)) + raise EasyBuildError("Sanity check failed: " + '\n'.join(self.sanity_check_fail_msgs)) else: self.log.debug("Sanity check passed!") @@ -3838,20 +3896,20 @@ def test_cases_step(self): for test in self.cfg['tests']: change_dir(self.orig_workdir) if os.path.isabs(test): - path = test + test_cmd = test else: for source_path in source_paths(): - path = os.path.join(source_path, self.name, test) - if os.path.exists(path): + test_cmd = os.path.join(source_path, self.name, test) + if os.path.exists(test_cmd): break - if not os.path.exists(path): - raise EasyBuildError("Test specifies invalid path: %s", path) + if not os.path.exists(test_cmd): + raise EasyBuildError(f"Test specifies invalid path: {test_cmd}") try: - self.log.debug("Running test %s" % path) - run_cmd(path, log_all=True, simple=True) + self.log.debug(f"Running test {test_cmd}") + run_shell_cmd(test_cmd) except EasyBuildError as err: - raise EasyBuildError("Running test %s failed: %s", path, err) + raise EasyBuildError(f"Running test {test_cmd} failed: {err}") def update_config_template_run_step(self): """Update the the easyconfig template dictionary with easyconfig.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP names""" @@ -4108,6 +4166,11 @@ def run_all_steps(self, run_test_cases): start_time = datetime.now() try: self.run_step(step_name, step_methods) + except RunShellCmdError as err: + err.print() + ec_path = os.path.basename(self.cfg.path) + error_msg = f"shell command '{err.cmd_name} ...' failed in {step_name} step for {ec_path}" + raise EasyBuildError(error_msg) finally: if not self.dry_run: step_duration = datetime.now() - start_time @@ -4165,6 +4228,10 @@ def build_and_install_one(ecdict, init_env): dry_run_msg('', silent=silent) print_msg("processing EasyBuild easyconfig %s" % spec, log=_log, silent=silent) + if ecdict['ec']['build_info_msg']: + msg = "This easyconfig provides the following build information:\n\n%s\n" + print_msg(msg % ecdict['ec']['build_info_msg'], log=_log, silent=silent) + if dry_run: # print note on interpreting dry run output (argument is reference to location of dry run messages) print_dry_run_note('below', silent=silent) @@ -4205,7 +4272,7 @@ def build_and_install_one(ecdict, init_env): app.cfg['skip'] = skip # build easyconfig - errormsg = '(no error)' + error_msg = '(no error)' # timing info start_time = time.time() try: @@ -4243,9 +4310,7 @@ def build_and_install_one(ecdict, init_env): adjust_permissions(app.installdir, stat.S_IWUSR, add=False, recursive=True) except EasyBuildError as err: - first_n = 300 - errormsg = "build failed (first %d chars): %s" % (first_n, err.msg[:first_n]) - _log.warning(errormsg) + error_msg = err.msg result = False ended = 'ended' @@ -4367,11 +4432,7 @@ def ensure_writable_log_dir(log_dir): # build failed success = False summary = 'FAILED' - - build_dir = '' - if app.builddir: - build_dir = " (build directory: %s)" % (app.builddir) - succ = "unsuccessfully%s: %s" % (build_dir, errormsg) + succ = "unsuccessfully: " + error_msg # cleanup logs app.close_log() @@ -4404,7 +4465,7 @@ def ensure_writable_log_dir(log_dir): del app - return (success, application_log, errormsg) + return (success, application_log, error_msg) def copy_easyblocks_for_reprod(easyblock_instances, reprod_dir): @@ -4627,8 +4688,14 @@ def inject_checksums(ecs, checksum_type): """ def make_list_lines(values, indent_level): """Make lines for list of values.""" + def to_str(s): + if isinstance(s, str): + return "'%s'" % s + else: + return str(s) + line_indent = INDENT_4SPACES * indent_level - return [line_indent + "'%s'," % x for x in values] + return [line_indent + to_str(x) + ',' for x in values] def make_checksum_lines(checksums, indent_level): """Make lines for list of checksums.""" diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index b37c3660f0..1b197ab912 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -130,6 +130,7 @@ 'sanity_check_paths': [{}, ("List of files and directories to check " "(format: {'files':, 'dirs':})"), BUILD], 'skip': [False, "Skip existing software", BUILD], + 'skip_mod_files_sanity_check': [False, "Skip the check for .mod files in a GCCcore level install", BUILD], 'skipsteps': [[], "Skip these steps", BUILD], 'source_urls': [[], "List of URLs for source files", BUILD], 'sources': [[], "List of source files", BUILD], @@ -189,10 +190,12 @@ 'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS], # MODULES easyconfig parameters + 'allow_append_abs_path': [False, "Allow specifying absolute paths to append in modextrapaths_append", MODULES], 'allow_prepend_abs_path': [False, "Allow specifying absolute paths to prepend in modextrapaths", MODULES], 'include_modpath_extensions': [True, "Include $MODULEPATH extensions specified by module naming scheme.", MODULES], 'modaliases': [{}, "Aliases to be defined in module file", MODULES], 'modextrapaths': [{}, "Extra paths to be prepended in module file", MODULES], + 'modextrapaths_append': [{}, "Extra paths to be appended in module file", MODULES], 'modextravars': [{}, "Extra environment variables to be added to module file", MODULES], 'modloadmsg': [{}, "Message that should be printed when generated module is loaded", MODULES], 'modunloadmsg': [{}, "Message that should be printed when generated module is unloaded", MODULES], @@ -228,6 +231,8 @@ 'buildstats': [None, "A list of dicts with build statistics", OTHER], 'deprecated': [False, "String specifying reason why this easyconfig file is deprecated " "and will be archived in the next major release of EasyBuild", OTHER], + 'build_info_msg': [None, "String with information to be printed to stdout and logged during the building " + "of the easyconfig", OTHER], } diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index b0704ca11a..6e0d2aa141 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -74,7 +74,7 @@ from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.module_naming_scheme.utilities import det_hidden_modname, is_valid_module_name -from easybuild.tools.modules import modules_tool +from easybuild.tools.modules import modules_tool, NoModulesTool from easybuild.tools.systemtools import check_os_dependency, pick_dep_version from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME, is_system_toolchain from easybuild.tools.toolchain.toolchain import TOOLCHAIN_CAPABILITIES, TOOLCHAIN_CAPABILITY_CUDA @@ -403,20 +403,6 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False): return toolchain_hierarchy -@contextmanager -def disable_templating(ec): - """Temporarily disable templating on the given EasyConfig - - Usage: - with disable_templating(ec): - # Do what you want without templating - # Templating set to previous value - """ - _log.deprecated("disable_templating(ec) was replaced by ec.disable_templating()", '5.0') - with ec.disable_templating() as old_value: - yield old_value - - class EasyConfig(object): """ Class which handles loading, reading, validation of easyconfigs @@ -1131,6 +1117,15 @@ def dependencies(self, build_only=False): return retained_deps + def dependency_names(self, build_only=False): + """ + Return a set of names of all (direct) dependencies after filtering. + Iterable builddependencies are flattened when not iterating. + + :param build_only: only return build dependencies, discard others + """ + return {dep['name'] for dep in self.dependencies(build_only=build_only) if dep['name']} + def builddependencies(self): """ Return a flat list of the parsed build dependencies @@ -1298,6 +1293,9 @@ def probe_external_module_metadata(self, mod_name, existing_metadata=None): :param existing_metadata: already available metadata for this external module (if any) """ res = {} + if isinstance(self.modules_tool, NoModulesTool): + self.log.debug('Ignoring request for external module data for %s as no modules tool is active', mod_name) + return res if existing_metadata is None: existing_metadata = {} @@ -1857,19 +1855,10 @@ def det_installversion(version, toolchain_name, toolchain_version, prefix, suffi _log.nosupport('Use det_full_ec_version from easybuild.tools.module_generator instead of %s' % old_fn, '2.0') -def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=None, **kwargs): +def get_easyblock_class(easyblock, name=None, error_on_failed_import=True, error_on_missing_easyblock=True, **kwargs): """ Get class for a particular easyblock (or use default) """ - if 'default_fallback' in kwargs: - msg = "Named argument 'default_fallback' for get_easyblock_class is deprecated, " - msg += "use 'error_on_missing_easyblock' instead" - _log.deprecated(msg, '4.0') - if error_on_missing_easyblock is None: - error_on_missing_easyblock = kwargs['default_fallback'] - elif error_on_missing_easyblock is None: - error_on_missing_easyblock = True - cls = None try: if easyblock: diff --git a/easybuild/framework/easyconfig/format/format.py b/easybuild/framework/easyconfig/format/format.py index e1c2dac2d0..da8bb97c89 100644 --- a/easybuild/framework/easyconfig/format/format.py +++ b/easybuild/framework/easyconfig/format/format.py @@ -72,7 +72,7 @@ ] LAST_PARAMS = ['exts_default_options', 'exts_list', 'sanity_check_paths', 'sanity_check_commands', - 'modextrapaths', 'modextravars', + 'modextrapaths', 'modextrapaths_append', 'modextravars', 'moduleclass'] SANITY_CHECK_PATHS_DIRS = 'dirs' @@ -369,7 +369,7 @@ def parse(self, configobj): for key, value in self.supported.items(): if key not in self.VERSION_OPERATOR_VALUE_TYPES: raise EasyBuildError('Unsupported key %s in %s section', key, self.SECTION_MARKER_SUPPORTED) - self.sections['%s' % key] = value + self.sections[key] = value for key, supported_key, fn_name in [('version', 'versions', 'get_version_str'), ('toolchain', 'toolchains', 'as_dict')]: diff --git a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py index b9fd31bf92..37e14b529e 100644 --- a/easybuild/framework/easyconfig/format/pyheaderconfigobj.py +++ b/easybuild/framework/easyconfig/format/pyheaderconfigobj.py @@ -233,12 +233,13 @@ def pyheader_env(self): current_builtins = globals()['__builtins__'] builtins = {} for name in self.PYHEADER_ALLOWED_BUILTINS: - if hasattr(current_builtins, name): + try: builtins[name] = getattr(current_builtins, name) - elif isinstance(current_builtins, dict) and name in current_builtins: - builtins[name] = current_builtins[name] - else: - self.log.warning('No builtin %s found.' % name) + except AttributeError: + if isinstance(current_builtins, dict) and name in current_builtins: + builtins[name] = current_builtins[name] + else: + self.log.warning('No builtin %s found.' % name) global_vars['__builtins__'] = builtins self.log.debug("Available builtins: %s" % global_vars['__builtins__']) diff --git a/easybuild/framework/easyconfig/templates.py b/easybuild/framework/easyconfig/templates.py index 728e5fcf52..0d8a4d5cf3 100644 --- a/easybuild/framework/easyconfig/templates.py +++ b/easybuild/framework/easyconfig/templates.py @@ -76,9 +76,9 @@ ('installdir', "Installation directory"), ('start_dir', "Directory in which the build process begins"), ] -# software names for which to define ver and shortver templates +# software names for which to define ver, majver and shortver templates TEMPLATE_SOFTWARE_VERSIONS = [ - # software name, prefix for *ver and *shortver + # software name, prefix for *ver, *majver and *shortver ('CUDA', 'cuda'), ('CUDAcore', 'cuda'), ('Java', 'java'), @@ -89,12 +89,17 @@ # template values which are only generated dynamically TEMPLATE_NAMES_DYNAMIC = [ ('arch', "System architecture (e.g. x86_64, aarch64, ppc64le, ...)"), + ('sysroot', "Location root directory of system, prefix for standard paths like /usr/lib and /usr/include" + "as specify by the --sysroot configuration option"), ('mpi_cmd_prefix', "Prefix command for running MPI programs (with default number of ranks)"), ('cuda_compute_capabilities', "Comma-separated list of CUDA compute capabilities, as specified via " "--cuda-compute-capabilities configuration option or via cuda_compute_capabilities easyconfig parameter"), ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"), ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"), ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"), + ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"), + ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"), + ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"), ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"), ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"), ] @@ -181,13 +186,10 @@ # versionmajor, versionminor, versionmajorminor (eg '.'.join(version.split('.')[:2])) ) -def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None): +def template_constant_dict(config, ignore=None, toolchain=None): """Create a dict for templating the values in the easyconfigs. - config is a dict with the structure of EasyConfig._config """ - if skip_lower is not None: - _log.deprecated("Use of 'skip_lower' named argument for template_constant_dict has no effect anymore", '4.0') - # TODO find better name # ignore if ignore is None: @@ -200,6 +202,9 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) # set 'arch' for system architecture based on 'machine' (4th) element of platform.uname() return value template_values['arch'] = platform.uname()[4] + # set 'sysroot' template based on 'sysroot' configuration option, using empty string as fallback + template_values['sysroot'] = build_option('sysroot') or '' + # step 1: add TEMPLATE_NAMES_EASYCONFIG for name in TEMPLATE_NAMES_EASYCONFIG: if name in ignore: @@ -363,6 +368,10 @@ def template_constant_dict(config, ignore=None, skip_lower=None, toolchain=None) template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities) template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities) template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities) + int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities] + template_values['cuda_int_comma_sep'] = ','.join(int_values) + template_values['cuda_int_space_sep'] = ' '.join(int_values) + template_values['cuda_int_semicolon_sep'] = ';'.join(int_values) sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities] template_values['cuda_sm_comma_sep'] = ','.join(sm_values) template_values['cuda_sm_space_sep'] = ' '.join(sm_values) @@ -426,6 +435,7 @@ def template_documentation(): # step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS doc.append("Template names/values for (short) software versions") for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (indent_l1, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name)) diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index ce22c81f62..d5c482fecd 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -465,7 +465,7 @@ def find_related_easyconfigs(path, ec): if len(parsed_version) >= 2: version_patterns.append(r'%s\.%s\.\w+' % tuple(parsed_version[:2])) # major/minor version match if parsed_version != parsed_version[0]: - version_patterns.append(r'%s\.[\d-]+\.\w+' % parsed_version[0]) # major version match + version_patterns.append(r'%s\.[\d-]+(\.\w+)*' % parsed_version[0]) # major version match version_patterns.append(r'[\w.]+') # any version regexes = [] @@ -694,7 +694,9 @@ def check_sha256_checksums(ecs, whitelist=None): continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) - checksum_issues.extend(eb_class(ec).check_checksums()) + eb = eb_class(ec) + checksum_issues.extend(eb.check_checksums()) + eb.close_log() return checksum_issues diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index b812f54cad..c27c8f8728 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -90,15 +90,7 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): tweaked_ecs_path, tweaked_ecs_deps_path = None, None if targetdirs is not None: tweaked_ecs_path, tweaked_ecs_deps_path = targetdirs - # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) - toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) - if len(toolchains) > 1: - raise EasyBuildError("Multiple toolchains featured in easyconfigs, --try-X not supported in that case: %s", - toolchains) - # Toolchain is unique, let's store it - source_toolchain = easyconfigs[-1]['ec']['toolchain'] modifying_toolchains_or_deps = False - target_toolchain = {} src_to_dst_tc_mapping = {} revert_to_regex = False @@ -116,6 +108,16 @@ def tweak(easyconfigs, build_specs, modtool, targetdirs=None): revert_to_regex = True if not revert_to_regex: + # make sure easyconfigs all feature the same toolchain (otherwise we *will* run into trouble) + toolchains = nub(['%(name)s/%(version)s' % ec['ec']['toolchain'] for ec in easyconfigs]) + if len(toolchains) > 1: + raise EasyBuildError("Multiple toolchains featured in easyconfigs, " + "--try-X not supported in that case: %s", + toolchains) + # Toolchain is unique, let's store it + source_toolchain = easyconfigs[-1]['ec']['toolchain'] + target_toolchain = {} + # we're doing something that involves the toolchain hierarchy; # obtain full dependency graph for specified easyconfigs; # easyconfigs will be ordered 'top-to-bottom' (toolchains and dependencies appearing first) @@ -321,7 +323,7 @@ def __repr__(self): newval = "%s + %s" % (fval, res.group('val')) _log.debug("Prepending %s to %s" % (fval, key)) else: - newval = "%s" % fval + newval = str(fval) _log.debug("Overwriting %s with %s" % (key, fval)) ectxt = regexp.sub("%s = %s" % (res.group('key'), newval), ectxt) _log.info("Tweaked %s list to '%s'" % (key, newval)) diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 8dc4669392..9f099eb74c 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -42,7 +42,7 @@ from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict from easybuild.tools.build_log import EasyBuildError, raise_nosupport from easybuild.tools.filetools import change_dir -from easybuild.tools.run import check_async_cmd, run_cmd +from easybuild.tools.run import run_shell_cmd def resolve_exts_filter_template(exts_filter, ext): @@ -150,12 +150,7 @@ def __init__(self, mself, ext, extra_params=None): self.sanity_check_module_loaded = False self.fake_mod_data = None - self.async_cmd_info = None - self.async_cmd_output = None - self.async_cmd_check_cnt = None - # initial read size should be relatively small, - # to avoid hanging for a long time until desired output is available in async_cmd_check - self.async_cmd_read_size = 1024 + self.async_cmd_task = None @property def name(self): @@ -195,44 +190,6 @@ def postrun(self): """ self.master.run_post_install_commands(commands=self.cfg.get('postinstallcmds', [])) - def async_cmd_start(self, cmd, inp=None): - """ - Start installation asynchronously using specified command. - """ - self.async_cmd_output = '' - self.async_cmd_check_cnt = 0 - self.async_cmd_info = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False, asynchronous=True) - - def async_cmd_check(self): - """ - Check progress of installation command that was started asynchronously. - - :return: True if command completed, False otherwise - """ - if self.async_cmd_info is None: - raise EasyBuildError("No installation command running asynchronously for %s", self.name) - elif self.async_cmd_info is False: - self.log.info("No asynchronous command was started for extension %s", self.name) - return True - else: - self.log.debug("Checking on installation of extension %s...", self.name) - # use small read size, to avoid waiting for a long time until sufficient output is produced - res = check_async_cmd(*self.async_cmd_info, output_read_size=self.async_cmd_read_size) - self.async_cmd_output += res['output'] - if res['done']: - self.log.info("Installation of extension %s completed!", self.name) - self.async_cmd_info = None - else: - self.async_cmd_check_cnt += 1 - self.log.debug("Installation of extension %s still running (checked %d times)", - self.name, self.async_cmd_check_cnt) - # increase read size after sufficient checks, - # to avoid that installation hangs due to output buffer filling up... - if self.async_cmd_check_cnt % 10 == 0 and self.async_cmd_read_size < (1024 ** 2): - self.async_cmd_read_size *= 2 - - return res['done'] - @property def required_deps(self): """Return list of required dependencies for this extension.""" @@ -273,15 +230,14 @@ def sanity_check_step(self): self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name) elif exts_filter: cmd, stdin = resolve_exts_filter_template(exts_filter, self) - # set log_ok to False so we can catch the error instead of run_cmd - (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin) + cmd_res = run_shell_cmd(cmd, fail_on_error=False, stdin=stdin) - if ec: + if cmd_res.exit_code: if stdin: fail_msg = 'command "%s" (stdin: "%s") failed' % (cmd, stdin) else: fail_msg = 'command "%s" failed' % cmd - fail_msg += "; output:\n%s" % output.strip() + fail_msg += "; output:\n%s" % cmd_res.output.strip() self.log.warning("Sanity check for '%s' extension failed: %s", self.name, fail_msg) res = (False, fail_msg) # keep track of all reasons of failure diff --git a/easybuild/main.py b/easybuild/main.py index 6e373f9fb0..9272c7abef 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -42,6 +42,7 @@ import os import stat import sys +import tempfile import traceback # IMPORTANT this has to be the first easybuild import as it customises the logging @@ -69,7 +70,8 @@ from easybuild.tools.github import add_pr_labels, install_github_token, list_prs, merge_pr, new_branch_github, new_pr from easybuild.tools.github import new_pr_from_branch from easybuild.tools.github import sync_branch_with_develop, sync_pr_with_develop, update_branch, update_pr -from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, load_hooks, run_hook +from easybuild.tools.hooks import BUILD_AND_INSTALL_LOOP, PRE_PREF, POST_PREF, START, END, CANCEL, CRASH, FAIL +from easybuild.tools.hooks import load_hooks, run_hook from easybuild.tools.modules import modules_tool from easybuild.tools.options import opts_dict_to_eb_opts, set_up_configuration, use_color from easybuild.tools.output import COLOR_GREEN, COLOR_RED, STATUS_BAR, colorize, print_checks, rich_live_cm @@ -147,11 +149,11 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): # keep track of success/total count if ec_res['success']: - test_msg = "Successfully built %s" % ec['spec'] + test_msg = "Successfully installed %s" % ec['spec'] else: - test_msg = "Build of %s failed" % ec['spec'] + test_msg = "Installation of %s failed" % os.path.basename(ec['spec']) if 'err' in ec_res: - test_msg += " (err: %s)" % ec_res['err'] + test_msg += ": %s" % ec_res['err'] # dump test report next to log file test_report_txt = create_test_report(test_msg, [(ec, ec_res)], init_session_state) @@ -167,8 +169,8 @@ def build_and_install_software(ecs, init_session_state, exit_on_failure=True): adjust_permissions(parent_dir, stat.S_IWUSR, add=False, recursive=False) if not ec_res['success'] and exit_on_failure: - if 'traceback' in ec_res: - raise EasyBuildError(ec_res['traceback']) + if not isinstance(ec_res['err'], EasyBuildError): + raise ec_res['err'] else: raise EasyBuildError(test_msg) @@ -252,8 +254,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state easyconfig._easyconfigs_cache.clear() easyconfig._easyconfig_files_cache.clear() - # restore environment + # restore environment and reset tempdir (to avoid tmpdir path getting progressively longer) restore_env(init_env) + tempfile.tempdir = None # If EasyConfig specific arguments were supplied in EasyStack file # merge arguments with original command line args @@ -329,11 +332,23 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session categorized_paths = categorize_files_by_type(eb_args) + set_pr_options = [opt for opt in ( + 'new_branch_github', + 'new_pr', + 'new_pr_from_branch', + 'preview_pr', + 'sync_branch_with_develop', + 'sync_pr_with_develop', + 'update_branch_github', + 'update_pr', + ) if getattr(options, opt) + ] + any_pr_option_set = len(set_pr_options) > 0 + if len(set_pr_options) > 1: + raise EasyBuildError("The following options are set but incompatible: %s.\nYou can only use one at a time!", + ', '.join(['--' + opt.replace('_', '-') for opt in set_pr_options])) # command line options that do not require any easyconfigs to be specified - pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr - pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop - pr_options = pr_options or options.update_branch_github or options.update_pr - no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] + no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) @@ -423,9 +438,10 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules - keep_available_modules = forced or dry_run_mode or options.extended_dry_run or pr_options or options.copy_ec - keep_available_modules = keep_available_modules or options.inject_checksums or options.sanity_check_only - keep_available_modules = keep_available_modules or options.inject_checksums_to_json + keep_available_modules = any(( + forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums, + options.sanity_check_only, options.inject_checksums_to_json) + ) # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not keep_available_modules: @@ -444,12 +460,12 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled - if options.robot and (not dry_run_mode or pr_options): + if options.robot and (not dry_run_mode or any_pr_option_set): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs - elif pr_options: + elif any_pr_option_set: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) @@ -468,7 +484,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return True # creating/updating PRs - if pr_options: + if any_pr_option_set: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: @@ -578,30 +594,20 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session return overall_success -def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): +def main(args=None, logfile=None, do_build=None, testing=False, modtool=None, prepared_cfg_data=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode + :param prepared_cfg_data: prepared configuration data for main function, as returned by prepare_main (or None) """ + if prepared_cfg_data is None or any([args, logfile, testing]): + init_session_state, eb_go, cfg_settings = prepare_main(args=args, logfile=logfile, testing=testing) + else: + init_session_state, eb_go, cfg_settings = prepared_cfg_data - register_lock_cleanup_signal_handlers() - - # if $CDPATH is set, unset it, it'll only cause trouble... - # see https://github.com/easybuilders/easybuild-framework/issues/2944 - if 'CDPATH' in os.environ: - del os.environ['CDPATH'] - - # When EB is run via `exec` the special bash variable $_ is not set - # So emulate this here to allow (module) scripts depending on that to work - if '_' not in os.environ: - os.environ['_'] = sys.executable - - # purposely session state very early, to avoid modules loaded by EasyBuild meddling in - init_session_state = session_state() - eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args global _log @@ -727,10 +733,49 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): cleanup(logfile, eb_tmpdir, testing, silent=False) -if __name__ == "__main__": +def prepare_main(args=None, logfile=None, testing=None): + """ + Prepare for calling main function by setting up the EasyBuild configuration + :param args: command line arguments to take into account when parsing the EasyBuild configuration settings + :param logfile: log file to use + :param testing: enable testing mode + :return: 3-tuple with initial session state data, EasyBuildOptions instance, and tuple with configuration settings + """ + register_lock_cleanup_signal_handlers() + + # if $CDPATH is set, unset it, it'll only cause trouble... + # see https://github.com/easybuilders/easybuild-framework/issues/2944 + if 'CDPATH' in os.environ: + del os.environ['CDPATH'] + + # When EB is run via `exec` the special bash variable $_ is not set + # So emulate this here to allow (module) scripts depending on that to work + if '_' not in os.environ: + os.environ['_'] = sys.executable + + # purposely session state very early, to avoid modules loaded by EasyBuild meddling in + init_session_state = session_state() + eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) + + return init_session_state, eb_go, cfg_settings + + +def main_with_hooks(args=None): + init_session_state, eb_go, cfg_settings = prepare_main(args=args) + hooks = load_hooks(eb_go.options.hooks) try: - main() + main(args=args, prepared_cfg_data=(init_session_state, eb_go, cfg_settings)) except EasyBuildError as err: - print_error(err.msg) + run_hook(FAIL, hooks, args=[err]) + print_error(err.msg, exit_on_error=True, exit_code=1) except KeyboardInterrupt as err: + run_hook(CANCEL, hooks, args=[err]) print_error("Cancelled by user: %s" % err) + except Exception as err: + run_hook(CRASH, hooks, args=[err]) + sys.stderr.write("EasyBuild crashed! Please consider reporting a bug, this should not happen...\n\n") + raise + + +if __name__ == "__main__": + main_with_hooks() diff --git a/easybuild/scripts/findPythonDeps.py b/easybuild/scripts/findPythonDeps.py index 663926a9d6..2d59d77bc0 100755 --- a/easybuild/scripts/findPythonDeps.py +++ b/easybuild/scripts/findPythonDeps.py @@ -48,23 +48,25 @@ def can_run(cmd, argument): return False -def run_cmd(arguments, action_desc, capture_stderr=True, **kwargs): +def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs): """Run the command and return the return code and output""" extra_args = kwargs or {} if sys.version_info[0] >= 3: extra_args['universal_newlines'] = True stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args) - out, _ = p.communicate() + out, err = p.communicate() if p.returncode != 0: - raise RuntimeError('Failed to %s: %s' % (action_desc, out)) + if err: + err = "\nSTDERR:\n" + err + raise RuntimeError('Failed to %s: %s%s' % (action_desc, out, err)) return out def run_in_venv(cmd, venv_path, action_desc): """Run the given command in the virtualenv at the given path""" cmd = 'source %s/bin/activate && %s' % (venv_path, cmd) - return run_cmd(cmd, action_desc, shell=True, executable='/bin/bash') + return run_shell_cmd(cmd, action_desc, shell=True, executable='/bin/bash') def get_dep_tree(package_spec, verbose): @@ -76,7 +78,7 @@ def get_dep_tree(package_spec, verbose): venv_dir = os.path.join(tmp_dir, 'venv') if verbose: print('Creating virtualenv at ' + venv_dir) - run_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') + run_shell_cmd(['virtualenv', '--system-site-packages', venv_dir], action_desc='create virtualenv') if verbose: print('Updating pip in virtualenv') run_in_venv('pip install --upgrade pip', venv_dir, action_desc='update pip') @@ -94,10 +96,13 @@ def get_dep_tree(package_spec, verbose): def find_deps(pkgs, dep_tree): """Recursively resolve dependencies of the given package(s) and return them""" res = [] - for pkg in pkgs: - pkg = canonicalize_name(pkg) + for orig_pkg in pkgs: + pkg = canonicalize_name(orig_pkg) matching_entries = [entry for entry in dep_tree if pkg in (entry['package']['package_name'], entry['package']['key'])] + if not matching_entries: + matching_entries = [entry for entry in dep_tree + if orig_pkg in (entry['package']['package_name'], entry['package']['key'])] if not matching_entries: raise RuntimeError("Found no installed package for '%s' in %s" % (pkg, dep_tree)) if len(matching_entries) > 1: @@ -167,24 +172,26 @@ def print_deps(package, verbose): sys.exit(1) if args.verbose: print('Checking with EasyBuild for missing dependencies') - missing_dep_out = run_cmd(['eb', args.ec, '--missing'], - capture_stderr=False, - action_desc='Get missing dependencies' - ) + missing_dep_out = run_shell_cmd(['eb', args.ec, '--missing'], + capture_stderr=False, + action_desc='Get missing dependencies') + excluded_dep = '(%s)' % os.path.basename(args.ec) missing_deps = [dep for dep in missing_dep_out.split('\n') - if dep.startswith('*') and '(%s)' % args.ec not in dep + if dep.startswith('*') and excluded_dep not in dep ] if missing_deps: print('You need to install all modules on which %s depends first!' % args.ec) print('\n\t'.join(['Missing:'] + missing_deps)) sys.exit(1) + # If the --ec argument is a (relative) existing path make it absolute so we can find it after the chdir + ec_arg = os.path.abspath(args.ec) if os.path.exists(args.ec) else args.ec with temporary_directory() as tmp_dir: old_dir = os.getcwd() os.chdir(tmp_dir) if args.verbose: print('Running EasyBuild to get build environment') - run_cmd(['eb', args.ec, '--dump-env', '--force'], action_desc='Dump build environment') + run_shell_cmd(['eb', ec_arg, '--dump-env', '--force'], action_desc='Dump build environment') os.chdir(old_dir) cmd = "source %s/*.env && python %s '%s'" % (tmp_dir, sys.argv[0], args.package) @@ -192,7 +199,7 @@ def print_deps(package, verbose): cmd += ' --verbose' print('Restarting script in new build environment') - out = run_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') + out = run_shell_cmd(cmd, action_desc='Run in new environment', shell=True, executable='/bin/bash') print(out) else: if not can_run('virtualenv', '--version'): diff --git a/easybuild/scripts/install-EasyBuild-develop.sh b/easybuild/scripts/install-EasyBuild-develop.sh index 4181d8c42a..f88ae280a0 100755 --- a/easybuild/scripts/install-EasyBuild-develop.sh +++ b/easybuild/scripts/install-EasyBuild-develop.sh @@ -14,7 +14,7 @@ print_usage() echo echo " github_username: username on GitHub for which the EasyBuild repositories should be cloned" echo - echo " install_dir: directory were all the EasyBuild files will be installed" + echo " install_dir: directory where all the EasyBuild files will be installed" echo } @@ -79,7 +79,7 @@ EOF # Check for 'help' argument -if [ "$1" = "-h" -o "$1" = "--help" ] ; then +if [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then print_usage exit 0 fi @@ -116,13 +116,14 @@ github_clone_branch "easybuild" "develop" EB_DEVEL_MODULE_NAME="EasyBuild-develop" MODULES_INSTALL_DIR=${INSTALL_DIR}/modules EB_DEVEL_MODULE="${MODULES_INSTALL_DIR}/${EB_DEVEL_MODULE_NAME}" -mkdir -p ${MODULES_INSTALL_DIR} +mkdir -p "${MODULES_INSTALL_DIR}" print_devel_module > "${EB_DEVEL_MODULE}" -echo +echo echo "=== Run 'module use ${MODULES_INSTALL_DIR}' and 'module load ${EB_DEVEL_MODULE_NAME}' to use your development version of EasyBuild." echo "=== (you can append ${MODULES_INSTALL_DIR} to your MODULEPATH to make this module always available for loading)" echo echo "=== To update each repository, run 'git pull origin' in each subdirectory of ${INSTALL_DIR}" +echo "=== Or run $(dirname "$0")/update-EasyBuild-develop.sh '${INSTALL_DIR}'" echo exit 0 diff --git a/easybuild/scripts/mk_tmpl_easyblock_for.py b/easybuild/scripts/mk_tmpl_easyblock_for.py index c5c7c0110a..1cd63393d6 100755 --- a/easybuild/scripts/mk_tmpl_easyblock_for.py +++ b/easybuild/scripts/mk_tmpl_easyblock_for.py @@ -122,7 +122,7 @@ import easybuild.tools.toolchain as toolchain %(parent_import)s from easybuild.framework.easyconfig import CUSTOM, MANDATORY -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class %(class_name)s(%(parent)s): @@ -150,7 +150,7 @@ def configure_step(self): env.setvar('CUSTOM_ENV_VAR', 'foo') cmd = "configure command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) # complete configuration with configure_method of parent super(%(class_name)s, self).configure_step() @@ -167,20 +167,20 @@ def build_step(self): # enable parallel build par = self.cfg['parallel'] cmd = "build command --parallel %%d --compiler-family %%s" %% (par, comp_fam) - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def test_step(self): \"\"\"Custom built-in test procedure for %(name)s.\"\"\" if self.cfg['runtest']: cmd = "test-command" - run_cmd(cmd, simple=True, log_all=True, log_ok=True) + run_shell_cmd(cmd) def install_step(self): \"\"\"Custom install procedure for %(name)s.\"\"\" cmd = "install command" - run_cmd(cmd, log_all=True, simple=True, log_ok=True) + run_shell_cmd(cmd) def sanity_check_step(self): \"\"\"Custom sanity check for %(name)s.\"\"\" diff --git a/easybuild/scripts/rpath_wrapper_template.sh.in b/easybuild/scripts/rpath_wrapper_template.sh.in index 55c3388c5b..73eb21f0e0 100644 --- a/easybuild/scripts/rpath_wrapper_template.sh.in +++ b/easybuild/scripts/rpath_wrapper_template.sh.in @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2016-2023 Ghent University # diff --git a/easybuild/scripts/update-EasyBuild-develop.sh b/easybuild/scripts/update-EasyBuild-develop.sh new file mode 100755 index 0000000000..c80289ad96 --- /dev/null +++ b/easybuild/scripts/update-EasyBuild-develop.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +# Stop in case of error +set -e + +# Print script help +print_usage() +{ + echo "Checkout develop branch of all EasyBuild repositories" + echo "and pull changes from the remote repository." + echo "To be used with the EasyBuild-develop module or a set git-working-dirs-path" + echo "Usage: $0 []" + echo + echo " git_dir: directory where all the EasyBuild repositories are installed." + echo " Automatically detected if not specified." + echo +} + +if [[ "$1" = "-h" ]] || [[ "$1" = "--help" ]]; then + print_usage + exit 0 +fi + +if [[ $# -gt 1 ]] ; then + echo "Error: invalid arguments" + echo + print_usage + exit 1 +fi + +if [[ $# -eq 1 ]]; then + git_dir=$1 +else + # Auto detect git_dir + git_dir="" + if ! which eb &> /dev/null; then + module load EasyBuild-develop || module load EasyBuild || true + if ! which eb &> /dev/null; then + echo 'Found neither the `eb` command nor a working module.' + echo 'Please specify the git_dir!' + exit 1 + fi + fi + if out=$(eb --show-config | grep -F 'git-working-dirs-path'); then + path=$(echo "$out" | awk '{print $NF}') + if [[ -n "$path" ]] && [[ -d "$path" ]]; then + git_dir=$path + echo "Using git_dir from git-working-dirs-path: $git_dir" + fi + fi + if [[ -z "$git_dir" ]]; then + eb_dir=$(dirname "$(which eb)") + if [[ "$(basename "$eb_dir")" == "easybuild-framework" ]] && [[ -d "$eb_dir/.git" ]]; then + git_dir=$(dirname "$eb_dir") + echo "Using git_dir from eb command: $git_dir" + else + echo 'Please specify the git_dir as auto-detection failed!' + exit 1 + fi + fi +fi + +cd "$git_dir" + +for folder in easybuild easybuild-framework easybuild-easyblocks easybuild-easyconfigs; do + echo # A newline + if [[ -d "$folder" ]]; then + echo "========= Checking ${folder} =========" + else + echo "========= Skipping non-existent ${folder} =========" + fi + cd "$folder" + git checkout "develop" + if git remote | grep -qF github_easybuilders; then + git pull "github_easybuilders" + else + git pull + fi + cd .. +done + +index_file="$git_dir/easybuild-easyconfigs/easybuild/easyconfigs/.eb-path-index" +if [[ -f "$index_file" ]]; then + echo -n "Trying to remove index from ${index_file}..." + if rm "$index_file"; then + echo "Done!" + echo "Recreate with 'eb --create-index \"$(dirname "$index_file")\"'" + else + echo "Failed!" + fi +fi diff --git a/easybuild/toolchains/compiler/gcc.py b/easybuild/toolchains/compiler/gcc.py index 84a612b132..9ef98f5038 100644 --- a/easybuild/toolchains/compiler/gcc.py +++ b/easybuild/toolchains/compiler/gcc.py @@ -50,7 +50,7 @@ class Gcc(Compiler): COMPILER_FAMILY = TC_CONSTANT_GCC COMPILER_UNIQUE_OPTS = { - 'loop': (False, "Automatic loop parallellisation"), + 'loop': (False, "Automatic loop parallelisation"), 'f2c': (False, "Generate code compatible with f2c and f77"), 'lto': (False, "Enable Link Time Optimization"), } diff --git a/easybuild/toolchains/compiler/intel_compilers.py b/easybuild/toolchains/compiler/intel_compilers.py index ef537d9315..ae97dfa87d 100644 --- a/easybuild/toolchains/compiler/intel_compilers.py +++ b/easybuild/toolchains/compiler/intel_compilers.py @@ -109,8 +109,9 @@ def set_variables(self): self.options.options_map['loose'] = ['fp-model fast'] # fp-model fast=2 gives "warning: overriding '-ffp-model=fast=2' option with '-ffp-model=fast'" self.options.options_map['veryloose'] = ['fp-model fast'] - # recommended in porting guide - self.options.options_map['openmp'] = ['fiopenmp'] + # recommended in porting guide: qopenmp, unlike fiopenmp, works for both classic and oneapi compilers + # https://www.intel.com/content/www/us/en/developer/articles/guide/porting-guide-for-ifort-to-ifx.html + self.options.options_map['openmp'] = ['qopenmp'] # -xSSE2 is not supported by Intel oneAPI compilers, # so use -march=x86-64 -mtune=generic when using optarch=GENERIC diff --git a/easybuild/toolchains/ifbf.py b/easybuild/toolchains/ifbf.py new file mode 100644 index 0000000000..3507e1eab9 --- /dev/null +++ b/easybuild/toolchains/ifbf.py @@ -0,0 +1,44 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for ifbf toolchain (includes Intel compilers, FlexiBLAS, and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.intel_compilers import IntelCompilersToolchain +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS + + +class Ifbf(IntelCompilersToolchain, FlexiBLAS, Fftw): + """ + Compiler toolchain with Intel compilers, FlexiBLAS, and FFTW + """ + NAME = 'ifbf' + SUBTOOLCHAIN = IntelCompilersToolchain.NAME + OPTIONAL = True diff --git a/easybuild/toolchains/iofbf.py b/easybuild/toolchains/iofbf.py new file mode 100644 index 0000000000..3410ffaf9f --- /dev/null +++ b/easybuild/toolchains/iofbf.py @@ -0,0 +1,47 @@ +## +# Copyright 2012-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for iofbf toolchain (includes Intel compilers, OpenMPI, +FlexiBLAS, LAPACK, ScaLAPACK and FFTW). + +Authors: + +* Sebastian Achilles (Juelich Supercomputing Centre) +""" + +from easybuild.toolchains.iompi import Iompi +from easybuild.toolchains.ifbf import Ifbf +from easybuild.toolchains.fft.fftw import Fftw +from easybuild.toolchains.linalg.flexiblas import FlexiBLAS +from easybuild.toolchains.linalg.scalapack import ScaLAPACK + + +class Iofbf(Iompi, FlexiBLAS, ScaLAPACK, Fftw): + """ + Compiler toolchain with Intel compilers (icc/ifort), OpenMPI, + FlexiBLAS, LAPACK, ScaLAPACK and FFTW. + """ + NAME = 'iofbf' + SUBTOOLCHAIN = [Iompi.NAME, Ifbf.NAME] diff --git a/easybuild/toolchains/linalg/flexiblas.py b/easybuild/toolchains/linalg/flexiblas.py index 43b450b3de..3726e7444f 100644 --- a/easybuild/toolchains/linalg/flexiblas.py +++ b/easybuild/toolchains/linalg/flexiblas.py @@ -34,7 +34,7 @@ from easybuild.tools.toolchain.linalg import LinAlg -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -48,11 +48,11 @@ def det_flexiblas_backend_libs(): # System-wide (config directory): # OPENBLAS # library = libflexiblas_openblas.so - out, _ = run_cmd("flexiblas list", simple=False, trace=False) + res = run_shell_cmd("flexiblas list", hidden=True) shlib_ext = get_shared_lib_ext() flexiblas_lib_regex = re.compile(r'library = (?Plib.*\.%s)' % shlib_ext, re.M) - flexiblas_libs = flexiblas_lib_regex.findall(out) + flexiblas_libs = flexiblas_lib_regex.findall(res.output) backend_libs = [] for flexiblas_lib in flexiblas_libs: @@ -70,6 +70,7 @@ class FlexiBLAS(LinAlg): """ BLAS_MODULE_NAME = ['FlexiBLAS'] BLAS_LIB = ['flexiblas'] + BLAS_LIB_MT = ['flexiblas'] BLAS_INCLUDE_DIR = [os.path.join('include', 'flexiblas')] BLAS_FAMILY = TC_CONSTANT_FLEXIBLAS diff --git a/easybuild/tools/build_log.py b/easybuild/tools/build_log.py index 215b8d38aa..f8529fff57 100644 --- a/easybuild/tools/build_log.py +++ b/easybuild/tools/build_log.py @@ -167,7 +167,7 @@ def nosupport(self, msg, ver): def error(self, msg, *args, **kwargs): """Print error message and raise an EasyBuildError.""" - ebmsg = "EasyBuild crashed with an error %s: " % self.caller_info() + ebmsg = "EasyBuild encountered an error %s: " % self.caller_info() fancylogger.FancyLogger.error(self, ebmsg + msg, *args, **kwargs) def devel(self, msg, *args, **kwargs): @@ -335,8 +335,18 @@ def print_error(msg, *args, **kwargs): if args: msg = msg % args + # grab exit code, if specified; + # also consider deprecated 'exitCode' option + exitCode = kwargs.pop('exitCode', None) + exit_code = kwargs.pop('exit_code', exitCode) + if exitCode is not None: + _init_easybuildlog.deprecated("'exitCode' option in print_error function is replaced with 'exit_code'", '6.0') + + # use 1 as defaut exit code + if exit_code is None: + exit_code = 1 + log = kwargs.pop('log', None) - exitCode = kwargs.pop('exitCode', 1) opt_parser = kwargs.pop('opt_parser', None) exit_on_error = kwargs.pop('exit_on_error', True) silent = kwargs.pop('silent', False) @@ -348,7 +358,7 @@ def print_error(msg, *args, **kwargs): if opt_parser: opt_parser.print_shorthelp() sys.stderr.write("ERROR: %s\n" % msg) - sys.exit(exitCode) + sys.exit(exit_code) elif log is not None: raise EasyBuildError(msg) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index e2f4dfff8d..ecd503ba1a 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -240,6 +240,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'job_polling_interval', 'job_target_resource', 'locks_dir', + 'module_cache_suffix', 'modules_footer', 'modules_header', 'mpi_cmd_template', @@ -277,6 +278,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'enforce_checksums', 'experimental', 'extended_dry_run', + 'fail_on_mod_files_gcccore', 'force', 'generate_devel_module', 'group_writable_installdir', @@ -300,6 +302,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'sequential', 'set_default_module', 'set_gid_bit', + 'silence_hook_trigger', 'skip_extensions', 'skip_test_cases', 'skip_test_step', diff --git a/easybuild/tools/configobj.py b/easybuild/tools/configobj.py index 8163726280..00381abfcc 100644 --- a/easybuild/tools/configobj.py +++ b/easybuild/tools/configobj.py @@ -1252,7 +1252,7 @@ def set_section(in_section, this_section): self.configspec = None return - elif getattr(infile, 'read', MISSING) is not MISSING: + elif hasattr(infile, 'read'): # This supports file like objects infile = infile.read() or [] # needs splitting into lines - but needs doing *after* decoding diff --git a/easybuild/tools/containers/apptainer.py b/easybuild/tools/containers/apptainer.py index 67a6db5bc3..8a294ce1ad 100644 --- a/easybuild/tools/containers/apptainer.py +++ b/easybuild/tools/containers/apptainer.py @@ -35,7 +35,7 @@ from easybuild.tools.config import CONT_IMAGE_FORMAT_SIF, CONT_IMAGE_FORMAT_SQUASHFS from easybuild.tools.config import build_option, container_path from easybuild.tools.filetools import remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class ApptainerContainer(SingularityContainer): @@ -48,15 +48,15 @@ class ApptainerContainer(SingularityContainer): def apptainer_version(): """Get Apptainer version.""" version_cmd = "apptainer --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Apptainer version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Apptainer version: {res.output}") - return res.group(0) + return regex_res.group(0) def build_image(self, recipe_path): """Build container image by calling out to 'sudo apptainer build'.""" @@ -108,5 +108,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, apptainer, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Apptainer image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index aa37a90873..5a1597d634 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -36,7 +36,7 @@ from easybuild.tools.containers.utils import det_os_deps from easybuild.tools.filetools import remove_dir from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd DOCKER_TMPL_HEADER = """\ @@ -164,7 +164,7 @@ def build_image(self, dockerfile): docker_cmd = ' '.join(['sudo', 'docker', 'build', '-f', dockerfile, '-t', container_name, '.']) print_msg("Running '%s', you may need to enter your 'sudo' password..." % docker_cmd) - run_cmd(docker_cmd, path=tempdir, stream_output=True) + run_shell_cmd(docker_cmd, work_dir=tempdir, stream_output=True) print_msg("Docker image created at %s" % container_name, log=self.log) remove_dir(tempdir) diff --git a/easybuild/tools/containers/singularity.py b/easybuild/tools/containers/singularity.py index 41ec9829c7..638a985ae5 100644 --- a/easybuild/tools/containers/singularity.py +++ b/easybuild/tools/containers/singularity.py @@ -40,7 +40,7 @@ from easybuild.tools.config import build_option, container_path from easybuild.tools.containers.base import ContainerGenerator from easybuild.tools.filetools import read_file, remove_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd ARCH = 'arch' # Arch Linux @@ -162,15 +162,15 @@ class SingularityContainer(ContainerGenerator): def singularity_version(): """Get Singularity version.""" version_cmd = "singularity --version" - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '%s': %s for tool {1} with output: {2}" % (version_cmd, out)) + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}': {res.output}") - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing Singularity version: %s" % out) + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing Singularity version: {res.output}") - return res.group(0) + return regex_res.group(0) def resolve_template(self): """Return template container recipe.""" @@ -403,5 +403,5 @@ def build_image(self, recipe_path): cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) - run_cmd(cmd, stream_output=True) + run_shell_cmd(cmd, stream_output=True) print_msg("Singularity image created at %s" % img_path, log=self.log) diff --git a/easybuild/tools/containers/utils.py b/easybuild/tools/containers/utils.py index e01a117427..d543ca77fc 100644 --- a/easybuild/tools/containers/utils.py +++ b/easybuild/tools/containers/utils.py @@ -36,7 +36,7 @@ from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def det_os_deps(easyconfigs): @@ -70,20 +70,23 @@ def check_tool(tool_name, min_tool_version=None): if not tool_path: return False - print_msg("{0} tool found at {1}".format(tool_name, tool_path)) + print_msg(f"{tool_name} tool found at {tool_path}") if not min_tool_version: return True - version_cmd = "{0} --version".format(tool_name) - out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) - if ec: - raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out)) - res = re.search(r"\d+\.\d+(\.\d+)?", out.strip()) - if not res: - raise EasyBuildError("Error parsing version for tool {0}".format(tool_name)) - tool_version = res.group(0) + version_cmd = f"{tool_name} --version" + res = run_shell_cmd(version_cmd, hidden=True, in_dry_run=True) + if res.exit_code: + raise EasyBuildError(f"Error running '{version_cmd}' for tool {tool_name} with output: {res.output}") + + regex_res = re.search(r"\d+\.\d+(\.\d+)?", res.output.strip()) + if not regex_res: + raise EasyBuildError(f"Error parsing version for tool {tool_name}") + + tool_version = regex_res.group(0) version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version) if version_ok: - print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version)) + print_msg(f"{tool_name} version '{tool_version}' is {min_tool_version} or higher ... OK") + return version_ok diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index cf43f66bad..5a41090166 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -38,6 +38,7 @@ """ import copy import inspect +import json import os from collections import OrderedDict from easybuild.tools import LooseVersion @@ -73,6 +74,7 @@ DETAILED = 'detailed' SIMPLE = 'simple' +FORMAT_JSON = 'json' FORMAT_MD = 'md' FORMAT_RST = 'rst' FORMAT_TXT = 'txt' @@ -116,6 +118,11 @@ def avail_cfgfile_constants(go_cfg_constants, output_format=FORMAT_TXT): return generate_doc('avail_cfgfile_constants_%s' % output_format, [go_cfg_constants]) +def avail_cfgfile_constants_json(go_cfg_constants): + """Generate documentation on constants for configuration files in json format""" + raise NotImplementedError("JSON output format not supported for avail_cfgfile_constants_json") + + def avail_cfgfile_constants_txt(go_cfg_constants): """Generate documentation on constants for configuration files in txt format""" doc = [ @@ -185,6 +192,11 @@ def avail_easyconfig_constants(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_constants_%s' % output_format, []) +def avail_easyconfig_constants_json(): + """Generate easyconfig constant documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_constants_json") + + def avail_easyconfig_constants_txt(): """Generate easyconfig constant documentation in txt format""" doc = ["Constants that can be used in easyconfigs"] @@ -243,6 +255,11 @@ def avail_easyconfig_licenses(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_licenses_%s' % output_format, []) +def avail_easyconfig_licenses_json(): + """Generate easyconfig license documentation in json format""" + raise NotImplementedError("JSON output format not supported for avail_easyconfig_licenses_json") + + def avail_easyconfig_licenses_txt(): """Generate easyconfig license documentation in txt format""" doc = ["License constants that can be used in easyconfigs"] @@ -290,7 +307,7 @@ def avail_easyconfig_licenses_md(): lics = sorted(EASYCONFIG_LICENSES_DICT.items()) table_values = [ ["``%s``" % lic().name for _, lic in lics], - ["%s" % lic().description for _, lic in lics], + [lic().description or '' for _, lic in lics], ["``%s``" % lic().version for _, lic in lics], ] @@ -355,6 +372,13 @@ def avail_easyconfig_params_rst(title, grouped_params): return '\n'.join(doc) +def avail_easyconfig_params_json(): + """ + Compose overview of available easyconfig parameters, in json format. + """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_params_json") + + def avail_easyconfig_params_txt(title, grouped_params): """ Compose overview of available easyconfig parameters, in plain text format. @@ -427,6 +451,11 @@ def avail_easyconfig_templates(output_format=FORMAT_TXT): return generate_doc('avail_easyconfig_templates_%s' % output_format, []) +def avail_easyconfig_templates_json(): + """ Returns template documentation in json text format """ + raise NotImplementedError("JSON output format not supported for avail_easyconfig_templates") + + def avail_easyconfig_templates_txt(): """ Returns template documentation in plain text format """ # This has to reflect the methods/steps used in easyconfig _generate_template_values @@ -441,6 +470,7 @@ def avail_easyconfig_templates_txt(): # step 2: add SOFTWARE_VERSIONS doc.append('Template names/values for (short) software versions') for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sshortver)s: short version for %s (.)" % (INDENT_4SPACES, pref, name)) doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name)) doc.append('') @@ -495,8 +525,10 @@ def avail_easyconfig_templates_rst(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (.)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] @@ -558,8 +590,10 @@ def avail_easyconfig_templates_md(): ver = [] ver_desc = [] for name, pref in TEMPLATE_SOFTWARE_VERSIONS: + ver.append('``%%(%smajver)s``' % pref) ver.append('``%%(%sshortver)s``' % pref) ver.append('``%%(%sver)s``' % pref) + ver_desc.append('major version for %s' % name) ver_desc.append('short version for %s (``.``)' % name) ver_desc.append('full version for %s' % name) table_values = [ver, ver_desc] @@ -636,6 +670,8 @@ def avail_classes_tree(classes, class_names, locations, detailed, format_strings def list_easyblocks(list_easyblocks=SIMPLE, output_format=FORMAT_TXT): + if output_format == FORMAT_JSON: + raise NotImplementedError("JSON output format not supported for list_easyblocks") format_strings = { FORMAT_MD: { 'det_root_templ': "- **%s** (%s%s)", @@ -1020,6 +1056,38 @@ def list_software_txt(software, detailed=False): return '\n'.join(lines) +def list_software_json(software, detailed=False): + """ + Return overview of supported software in json + + :param software: software information (strucuted like list_software does) + :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info) + :return: multi-line string presenting requested info + """ + lines = ['['] + for key in sorted(software, key=lambda x: x.lower()): + for entry in software[key]: + if detailed: + # deep copy here to avoid modifying the original dict + entry = copy.deepcopy(entry) + entry['description'] = ' '.join(entry['description'].split('\n')).strip() + else: + entry = {} + entry['name'] = key + + lines.append(json.dumps(entry, indent=4, sort_keys=True, separators=(',', ': ')) + ",") + if not detailed: + break + + # remove trailing comma on last line + if len(lines) > 1: + lines[-1] = lines[-1].rstrip(',') + + lines.append(']') + + return '\n'.join(lines) + + def list_toolchains(output_format=FORMAT_TXT): """Show list of known toolchains.""" _, all_tcs = search_toolchain('') @@ -1168,6 +1236,11 @@ def list_toolchains_txt(tcs): return '\n'.join(doc) +def list_toolchains_json(tcs): + """ Returns overview of all toolchains in json format """ + raise NotImplementedError("JSON output not implemented yet for --list-toolchains") + + def avail_toolchain_opts(name, output_format=FORMAT_TXT): """Show list of known options for given toolchain.""" tc_class, _ = search_toolchain(name) @@ -1177,10 +1250,9 @@ def avail_toolchain_opts(name, output_format=FORMAT_TXT): tc_dict = {} for cst in ['COMPILER_SHARED_OPTS', 'COMPILER_UNIQUE_OPTS', 'MPI_SHARED_OPTS', 'MPI_UNIQUE_OPTS']: - if hasattr(tc, cst): - opts = getattr(tc, cst) - if opts is not None: - tc_dict.update(opts) + opts = getattr(tc, cst, None) + if opts is not None: + tc_dict.update(opts) return generate_doc('avail_toolchain_opts_%s' % output_format, [name, tc_dict]) @@ -1194,7 +1266,7 @@ def avail_toolchain_opts_md(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] @@ -1212,7 +1284,7 @@ def avail_toolchain_opts_rst(name, tc_dict): tc_items = sorted(tc_dict.items()) table_values = [ ['``%s``' % val[0] for val in tc_items], - ['%s' % val[1][1] for val in tc_items], + [val[1][1] for val in tc_items], ['``%s``' % val[1][0] for val in tc_items], ] @@ -1221,6 +1293,11 @@ def avail_toolchain_opts_rst(name, tc_dict): return '\n'.join(doc) +def avail_toolchain_opts_json(name, tc_dict): + """ Returns overview of toolchain options in jsonformat """ + raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts") + + def avail_toolchain_opts_txt(name, tc_dict): """ Returns overview of toolchain options in txt format """ doc = ["Available options for %s toolchain:" % name] @@ -1247,6 +1324,13 @@ def get_easyblock_classes(package_name): return easyblocks +def gen_easyblocks_overview_json(package_name, path_to_examples, common_params=None, doc_functions=None): + """ + Compose overview of all easyblocks in the given package in json format + """ + raise NotImplementedError("JSON output not implemented yet for gen_easyblocks_overview") + + def gen_easyblocks_overview_md(package_name, path_to_examples, common_params=None, doc_functions=None): """ Compose overview of all easyblocks in the given package in MarkDown format diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 7a431c8ee5..470e347e88 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -58,15 +58,15 @@ import zlib from functools import partial from html.parser import HTMLParser -from importlib.util import spec_from_file_location, module_from_spec import urllib.request as std_urllib from easybuild.base import fancylogger -from easybuild.tools import run # import build_log must stay, to use of EasyBuildLog from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning from easybuild.tools.config import ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN, build_option, install_path from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ONE, start_progress_bar, stop_progress_bar, update_progress_bar +from easybuild.tools.hooks import load_source +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import natural_keys, nub, remove_unwanted_chars, trace_msg try: @@ -445,7 +445,7 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced """ if not os.path.isfile(fn) and not build_option('extended_dry_run'): - raise EasyBuildError("Can't extract file %s: no such file", fn) + raise EasyBuildError(f"Can't extract file {fn}: no such file") mkdir(dest, parents=True) @@ -453,24 +453,24 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced abs_dest = os.path.abspath(dest) # change working directory - _log.debug("Unpacking %s in directory %s", fn, abs_dest) + _log.debug(f"Unpacking {fn} in directory {abs_dest}") cwd = change_dir(abs_dest) if cmd: # complete command template with filename cmd = cmd % fn - _log.debug("Using specified command to unpack %s: %s", fn, cmd) + _log.debug("Using specified command to unpack {fn}: {cmd}") else: cmd = extract_cmd(fn, overwrite=overwrite) - _log.debug("Using command derived from file extension to unpack %s: %s", fn, cmd) + _log.debug("Using command derived from file extension to unpack {fn}: {cmd}") if not cmd: - raise EasyBuildError("Can't extract file %s with unknown filetype", fn) + raise EasyBuildError("Can't extract file {fn} with unknown filetype") if extra_options: - cmd = "%s %s" % (cmd, extra_options) + cmd = f"{cmd} {extra_options}" - run.run_cmd(cmd, simple=True, force_in_dry_run=forced, trace=trace) + run_shell_cmd(cmd, in_dry_run=forced, hidden=not trace) # note: find_base_dir also changes into the base dir! base_dir = find_base_dir() @@ -479,14 +479,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced # change back to where we came from (unless that was a non-existing directory) if not change_into_dir: if cwd is None: - raise EasyBuildError("Can't change back to non-existing directory after extracting %s in %s", fn, dest) + raise EasyBuildError(f"Can't change back to non-existing directory after extracting {fn} in {dest}") else: change_dir(cwd) return base_dir -def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, on_error=None): +def which(cmd, retain_all=False, check_perms=True, log_ok=True, on_error=WARN): """ Return (first) path in $PATH for specified command, or None if command is not found @@ -495,17 +495,6 @@ def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=None, :param log_ok: Log an info message where the command has been found (if any) :param on_error: What to do if the command was not found, default: WARN. Possible values: IGNORE, WARN, ERROR """ - if log_error is not None: - _log.deprecated("'log_error' named argument in which function has been replaced by 'on_error'", '5.0') - # If set, make sure on_error is at least WARN - if log_error and on_error == IGNORE: - on_error = WARN - elif not log_error and on_error is None: # If set to False, use IGNORE unless on_error is also set - on_error = IGNORE - # Set default - # TODO: After removal of log_error from the parameters, on_error=WARN can be used instead of this - if on_error is None: - on_error = WARN if on_error not in (IGNORE, WARN, ERROR): raise EasyBuildError("Invalid value for 'on_error': %s", on_error) @@ -1263,14 +1252,11 @@ def verify_checksum(path, checksums): for checksum in checksums: if isinstance(checksum, dict): - if filename in checksum: + try: # Set this to a string-type checksum checksum = checksum[filename] - elif build_option('enforce_checksums'): - raise EasyBuildError("Missing checksum for %s", filename) - else: - # Set to None and allow to fail elsewhere - checksum = None + except KeyError: + raise EasyBuildError("Missing checksum for %s in %s", filename, checksum) if isinstance(checksum, str): # if no checksum type is specified, it is assumed to be MD5 (32 characters) or SHA256 (64 characters) @@ -1300,7 +1286,8 @@ def verify_checksum(path, checksums): # no matching checksums return False else: - raise EasyBuildError("Invalid checksum spec '%s', should be a string (MD5) or 2-tuple (type, value).", + raise EasyBuildError("Invalid checksum spec '%s': should be a string (MD5 or SHA256), " + "2-tuple (type, value), or tuple of alternative checksum specs.", checksum) actual_checksum = compute_checksum(path, typ) @@ -1547,27 +1534,27 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False if build_option('extended_dry_run'): # skip checking of files in dry run mode patch_filename = os.path.basename(patch_file) - dry_run_msg("* applying patch file %s" % patch_filename, silent=build_option('silent')) + dry_run_msg(f"* applying patch file {patch_filename}", silent=build_option('silent')) elif not os.path.isfile(patch_file): - raise EasyBuildError("Can't find patch %s: no such file", patch_file) + raise EasyBuildError(f"Can't find patch {patch_file}: no such file") elif fn and not os.path.isfile(fn): - raise EasyBuildError("Can't patch file %s: no such file", fn) + raise EasyBuildError(f"Can't patch file {fn}: no such file") # copy missing files if copy: if build_option('extended_dry_run'): - dry_run_msg(" %s copied to %s" % (patch_file, dest), silent=build_option('silent')) + dry_run_msg(f" {patch_file} copied to {dest}", silent=build_option('silent')) else: copy_file(patch_file, dest) - _log.debug("Copied patch %s to dir %s" % (patch_file, dest)) + _log.debug(f"Copied patch {patch_file} to dir {dest}") # early exit, work is done after copying return True elif not os.path.isdir(dest): - raise EasyBuildError("Can't patch directory %s: no such directory", dest) + raise EasyBuildError(f"Can't patch directory {dest}: no such directory") # use absolute paths abs_patch_file = os.path.abspath(patch_file) @@ -1582,14 +1569,14 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False patch_subextension = os.path.splitext(patch_stem)[1] if patch_subextension == ".patch": workdir = tempfile.mkdtemp(prefix='eb-patch-') - _log.debug("Extracting the patch to: %s", workdir) + _log.debug(f"Extracting the patch to: {workdir}") # extracting the patch extracted_dir = extract_file(abs_patch_file, workdir, change_into_dir=False) abs_patch_file = os.path.join(extracted_dir, patch_stem) if use_git: verbose = '--verbose ' if build_option('debug') else '' - patch_cmd = "git apply %s%s" % (verbose, abs_patch_file) + patch_cmd = f"git apply {verbose}{abs_patch_file}" else: if level is None and build_option('extended_dry_run'): level = '' @@ -1602,27 +1589,30 @@ def apply_patch(patch_file, dest, fn=None, copy=False, level=None, use_git=False patched_files = det_patched_files(path=abs_patch_file) if not patched_files: - raise EasyBuildError("Can't guess patchlevel from patch %s: no testfile line found in patch", - abs_patch_file) + msg = f"Can't guess patchlevel from patch {abs_patch_file}: no testfile line found in patch" + raise EasyBuildError(msg) level = guess_patch_level(patched_files, abs_dest) if level is None: # level can also be 0 (zero), so don't use "not level" # no match - raise EasyBuildError("Can't determine patch level for patch %s from directory %s", patch_file, abs_dest) + raise EasyBuildError(f"Can't determine patch level for patch {patch_file} from directory {abs_dest}") else: - _log.debug("Guessed patch level %d for patch %s" % (level, patch_file)) + _log.debug(f"Guessed patch level {level} for patch {patch_file}") else: - _log.debug("Using specified patch level %d for patch %s" % (level, patch_file)) + _log.debug(f"Using specified patch level {level} for patch {patch_file}") backup_option = '-b ' if build_option('backup_patched_files') else '' - patch_cmd = "patch " + backup_option + "-p%s -i %s" % (level, abs_patch_file) + patch_cmd = f"patch {backup_option} -p{level} -i {abs_patch_file}" - out, ec = run.run_cmd(patch_cmd, simple=False, path=abs_dest, log_ok=False, trace=False) + res = run_shell_cmd(patch_cmd, fail_on_error=False, hidden=True, work_dir=abs_dest) + + if res.exit_code: + msg = f"Couldn't apply patch file {patch_file}. " + msg += f"Process exited with code {res.exit_code}: {res.output}" + raise EasyBuildError(msg) - if ec: - raise EasyBuildError("Couldn't apply patch file %s. Process exited with code %s: %s", patch_file, ec, out) return True @@ -1731,7 +1721,7 @@ def convert_name(name, upper=False): def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False, onlydirs=False, recursive=True, - group_id=None, relative=True, ignore_errors=False, skip_symlinks=None): + group_id=None, relative=True, ignore_errors=False): """ Change permissions for specified path, using specified permission bits @@ -1748,11 +1738,6 @@ def adjust_permissions(provided_path, permission_bits, add=True, onlyfiles=False and directories (if onlyfiles is False) in path """ - if skip_symlinks is not None: - depr_msg = "Use of 'skip_symlinks' argument for 'adjust_permissions' is deprecated " - depr_msg += "(symlinks are never followed anymore)" - _log.deprecated(depr_msg, '4.0') - provided_path = os.path.abspath(provided_path) if recursive: @@ -1912,7 +1897,7 @@ def mkdir(path, parents=False, set_gid=None, sticky=None): # climb up until we hit an existing path or the empty string (for relative paths) while existing_parent_path and not os.path.exists(existing_parent_path): existing_parent_path = os.path.dirname(existing_parent_path) - os.makedirs(path) + os.makedirs(path, exist_ok=True) else: os.mkdir(path) except FileExistsError as err: @@ -2088,13 +2073,6 @@ def path_matches(path, paths): return False -def rmtree2(path, n=3): - """Wrapper around shutil.rmtree to make it more robust when used on NFS mounted file systems.""" - - _log.deprecated("Use 'remove_dir' rather than 'rmtree2'", '5.0') - remove_dir(path) - - def find_backup_name_candidate(src_file): """Returns a non-existing file to be used as destination for backup files""" @@ -2148,7 +2126,7 @@ def move_logs(src_logfile, target_logfile): try: # there may be multiple log files, due to log rotation - app_logs = glob.glob('%s*' % src_logfile) + app_logs = glob.glob(f'{src_logfile}*') for app_log in app_logs: # retain possible suffix new_log_path = target_logfile + app_log[src_logfile_len:] @@ -2159,11 +2137,11 @@ def move_logs(src_logfile, target_logfile): # move log to target path move_file(app_log, new_log_path) - _log.info("Moved log file %s to %s" % (src_logfile, new_log_path)) + _log.info(f"Moved log file {src_logfile} to {new_log_path}") if zip_log_cmd: - run.run_cmd("%s %s" % (zip_log_cmd, new_log_path)) - _log.info("Zipped log %s using '%s'", new_log_path, zip_log_cmd) + run_shell_cmd(f"{zip_log_cmd} {new_log_path}") + _log.info(f"Zipped log {new_log_path} using '{zip_log_cmd}'") except (IOError, OSError) as err: raise EasyBuildError("Failed to move log file(s) %s* to new log file %s*: %s", @@ -2201,11 +2179,6 @@ def cleanup(logfile, tempdir, testing, silent=False): print_msg(msg, log=None, silent=testing or silent) -def copytree(src, dst, symlinks=False, ignore=None): - """DEPRECATED and removed. Use copy_dir""" - _log.deprecated("Use 'copy_dir' rather than 'copytree'", '4.0') - - def encode_string(name): """ This encoding function handles funky software names ad infinitum, like: @@ -2253,21 +2226,6 @@ def decode_class_name(name): return decode_string(name) -def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None): - """NO LONGER SUPPORTED: use run_cmd from easybuild.tools.run instead""" - _log.nosupport("run_cmd was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def run_cmd_qa(cmd, qa, no_qa=None, log_ok=True, log_all=False, simple=False, regexp=True, std_qa=None, path=None): - """NO LONGER SUPPORTED: use run_cmd_qa from easybuild.tools.run instead""" - _log.nosupport("run_cmd_qa was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - -def parse_log_for_error(txt, regExp=None, stdout=True, msg=None): - """NO LONGER SUPPORTED: use parse_log_for_error from easybuild.tools.run instead""" - _log.nosupport("parse_log_for_error was moved from easybuild.tools.filetools to easybuild.tools.run", '2.0') - - def det_size(path): """ Determine total size of given filepath (in bytes). @@ -2582,17 +2540,17 @@ def copy(paths, target_path, force_in_dry_run=False, **kwargs): raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path) -def get_source_tarball_from_git(filename, targetdir, git_config): +def get_source_tarball_from_git(filename, target_dir, git_config): """ Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it :param filename: name of the archive to save the code to (must be .tar.gz) - :param targetdir: target directory where to save the archive to + :param target_dir: target directory where to save the archive to :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit """ # sanity check on git_config value being passed if not isinstance(git_config, dict): - raise EasyBuildError("Found unexpected type of value for 'git_config' argument: %s" % type(git_config)) + raise EasyBuildError("Found unexpected type of value for 'git_config' argument: {type(git_config)}") # Making a copy to avoid modifying the object with pops git_config = git_config.copy() @@ -2603,10 +2561,12 @@ def get_source_tarball_from_git(filename, targetdir, git_config): recursive = git_config.pop('recursive', False) clone_into = git_config.pop('clone_into', False) keep_git_dir = git_config.pop('keep_git_dir', False) + extra_config_params = git_config.pop('extra_config_params', None) + recurse_submodules = git_config.pop('recurse_submodules', None) # input validation of git_config dict if git_config: - raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: %s", git_config) + raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: {git_config}") if not repo_name: raise EasyBuildError("repo_name not specified in git_config parameter") @@ -2624,11 +2584,14 @@ def get_source_tarball_from_git(filename, targetdir, git_config): raise EasyBuildError("git_config currently only supports filename ending in .tar.gz") # prepare target directory and clone repository - mkdir(targetdir, parents=True) - targetpath = os.path.join(targetdir, filename) + mkdir(target_dir, parents=True) # compose 'git clone' command, and run it - clone_cmd = ['git', 'clone'] + if extra_config_params: + git_cmd = 'git ' + ' '.join(['-c %s' % param for param in extra_config_params]) + else: + git_cmd = 'git' + clone_cmd = [git_cmd, 'clone'] if not keep_git_dir and not commit: # Speed up cloning by only fetching the most recent commit, not the whole history @@ -2639,18 +2602,20 @@ def get_source_tarball_from_git(filename, targetdir, git_config): clone_cmd.extend(['--branch', tag]) if recursive: clone_cmd.append('--recursive') + if recurse_submodules: + clone_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) else: # checkout is done separately below for specific commits clone_cmd.append('--no-checkout') - clone_cmd.append('%s/%s.git' % (url, repo_name)) + clone_cmd.append(f'{url}/{repo_name}.git') if clone_into: - clone_cmd.append('%s' % clone_into) + clone_cmd.append(clone_into) tmpdir = tempfile.mkdtemp() - cwd = change_dir(tmpdir) - run.run_cmd(' '.join(clone_cmd), log_all=True, simple=True, regexp=False, trace=False) + + run_shell_cmd(' '.join(clone_cmd), hidden=True, verbose_dry_run=True, work_dir=tmpdir) # If the clone is done into a specified name, change repo_name if clone_into: @@ -2658,54 +2623,78 @@ def get_source_tarball_from_git(filename, targetdir, git_config): # if a specific commit is asked for, check it out if commit: - checkout_cmd = ['git', 'checkout', commit] - if recursive: - checkout_cmd.extend(['&&', 'git', 'submodule', 'update', '--init', '--recursive']) + checkout_cmd = [git_cmd, 'checkout', commit] - run.run_cmd(' '.join(checkout_cmd), log_all=True, simple=True, regexp=False, path=repo_name, trace=False) + if recursive or recurse_submodules: + checkout_cmd.extend(['&&', git_cmd, 'submodule', 'update', '--init']) + if recursive: + checkout_cmd.append('--recursive') + if recurse_submodules: + checkout_cmd.extend(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) + + work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir + run_shell_cmd(' '.join(checkout_cmd), work_dir=work_dir, hidden=True, verbose_dry_run=True) elif not build_option('extended_dry_run'): # If we wanted to get a tag make sure we actually got a tag and not a branch with the same name # This doesn't make sense in dry-run mode as we don't have anything to check - cmd = 'git describe --exact-match --tags HEAD' - # Note: Disable logging to also disable the error handling in run_cmd - (out, ec) = run.run_cmd(cmd, log_ok=False, log_all=False, regexp=False, path=repo_name, trace=False) - if ec != 0 or tag not in out.splitlines(): - print_warning('Tag %s was not downloaded in the first try due to %s/%s containing a branch' - ' with the same name. You might want to alert the maintainers of %s about that issue.', - tag, url, repo_name, repo_name) + cmd = f"{git_cmd} describe --exact-match --tags HEAD" + work_dir = os.path.join(tmpdir, repo_name) if repo_name else tmpdir + res = run_shell_cmd(cmd, fail_on_error=False, work_dir=work_dir, hidden=True, verbose_dry_run=True) + + if res.exit_code != 0 or tag not in res.output.splitlines(): + msg = f"Tag {tag} was not downloaded in the first try due to {url}/{repo_name} containing a branch" + msg += f" with the same name. You might want to alert the maintainers of {repo_name} about that issue." + print_warning(msg) + cmds = [] if not keep_git_dir: # make the repo unshallow first; # this is equivalent with 'git fetch -unshallow' in Git 1.8.3+ # (first fetch seems to do nothing, unclear why) - cmds.append('git fetch --depth=2147483647 && git fetch --depth=2147483647') + cmds.append(f"{git_cmd} fetch --depth=2147483647 && git fetch --depth=2147483647") - cmds.append('git checkout refs/tags/' + tag) + cmds.append(f"{git_cmd} checkout refs/tags/{tag}") # Clean all untracked files, e.g. from left-over submodules - cmds.append('git clean --force -d -x') + cmds.append(f"{git_cmd} clean --force -d -x") if recursive: - cmds.append('git submodule update --init --recursive') + cmds.append(f"{git_cmd} submodule update --init --recursive") + elif recurse_submodules: + cmds.append(f"{git_cmd} submodule update --init ") + cmds[-1] += ' '.join(["--recurse-submodules='%s'" % pat for pat in recurse_submodules]) + for cmd in cmds: - run.run_cmd(cmd, log_all=True, simple=True, regexp=False, path=repo_name, trace=False) - - # When CentOS 7 is phased out and tar>1.28 is everywhere, replace find-sort-pipe with tar-flag - # '--sort=name' and place LC_ALL in front of tar. Also remove flags --null, --no-recursion, and - # --files-from - from the flags to tar. See https://reproducible-builds.org/docs/archives/ - tar_cmd = ['find', repo_name, '-print0', '-path \'*/.git\' -prune' if not keep_git_dir else '', '|', - 'LC_ALL=C', 'sort', '--zero-terminated', '|', - 'GZIP=--no-name', 'tar', '--create', '--file', targetpath, '--no-recursion', - '--gzip', '--mtime="1970-01-01 00:00Z"', '--owner=0', '--group=0', - '--numeric-owner', '--format=gnu', '--null', - '--no-recursion', '--files-from -'] - run.run_cmd(' '.join(tar_cmd), log_all=True, simple=True, regexp=False, trace=False) + run_shell_cmd(cmd, work_dir=work_dir, hidden=True, verbose_dry_run=True) + + # Create archive + archive_path = os.path.join(target_dir, filename) + + if keep_git_dir: + # create archive of git repo including .git directory + tar_cmd = ['tar', 'cfvz', archive_path, repo_name] + else: + # create reproducible archive + # see https://reproducible-builds.org/docs/archives/ + tar_cmd = [ + # print names of all files and folders excluding .git directory + 'find', repo_name, '-name ".git"', '-prune', '-o', '-print0', + # reset access and modification timestamps + '-exec', 'touch', '-t 197001010100', '{}', '\;', '|', + # sort file list + 'LC_ALL=C', 'sort', '--zero-terminated', '|', + # create tarball in GNU format with ownership reset + 'tar', '--create', '--no-recursion', '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', + '--null', '--files-from', '-', '|', + # compress tarball with gzip without original file name and timestamp + 'gzip', '--no-name', '>', archive_path + ] + run_shell_cmd(' '.join(tar_cmd), work_dir=tmpdir, hidden=True, verbose_dry_run=True) # cleanup (repo_name dir does not exist in dry run mode) - change_dir(cwd) remove(tmpdir) - return targetpath + return archive_path def move_file(path, target_path, force_in_dry_run=False): @@ -2785,14 +2774,6 @@ def install_fake_vsc(): return fake_vsc_path -def load_source(filename, path): - """Load file as Python module""" - spec = spec_from_file_location(filename, path) - module = module_from_spec(spec) - spec.loader.exec_module(module) - return module - - def get_easyblock_class_name(path): """Make sure file is an easyblock and get easyblock class name""" fn = os.path.basename(path).split('.')[0] diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 6fff6c2e53..0d9c85baf3 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -720,9 +720,9 @@ def setup_repo_from(git_repo, github_url, target_account, branch_name, silent=Fa raise EasyBuildError("Fetching branch '%s' from remote %s failed: empty result", branch_name, origin) # git checkout -b ; git pull - if hasattr(origin.refs, branch_name): + try: origin_branch = getattr(origin.refs, branch_name) - else: + except AttributeError: raise EasyBuildError("Branch '%s' not found at %s", branch_name, github_url) _log.debug("Checking out branch '%s' from remote %s", branch_name, github_url) @@ -2009,17 +2009,18 @@ def check_github(): github_user = build_option('github_user') github_account = build_option('github_org') or build_option('github_user') - if github_user is None: - check_res = "(none available) => FAIL" - status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False - else: + if github_user: check_res = "%s => OK" % github_user + else: + check_res = "%s => FAIL" % ('(none available)' if github_user is None else '(empty)') + status['--new-pr'] = status['--update-pr'] = status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check GitHub token print_msg("* GitHub token...", log=_log, prefix=False, newline=False) github_token = fetch_github_token(github_user) + github_token_valid = False if github_token is None: check_res = "(no token found) => FAIL" else: @@ -2028,6 +2029,7 @@ def check_github(): token_descr = partial_token + " (len: %d)" % len(github_token) if validate_github_token(github_token, github_user): check_res = "%s => OK (validated)" % token_descr + github_token_valid = True else: check_res = "%s => FAIL (validation failed)" % token_descr @@ -2120,7 +2122,7 @@ def check_github(): try: getattr(git_repo.remotes, remote_name).push(branch_name, delete=True) except GitCommandError as err: - sys.stderr.write("WARNING: failed to delete test branch from GitHub: %s\n" % err) + print_warning("failed to delete test branch from GitHub: %s" % err, log=_log) # test creating a gist print_msg("* creating gists...", log=_log, prefix=False, newline=False) @@ -2138,17 +2140,33 @@ def check_github(): if gist_url and re.match('https://gist.github.com/%s/[0-9a-f]+$' % github_user, gist_url): check_res = "OK" - else: + elif not github_user: + check_res = "FAIL (no GitHub user specified)" + elif not github_token: + check_res = "FAIL (missing github token)" + elif not github_token_valid: + check_res = "FAIL (invalid github token)" + elif gist_url: check_res = "FAIL (gist_url: %s)" % gist_url - status['--upload-test-report'] = False + else: + check_res = "FAIL" + if 'FAIL' in check_res: + status['--upload-test-report'] = False print_msg(check_res, log=_log, prefix=False) # check whether location to local working directories for Git repositories is available (not strictly needed) print_msg("* location to Git working dirs... ", log=_log, prefix=False, newline=False) git_working_dirs_path = build_option('git_working_dirs_path') if git_working_dirs_path: - check_res = "OK (%s)" % git_working_dirs_path + repos = [GITHUB_EASYCONFIGS_REPO, GITHUB_EASYBLOCKS_REPO, GITHUB_FRAMEWORK_REPO] + missing_repos = [repo for repo in repos if not os.path.exists(os.path.join(git_working_dirs_path, repo))] + if not missing_repos: + check_res = "OK (%s)" % git_working_dirs_path + elif missing_repos != repos: + check_res = "OK (%s) but missing %s (suboptimal)" % (git_working_dirs_path, ', '.join(missing_repos)) + else: + check_res = "set (%s) but not populated (suboptimal)" % git_working_dirs_path else: check_res = "not found (suboptimal)" @@ -2251,15 +2269,18 @@ def install_github_token(github_user, silent=False): def validate_github_token(token, github_user): """ Check GitHub token: - * see if it conforms expectations (only [a-f]+[0-9] characters, length of 40) - * see if it can be used for authenticated access + * see if it conforms expectations (classic GitHub token with only [0-9a-f] characters + and length of 40 starting with 'ghp_', or fine-grained GitHub token with only + alphanumeric ([a-zA-Z0-9]) characters + '_' and length of 93 starting with 'github_pat_'), + * see if it can be used for authenticated access. """ # cfr. https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/ token_regex = re.compile('^ghp_[a-zA-Z0-9]{36}$') token_regex_old_format = re.compile('^[0-9a-f]{40}$') + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/about-authentication-to-github#githubs-token-formats + token_regex_fine_grained = re.compile('github_pat_[a-zA-Z0-9_]{82}') - # token should be 40 characters long, and only contain characters in [0-9a-f] - sanity_check = bool(token_regex.match(token)) + sanity_check = bool(token_regex.match(token)) or bool(token_regex_fine_grained.match(token)) if sanity_check: _log.info("Sanity check on token passed") else: diff --git a/easybuild/tools/hooks.py b/easybuild/tools/hooks.py index 9f1dcec311..2bb329027b 100644 --- a/easybuild/tools/hooks.py +++ b/easybuild/tools/hooks.py @@ -35,7 +35,7 @@ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from easybuild.tools.filetools import load_source +from importlib.util import spec_from_file_location, module_from_spec _log = fancylogger.getLogger('hooks', fname=False) @@ -66,6 +66,12 @@ MODULE_WRITE = 'module_write' END = 'end' +CANCEL = 'cancel' +CRASH = 'crash' +FAIL = 'fail' + +RUN_SHELL_CMD = 'run_shell_cmd' + PRE_PREF = 'pre_' POST_PREF = 'post_' HOOK_SUFF = '_hook' @@ -101,6 +107,11 @@ for p in [PRE_PREF, POST_PREF]] + [ POST_PREF + BUILD_AND_INSTALL_LOOP, END, + CANCEL, + CRASH, + FAIL, + PRE_PREF + RUN_SHELL_CMD, + POST_PREF + RUN_SHELL_CMD, ] KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES] @@ -109,6 +120,14 @@ _cached_hooks = {} +def load_source(filename, path): + """Load file as Python module""" + spec = spec_from_file_location(filename, path) + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def load_hooks(hooks_path): """Load defined hooks (if any).""" @@ -198,7 +217,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False): return res -def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None): +def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, kwargs=None, msg=None): """ Run hook with specified label and return result of calling the hook or None. @@ -214,6 +233,8 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if hook: if args is None: args = [] + if kwargs is None: + kwargs = {} if pre_step_hook: label = 'pre-' + label @@ -222,9 +243,9 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, if msg is None: msg = "Running %s hook..." % label - if build_option('debug'): + if build_option('debug') and not build_option('silence_hook_trigger'): print_msg(msg) - _log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args) - res = hook(*args) + _log.info("Running '%s' hook function (args: %s, keyword args: %s)...", hook.__name__, args, kwargs) + res = hook(*args, **kwargs) return res diff --git a/easybuild/tools/job/slurm.py b/easybuild/tools/job/slurm.py index 1f2dea776d..97a925bd35 100644 --- a/easybuild/tools/job/slurm.py +++ b/easybuild/tools/job/slurm.py @@ -37,7 +37,7 @@ from easybuild.tools.config import JOB_DEPS_TYPE_ABORT_ON_ERROR, JOB_DEPS_TYPE_ALWAYS_RUN, build_option from easybuild.tools.job.backend import JobBackend from easybuild.tools.filetools import which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd _log = fancylogger.getLogger('slurm', fname=False) @@ -78,8 +78,8 @@ def __init__(self, *args, **kwargs): def _check_version(self): """Check whether version of Slurm complies with required version.""" - (out, _) = run_cmd("sbatch --version", trace=False) - slurm_ver = out.strip().split(' ')[-1] + res = run_shell_cmd("sbatch --version", hidden=True) + slurm_ver = res.output.strip().split(' ')[-1] self.log.info("Found Slurm version %s", slurm_ver) if LooseVersion(slurm_ver) < LooseVersion(self.REQ_VERSION): @@ -116,16 +116,16 @@ def queue(self, job, dependencies=frozenset()): else: submit_cmd += ' --%s "%s"' % (key, job.job_specs[key]) - (out, _) = run_cmd(submit_cmd, trace=False) + cmd_res = run_shell_cmd(submit_cmd, hidden=True) jobid_regex = re.compile("^Submitted batch job (?P[0-9]+)") - res = jobid_regex.search(out) - if res: - job.jobid = res.group('jobid') + regex_res = jobid_regex.search(cmd_res.output) + if regex_res: + job.jobid = regex_res.group('jobid') self.log.info("Job submitted, got job ID %s", job.jobid) else: - raise EasyBuildError("Failed to determine job ID from output of submission command: %s", out) + raise EasyBuildError("Failed to determine job ID from output of submission command: %s", cmd_res.output) self._submitted.append(job) @@ -142,7 +142,7 @@ def complete(self): job_ids.append(job.jobid) if job_ids: - run_cmd("scontrol release %s" % ' '.join(job_ids), trace=False) + run_shell_cmd("scontrol release %s" % ' '.join(job_ids), hidden=True) submitted_jobs = '; '.join(["%s (%s): %s" % (job.name, job.module, job.jobid) for job in self._submitted]) print_msg("List of submitted jobs (%d): %s" % (len(self._submitted), submitted_jobs), log=self.log) diff --git a/easybuild/tools/loose_version.py b/easybuild/tools/loose_version.py index e5594fc5fb..1855fee74a 100644 --- a/easybuild/tools/loose_version.py +++ b/easybuild/tools/loose_version.py @@ -81,6 +81,9 @@ def _cmp(self, other): def __eq__(self, other): return self._cmp(other) == 0 + def __ne__(self, other): + return self._cmp(other) != 0 + def __lt__(self, other): return self._cmp(other) < 0 diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index e5a993cb15..851d4131ff 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -48,7 +48,7 @@ from easybuild.tools.config import build_option, get_module_syntax, install_path from easybuild.tools.filetools import convert_name, mkdir, read_file, remove_file, resolve_path, symlink, write_file from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, EnvironmentModulesC, Lmod, modules_tool -from easybuild.tools.utilities import get_subclasses, quote_str +from easybuild.tools.utilities import get_subclasses, nub, quote_str _log = fancylogger.getLogger('module_generator', fname=False) @@ -621,7 +621,8 @@ def _generate_extensions_list(self): """ Generate a list of all extensions in name/version format """ - return self.app.make_extension_string(name_version_sep='/', ext_sep=',').split(',') + exts_str = self.app.make_extension_string(name_version_sep='/', ext_sep=',') + return exts_str.split(',') if exts_str else [] def _generate_help_text(self): """ @@ -665,7 +666,7 @@ def _generate_help_text(self): if multi_deps: compatible_modules_txt = '\n'.join([ "This module is compatible with the following modules, one of each line is required:", - ] + ['* %s' % d for d in multi_deps]) + ] + ['* %s' % d for d in nub(multi_deps)]) lines.extend(self._generate_section("Compatible modules", compatible_modules_txt)) # Extensions (if any) @@ -1287,7 +1288,7 @@ def get_description(self, conflict=True): extensions_list = self._generate_extensions_list() if extensions_list: - extensions_stmt = 'extensions("%s")' % ','.join(['%s' % x for x in extensions_list]) + extensions_stmt = 'extensions("%s")' % ','.join([str(x) for x in extensions_list]) # put this behind a Lmod version check as 'extensions' is only (well) supported since Lmod 8.2.8, # see https://lmod.readthedocs.io/en/latest/330_extensions.html#module-extensions and # https://github.com/TACC/Lmod/issues/428 diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 2a32e7fdd4..75a1022b50 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -50,7 +50,7 @@ from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import get_subclasses, nub # software root/version environment variable name prefixes @@ -272,11 +272,7 @@ def set_and_check_version(self): if StrictVersion(self.version) < StrictVersion(self.DEPR_VERSION): depr_msg = "Support for %s version < %s is deprecated, " % (self.NAME, self.DEPR_VERSION) depr_msg += "found version %s" % self.version - - if self.version.startswith('6') and 'Lmod6' in build_option('silence_deprecation_warnings'): - self.log.warning(depr_msg) - else: - self.log.deprecated(depr_msg, '5.0') + self.log.deprecated(depr_msg, '6.0') if self.MAX_VERSION is not None: self.log.debug("Maximum allowed %s version defined: %s", self.NAME, self.MAX_VERSION) @@ -312,7 +308,7 @@ def check_module_function(self, allow_mismatch=False, regex=None): output, exit_code = None, 1 else: cmd = "type module" - res = run(cmd, fail_on_error=False, in_dry_run=False, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False) output, exit_code = res.output, res.exit_code if regex is None: @@ -549,18 +545,14 @@ def module_wrapper_exists(self, mod_name, modulerc_fn='.modulerc', mod_wrapper_r return wrapped_mod - def exist(self, mod_names, mod_exists_regex_template=None, skip_avail=False, maybe_partial=True): + def exist(self, mod_names, skip_avail=False, maybe_partial=True): """ Check if modules with specified names exists. :param mod_names: list of module names - :param mod_exists_regex_template: DEPRECATED and unused :param skip_avail: skip checking through 'module avail', only check via 'module show' :param maybe_partial: indicates if the module name may be a partial module name """ - if mod_exists_regex_template is not None: - self.log.deprecated('mod_exists_regex_template is no longer used', '5.0') - def mod_exists_via_show(mod_name): """ Helper function to check whether specified module name exists through 'module show'. @@ -824,8 +816,8 @@ def run_module(self, *args, **kwargs): cmd_list = self.compose_cmd_list(args) cmd = ' '.join(cmd_list) # note: module commands are always run in dry mode, and are kept hidden in trace and dry run output - res = run(cmd_list, env=environ, fail_on_error=False, shell=False, split_stderr=True, - hidden=True, in_dry_run=True) + res = run_shell_cmd(cmd_list, env=environ, fail_on_error=False, use_bash=False, split_stderr=True, + hidden=True, in_dry_run=True, output_file=False) # stdout will contain python code (to change environment etc) # stderr will contain text (just like the normal module command) @@ -1189,6 +1181,7 @@ class EnvironmentModulesC(ModulesTool): COMMAND = "modulecmd" REQ_VERSION = '3.2.10' MAX_VERSION = '3.99' + DEPR_VERSION = '3.999' VERSION_REGEXP = r'^\s*(VERSION\s*=\s*)?(?P\d\S*)\s*' def run_module(self, *args, **kwargs): @@ -1250,6 +1243,7 @@ class EnvironmentModulesTcl(EnvironmentModulesC): COMMAND_SHELL = ['tclsh'] VERSION_OPTION = '' REQ_VERSION = None + DEPR_VERSION = '9999.9' VERSION_REGEXP = r'^Modules\s+Release\s+Tcl\s+(?P\d\S*)\s' def set_path_env_var(self, key, paths): @@ -1282,14 +1276,14 @@ def tweak_stdout(txt): return super(EnvironmentModulesTcl, self).run_module(*args, **kwargs) - def available(self, mod_name=None): + def available(self, mod_name=None, extra_args=None): """ Return a list of available modules for the given (partial) module name; use None to obtain a list of all available modules. :param mod_name: a (partial) module name for filtering (default: None) """ - mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name) + mods = super(EnvironmentModulesTcl, self).available(mod_name=mod_name, extra_args=extra_args) # strip off slash at beginning, if it's there # under certain circumstances, 'modulecmd.tcl avail' (DEISA variant) spits out available modules like this clean_mods = [mod.lstrip(os.path.sep) for mod in mods] @@ -1324,8 +1318,60 @@ class EnvironmentModules(EnvironmentModulesTcl): COMMAND = os.path.join(os.getenv('MODULESHOME', 'MODULESHOME_NOT_DEFINED'), 'libexec', 'modulecmd.tcl') COMMAND_ENVIRONMENT = 'MODULES_CMD' REQ_VERSION = '4.0.0' + DEPR_VERSION = '4.0.0' # needs to be set as EnvironmentModules inherits from EnvironmentModulesTcl MAX_VERSION = None - VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d\S*)\s' + VERSION_REGEXP = r'^Modules\s+Release\s+(?P\d[^+\s]*)(\+\S*)?\s' + + SHOW_HIDDEN_OPTION = '--all' + + def __init__(self, *args, **kwargs): + """Constructor, set Environment Modules-specific class variable values.""" + # ensure in-depth modulepath search (MODULES_AVAIL_INDEPTH has been introduced in v4.3) + setvar('MODULES_AVAIL_INDEPTH', '1', verbose=False) + # match against module name start (MODULES_SEARCH_MATCH has been introduced in v4.3) + setvar('MODULES_SEARCH_MATCH', 'starts_with', verbose=False) + # ensure no debug message (MODULES_VERBOSITY has been introduced in v4.3) + setvar('MODULES_VERBOSITY', 'normal', verbose=False) + # make module search case sensitive (search is case insensitive by default since v5.0) + setvar('MODULES_ICASE', 'never', verbose=False) + # disable extended default (introduced in v4.4 and enabled by default in v5.0) + setvar('MODULES_EXTENDED_DEFAULT', '0', verbose=False) + # hard disable output redirection, output messages are expected on stderr + setvar('MODULES_REDIRECT_OUTPUT', '0', verbose=False) + # make sure modulefile cache is ignored (cache mechanism supported since v5.3) + setvar('MODULES_IGNORE_CACHE', '1', verbose=False) + # ensure only module names are returned on avail (MODULES_AVAIL_TERSE_OUTPUT added in v4.7) + setvar('MODULES_AVAIL_TERSE_OUTPUT', '', verbose=False) + # ensure only module names are returned on list (MODULES_LIST_TERSE_OUTPUT added in v4.7) + setvar('MODULES_LIST_TERSE_OUTPUT', '', verbose=False) + + super(EnvironmentModules, self).__init__(*args, **kwargs) + + def check_module_function(self, allow_mismatch=False, regex=None): + """Check whether selected module tool matches 'module' function definition.""" + # Modules 5.1.0+: module command is called from _module_raw shell function + # Modules 4.2.0..5.0.1: module command is called from _module_raw shell function if it has + # been initialized in an interactive shell session (i.e., a session attached to a tty) + if self.testing: + if '_module_raw' in os.environ: + out, ec = os.environ['_module_raw'], 0 + else: + out, ec = None, 1 + else: + cmd = "type _module_raw" + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=False, hidden=True, output_file=False) + out, ec = res.output, res.exit_code + + if regex is None: + regex = r".*%s" % os.path.basename(self.cmd) + mod_cmd_re = re.compile(regex, re.M) + + if ec == 0 and mod_cmd_re.search(out): + self.log.debug("Found pattern '%s' in defined '_module_raw' function." % mod_cmd_re.pattern) + else: + self.log.debug("Pattern '%s' not found in '_module_raw' function, falling back to 'module' function", + mod_cmd_re.pattern) + super(EnvironmentModules, self).check_module_function(allow_mismatch, regex) def check_module_output(self, cmd, stdout, stderr): """Check output of 'module' command, see if if is potentially invalid.""" @@ -1334,17 +1380,51 @@ def check_module_output(self, cmd, stdout, stderr): else: self.log.debug("No errors detected when running module command '%s'", cmd) + def available(self, mod_name=None, extra_args=None): + """ + Return a list of available modules for the given (partial) module name; + use None to obtain a list of all available modules. + + :param mod_name: a (partial) module name for filtering (default: None) + """ + if extra_args is None: + extra_args = [] + # make hidden modules visible (requires Environment Modules 4.6.0) + if StrictVersion(self.version) >= StrictVersion('4.6.0'): + extra_args.append(self.SHOW_HIDDEN_OPTION) + + return super(EnvironmentModules, self).available(mod_name=mod_name, extra_args=extra_args) + + def get_setenv_value_from_modulefile(self, mod_name, var_name): + """ + Get value for specific 'setenv' statement from module file for the specified module. + + :param mod_name: module name + :param var_name: name of the variable being set for which value should be returned + """ + # Tcl-based module tools produce "module show" output with setenv statements like: + # "setenv GCC_PATH /opt/gcc/8.3.0" + # "setenv VAR {some text} + # - line starts with 'setenv' + # - whitespace (spaces & tabs) around variable name + # - curly braces around value if it contain spaces + value = super(EnvironmentModules, self).get_setenv_value_from_modulefile(mod_name=mod_name, + var_name=var_name) + + if value: + value = value.strip('{}') + + return value + class Lmod(ModulesTool): """Interface to Lmod.""" NAME = "Lmod" COMMAND = 'lmod' COMMAND_ENVIRONMENT = 'LMOD_CMD' - REQ_VERSION = '6.5.1' - DEPR_VERSION = '7.0.0' - REQ_VERSION_DEPENDS_ON = '7.6.1' + REQ_VERSION = '8.0.0' + DEPR_VERSION = '8.0.0' VERSION_REGEXP = r"^Modules\s+based\s+on\s+Lua:\s+Version\s+(?P\d\S*)\s" - USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') SHOW_HIDDEN_OPTION = '--show-hidden' @@ -1360,7 +1440,14 @@ def __init__(self, *args, **kwargs): setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False) super(Lmod, self).__init__(*args, **kwargs) - self.supports_depends_on = StrictVersion(self.version) >= StrictVersion(self.REQ_VERSION_DEPENDS_ON) + version = StrictVersion(self.version) + + self.supports_depends_on = True + # See https://lmod.readthedocs.io/en/latest/125_personal_spider_cache.html + if version >= '8.7.12': + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.cache', 'lmod') + else: + self.USER_CACHE_DIR = os.path.join(os.path.expanduser('~'), '.lmod.d', '.cache') def check_module_function(self, *args, **kwargs): """Check whether selected module tool matches 'module' function definition.""" @@ -1426,7 +1513,8 @@ def update(self): cmd = ' '.join(cmd_list) self.log.debug("Running command '%s'...", cmd) - res = run(cmd_list, env=os.environ, fail_on_error=False, shell=False, split_stderr=True, hidden=True) + res = run_shell_cmd(cmd_list, env=os.environ, fail_on_error=False, use_bash=False, split_stderr=True, + hidden=True) stdout, stderr = res.output, res.stderr if stderr: @@ -1436,7 +1524,8 @@ def update(self): # don't actually update local cache when testing, just return the cache contents return stdout else: - cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT.lua') + suffix = build_option('module_cache_suffix') or '' + cache_fp = os.path.join(self.USER_CACHE_DIR, 'moduleT%s.lua' % suffix) self.log.debug("Updating Lmod spider cache %s with output from '%s'", cache_fp, cmd) cache_dir = os.path.dirname(cache_fp) if not os.path.exists(cache_dir): @@ -1512,13 +1601,9 @@ def module_wrapper_exists(self, mod_name): Determine whether a module wrapper with specified name exists. First check for wrapper defined in .modulerc.lua, fall back to also checking .modulerc (Tcl syntax). """ - res = None - - # first consider .modulerc.lua with Lmod 7.8 (or newer) - if StrictVersion(self.version) >= StrictVersion('7.8'): - mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' - res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', - mod_wrapper_regex_template=mod_wrapper_regex_template) + mod_wrapper_regex_template = r'^module_version\("(?P.*)", "%s"\)$' + res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua', + mod_wrapper_regex_template=mod_wrapper_regex_template) # fall back to checking for .modulerc in Tcl syntax if res is None: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index c32a2f9a85..197514eb07 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -79,7 +79,7 @@ from easybuild.tools.config import get_pretend_installpath, init, init_build_options, mk_full_default_path from easybuild.tools.config import BuildOptions, ConfigurationVariables from easybuild.tools.configobj import ConfigObj, ConfigObjError -from easybuild.tools.docs import FORMAT_MD, FORMAT_RST, FORMAT_TXT +from easybuild.tools.docs import FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT from easybuild.tools.docs import avail_cfgfile_constants, avail_easyconfig_constants, avail_easyconfig_licenses from easybuild.tools.docs import avail_toolchain_opts, avail_easyconfig_params, avail_easyconfig_templates from easybuild.tools.docs import list_easyblocks, list_toolchains @@ -98,13 +98,13 @@ from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes from easybuild.tools.modules import Lmod from easybuild.tools.robot import det_robot_path -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.package.utilities import avail_package_naming_schemes from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME from easybuild.tools.repository.repository import avail_repositories -from easybuild.tools.systemtools import UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family -from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_system_info +from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family +from easybuild.tools.systemtools import get_cpu_features, get_gpu_info, get_os_type, get_system_info from easybuild.tools.version import this_is_easybuild @@ -131,6 +131,8 @@ def terminal_supports_colors(stream): DEFAULT_LIST_PR_ORDER = GITHUB_PR_ORDER_CREATED DEFAULT_LIST_PR_DIREC = GITHUB_PR_DIRECTION_DESC +RPATH_DEFAULT = False if get_os_type() == DARWIN else True + _log = fancylogger.getLogger('options', fname=False) @@ -275,8 +277,8 @@ def basic_options(self): 'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}), 'rebuild': ("Rebuild software, even if module already exists (don't skip OS dependencies checks)", None, 'store_true', False), - 'robot': ("Enable dependency resolution, using easyconfigs in specified paths", - 'pathlist', 'store_or_None', [], 'r', {'metavar': 'PATH[%sPATH]' % os.pathsep}), + 'robot': ("Enable dependency resolution, optionally consider additional paths to search for easyconfigs", + 'pathlist', 'store_or_None', [], 'r', {'metavar': '[PATH[%sPATH]]' % os.pathsep}), 'robot-paths': ("Additional paths to consider by robot for easyconfigs (--robot paths get priority)", 'pathlist', 'add_flex', self.default_robot_paths, {'metavar': 'PATH[%sPATH]' % os.pathsep}), 'search-paths': ("Additional locations to consider in --search (next to --robot and --robot-paths paths)", @@ -400,6 +402,8 @@ def override_options(self): None, 'store_true', False), 'extra-modules': ("List of extra modules to load after setting up the build environment", 'strlist', 'extend', None), + 'fail-on-mod-files-gcccore': ("Fail if .mod files are detected in a GCCcore install", None, 'store_true', + False), 'fetch': ("Allow downloading sources ignoring OS and modules tool dependencies, " "implies --stop=fetch, --ignore-osdeps and ignore modules tool", None, 'store_true', False), 'filter-deps': ("List of dependencies that you do *not* want to install with EasyBuild, " @@ -455,6 +459,9 @@ def override_options(self): "environment variable and its value separated by a colon (':')", None, 'store', DEFAULT_MINIMAL_BUILD_ENV), 'minimal-toolchains': ("Use minimal toolchain when resolving dependencies", None, 'store_true', False), + 'module-cache-suffix': ("Suffix to add to the cache file name (before the extension) " + "when updating the modules tool cache", + None, 'store', None), 'module-only': ("Only generate module file(s); skip all steps except for %s" % ', '.join(MODULE_ONLY_STEPS), None, 'store_true', False), 'modules-tool-version-check': ("Check version of modules tool being used", None, 'store_true', True), @@ -463,11 +470,12 @@ def override_options(self): 'mpi-tests': ("Run MPI tests (when relevant)", None, 'store_true', True), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), - 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, [FORMAT_MD, FORMAT_RST, FORMAT_TXT]), + 'output-format': ("Set output format", 'choice', 'store', FORMAT_TXT, + [FORMAT_JSON, FORMAT_MD, FORMAT_RST, FORMAT_TXT]), 'output-style': ("Control output style; auto implies using Rich if available to produce rich output, " "with fallback to basic colored output", 'choice', 'store', OUTPUT_STYLE_AUTO, OUTPUT_STYLES), - 'parallel': ("Specify (maximum) level of parallellism used during build procedure", + 'parallel': ("Specify (maximum) level of parallelism used during build procedure", 'int', 'store', None), 'parallel-extensions-install': ("Install list of extensions in parallel (if supported)", None, 'store_true', False), @@ -483,7 +491,7 @@ def override_options(self): 'required-linked-shared-libs': ("Comma-separated list of shared libraries (names, file names, or paths) " "which must be linked in all installed binaries/libraries", 'strlist', 'extend', None), - 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', False), + 'rpath': ("Enable use of RPATH for linking with libraries", None, 'store_true', RPATH_DEFAULT), 'rpath-filter': ("List of regex patterns to use for filtering out RPATH paths", 'strlist', 'store', None), 'rpath-override-dirs': ("Path(s) to be prepended when linking with RPATH (string, colon-separated)", None, 'store', None), @@ -495,6 +503,8 @@ def override_options(self): 'silence-deprecation-warnings': ( "Silence specified deprecation warnings out of (%s)" % ', '.join(all_deprecations), 'strlist', 'extend', []), + 'silence-hook-trigger': ("Suppress printing of debug message every time a hook is triggered", + None, 'store_true', False), 'skip-extensions': ("Skip installation of extensions", None, 'store_true', False), 'skip-test-cases': ("Skip running test cases", None, 'store_true', False, 't'), 'skip-test-step': ("Skip running the test step (e.g. unit tests)", None, 'store_true', False), @@ -1723,11 +1733,13 @@ def process_software_build_specs(options): }) # provide both toolchain and toolchain_name/toolchain_version keys - if 'toolchain_name' in build_specs: + try: build_specs['toolchain'] = { 'name': build_specs['toolchain_name'], - 'version': build_specs.get('toolchain_version', None), + 'version': build_specs['toolchain_version'], } + except KeyError: + pass # Don't set toolchain key if we don't have both keys # process --amend and --try-amend if options.amend or options.try_amend: @@ -1883,8 +1895,9 @@ def set_tmpdir(tmpdir=None, raise_error=False): fd, tmptest_file = tempfile.mkstemp() os.close(fd) os.chmod(tmptest_file, 0o700) - if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False, - stream_output=False): + res = run_shell_cmd(tmptest_file, fail_on_error=False, in_dry_run=True, hidden=True, stream_output=False, + with_hooks=False) + if res.exit_code: msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir() msg += "This can cause problems in the build process, consider using --tmpdir." if raise_error: diff --git a/easybuild/tools/package/utilities.py b/easybuild/tools/package/utilities.py index 9a695e33da..adaa0814fc 100644 --- a/easybuild/tools/package/utilities.py +++ b/easybuild/tools/package/utilities.py @@ -45,7 +45,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, which from easybuild.tools.package.package_naming_scheme.pns import PackageNamingScheme -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.utilities import get_subclasses, import_available_modules @@ -145,7 +145,7 @@ def package_with_fpm(easyblock): ]) cmd = ' '.join(cmdlist) _log.debug("The flattened cmdlist looks like: %s", cmd) - run_cmd(cmdlist, log_all=True, simple=True, shell=False) + run_shell_cmd(cmdlist, use_bash=False) _log.info("Created %s package(s) in %s", pkgtype, workdir) diff --git a/easybuild/tools/py2vs3/py3.py b/easybuild/tools/py2vs3/py3.py index f37cbf0f08..21dfc49e10 100644 --- a/easybuild/tools/py2vs3/py3.py +++ b/easybuild/tools/py2vs3/py3.py @@ -43,6 +43,7 @@ from html.parser import HTMLParser # noqa from itertools import zip_longest from io import StringIO # noqa +from os import makedirs # noqa from string import ascii_letters, ascii_lowercase # noqa from urllib.request import HTTPError, HTTPSHandler, Request, URLError, build_opener, urlopen # noqa from urllib.parse import urlencode # noqa diff --git a/easybuild/tools/run.py b/easybuild/tools/run.py index c972277b00..2592e198f1 100644 --- a/easybuild/tools/run.py +++ b/easybuild/tools/run.py @@ -37,9 +37,12 @@ """ import contextlib import functools +import inspect import os import re import signal +import shutil +import string import subprocess import sys import tempfile @@ -47,11 +50,19 @@ from collections import namedtuple from datetime import datetime +try: + # get_native_id is only available in Python >= 3.8 + from threading import get_native_id as get_thread_id +except ImportError: + # get_ident is available in Python >= 3.3 + from threading import get_ident as get_thread_id + import easybuild.tools.asyncprocess as asyncprocess from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, time_str_since from easybuild.tools.config import ERROR, IGNORE, WARN, build_option -from easybuild.tools.utilities import trace_msg +from easybuild.tools.hooks import RUN_SHELL_CMD, load_hooks, run_hook +from easybuild.tools.utilities import nub, trace_msg _log = fancylogger.getLogger('run', fname=False) @@ -70,11 +81,82 @@ "sysctl -n machdep.cpu.brand_string", # used in get_cpu_model (OS X) "sysctl -n machdep.cpu.vendor", # used in get_cpu_vendor (OS X) "type module", # used in ModulesTool.check_module_function + "type _module_raw", # used in EnvironmentModules.check_module_function "ulimit -u", # used in det_parallelism ] -RunResult = namedtuple('RunResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir')) +RunShellCmdResult = namedtuple('RunShellCmdResult', ('cmd', 'exit_code', 'output', 'stderr', 'work_dir', + 'out_file', 'err_file', 'thread_id', 'task_id')) + + +class RunShellCmdError(BaseException): + + def __init__(self, cmd_result, caller_info, *args, **kwargs): + """Constructor for RunShellCmdError.""" + self.cmd = cmd_result.cmd + self.cmd_name = os.path.basename(self.cmd.split(' ')[0]) + self.exit_code = cmd_result.exit_code + self.work_dir = cmd_result.work_dir + self.output = cmd_result.output + self.out_file = cmd_result.out_file + self.stderr = cmd_result.stderr + self.err_file = cmd_result.err_file + + self.caller_info = caller_info + + msg = f"Shell command '{self.cmd_name}' failed!" + super(RunShellCmdError, self).__init__(msg, *args, **kwargs) + + def print(self): + """ + Report failed shell command for this RunShellCmdError instance + """ + + def pad_4_spaces(msg): + return ' ' * 4 + msg + + error_info = [ + '', + "ERROR: Shell command failed!", + pad_4_spaces(f"full command -> {self.cmd}"), + pad_4_spaces(f"exit code -> {self.exit_code}"), + pad_4_spaces(f"working directory -> {self.work_dir}"), + ] + + if self.out_file is not None: + # if there's no separate file for error/warnings, then out_file includes both stdout + stderr + out_info_msg = "output (stdout + stderr)" if self.err_file is None else "output (stdout) " + error_info.append(pad_4_spaces(f"{out_info_msg} -> {self.out_file}")) + + if self.err_file is not None: + error_info.append(pad_4_spaces(f"error/warnings (stderr) -> {self.err_file}")) + + caller_file_name, caller_line_nr, caller_function_name = self.caller_info + called_from_info = f"'{caller_function_name}' function in {caller_file_name} (line {caller_line_nr})" + error_info.extend([ + pad_4_spaces(f"called from -> {called_from_info}"), + '', + ]) + + sys.stderr.write('\n'.join(error_info) + '\n') + + +def raise_run_shell_cmd_error(cmd_res): + """ + Raise RunShellCmdError for failed shell command, after collecting additional caller info + """ + + # figure out where failing command was run + # need to go 3 levels down: + # 1) this function + # 2) run_shell_cmd function + # 3) run_cmd_cache decorator + # 4) actual caller site + frameinfo = inspect.getouterframes(inspect.currentframe())[3] + caller_info = (frameinfo.filename, frameinfo.lineno, frameinfo.function) + + raise RunShellCmdError(cmd_res, caller_info) def run_cmd_cache(func): @@ -103,14 +185,27 @@ def cache_aware_func(cmd, *args, **kwargs): return cache_aware_func -run_cache = run_cmd_cache +run_shell_cmd_cache = run_cmd_cache -@run_cache -def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - hidden=False, in_dry_run=False, work_dir=None, shell=True, - output_file=False, stream_output=False, asynchronous=False, - qa_patterns=None, qa_wait_patterns=None): +def fileprefix_from_cmd(cmd, allowed_chars=False): + """ + Simplify the cmd to only the allowed_chars we want in a filename + + :param cmd: the cmd (string) + :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-") + """ + if not allowed_chars: + allowed_chars = f"{string.ascii_letters}{string.digits}_-" + + return ''.join([c for c in cmd if c in allowed_chars]) + + +@run_shell_cmd_cache +def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, + hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True, + output_file=True, stream_output=None, asynchronous=False, task_id=None, with_hooks=True, + qa_patterns=None, qa_wait_patterns=None): """ Run specified (interactive) shell command, and capture output + exit code. @@ -120,11 +215,14 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, :param env: environment to use to run command (if None, inherit current process environment) :param hidden: do not show command in terminal output (when using --trace, or with --extended-dry-run / -x) :param in_dry_run: also run command in dry run mode + :param verbose_dry_run: show that command is run in dry run mode (overrules 'hidden') :param work_dir: working directory to run command in (current working directory if None) - :param shell: execute command through bash shell (enabled by default) + :param use_bash: execute command through bash shell (enabled by default) :param output_file: collect command output in temporary output file - :param stream_output: stream command output to stdout - :param asynchronous: run command asynchronously + :param stream_output: stream command output to stdout (auto-enabled with --logtostdout if None) + :param asynchronous: indicate that command is being run asynchronously + :param task_id: task ID for specified shell command (included in return value) + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) :param qa_patterns: list of 2-tuples with patterns for questions + corresponding answers :param qa_wait_patterns: list of 2-tuples with patterns for non-questions and number of iterations to allow these patterns to match with end out command output @@ -133,74 +231,165 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, - exit_code: exit code of command (integer) - stderr: stderr output if split_stderr is enabled, None otherwise """ + def to_cmd_str(cmd): + """ + Helper function to create string representation of specified command. + """ + if isinstance(cmd, str): + cmd_str = cmd.strip() + elif isinstance(cmd, list): + cmd_str = ' '.join(cmd) + else: + raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}") - # temporarily raise a NotImplementedError until all options are implemented - if any((work_dir, stream_output, asynchronous)): - raise NotImplementedError + return cmd_str + # temporarily raise a NotImplementedError until all options are implemented if qa_patterns or qa_wait_patterns: raise NotImplementedError - if isinstance(cmd, str): - cmd_str = cmd.strip() - elif isinstance(cmd, list): - cmd_str = ' '.join(cmd) - else: - raise EasyBuildError(f"Unknown command type ('{type(cmd)}'): {cmd}") - if work_dir is None: work_dir = os.getcwd() - # temporary output file for command output, if requested - if output_file or not hidden: - # collect output of running command in temporary log file, if desired - fd, cmd_out_fp = tempfile.mkstemp(suffix='.log', prefix='easybuild-run-') - os.close(fd) - _log.info(f'run_cmd: Output of "{cmd}" will be logged to {cmd_out_fp}') + cmd_str = to_cmd_str(cmd) + + thread_id = None + if asynchronous: + thread_id = get_thread_id() + _log.info(f"Initiating running of shell command '{cmd_str}' via thread with ID {thread_id}") + + # auto-enable streaming of command output under --logtostdout/-l, unless it was disabled explicitely + if stream_output is None and build_option('logtostdout'): + _log.info(f"Auto-enabling streaming output of '{cmd_str}' command because logging to stdout is enabled") + stream_output = True + + # temporary output file(s) for command output + if output_file: + toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output') + os.makedirs(toptmpdir, exist_ok=True) + cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0])) + tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-') + cmd_out_fp = os.path.join(tmpdir, 'out.txt') + _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}') + if split_stderr: + cmd_err_fp = os.path.join(tmpdir, 'err.txt') + _log.info(f'run_cmd: Errors and warnings of "{cmd_str}" will be logged to {cmd_err_fp}') + else: + cmd_err_fp = None else: - cmd_out_fp = None + cmd_out_fp, cmd_err_fp = None, None # early exit in 'dry run' mode, after printing the command that would be run (unless 'hidden' is enabled) if not in_dry_run and build_option('extended_dry_run'): - if not hidden: + if not hidden or verbose_dry_run: silent = build_option('silent') - msg = f" running command \"{cmd_str}\"\n" + msg = f" running shell command \"{cmd_str}\"\n" msg += f" (in {work_dir})" dry_run_msg(msg, silent=silent) - return RunResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir) + return RunShellCmdResult(cmd=cmd_str, exit_code=0, output='', stderr=None, work_dir=work_dir, + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) start_time = datetime.now() if not hidden: - cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp) + _cmd_trace_msg(cmd_str, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id) - if stdin: - # 'input' value fed to subprocess.run must be a byte sequence - stdin = stdin.encode() + if stream_output: + print_msg(f"(streaming) output for command '{cmd_str}':") # use bash as shell instead of the default /bin/sh used by subprocess.run # (which could be dash instead of bash, like on Ubuntu, see https://wiki.ubuntu.com/DashAsBinSh) # stick to None (default value) when not running command via a shell - executable = '/bin/bash' if shell else None + if use_bash: + bash = shutil.which('bash') + _log.info(f"Path to bash that will be used to run shell commands: {bash}") + executable, shell = bash, True + else: + executable, shell = None, False + + if with_hooks: + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': work_dir}) + if hook_res: + cmd, old_cmd = hook_res, cmd + cmd_str = to_cmd_str(cmd) + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) stderr = subprocess.PIPE if split_stderr else subprocess.STDOUT - _log.info(f"Running command '{cmd_str}' in {work_dir}") - proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=stderr, check=fail_on_error, - env=env, input=stdin, shell=shell, executable=executable) + log_msg = f"Running shell command '{cmd_str}' in {work_dir}" + if thread_id: + log_msg += f" (via thread with ID {thread_id})" + _log.info(log_msg) + + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, stdin=subprocess.PIPE, + cwd=work_dir, env=env, shell=shell, executable=executable) + + # 'input' value fed to subprocess.run must be a byte sequence + if stdin: + stdin = stdin.encode() + + if stream_output: + if stdin: + proc.stdin.write(stdin) + + exit_code = None + stdout, stderr = b'', b'' + + while exit_code is None: + exit_code = proc.poll() + + # use small read size (128 bytes) when streaming output, to make it stream more fluently + # -1 means reading until EOF + read_size = 128 if exit_code is None else -1 + + stdout += proc.stdout.read(read_size) + if split_stderr: + stderr += proc.stderr.read(read_size) + else: + (stdout, stderr) = proc.communicate(input=stdin) # return output as a regular string rather than a byte sequence (and non-UTF-8 characters get stripped out) - output = proc.stdout.decode('utf-8', 'ignore') - stderr_output = proc.stderr.decode('utf-8', 'ignore') if split_stderr else None + output = stdout.decode('utf-8', 'ignore') + stderr = stderr.decode('utf-8', 'ignore') if split_stderr else None - res = RunResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr_output, work_dir=work_dir) + # store command output to temporary file(s) + if output_file: + try: + with open(cmd_out_fp, 'w') as fp: + fp.write(output) + if split_stderr: + with open(cmd_err_fp, 'w') as fp: + fp.write(stderr) + except IOError as err: + raise EasyBuildError(f"Failed to dump command output to temporary file: {err}") + + res = RunShellCmdResult(cmd=cmd_str, exit_code=proc.returncode, output=output, stderr=stderr, work_dir=work_dir, + out_file=cmd_out_fp, err_file=cmd_err_fp, thread_id=thread_id, task_id=task_id) + # always log command output + cmd_name = cmd_str.split(' ')[0] if split_stderr: - log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code}, " - log_msg += f"with stdout:\n{res.output}\nstderr:\n{res.stderr}" + _log.info(f"Output of '{cmd_name} ...' shell command (stdout only):\n{res.output}") + _log.info(f"Warnings and errors of '{cmd_name} ...' shell command (stderr only):\n{res.stderr}") else: - log_msg = f"Command '{cmd_str}' exited with exit code {res.exit_code} and output:\n{res.output}" - _log.info(log_msg) + _log.info(f"Output of '{cmd_name} ...' shell command (stdout + stderr):\n{res.output}") + + if res.exit_code == 0: + _log.info(f"Shell command completed successfully (see output above): {cmd_str}") + else: + _log.warning(f"Shell command FAILED (exit code {res.exit_code}, see output above): {cmd_str}") + if fail_on_error: + raise_run_shell_cmd_error(res) + + if with_hooks: + run_hook_kwargs = { + 'exit_code': res.exit_code, + 'output': res.output, + 'stderr': res.stderr, + 'work_dir': res.work_dir, + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) if not hidden: time_since_start = time_str_since(start_time) @@ -209,7 +398,7 @@ def run(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None, return res -def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp): +def _cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp, cmd_err_fp, thread_id): """ Helper function to construct and print trace message for command being run @@ -217,21 +406,29 @@ def cmd_trace_msg(cmd, start_time, work_dir, stdin, cmd_out_fp): :param start_time: datetime object indicating when command was started :param work_dir: path of working directory in which command is run :param stdin: stdin input value for command - :param cmd_out_fp: path to output log file for command + :param cmd_out_fp: path to output file for command + :param cmd_err_fp: path to errors/warnings output file for command + :param thread_id: thread ID (None when not running shell command asynchronously) """ start_time = start_time.strftime('%Y-%m-%d %H:%M:%S') + if thread_id: + run_cmd_msg = f"running shell command (asynchronously, thread ID: {thread_id}):" + else: + run_cmd_msg = "running shell command:" + lines = [ - "running command:", + run_cmd_msg, + f"\t{cmd}", f"\t[started at: {start_time}]", f"\t[working dir: {work_dir}]", ] if stdin: lines.append(f"\t[input: {stdin}]") if cmd_out_fp: - lines.append(f"\t[output logged in {cmd_out_fp}]") - - lines.append('\t' + cmd) + lines.append(f"\t[output saved to {cmd_out_fp}]") + if cmd_err_fp: + lines.append(f"\t[errors/warnings saved to {cmd_err_fp}]") trace_msg('\n'.join(lines)) @@ -267,7 +464,8 @@ def get_output_from_process(proc, read_size=None, asynchronous=False): @run_cmd_cache def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None, - force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False): + force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False, + with_hooks=True): """ Run specified command (in a subshell) :param cmd: command to run @@ -284,6 +482,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True :param trace: print command being executed as part of trace output :param stream_output: enable streaming command output to stdout :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True) + :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined) """ cwd = os.getcwd() @@ -369,6 +568,13 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True else: raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd)) + if with_hooks: + hooks = load_hooks(build_option('hooks')) + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()}) + if isinstance(hook_res, str): + cmd, old_cmd = hook_res, cmd + _log.info("Command to run was changed by pre-%s hook: '%s' (was: '%s')", RUN_SHELL_CMD, cmd, old_cmd) + _log.info('running cmd: %s ' % cmd) try: proc = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -384,7 +590,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True return (proc, cmd, cwd, start_time, cmd_log) else: return complete_cmd(proc, cmd, cwd, start_time, cmd_log, log_ok=log_ok, log_all=log_all, simple=simple, - regexp=regexp, stream_output=stream_output, trace=trace) + regexp=regexp, stream_output=stream_output, trace=trace, with_hook=with_hooks) def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, output_read_size=1024, output=''): @@ -429,7 +635,7 @@ def check_async_cmd(proc, cmd, owd, start_time, cmd_log, fail_on_error=True, out def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False, simple=False, - regexp=True, stream_output=None, trace=True, output=''): + regexp=True, stream_output=None, trace=True, output='', with_hook=True): """ Complete running of command represented by passed subprocess.Popen instance. @@ -444,6 +650,7 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False :param regexp: regex used to check the output for errors; if True it will use the default (see parse_log_for_error) :param stream_output: enable streaming command output to stdout :param trace: print command being executed as part of trace output + :param with_hook: trigger post run_shell_cmd hooks (if defined) """ # use small read size when streaming output, to make it stream more fluently # read size should not be too small though, to avoid too much overhead @@ -479,6 +686,15 @@ def complete_cmd(proc, cmd, owd, start_time, cmd_log, log_ok=True, log_all=False sys.stdout.write(output) stdouterr += output + if with_hook: + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'exit_code': ec, + 'output': stdouterr, + 'work_dir': os.getcwd(), + } + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if trace: trace_msg("command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) @@ -621,6 +837,17 @@ def check_answers_list(answers): # Part 2: Run the command and answer questions # - this needs asynchronous stdout + hooks = load_hooks(build_option('hooks')) + run_hook_kwargs = { + 'interactive': True, + 'work_dir': os.getcwd(), + } + hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if isinstance(hook_res, str): + cmd, old_cmd = hook_res, cmd + _log.info("Interactive command to run was changed by pre-%s hook: '%s' (was: '%s')", + RUN_SHELL_CMD, cmd, old_cmd) + # # Log command output if cmd_log: cmd_log.write("# output for interactive command: %s\n\n" % cmd) @@ -735,6 +962,13 @@ def get_proc(): except IOError as err: _log.debug("runqanda cmd %s: remaining data read failed: %s", cmd, err) + run_hook_kwargs.update({ + 'interactive': True, + 'exit_code': ec, + 'output': stdout_err, + }) + run_hook(RUN_SHELL_CMD, hooks, post_step_hook=True, args=[cmd], kwargs=run_hook_kwargs) + if trace: trace_msg("interactive command completed: exit %s, ran in %s" % (ec, time_str_since(start_time))) @@ -888,7 +1122,7 @@ def extract_errors_from_log(log_txt, reg_exps): elif action == WARN: warnings.append(line) break - return warnings, errors + return nub(warnings), nub(errors) def check_log_for_errors(log_txt, reg_exps): @@ -903,10 +1137,10 @@ def check_log_for_errors(log_txt, reg_exps): errors_found_in_log += len(warnings) + len(errors) if warnings: - _log.warning("Found %s potential error(s) in command output (output: %s)", + _log.warning("Found %s potential error(s) in command output:\n\t%s", len(warnings), "\n\t".join(warnings)) if errors: - raise EasyBuildError("Found %s error(s) in command output (output: %s)", + raise EasyBuildError("Found %s error(s) in command output:\n\t%s", len(errors), "\n\t".join(errors)) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index e6e3de7067..657f7ec9f4 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -45,7 +45,6 @@ from collections import OrderedDict from ctypes.util import find_library from socket import gethostname -from easybuild.tools.run import subprocess_popen_text # pkg_resources is provided by the setuptools Python package, # which we really want to keep as an *optional* dependency @@ -65,7 +64,7 @@ from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import IGNORE from easybuild.tools.filetools import is_readable, read_file, which -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd, subprocess_popen_text _log = fancylogger.getLogger('systemtools', fname=False) @@ -274,7 +273,8 @@ def get_avail_core_count(): core_cnt = int(sum(sched_getaffinity())) else: # BSD-type systems - res = run('sysctl -n hw.ncpu', in_dry_run=True, hidden=True) + res = run_shell_cmd('sysctl -n hw.ncpu', in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) try: if int(res.output) > 0: core_cnt = int(res.output) @@ -311,7 +311,7 @@ def get_total_memory(): elif os_type == DARWIN: cmd = "sysctl -n hw.memsize" _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: memtotal = int(res.output.strip()) // (1024**2) @@ -393,14 +393,16 @@ def get_cpu_vendor(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.vendor" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) out = res.output.strip() if res.exit_code == 0 and out in VENDOR_IDS: vendor = VENDOR_IDS[out] _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd)) else: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) out = res.output.strip().split(' ')[0] if res.exit_code == 0 and out in CPU_VENDORS: vendor = out @@ -503,7 +505,7 @@ def get_cpu_model(): elif os_type == DARWIN: cmd = "sysctl -n machdep.cpu.brand_string" - res = run(cmd, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) if res.exit_code == 0: model = res.output.strip() _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model)) @@ -548,7 +550,7 @@ def get_cpu_speed(): elif os_type == DARWIN: cmd = "sysctl -n hw.cpufrequency_max" _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd) - res = run(cmd, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, with_hooks=False, output_file=False, stream_output=False) out = res.output.strip() cpu_freq = None if res.exit_code == 0 and out: @@ -596,7 +598,8 @@ def get_cpu_features(): for feature_set in ['extfeatures', 'features', 'leaf7_features']: cmd = "sysctl -n machdep.cpu.%s" % feature_set _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd) - res = run(cmd, in_dry_run=True, hidden=True, fail_on_error=False) + res = run_shell_cmd(cmd, in_dry_run=True, hidden=True, fail_on_error=False, with_hooks=False, + output_file=False, stream_output=False) if res.exit_code == 0: cpu_feat.extend(res.output.strip().lower().split()) @@ -623,7 +626,8 @@ def get_gpu_info(): try: cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader" _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n'): nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {}) @@ -641,13 +645,15 @@ def get_gpu_info(): try: cmd = "rocm-smi --showdriverversion --csv" _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) if res.exit_code == 0: amd_driver = res.output.strip().split('\n')[1].split(',')[1] cmd = "rocm-smi --showproductname --csv" _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, with_hooks=False, + output_file=False, stream_output=False) if res.exit_code == 0: for line in res.output.strip().split('\n')[1:]: amd_card_series = line.split(',')[1] @@ -866,7 +872,8 @@ def check_os_dependency(dep): pkg_cmd_flag.get(pkg_cmd), dep, ]) - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) found = res.exit_code == 0 if found: break @@ -878,7 +885,8 @@ def check_os_dependency(dep): # try locate if it's available if not found and which('locate'): cmd = 'locate -c --regexp "/%s$"' % dep - res = run(cmd, fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(cmd, fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) try: found = (res.exit_code == 0 and int(res.output.strip()) > 0) except ValueError: @@ -893,7 +901,8 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False): Get output of running version option for specific command line tool. Output is returned as a single-line string (newlines are replaced by '; '). """ - res = run(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd(' '.join([tool, version_option]), fail_on_error=False, in_dry_run=True, + hidden=True, with_hooks=False, output_file=False, stream_output=False) if not ignore_ec and res.exit_code: _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, res.output)) return UNKNOWN @@ -905,7 +914,8 @@ def get_gcc_version(): """ Process `gcc --version` and return the GCC version. """ - res = run('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True) + res = run_shell_cmd('gcc --version', fail_on_error=False, in_dry_run=True, hidden=True, + output_file=False, stream_output=False) gcc_ver = None if res.exit_code: _log.warning("Failed to determine the version of GCC: %s", res.output) @@ -961,7 +971,7 @@ def get_linked_libs_raw(path): or None for other types of files. """ - res = run("file %s" % path, fail_on_error=False, hidden=True) + res = run_shell_cmd("file %s" % path, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code: fail_msg = "Failed to run 'file %s': %s" % (path, res.output) _log.warning(fail_msg) @@ -996,7 +1006,7 @@ def get_linked_libs_raw(path): # take into account that 'ldd' may fail for strange reasons, # like printing 'not a dynamic executable' when not enough memory is available # (see also https://bugzilla.redhat.com/show_bug.cgi?id=1817111) - res = run(linked_libs_cmd, fail_on_error=False, hidden=True) + res = run_shell_cmd(linked_libs_cmd, fail_on_error=False, hidden=True, output_file=False, stream_output=False) if res.exit_code == 0: linked_libs_out = res.output else: @@ -1178,7 +1188,7 @@ def get_default_parallelism(): # No cache -> Calculate value from current system values par = get_avail_core_count() # determine max user processes via ulimit -u - res = run("ulimit -u", in_dry_run=True, hidden=True) + res = run_shell_cmd("ulimit -u", in_dry_run=True, hidden=True, output_file=False, stream_output=False) try: if res.output.startswith("unlimited"): maxuserproc = 2 ** 32 - 1 @@ -1205,7 +1215,7 @@ def get_default_parallelism(): raise EasyBuildError("Specified level of parallelism '%s' is not an integer value: %s", par, err) if maxpar is not None and maxpar < par: - _log.info("Limiting parallellism from %s to %s", par, maxpar) + _log.info("Limiting parallelism from %s to %s", par, maxpar) par = maxpar return par @@ -1329,8 +1339,8 @@ def det_pypkg_version(pkg_name, imported_pkg, import_name=None): except pkg_resources.DistributionNotFound as err: _log.debug("%s Python package not found: %s", pkg_name, err) - if version is None and hasattr(imported_pkg, '__version__'): - version = imported_pkg.__version__ + if version is None: + version = getattr(imported_pkg, '__version__', None) return version diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 68eedc23ce..ffd8ce580b 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -178,7 +178,7 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ ]) test_report.extend([ "#### Test result", - "%s" % msg, + msg, "", ]) diff --git a/easybuild/tools/toolchain/linalg.py b/easybuild/tools/toolchain/linalg.py index b7ce694ff7..907993571c 100644 --- a/easybuild/tools/toolchain/linalg.py +++ b/easybuild/tools/toolchain/linalg.py @@ -212,9 +212,9 @@ def _set_blacs_variables(self): """Set BLACS related variables""" lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) # BLACS @@ -254,11 +254,11 @@ def _set_scalapack_variables(self): raise EasyBuildError("_set_blas_variables: SCALAPACK_LIB not set") lib_map = {} - if hasattr(self, 'BLAS_LIB_MAP') and self.BLAS_LIB_MAP is not None: + if getattr(self, 'BLAS_LIB_MAP', None) is not None: lib_map.update(self.BLAS_LIB_MAP) - if hasattr(self, 'BLACS_LIB_MAP') and self.BLACS_LIB_MAP is not None: + if getattr(self, 'BLACS_LIB_MAP', None) is not None: lib_map.update(self.BLACS_LIB_MAP) - if hasattr(self, 'SCALAPACK_LIB_MAP') and self.SCALAPACK_LIB_MAP is not None: + if getattr(self, 'SCALAPACK_LIB_MAP', None) is not None: lib_map.update(self.SCALAPACK_LIB_MAP) self.SCALAPACK_LIB = self.variables.nappend('LIBSCALAPACK_ONLY', [x % lib_map for x in self.SCALAPACK_LIB]) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index f94eb05d9d..efba3d2e02 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -157,13 +157,13 @@ def _is_toolchain_for(cls, name): """see if this class can provide support for toolchain named name""" # TODO report later in the initialization the found version if name: - if hasattr(cls, 'NAME') and name == cls.NAME: - return True - else: + try: + return name == cls.NAME + except AttributeError: return False else: # is no name is supplied, check whether class can be used as a toolchain - return hasattr(cls, 'NAME') and cls.NAME + return bool(getattr(cls, 'NAME', None)) _is_toolchain_for = classmethod(_is_toolchain_for) @@ -320,10 +320,12 @@ def _copy_class_constants(self): if key not in self.CLASS_CONSTANT_COPIES: self.CLASS_CONSTANT_COPIES[key] = {} for cst in self.CLASS_CONSTANTS_TO_RESTORE: - if hasattr(self, cst): - self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(getattr(self, cst)) - else: + try: + value = getattr(self, cst) + except AttributeError: raise EasyBuildError("Class constant '%s' to be restored does not exist in %s", cst, self) + else: + self.CLASS_CONSTANT_COPIES[key][cst] = copy.deepcopy(value) self.log.devel("Copied class constants: %s", self.CLASS_CONSTANT_COPIES[key]) @@ -332,10 +334,12 @@ def _restore_class_constants(self): key = self.__class__ for cst in self.CLASS_CONSTANT_COPIES[key]: newval = copy.deepcopy(self.CLASS_CONSTANT_COPIES[key][cst]) - if hasattr(self, cst): - self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, getattr(self, cst)) - else: + try: + oldval = getattr(self, cst) + except AttributeError: self.log.devel("Restoring (currently undefined) class constant '%s' to %s", cst, newval) + else: + self.log.devel("Restoring class constant '%s' to %s (was: %s)", cst, newval, oldval) setattr(self, cst, newval) @@ -550,16 +554,6 @@ def _check_dependencies(self, dependencies): return deps - def add_dependencies(self, dependencies): - """ - [DEPRECATED] Verify if the given dependencies exist, and return them. - - This method is deprecated. - You should pass the dependencies to the 'prepare' method instead, via the 'deps' named argument. - """ - self.log.deprecated("use of 'Toolchain.add_dependencies' method", '4.0') - self.dependencies = self._check_dependencies(dependencies) - def is_required(self, name): """Determine whether this is a required toolchain element.""" # default: assume every element is required @@ -1117,8 +1111,11 @@ def _setenv_variables(self, donotset=None, verbose=True): setvar("EBVAR%s" % key, val, verbose=False) def get_flag(self, name): - """Get compiler flag for a certain option.""" - return "-%s" % self.options.option(name) + """Get compiler flag(s) for a certain option.""" + if isinstance(self.options.option(name), list): + return " ".join("-%s" % x for x in list(self.options.option(name))) + else: + return "-%s" % self.options.option(name) def toolchain_family(self): """Return toolchain family for this toolchain.""" diff --git a/easybuild/tools/toolchain/utilities.py b/easybuild/tools/toolchain/utilities.py index 48a586c567..56242b8fe4 100644 --- a/easybuild/tools/toolchain/utilities.py +++ b/easybuild/tools/toolchain/utilities.py @@ -63,7 +63,7 @@ def search_toolchain(name): package = easybuild.tools.toolchain check_attr_name = '%s_PROCESSED' % TC_CONST_PREFIX - if not hasattr(package, check_attr_name) or not getattr(package, check_attr_name): + if not getattr(package, check_attr_name, None): # import all available toolchains, so we know about them tc_modules = import_available_modules('easybuild.toolchains') @@ -76,7 +76,7 @@ def search_toolchain(name): if hasattr(elem, '__module__'): # exclude the toolchain class defined in that module if not tc_mod.__file__ == sys.modules[elem.__module__].__file__: - elem_name = elem.__name__ if hasattr(elem, '__name__') else elem + elem_name = getattr(elem, '__name__', elem) _log.debug("Adding %s to list of imported classes used for looking for constants", elem_name) mod_classes.append(elem) @@ -89,13 +89,14 @@ def search_toolchain(name): tc_const_value = getattr(mod_class_mod, elem) _log.debug("Found constant %s ('%s') in module %s, adding it to %s", tc_const_name, tc_const_value, mod_class_mod.__name__, package.__name__) - if hasattr(package, tc_const_name): + try: cur_value = getattr(package, tc_const_name) + except AttributeError: + setattr(package, tc_const_name, tc_const_value) + else: if not tc_const_value == cur_value: raise EasyBuildError("Constant %s.%s defined as '%s', can't set it to '%s'.", package.__name__, tc_const_name, cur_value, tc_const_value) - else: - setattr(package, tc_const_name, tc_const_value) # indicate that processing of toolchain constants is done, so it's not done again setattr(package, check_attr_name, True) diff --git a/easybuild/tools/version.py b/easybuild/tools/version.py index b5e82834cc..2ea349637d 100644 --- a/easybuild/tools/version.py +++ b/easybuild/tools/version.py @@ -92,7 +92,7 @@ def get_git_revision(): def this_is_easybuild(): """Standard starting message""" - top_version = max(FRAMEWORK_VERSION, EASYBLOCKS_VERSION) + top_version = max(FRAMEWORK_VERSION, LooseVersion(EASYBLOCKS_VERSION)) msg = "This is EasyBuild %s (framework: %s, easyblocks: %s) on host %s." msg = msg % (top_version, FRAMEWORK_VERSION, EASYBLOCKS_VERSION, gethostname()) diff --git a/eb b/eb index 402bb87d0a..f427c5e7e7 100755 --- a/eb +++ b/eb @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash ## # Copyright 2009-2023 Ghent University # diff --git a/test/framework/build_log.py b/test/framework/build_log.py index a1792b998f..21755b62b3 100644 --- a/test/framework/build_log.py +++ b/test/framework/build_log.py @@ -139,8 +139,8 @@ def test_easybuildlog(self): r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*onemorewarning.*", r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*lastwarning.*", r"fancyroot.test_easybuildlog \[WARNING\] :: Deprecated functionality.*thisisnotprinted.*", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): err: msg: %s", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): err: msg: %s", r"fancyroot.test_easybuildlog \[ERROR\] :: .*EasyBuild encountered an exception \(at .* in .*\): oops", '', ]) @@ -168,7 +168,7 @@ def test_easybuildlog(self): r"fancyroot.test_easybuildlog \[WARNING\] :: bleh", r"fancyroot.test_easybuildlog \[INFO\] :: 4\+2 = 42", r"fancyroot.test_easybuildlog \[DEBUG\] :: this is just a test", - r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): foo baz baz", + r"fancyroot.test_easybuildlog \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): foo baz baz", '', ]) logtxt_regex = re.compile(r'^%s' % expected_logtxt, re.M) @@ -223,7 +223,7 @@ def test_log_levels(self): info_msg = r"%s \[INFO\] :: fyi" % prefix warning_msg = r"%s \[WARNING\] :: this is a warning" % prefix deprecated_msg = r"%s \[WARNING\] :: Deprecated functionality, .*: almost kaput; see .*" % prefix - error_msg = r"%s \[ERROR\] :: EasyBuild crashed with an error \(at .* in .*\): kaput" % prefix + error_msg = r"%s \[ERROR\] :: EasyBuild encountered an error \(at .* in .*\): kaput" % prefix expected_logtxt = '\n'.join([ error_msg, diff --git a/test/framework/docs.py b/test/framework/docs.py index 146d05b24c..70280892e4 100644 --- a/test/framework/docs.py +++ b/test/framework/docs.py @@ -405,6 +405,104 @@ ``1.4``|``GCC/4.6.3``, ``system`` ``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} +LIST_SOFTWARE_SIMPLE_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + +* GCC +* gzip""" + +LIST_SOFTWARE_DETAILED_MD = """# List of supported software + +EasyBuild supports 2 different software packages (incl. toolchains, bundles): + +[g](#g) + + +## G + + +[GCC](#gcc) - [gzip](#gzip) + + +### GCC + +%(gcc_descr)s + +*homepage*: + +version |toolchain +---------|---------- +``4.6.3``|``system`` + +### gzip + +%(gzip_descr)s + +*homepage*: + +version|toolchain +-------|------------------------------- +``1.4``|``GCC/4.6.3``, ``system`` +``1.5``|``foss/2018a``, ``intel/2018a``""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + +LIST_SOFTWARE_SIMPLE_JSON = """[ +{ + "name": "GCC" +}, +{ + "name": "gzip" +} +]""" + +LIST_SOFTWARE_DETAILED_JSON = """[ +{ + "description": "%(gcc_descr)s", + "homepage": "http://gcc.gnu.org/", + "name": "GCC", + "toolchain": "system", + "version": "4.6.3", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "GCC/4.6.3", + "version": "1.4", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "system", + "version": "1.4", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "foss/2018a", + "version": "1.5", + "versionsuffix": "" +}, +{ + "description": "%(gzip_descr)s", + "homepage": "http://www.gzip.org/", + "name": "gzip", + "toolchain": "intel/2018a", + "version": "1.5", + "versionsuffix": "" +} +]""" % {'gcc_descr': GCC_DESCR, 'gzip_descr': GZIP_DESCR} + class DocsTest(EnhancedTestCase): @@ -541,6 +639,9 @@ def test_license_docs(self): regex = re.compile(r"^``GPLv3``\s*|The GNU General Public License", re.M) self.assertTrue(regex.search(lic_docs), "%s found in: %s" % (regex.pattern, lic_docs)) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, avail_easyconfig_licenses, output_format='json') + def test_list_easyblocks(self): """ Tests for list_easyblocks function @@ -569,6 +670,9 @@ def test_list_easyblocks(self): txt = list_easyblocks(list_easyblocks='detailed', output_format='md') self.assertEqual(txt, LIST_EASYBLOCKS_DETAILED_MD % {'topdir': topdir_easyblocks}) + # expect NotImplementedError for JSON output + self.assertRaises(NotImplementedError, list_easyblocks, output_format='json') + def test_list_software(self): """Test list_software* functions.""" build_options = { @@ -587,6 +691,9 @@ def test_list_software(self): self.assertEqual(list_software(output_format='md'), LIST_SOFTWARE_SIMPLE_MD) self.assertEqual(list_software(output_format='md', detailed=True), LIST_SOFTWARE_DETAILED_MD) + self.assertEqual(list_software(output_format='json'), LIST_SOFTWARE_SIMPLE_JSON) + self.assertEqual(list_software(output_format='json', detailed=True), LIST_SOFTWARE_DETAILED_JSON) + # GCC/4.6.3 is installed, no gzip module installed txt = list_software(output_format='txt', detailed=True, only_installed=True) self.assertTrue(re.search(r'^\* GCC', txt, re.M)) @@ -690,6 +797,10 @@ def test_list_toolchains(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + list_toolchains(output_format='json') + def test_avail_cfgfile_constants(self): """ Test avail_cfgfile_constants to generate overview of constants that can be used in a configuration file. @@ -734,6 +845,10 @@ def test_avail_cfgfile_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_cfgfile_constants(option_parser.go_cfg_constants, output_format='json') + def test_avail_easyconfig_constants(self): """ Test avail_easyconfig_constants to generate overview of constants that can be used in easyconfig files. @@ -777,6 +892,10 @@ def test_avail_easyconfig_constants(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_constants(output_format='json') + def test_avail_easyconfig_templates(self): """ Test avail_easyconfig_templates to generate overview of templates that can be used in easyconfig files. @@ -785,6 +904,7 @@ def test_avail_easyconfig_templates(self): r"^Template names/values derived from easyconfig instance", r"^\s+%\(version_major\)s: Major version", r"^Template names/values for \(short\) software versions", + r"^\s+%\(pymajver\)s: major version for Python", r"^\s+%\(pyshortver\)s: short version for Python \(\.\)", r"^Template constants that can be used in easyconfigs", r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)", @@ -826,6 +946,10 @@ def test_avail_easyconfig_templates(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_easyconfig_templates(output_format='json') + def test_avail_toolchain_opts(self): """ Test avail_toolchain_opts to generate overview of supported toolchain options. @@ -910,6 +1034,12 @@ def test_avail_toolchain_opts(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(txt_rst), "Pattern '%s' should be found in: %s" % (regex.pattern, txt_rst)) + # expect NotImplementedError for json output format + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('foss', output_format='json') + with self.assertRaises(NotImplementedError): + avail_toolchain_opts('intel', output_format='json') + def test_mk_table(self): """ Tests for mk_*_table functions. diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d3d6a32dbd..93d9fed793 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -39,6 +39,7 @@ from unittest import TextTestRunner import easybuild.tools.systemtools as st +from easybuild.base import fancylogger from easybuild.framework.easyblock import EasyBlock, get_easyblock_instance from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.easyconfig.easyconfig import EasyConfig @@ -642,6 +643,7 @@ def test_make_module_extra(self): # also check how absolute paths specified in modexself.contents = '\n'.join([ self.contents += "\nmodextrapaths = {'TEST_PATH_VAR': ['foo', '/test/absolute/path', 'bar']}" + self.contents += "\nmodextrapaths_append = {'TEST_PATH_VAR_APPEND': ['foo', '/test/absolute/path', 'bar']}" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -655,6 +657,7 @@ def test_make_module_extra(self): # allow use of absolute paths, and verify contents of module self.contents += "\nallow_prepend_abs_path = True" + self.contents += "\nallow_append_abs_path = True" self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) @@ -674,6 +677,9 @@ def test_make_module_extra(self): r"^prepend[-_]path.*TEST_PATH_VAR.*root.*foo", r"^prepend[-_]path.*TEST_PATH_VAR.*/test/absolute/path", r"^prepend[-_]path.*TEST_PATH_VAR.*root.*bar", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*foo", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*/test/absolute/path", + r"^append[-_]path.*TEST_PATH_VAR_APPEND.*root.*bar", ] for pattern in patterns: self.assertTrue(re.search(pattern, txt, re.M), "Pattern '%s' found in: %s" % (pattern, txt)) @@ -1179,6 +1185,7 @@ def test_make_module_step(self): 'PATH': ('xbin', 'pibin'), 'CPATH': 'pi/include', } + modextrapaths_append = {'APPEND_PATH': 'append_path'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, @@ -1192,6 +1199,7 @@ def test_make_module_step(self): "hiddendependencies = [('test', '1.2.3'), ('OpenMPI', '2.1.2-GCC-6.4.0-2.28')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), + "modextrapaths_append = %s" % str(modextrapaths_append), ]) # test if module is generated correctly @@ -1264,6 +1272,18 @@ def test_make_module_step(self): num_prepends = len(regex.findall(txt)) self.assertEqual(num_prepends, 1, "Expected exactly 1 %s command in %s" % (regex.pattern, txt)) + for (key, vals) in modextrapaths_append.items(): + if isinstance(vals, str): + vals = [vals] + for val in vals: + if get_module_syntax() == 'Tcl': + regex = re.compile(r'^append-path\s+%s\s+\$root/%s$' % (key, val), re.M) + elif get_module_syntax() == 'Lua': + regex = re.compile(r'^append_path\("%s", pathJoin\(root, "%s"\)\)$' % (key, val), re.M) + else: + self.fail("Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) + for (name, ver) in [('GCC', '6.4.0-2.28')]: if get_module_syntax() == 'Tcl': regex = re.compile(r'^\s*module load %s\s*$' % os.path.join(name, ver), re.M) @@ -1526,6 +1546,7 @@ def test_fetch_sources(self): def test_download_instructions(self): """Test use of download_instructions easyconfig parameter.""" + orig_test_ec = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'software_with_missing_sources'", @@ -1561,7 +1582,8 @@ def test_download_instructions(self): self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step) stderr = self.get_stderr().strip() self.mock_stderr(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '') @@ -1575,7 +1597,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nManual download from example.com required", stderr) + self.assertIn("Download instructions:\n\n Manual download from example.com required", stderr) + self.assertIn("Make the files available in the active source path", stderr) # wipe top-level download instructions, try again self.contents = self.contents.replace(download_instructions, '') @@ -1604,7 +1627,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # download instructions should also be printed if 'source_tmpl' is used to specify extension sources self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,") @@ -1617,7 +1641,8 @@ def test_download_instructions(self): stderr = self.get_stderr().strip() self.mock_stderr(False) self.mock_stdout(False) - self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr) + self.assertIn("Download instructions:\n\n Extension sources must be downloaded via example.com", stderr) + self.assertIn("Make the files available in the active source path", stderr) # create dummy source file for extension write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '') @@ -2080,7 +2105,7 @@ def test_extensions_sanity_check(self): eb.silent = True error_pattern = r"Sanity check failed: extensions sanity check failed for 1 extensions: toy\n" error_pattern += r"failing sanity check for 'toy' extension: " - error_pattern += r'command "thisshouldfail" failed; output:\n/bin/bash:.* thisshouldfail: command not found' + error_pattern += r'command "thisshouldfail" failed; output:\n.* thisshouldfail: command not found' with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, eb.run_all_steps, True) @@ -2095,7 +2120,7 @@ def test_extensions_sanity_check(self): eb.run_all_steps(True) def test_parallel(self): - """Test defining of parallellism.""" + """Test defining of parallelism.""" topdir = os.path.abspath(os.path.dirname(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') toytxt = read_file(toy_ec) @@ -2112,7 +2137,7 @@ def test_parallel(self): os.close(handle) write_file(toy_ec3, toytxt + "\nparallel = False") - # default: parallellism is derived from # available cores + ulimit + # default: parallelism is derived from # available cores + ulimit test_eb = EasyBlock(EasyConfig(toy_ec)) test_eb.check_readiness_step() self.assertTrue(isinstance(test_eb.cfg['parallel'], int) and test_eb.cfg['parallel'] > 0) @@ -2456,6 +2481,29 @@ def test_checksum_step(self): eb.fetch_sources() eb.checksum_step() + with self.mocked_stdout_stderr() as (stdout, stderr): + + # using checksum-less test easyconfig in location that does not provide checksums.json + test_ec = os.path.join(self.test_prefix, 'test-no-checksums.eb') + copy_file(toy_ec, test_ec) + write_file(test_ec, 'checksums = []', append=True) + ec = process_easyconfig(test_ec)[0] + + # enable logging to screen, so we can check whether error is logged when checksums.json is not found + fancylogger.logToScreen(enable=True, stdout=True) + + eb = get_easyblock_instance(ec) + eb.fetch_sources() + eb.checksum_step() + + fancylogger.logToScreen(enable=False, stdout=True) + stdout = self.get_stdout() + + # make sure there's no error logged for not finding checksums.json, + # see also https://github.com/easybuilders/easybuild-framework/issues/4301 + regex = re.compile("ERROR .*Couldn't find file checksums.json anywhere", re.M) + self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in log" % regex.pattern) + # fiddle with checksum to check whether faulty checksum is catched copy_file(toy_ec, self.test_prefix) toy_ec = os.path.join(self.test_prefix, os.path.basename(toy_ec)) diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e9266e0aa8..141e9b0f77 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -59,10 +59,10 @@ from easybuild.framework.easyconfig.parser import EasyConfigParser, fetch_parameters_from_easyconfig from easybuild.framework.easyconfig.templates import template_constant_dict, to_template_str from easybuild.framework.easyconfig.style import check_easyconfigs_style -from easybuild.framework.easyconfig.tools import categorize_files_by_type, check_sha256_checksums, dep_graph -from easybuild.framework.easyconfig.tools import det_copy_ec_specs, find_related_easyconfigs, get_paths_for +from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, check_sha256_checksums +from easybuild.framework.easyconfig.tools import dep_graph, det_copy_ec_specs, find_related_easyconfigs, get_paths_for from easybuild.framework.easyconfig.tools import parse_easyconfigs -from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak_one +from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak, tweak_one from easybuild.framework.extension import resolve_exts_filter_template from easybuild.toolchains.system import SystemToolchain from easybuild.tools.build_log import EasyBuildError @@ -74,7 +74,7 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compilers, det_toolchain_mpi from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.options import parse_external_modules_metadata -from easybuild.tools.robot import resolve_dependencies +from easybuild.tools.robot import det_robot_path, resolve_dependencies from easybuild.tools.systemtools import AARCH64, KNOWN_ARCH_CONSTANTS, POWER, X86_64 from easybuild.tools.systemtools import get_cpu_architecture, get_shared_lib_ext, get_os_name, get_os_version @@ -302,7 +302,9 @@ def test_dependency(self): self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') + self.assertEqual(eb.dependency_names(), {'first', 'second', 'foo', 'bar'}) # same tests for builddependencies + self.assertEqual(eb.dependency_names(build_only=True), {'first', 'second'}) first = eb.builddependencies()[0] second = eb.builddependencies()[1] @@ -355,6 +357,7 @@ def test_false_dep_version(self): self.assertEqual(len(deps), 2) self.assertEqual(deps[0]['name'], 'second_build') self.assertEqual(deps[1]['name'], 'first') + self.assertEqual(eb.dependency_names(), {'first', 'second_build'}) # more realistic example: only filter dep for POWER self.contents = '\n'.join([ @@ -378,12 +381,14 @@ def test_false_dep_version(self): deps = eb.dependencies() self.assertEqual(len(deps), 1) self.assertEqual(deps[0]['name'], 'not_on_power') + self.assertEqual(eb.dependency_names(), {'not_on_power'}) # only power, dependency gets filtered st.get_cpu_architecture = lambda: POWER eb = EasyConfig(self.eb_file) deps = eb.dependencies() self.assertEqual(deps, []) + self.assertEqual(eb.dependency_names(), set()) def test_extra_options(self): """ extra_options should allow other variables to be stored """ @@ -732,6 +737,38 @@ def test_tweaking(self): # cleanup os.remove(tweaked_fn) + def test_tweak_multiple_tcs(self): + """Test that tweaking variables of ECs from multiple toolchains works""" + test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + + # Create directories to store the tweaked easyconfigs + tweaked_ecs_paths, pr_path = alt_easyconfig_paths(self.test_prefix, tweaked_ecs=True) + robot_path = det_robot_path([test_easyconfigs], tweaked_ecs_paths, pr_path, auto_robot=True) + + init_config(build_options={ + 'valid_module_classes': module_classes(), + 'robot_path': robot_path, + 'check_osdeps': False, + }) + + # Allow tweaking of non-toolchain values for multiple ECs of different toolchains + untweaked_openmpi_1 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-2.1.2-GCC-4.6.4.eb') + untweaked_openmpi_2 = os.path.join(test_easyconfigs, 'o', 'OpenMPI', 'OpenMPI-3.1.1-GCC-7.3.0-2.30.eb') + easyconfigs, _ = parse_easyconfigs([(untweaked_openmpi_1, False), (untweaked_openmpi_2, False)]) + tweak_specs = {'moduleclass': 'debugger'} + easyconfigs = tweak(easyconfigs, tweak_specs, self.modtool, targetdirs=tweaked_ecs_paths) + # Check that all expected tweaked easyconfigs exists + tweaked_openmpi_1 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_1)) + tweaked_openmpi_2 = os.path.join(tweaked_ecs_paths[0], os.path.basename(untweaked_openmpi_2)) + self.assertTrue(os.path.isfile(tweaked_openmpi_1)) + self.assertTrue(os.path.isfile(tweaked_openmpi_2)) + tweaked_openmpi_content_1 = read_file(tweaked_openmpi_1) + tweaked_openmpi_content_2 = read_file(tweaked_openmpi_2) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_1, + "Tweaked value not found in " + tweaked_openmpi_content_1) + self.assertTrue('moduleclass = "debugger"' in tweaked_openmpi_content_2, + "Tweaked value not found in " + tweaked_openmpi_content_2) + def test_installversion(self): """Test generation of install version.""" @@ -1118,6 +1155,7 @@ def test_templating_constants(self): 'R: %%(rver)s, %%(rmajver)s, %%(rminver)s, %%(rshortver)s', ]), 'modextrapaths = {"PI_MOD_NAME": "%%(module_name)s"}', + 'modextrapaths_append = {"PATH_APPEND": "appended_path"}', 'license_file = HOME + "/licenses/PI/license.txt"', "github_account = 'easybuilders'", ]) % inp @@ -1158,6 +1196,7 @@ def test_templating_constants(self): self.assertEqual(ec['modloadmsg'], expected) self.assertEqual(ec['modunloadmsg'], expected) self.assertEqual(ec['modextrapaths'], {'PI_MOD_NAME': 'PI/3.04-Python-2.7.10'}) + self.assertEqual(ec['modextrapaths_append'], {'PATH_APPEND': 'appended_path'}) self.assertEqual(ec['license_file'], os.path.join(os.environ['HOME'], 'licenses', 'PI', 'license.txt')) # test the escaping insanity here (ie all the crap we allow in easyconfigs) @@ -1294,7 +1333,7 @@ def test_templating_doc(self): # expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between) temps = [ easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG, - easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2, + easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3, easyconfig.templates.TEMPLATE_NAMES_CONFIG, easyconfig.templates.TEMPLATE_NAMES_LOWER, easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, @@ -1350,6 +1389,34 @@ def test_start_dir_template(self): self.assertIn('start_dir in extension configure is %s &&' % ext_start_dir, logtxt) self.assertIn('start_dir in extension build is %s &&' % ext_start_dir, logtxt) + def test_sysroot_template(self): + """Test the %(sysroot)s template""" + + test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nconfigopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\nbuildopts = "--some-opt=%(sysroot)s/"' + test_ec_txt += '\ninstallopts = "--some-opt=%(sysroot)s/"' + write_file(test_ec, test_ec_txt) + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=/") + self.assertEqual(ec['buildopts'], "--some-opt=/") + self.assertEqual(ec['installopts'], "--some-opt=/") + + # Validate the value of the sysroot template if sysroot is unset (i.e. the build option is None) + # As a test, we'll set the sysroot to self.test_prefix, as it has to be a directory that is guaranteed to exist + update_build_option('sysroot', self.test_prefix) + + ec = EasyConfig(test_ec) + self.assertEqual(ec['configopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['buildopts'], "--some-opt=%s/" % self.test_prefix) + self.assertEqual(ec['installopts'], "--some-opt=%s/" % self.test_prefix) + def test_constant_doc(self): """test constant documentation""" doc = avail_easyconfig_constants() @@ -1538,17 +1605,6 @@ def test_get_easyblock_class(self): self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None) - # also test deprecated default_fallback named argument - self.assertErrorRegex(EasyBuildError, "DEPRECATED", get_easyblock_class, None, name='gzip', - default_fallback=False) - - orig_value = easybuild.tools.build_log.CURRENT_VERSION - easybuild.tools.build_log.CURRENT_VERSION = '3.9' - self.mock_stderr(True) - self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) - self.mock_stderr(False) - easybuild.tools.build_log.CURRENT_VERSION = orig_value - def test_letter_dir(self): """Test letter_dir_for function.""" test_cases = { @@ -1613,18 +1669,15 @@ def test_filter_deps(self): test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') ec_file = os.path.join(test_ecs_dir, 'f', 'foss', 'foss-2018a.eb') ec = EasyConfig(ec_file) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test filtering multiple deps init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI']) + self.assertEqual(ec.dependency_names(), {'GCC', 'OpenBLAS', 'OpenMPI'}) # test filtering of non-existing dep init_config(build_options={'filter_deps': ['zlib']}) - deps = sorted([dep['name'] for dep in ec.dependencies()]) - self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) + self.assertEqual(ec.dependency_names(), {'FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK'}) # test parsing of value passed to --filter-deps opts = init_config(args=[]) @@ -1658,6 +1711,7 @@ def test_filter_deps(self): init_config(build_options=build_options) ec = EasyConfig(ec_file, validate=False) self.assertEqual(ec.dependencies(), []) + self.assertEqual(ec.dependency_names(), set()) def test_replaced_easyconfig_parameters(self): """Test handling of replaced easyconfig parameters.""" @@ -1846,6 +1900,9 @@ def test_external_dependencies(self): } self.assertEqual(deps[7]['external_module_metadata'], cray_netcdf_metadata) + # External module names are omitted + self.assertEqual(ec.dependency_names(), {'intel'}) + # provide file with partial metadata for some external modules; # metadata obtained from probing modules should be added to it... metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') @@ -1874,6 +1931,7 @@ def test_external_dependencies(self): deps = ec.dependencies() self.assertEqual(len(deps), 8) + self.assertEqual(ec.dependency_names(), {'intel'}) for idx in [0, 1, 2, 6]: self.assertEqual(deps[idx]['external_module_metadata'], {}) @@ -2151,12 +2209,15 @@ def eval_quoted_string(quoted_val, val): Helper function to sanity check we can use the quoted string in Python contexts. Returns the evaluated (i.e. unquoted) string """ - globals = dict() + scope = dict() try: - exec('res = %s' % quoted_val, globals) - except Exception as e: # pylint: disable=broad-except - self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e)) - return globals['res'] + # this is needlessly complicated because we can't use 'exec' here without potentially running + # into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9) + # cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction + eval(compile('res = %s' % quoted_val, '', 'exec'), dict(), scope) + except Exception as err: # pylint: disable=broad-except + self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, err)) + return scope['res'] def assertEqual_unquoted(quoted_val, val): """Assert that evaluating the quoted_val yields the val""" @@ -3237,6 +3298,7 @@ def test_template_constant_dict(self): 'nameletter': 'g', 'nameletterlower': 'g', 'parallel': None, + 'sysroot': '', 'toolchain_name': 'foss', 'toolchain_version': '2018a', 'version': '1.5', @@ -3319,6 +3381,7 @@ def test_template_constant_dict(self): 'pyminver': '7', 'pyshortver': '3.7', 'pyver': '3.7.2', + 'sysroot': '', 'version': '0.01', 'version_major': '0', 'version_major_minor': '0.01', @@ -3383,6 +3446,7 @@ def test_template_constant_dict(self): 'namelower': 'foo', 'nameletter': 'f', 'nameletterlower': 'f', + 'sysroot': '', 'version': '1.2.3', 'version_major': '1', 'version_major_minor': '1.2', @@ -4374,31 +4438,39 @@ def test_cuda_compute_capabilities(self): description = 'test' toolchain = SYSTEM cuda_compute_capabilities = ['5.1', '7.0', '7.1'] - installopts = '%(cuda_compute_capabilities)s' - preinstallopts = '%(cuda_cc_space_sep)s' - prebuildopts = '%(cuda_cc_semicolon_sep)s' - configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"' + configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"' + prebuildopts = '%(cuda_cc_semicolon_sep)s' + buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" ' + 'semi="%(cuda_int_semicolon_sep)s"') + preinstallopts = '%(cuda_cc_space_sep)s' + installopts = '%(cuda_compute_capabilities)s' """) self.prep() ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '5.1,7.0,7.1') - self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') - self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" ' 'space="sm_51 sm_70 sm_71"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"') + self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1') + self.assertEqual(ec['buildopts'], 'comma="51,70,71" ' + 'space="51 70 71" ' + 'semi="51;70;71"') + self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1') + self.assertEqual(ec['installopts'], '5.1,7.0,7.1') # build options overwrite it init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']}) ec = EasyConfig(self.eb_file) - self.assertEqual(ec['installopts'], '4.2,6.3') - self.assertEqual(ec['preinstallopts'], '4.2 6.3') - self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" ' 'space="sm_42 sm_63"') - self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"') + self.assertEqual(ec['buildopts'], 'comma="42,63" ' + 'space="42 63" ' + 'semi="42;63"') + self.assertEqual(ec['prebuildopts'], '4.2;6.3') + self.assertEqual(ec['preinstallopts'], '4.2 6.3') + self.assertEqual(ec['installopts'], '4.2,6.3') def test_det_copy_ec_specs(self): """Test det_copy_ec_specs function.""" @@ -4661,6 +4733,9 @@ def test_get_cuda_cc_template_value(self): 'cuda_compute_capabilities': '6.5,7.0', 'cuda_cc_space_sep': '6.5 7.0', 'cuda_cc_semicolon_sep': '6.5;7.0', + 'cuda_int_comma_sep': '65,70', + 'cuda_int_space_sep': '65 70', + 'cuda_int_semicolon_sep': '65;70', 'cuda_sm_comma_sep': 'sm_65,sm_70', 'cuda_sm_space_sep': 'sm_65 sm_70', } diff --git a/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb new file mode 100644 index 0000000000..008b76a1b4 --- /dev/null +++ b/test/framework/easyconfigs/test_ecs/i/iimpi/iimpi-2018a.eb @@ -0,0 +1,20 @@ +# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild +easyblock = 'Toolchain' + +name = 'iimpi' +version = '2018a' + +homepage = 'https://software.intel.com/parallel-studio-xe' +description = """Intel C/C++ and Fortran compilers, alongside Intel MPI.""" + +toolchain = SYSTEM + +local_compver = '2016.1.150' +local_suff = '-GCC-4.9.3-2.25' +dependencies = [ + ('icc', local_compver, local_suff), + ('ifort', local_compver, local_suff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_suff))), +] + +moduleclass = 'toolchain' diff --git a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb index af32ecb585..8fdf23171b 100644 --- a/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb +++ b/test/framework/easyconfigs/test_ecs/i/intel/intel-2018a.eb @@ -8,14 +8,17 @@ description = """Intel Cluster Toolkit Compiler Edition provides Intel C/C++ and toolchain = SYSTEM +local_compver = '2016.1.150' +local_gccver = '4.9.3' +local_binutilsver = '2.25' +local_gccsuff = '-GCC-%s-%s' % (local_gccver, local_binutilsver) # fake intel toolchain easyconfig, no dependencies (good enough for testing) local_fake_dependencies = [ - ('GCCcore', '6.4.0'), - ('binutils', '2.28', '-GCCcore-6.4.0'), - ('icc', '2018.1.163', '-GCCcore-6.4.0'), - ('ifort', '2018.1.163', '-GCCcore-6.4.0'), - ('impi', '2018.1.163', '', ('iccifort', '2018.1.163-GCCcore-6.4.0')), - ('imkl', '2018.1.163', '', ('iimpi', version)), + ('GCCcore', local_gccver), + ('binutils', local_binutilsver, '-GCCcore-%s' % local_gccver), + ('icc', local_compver, local_gccsuff), + ('ifort', local_compver, local_gccsuff), + ('impi', '5.1.2.150', '', ('iccifort', '%s%s' % (local_compver, local_gccsuff))), ] moduleclass = 'toolchain' diff --git a/test/framework/easystack.py b/test/framework/easystack.py index e131964df6..198350a0e7 100644 --- a/test/framework/easystack.py +++ b/test/framework/easystack.py @@ -31,6 +31,7 @@ import os import re import sys +import tempfile from unittest import TextTestRunner import easybuild.tools.build_log @@ -129,11 +130,22 @@ def test_easystack_invalid_key2(self): self.assertErrorRegex(EasyBuildError, error_pattern, parse_easystack, test_easystack) def test_easystack_restore_env_after_each_build(self): - """Test that the build environment is reset for each easystack item""" + """Test that the build environment and tmpdir is reset for each easystack item""" + + orig_tmpdir_tempfile = tempfile.gettempdir() + orig_tmpdir_env = os.getenv('TMPDIR') + orig_tmpdir_tempfile_len = len(orig_tmpdir_env.split(os.path.sep)) + orig_tmpdir_env_len = len(orig_tmpdir_env.split(os.path.sep)) + test_es_txt = '\n'.join([ "easyconfigs:", " - toy-0.0-gompi-2018a.eb:", " - libtoy-0.0.eb:", + # also include a couple of easyconfigs for which a module is already available in test environment, + # see test/framework/modules + " - GCC-7.3.0-2.30", + " - FFTW-3.3.7-gompi-2018a", + " - foss-2018a", ]) test_es_path = os.path.join(self.test_prefix, 'test.yml') write_file(test_es_path, test_es_txt) @@ -145,10 +157,25 @@ def test_easystack_restore_env_after_each_build(self): ] self.mock_stdout(True) stdout = self.eb_main(args, do_build=True, raise_error=True) + stdout = self.eb_main(args, do_build=True, raise_error=True, reset_env=False, redo_init_config=False) self.mock_stdout(False) regex = re.compile(r"WARNING Loaded modules detected: \[.*gompi/2018.*\]\n") self.assertFalse(regex.search(stdout), "Pattern '%s' should not be found in: %s" % (regex.pattern, stdout)) + # temporary directory after run should be exactly 2 levels deeper than original one: + # - 1 level added by setting up configuration in EasyBuild main function + # - 1 extra level added by first re-configuration for easystack item + # (because $TMPDIR set by configuration done in main function is retained) + tmpdir_tempfile = tempfile.gettempdir() + tmpdir_env = os.getenv('TMPDIR') + tmpdir_tempfile_len = len(tmpdir_tempfile.split(os.path.sep)) + tmpdir_env_len = len(tmpdir_env.split(os.path.sep)) + + self.assertEqual(tmpdir_tempfile_len, orig_tmpdir_tempfile_len + 2) + self.assertEqual(tmpdir_env_len, orig_tmpdir_env_len + 2) + self.assertTrue(tmpdir_tempfile.startswith(orig_tmpdir_tempfile)) + self.assertTrue(tmpdir_env.startswith(orig_tmpdir_env)) + def test_missing_easyconfigs_key(self): """Test that EasyStack file that doesn't contain an EasyConfigs key will fail with sane error message""" topdir = os.path.dirname(os.path.abspath(__file__)) diff --git a/test/framework/filetools.py b/test/framework/filetools.py index 5e2d0412fe..8d823ef59c 100644 --- a/test/framework/filetools.py +++ b/test/framework/filetools.py @@ -351,6 +351,14 @@ def test_checksums(self): alt_checksums = ('7167b64b1ca062b9674ffef46f9325db7167b64b1ca062b9674ffef46f9325db', broken_checksums['sha256']) self.assertFalse(ft.verify_checksum(fp, alt_checksums)) + # Check dictionary + alt_checksums = (known_checksums['sha256'],) + self.assertTrue(ft.verify_checksum(fp, {os.path.basename(fp): known_checksums['sha256']})) + faulty_dict = {'wrong-name': known_checksums['sha256']} + self.assertErrorRegex(EasyBuildError, + "Missing checksum for " + os.path.basename(fp) + " in .*wrong-name.*", + ft.verify_checksum, fp, faulty_dict) + # check whether missing checksums are enforced build_options = { 'enforce_checksums': True, @@ -365,6 +373,8 @@ def test_checksums(self): for checksum in [known_checksums[x] for x in ('md5', 'sha256')]: dict_checksum = {os.path.basename(fp): checksum, 'foo': 'baa'} self.assertTrue(ft.verify_checksum(fp, dict_checksum)) + del dict_checksum[os.path.basename(fp)] + self.assertErrorRegex(EasyBuildError, "Missing checksum for", ft.verify_checksum, fp, dict_checksum) def test_common_path_prefix(self): """Test get common path prefix for a list of paths.""" @@ -2280,7 +2290,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertNotExists(os.path.join(self.test_prefix, 'toy-0.0')) - self.assertTrue(re.search('running command "tar xzf .*/toy-0.0.tar.gz"', txt)) + self.assertTrue(re.search('running shell command "tar xzf .*/toy-0.0.tar.gz"', txt)) with self.mocked_stdout_stderr(): path = ft.extract_file(toy_tarball, self.test_prefix, forced=True, change_into_dir=False) @@ -2304,7 +2314,7 @@ def test_extract_file(self): self.assertTrue(os.path.samefile(path, self.test_prefix)) self.assertTrue(os.path.samefile(os.getcwd(), self.test_prefix)) self.assertFalse(stderr) - self.assertTrue("running command" in stdout) + self.assertTrue("running shell command" in stdout) # check whether disabling trace output works with self.mocked_stdout_stderr(): @@ -2397,7 +2407,7 @@ def test_index_functions(self): # test with specified path with and without trailing '/'s for path in [test_ecs, test_ecs + '/', test_ecs + '//']: index = ft.create_index(path) - self.assertEqual(len(index), 91) + self.assertEqual(len(index), 92) expected = [ os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'), @@ -2411,12 +2421,13 @@ def test_index_functions(self): self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json') # set up some files to create actual index file for - ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g')) + ecs_dir = os.path.join(self.test_prefix, 'easyconfigs') + ft.copy_dir(os.path.join(test_ecs, 'g'), ecs_dir) # test dump_index function - index_fp = ft.dump_index(self.test_prefix) + index_fp = ft.dump_index(ecs_dir) self.assertExists(index_fp) - self.assertTrue(os.path.samefile(self.test_prefix, os.path.dirname(index_fp))) + self.assertTrue(os.path.samefile(ecs_dir, os.path.dirname(index_fp))) datestamp_pattern = r"[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+" expected_header = [ @@ -2424,9 +2435,9 @@ def test_index_functions(self): "# valid until: " + datestamp_pattern, ] expected = [ - os.path.join('g', 'gzip', 'gzip-1.4.eb'), - os.path.join('g', 'GCC', 'GCC-7.3.0-2.30.eb'), - os.path.join('g', 'gompic', 'gompic-2018a.eb'), + os.path.join('gzip', 'gzip-1.4.eb'), + os.path.join('GCC', 'GCC-7.3.0-2.30.eb'), + os.path.join('gompic', 'gompic-2018a.eb'), ] index_txt = ft.read_file(index_fp) for fn in expected_header + expected: @@ -2436,28 +2447,28 @@ def test_index_functions(self): # test load_index function self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) # dump_index will not overwrite existing index without force error_pattern = "File exists, not overwriting it without --force" - self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, self.test_prefix) + self.assertErrorRegex(EasyBuildError, error_pattern, ft.dump_index, ecs_dir) ft.remove_file(index_fp) # test creating index file that's infinitely valid - index_fp = ft.dump_index(self.test_prefix, max_age_sec=0) + index_fp = ft.dump_index(ecs_dir, max_age_sec=0) index_txt = ft.read_file(index_fp) expected_header[1] = r"# valid until: 9999-12-31 23:59:59\.9+" for fn in expected_header + expected: @@ -2466,40 +2477,40 @@ def test_index_functions(self): self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertFalse(stderr) - regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % self.test_prefix) + regex = re.compile(r"^== found valid index for %s, so using it\.\.\.$" % ecs_dir) self.assertTrue(regex.match(stdout.strip()), "Pattern '%s' matches with: %s" % (regex.pattern, stdout)) - self.assertEqual(len(index), 26) + self.assertEqual(len(index), 25) for fn in expected: self.assertIn(fn, index) ft.remove_file(index_fp) # test creating index file that's only valid for a (very) short amount of time - index_fp = ft.dump_index(self.test_prefix, max_age_sec=1) + index_fp = ft.dump_index(ecs_dir, max_age_sec=1) time.sleep(3) self.mock_stderr(True) self.mock_stdout(True) - index = ft.load_index(self.test_prefix) + index = ft.load_index(ecs_dir) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) self.mock_stdout(False) self.assertIsNone(index) self.assertFalse(stdout) - regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % self.test_prefix) + regex = re.compile(r"WARNING: Index for %s is no longer valid \(too old\), so ignoring it" % ecs_dir) self.assertTrue(regex.search(stderr), "Pattern '%s' found in: %s" % (regex.pattern, stderr)) # check whether load_index takes into account --ignore-index init_config(build_options={'ignore_index': True}) - self.assertEqual(ft.load_index(self.test_prefix), None) + self.assertEqual(ft.load_index(ecs_dir), None) def test_search_file(self): """Test search_file function.""" @@ -2791,26 +2802,25 @@ def run_check(): 'git_repo': 'git@github.com:easybuilders/testrepository.git', 'test_prefix': self.test_prefix, } + reprod_tar_cmd_pattern = ( + r' running shell command "find {} -name \".git\" -prune -o -print0 -exec touch -t 197001010100 {{}} \; |' + r' LC_ALL=C sort --zero-terminated | tar --create --no-recursion --owner=0 --group=0 --numeric-owner' + r' --format=gnu --null --files-from - | gzip --no-name > %(test_prefix)s/target/test.tar.gz' + ) expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file %(test_prefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() git_config['clone_into'] = 'test123' expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', + r' running shell command "git clone --depth 1 --branch tag_for_tests %(git_repo)s test123"', r" \(in .*/tmp.*\)", - r' running command "find test123 -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("test123"), r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2818,24 +2828,44 @@ def run_check(): git_config['recursive'] = True expected = '\n'.join([ - r' running command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --depth 1 --branch tag_for_tests --recursive %(git_repo)s"', + r" \(in .*/tmp.*\)", + reprod_tar_cmd_pattern.format("testrepository"), + r" \(in .*/tmp.*\)", + ]) % string_args + run_check() + + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] + expected = '\n'.join([ + ' running shell command "git clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() + git_config['extra_config_params'] = [ + 'submodule."fastahack".active=false', + 'submodule."sha1".active=false', + ] + expected = '\n'.join([ + ' running shell command "git -c submodule."fastahack".active=false -c submodule."sha1".active=false' + + ' clone --depth 1 --branch tag_for_tests --recursive' + + ' --recurse-submodules=\'!vcflib\' --recurse-submodules=\'!sdsl-lite\' %(git_repo)s"', + r" \(in .*/tmp.*\)", + reprod_tar_cmd_pattern.format("testrepository"), + r" \(in .*/tmp.*\)", + ]) % string_args + run_check() + del git_config['recurse_submodules'] + del git_config['extra_config_params'] + git_config['keep_git_dir'] = True expected = '\n'.join([ - r' running command "git clone --branch tag_for_tests --recursive %(git_repo)s"', + r' running shell command "git clone --branch tag_for_tests --recursive %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "find testrepository -print0 | LC_ALL=C sort --zero-terminated | GZIP=--no-name tar' - r' --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion --gzip' - r' --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu --null --no-recursion' - r' --files-from -"', + r' running shell command "tar cfvz .*/target/test.tar.gz testrepository"', r" \(in .*/tmp.*\)", ]) % string_args run_check() @@ -2844,32 +2874,38 @@ def run_check(): del git_config['tag'] git_config['commit'] = '8456f86' expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86 && git submodule update --init --recursive"', + r' running shell command "git checkout 8456f86 && git submodule update --init --recursive"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() - del git_config['recursive'] + git_config['recurse_submodules'] = ['!vcflib', '!sdsl-lite'] expected = '\n'.join([ - r' running command "git clone --no-checkout %(git_repo)s"', + r' running shell command "git clone --no-checkout %(git_repo)s"', r" \(in .*/tmp.*\)", - r' running command "git checkout 8456f86"', + r' running shell command "git checkout 8456f86"', r" \(in testrepository\)", - r' running command "find testrepository -print0 -path \'*/.git\' -prune | LC_ALL=C sort --zero-terminated' - r' | GZIP=--no-name tar --create --file #(test_fprefix)s/target/test.tar.gz --no-recursion' - r' --gzip --mtime="1970-01-01 00:00Z" --owner=0 --group=0 --numeric-owner --format=gnu' - r' --null --no-recursion --files-from -"', + reprod_tar_cmd_pattern.format("testrepository"), r" \(in .*/tmp.*\)", ]) % string_args run_check() + del git_config['recursive'] + del git_config['recurse_submodules'] + expected = '\n'.join([ + r' running shell command "git clone --no-checkout %(git_repo)s"', + r" \(in /.*\)", + r' running shell command "git checkout 8456f86"', + r" \(in /.*/testrepository\)", + reprod_tar_cmd_pattern.format("testrepository"), + r" \(in /.*\)", + ]) % string_args + run_check() + # Test with real data. init_config() git_config = { diff --git a/test/framework/github.py b/test/framework/github.py index ec7b19181f..890ccfa28a 100644 --- a/test/framework/github.py +++ b/test/framework/github.py @@ -347,7 +347,6 @@ def test_github_reasons_for_closing(self): self.assertIsInstance(res, list) self.assertEqual(stderr.strip(), "WARNING: Using easyconfigs from closed PR #16080") patterns = [ - "Status of last commit is SUCCESS", "Last comment on", "No activity since", "* c-ares-1.18.1", @@ -655,6 +654,11 @@ def test_validate_github_token(self): if token_old_format: self.assertTrue(gh.validate_github_token(token_old_format, GITHUB_TEST_ACCOUNT)) + # if a fine-grained token is available, test with that too + finegrained_token = os.getenv('TEST_GITHUB_TOKEN_FINEGRAINED') + if finegrained_token: + self.assertTrue(gh.validate_github_token(finegrained_token, GITHUB_TEST_ACCOUNT)) + def test_github_find_easybuild_easyconfig(self): """Test for find_easybuild_easyconfig function""" if self.skip_github_tests: diff --git a/test/framework/hooks.py b/test/framework/hooks.py index fad251b040..c8e5d34583 100644 --- a/test/framework/hooks.py +++ b/test/framework/hooks.py @@ -32,8 +32,9 @@ from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner -import easybuild.tools.hooks +import easybuild.tools.hooks # so we can reset cached hooks from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import update_build_option from easybuild.tools.filetools import remove_file, write_file from easybuild.tools.hooks import find_hook, load_hooks, run_hook, verify_hooks @@ -52,6 +53,9 @@ def setUp(self): 'def parse_hook(ec):', ' print("Parse hook with argument %s" % ec)', '', + 'def pre_build_and_install_loop_hook(ecs):', + ' print("About to start looping for %d easyconfigs!" % len(ecs))', + '', 'def foo():', ' print("running foo helper method")', '', @@ -64,9 +68,31 @@ def setUp(self): '', 'def pre_single_extension_hook(ext):', ' print("this is run before installing an extension")', + '', + 'def pre_run_shell_cmd_hook(cmd, interactive=False):', + ' if interactive:', + ' print("this is run before running interactive command \'%s\'" % cmd)', + ' else:', + ' print("this is run before running command \'%s\'" % cmd)', + ' if cmd == "make install":', + ' return "sudo " + cmd', + '', + 'def fail_hook(err):', + ' print("EasyBuild FAIL: %s" % err)', + '', + 'def crash_hook(err):', + ' print("EasyBuild CRASHED, oh no! => %s" % err)', ]) write_file(self.test_hooks_pymod, test_hooks_pymod_txt) + def tearDown(self): + """Cleanup.""" + + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + + super(HooksTest, self).tearDown() + def test_load_hooks(self): """Test for load_hooks function.""" @@ -74,11 +100,15 @@ def test_load_hooks(self): hooks = load_hooks(self.test_hooks_pymod) - self.assertEqual(len(hooks), 5) + self.assertEqual(len(hooks), 9) expected = [ + 'crash_hook', + 'fail_hook', 'parse_hook', 'post_configure_hook', + 'pre_build_and_install_loop_hook', 'pre_install_hook', + 'pre_run_shell_cmd_hook', 'pre_single_extension_hook', 'start_hook', ] @@ -113,6 +143,10 @@ def test_find_hook(self): pre_install_hook = [hooks[k] for k in hooks if k == 'pre_install_hook'][0] pre_single_extension_hook = [hooks[k] for k in hooks if k == 'pre_single_extension_hook'][0] start_hook = [hooks[k] for k in hooks if k == 'start_hook'][0] + pre_run_shell_cmd_hook = [hooks[k] for k in hooks if k == 'pre_run_shell_cmd_hook'][0] + crash_hook = [hooks[k] for k in hooks if k == 'crash_hook'][0] + fail_hook = [hooks[k] for k in hooks if k == 'fail_hook'][0] + pre_build_and_install_loop_hook = [hooks[k] for k in hooks if k == 'pre_build_and_install_loop_hook'][0] self.assertEqual(find_hook('configure', hooks), None) self.assertEqual(find_hook('configure', hooks, pre_step_hook=True), None) @@ -138,6 +172,23 @@ def test_find_hook(self): self.assertEqual(find_hook('start', hooks, pre_step_hook=True), None) self.assertEqual(find_hook('start', hooks, post_step_hook=True), None) + self.assertEqual(find_hook('run_shell_cmd', hooks), None) + self.assertEqual(find_hook('run_shell_cmd', hooks, pre_step_hook=True), pre_run_shell_cmd_hook) + self.assertEqual(find_hook('run_shell_cmd', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('fail', hooks), fail_hook) + self.assertEqual(find_hook('fail', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('fail', hooks, post_step_hook=True), None) + + self.assertEqual(find_hook('crash', hooks), crash_hook) + self.assertEqual(find_hook('crash', hooks, pre_step_hook=True), None) + self.assertEqual(find_hook('crash', hooks, post_step_hook=True), None) + + hook_name = 'build_and_install_loop' + self.assertEqual(find_hook(hook_name, hooks), None) + self.assertEqual(find_hook(hook_name, hooks, pre_step_hook=True), pre_build_and_install_loop_hook) + self.assertEqual(find_hook(hook_name, hooks, post_step_hook=True), None) + def test_run_hook(self): """Test for run_hook function.""" @@ -145,43 +196,77 @@ def test_run_hook(self): init_config(build_options={'debug': True}) - self.mock_stdout(True) - self.mock_stderr(True) - run_hook('start', hooks) - run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") - run_hook('configure', hooks, pre_step_hook=True, args=[None]) - run_hook('configure', hooks, post_step_hook=True, args=[None]) - run_hook('build', hooks, pre_step_hook=True, args=[None]) - run_hook('build', hooks, post_step_hook=True, args=[None]) - run_hook('install', hooks, pre_step_hook=True, args=[None]) - run_hook('install', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, pre_step_hook=True, args=[None]) - for _ in range(3): - run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) - run_hook('single_extension', hooks, post_step_hook=True, args=[None]) - run_hook('extensions', hooks, post_step_hook=True, args=[None]) - stdout = self.get_stdout() - stderr = self.get_stderr() - self.mock_stdout(False) - self.mock_stderr(False) - - expected_stdout = '\n'.join([ + def run_hooks(): + self.mock_stdout(True) + self.mock_stderr(True) + run_hook('start', hooks) + run_hook('parse', hooks, args=[''], msg="Running parse hook for example.eb...") + run_hook('build_and_install_loop', hooks, args=[['ec1', 'ec2']], pre_step_hook=True) + run_hook('configure', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["configure.sh"], kwargs={'interactive': True}) + run_hook('configure', hooks, post_step_hook=True, args=[None]) + run_hook('build', hooks, pre_step_hook=True, args=[None]) + run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make -j 3"]) + run_hook('build', hooks, post_step_hook=True, args=[None]) + run_hook('install', hooks, pre_step_hook=True, args=[None]) + res = run_hook('run_shell_cmd', hooks, pre_step_hook=True, args=["make install"], kwargs={}) + self.assertEqual(res, "sudo make install") + run_hook('install', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, pre_step_hook=True, args=[None]) + for _ in range(3): + run_hook('single_extension', hooks, pre_step_hook=True, args=[None]) + run_hook('single_extension', hooks, post_step_hook=True, args=[None]) + run_hook('extensions', hooks, post_step_hook=True, args=[None]) + run_hook('fail', hooks, args=[EasyBuildError('oops')]) + run_hook('crash', hooks, args=[RuntimeError('boom!')]) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + return stdout, stderr + + stdout, stderr = run_hooks() + + expected_stdout_lines = [ "== Running start hook...", "this is triggered at the very beginning", "== Running parse hook for example.eb...", "Parse hook with argument ", + "== Running pre-build_and_install_loop hook...", + "About to start looping for 2 easyconfigs!", + "== Running pre-run_shell_cmd hook...", + "this is run before running interactive command 'configure.sh'", "== Running post-configure hook...", "this is run after configure step", "running foo helper method", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make -j 3'", "== Running pre-install hook...", "this is run before install step", + "== Running pre-run_shell_cmd hook...", + "this is run before running command 'make install'", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", "== Running pre-single_extension hook...", "this is run before installing an extension", - ]) + "== Running fail hook...", + "EasyBuild FAIL: 'oops'", + "== Running crash hook...", + "EasyBuild CRASHED, oh no! => boom!", + ] + expected_stdout = '\n'.join(expected_stdout_lines) + + self.assertEqual(stdout.strip(), expected_stdout) + self.assertEqual(stderr, '') + + # test silencing of hook trigger + update_build_option('silence_hook_trigger', True) + stdout, stderr = run_hooks() + + expected_stdout = '\n'.join(x for x in expected_stdout_lines if not x.startswith('== Running')) self.assertEqual(stdout.strip(), expected_stdout) self.assertEqual(stderr, '') diff --git a/test/framework/lib.py b/test/framework/lib.py index 9eed88632f..a6459b28c3 100644 --- a/test/framework/lib.py +++ b/test/framework/lib.py @@ -43,7 +43,7 @@ from easybuild.tools.options import set_up_configuration from easybuild.tools.filetools import mkdir from easybuild.tools.modules import modules_tool -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd, run_cmd class EasyBuildLibTest(TestCase): @@ -67,14 +67,7 @@ def tearDown(self): def configure(self): """Utility function to set up EasyBuild configuration.""" - - # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called - if BuildOptions in BuildOptions._instances: - del BuildOptions._instances[BuildOptions] - - self.assertNotIn(BuildOptions, BuildOptions._instances) - set_up_configuration(silent=True) - self.assertIn(BuildOptions, BuildOptions._instances) + set_up_configuration(silent=True, reconfigure=True) def test_run_cmd(self): """Test use of run_cmd function in the context of using EasyBuild framework as a library.""" @@ -92,8 +85,24 @@ def test_run_cmd(self): self.assertEqual(ec, 0) self.assertEqual(out, 'hello\n') + def test_run_shell_cmd(self): + """Test use of run_shell_cmd function in the context of using EasyBuild framework as a library.""" + + error_pattern = r"Undefined build option: .*" + error_pattern += r" Make sure you have set up the EasyBuild configuration using set_up_configuration\(\)" + self.assertErrorRegex(EasyBuildError, error_pattern, run_shell_cmd, "echo hello") + + self.configure() + + # runworks fine if set_up_configuration was called first + self.mock_stdout(True) + res = run_shell_cmd("echo hello") + self.mock_stdout(False) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'hello\n') + def test_mkdir(self): - """Test use of run_cmd function in the context of using EasyBuild framework as a library.""" + """Test use of mkdir function in the context of using EasyBuild framework as a library.""" test_dir = os.path.join(self.tmpdir, 'test123') diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index b6bec17093..d2fbb6f642 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -82,7 +82,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -107,7 +107,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -137,7 +137,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -161,7 +161,7 @@ def test_descr(self): '', 'Description', '===========', - "%s" % descr, + descr, '', '', "More information", @@ -756,6 +756,16 @@ def test_module_extensions(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(desc), "Pattern '%s' found in: %s" % (regex.pattern, desc)) + # check if the extensions is missing if there are no extensions + test_ec = os.path.join(test_dir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-test.eb') + + ec = EasyConfig(test_ec) + eb = EasyBlock(ec) + modgen = self.MODULE_GENERATOR_CLASS(eb) + desc = modgen.get_description() + + self.assertFalse(re.search(r"\s*extensions\(", desc), "No extensions found in: %s" % desc) + def test_prepend_paths(self): """Test generating prepend-paths statements.""" # test prepend_paths diff --git a/test/framework/modules.py b/test/framework/modules.py index d0c330e622..a849148bdf 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -50,11 +50,11 @@ from easybuild.tools.modules import EnvironmentModules, EnvironmentModulesC, EnvironmentModulesTcl, Lmod, NoModulesTool from easybuild.tools.modules import curr_module_paths, get_software_libdir, get_software_root, get_software_version from easybuild.tools.modules import invalidate_module_caches_for, modules_tool, reset_module_caches -from easybuild.tools.run import run +from easybuild.tools.run import run_shell_cmd # number of modules included for testing purposes -TEST_MODULES_COUNT = 92 +TEST_MODULES_COUNT = 110 class ModulesTest(EnhancedTestCase): @@ -212,12 +212,12 @@ def test_avail(self): # test modules include 3 GCC modules and one GCCcore module ms = self.modtool.available('GCC') - expected = ['GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] + expected = ['GCC/12.3.0', 'GCC/4.6.3', 'GCC/4.6.4', 'GCC/6.4.0-2.28', 'GCC/7.3.0-2.30'] # Tcl-only modules tool does an exact match on module name, Lmod & Tcl/C do prefix matching # EnvironmentModules is a subclass of EnvironmentModulesTcl, but Modules 4+ behaves similarly to Tcl/C impl., # so also append GCCcore/6.2.0 if we are an instance of EnvironmentModules if not isinstance(self.modtool, EnvironmentModulesTcl) or isinstance(self.modtool, EnvironmentModules): - expected.append('GCCcore/6.2.0') + expected.extend(['GCCcore/12.3.0', 'GCCcore/6.2.0']) self.assertEqual(ms, expected) # test modules include one GCC/4.6.3 module @@ -233,6 +233,12 @@ def test_avail(self): self.assertIn('bzip2/.1.0.6', ms) self.assertIn('toy/.0.0-deps', ms) self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) + elif (isinstance(self.modtool, EnvironmentModules) + and StrictVersion(self.modtool.version) >= StrictVersion('4.6.0')): + # bzip2/.1.0.6 is not there, since that's a module file in Lua syntax + self.assertEqual(len(ms), TEST_MODULES_COUNT + 2) + self.assertIn('toy/.0.0-deps', ms) + self.assertIn('OpenMPI/.2.1.2-GCC-6.4.0-2.28', ms) else: self.assertEqual(len(ms), TEST_MODULES_COUNT) @@ -452,9 +458,8 @@ def test_load(self): # if GCC is loaded again, $EBROOTGCC should be set again, and GCC should be listed last self.modtool.load(['GCC/6.4.0-2.28']) - # environment modules v4.0 does not reload already loaded modules, will be changed in v4.2 - modtool_ver = StrictVersion(self.modtool.version) - if not isinstance(self.modtool, EnvironmentModules) or modtool_ver >= StrictVersion('4.2'): + # environment modules v4+ does not reload already loaded modules + if not isinstance(self.modtool, EnvironmentModules): self.assertTrue(os.environ.get('EBROOTGCC')) if isinstance(self.modtool, Lmod): @@ -1332,7 +1337,7 @@ def test_module_use_bash(self): self.assertIn(modules_dir, modulepath) with self.mocked_stdout_stderr(): - res = run("bash -c 'echo MODULEPATH: $MODULEPATH'") + res = run_shell_cmd("bash -c 'echo MODULEPATH: $MODULEPATH'") self.assertEqual(res.output.strip(), f"MODULEPATH: {modulepath}") self.assertIn(modules_dir, res.output) @@ -1538,8 +1543,10 @@ def test_modulecmd_strip_source(self): os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.getenv('PATH')) - modtool = EnvironmentModulesC() - modtool.run_module('load', 'test123') + self.allow_deprecated_behaviour() + with self.mocked_stdout_stderr(): + modtool = EnvironmentModulesC() + modtool.run_module('load', 'test123') self.assertEqual(os.getenv('TEST123'), 'test123') def test_get_setenv_value_from_modulefile(self): diff --git a/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a new file mode 100644 index 0000000000..e35c12d348 --- /dev/null +++ b/test/framework/modules/FFTW.MPI/3.3.10-gompi-2023a @@ -0,0 +1,42 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW.MPI/3.3.10-gompi-2023a + +conflict FFTW.MPI + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +setenv EBROOTFFTWMPI "$root" +setenv EBVERSIONFFTWMPI "3.3.10" +setenv EBDEVELFFTWMPI "$root/easybuild/FFTW.MPI-3.3.10-gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 new file mode 100644 index 0000000000..2ae82a0193 --- /dev/null +++ b/test/framework/modules/FFTW/3.3.10-GCC-12.3.0 @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data. + + +More information +================ + - Homepage: https://www.fftw.org + } +} + +module-whatis {Description: FFTW is a C subroutine library for computing the discrete Fourier transform (DFT) +in one or more dimensions, of arbitrary input size, and of both real and complex data.} +module-whatis {Homepage: https://www.fftw.org} +module-whatis {URL: https://www.fftw.org} + +set root /prefix/software/FFTW/3.3.10-GCC-12.3.0 + +conflict FFTW + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFFTW "$root" +setenv EBVERSIONFFTW "3.3.10" +setenv EBDEVELFFTW "$root/easybuild/FFTW-3.3.10-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 new file mode 100644 index 0000000000..00f02d0460 --- /dev/null +++ b/test/framework/modules/FlexiBLAS/3.3.1-GCC-12.3.0 @@ -0,0 +1,47 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it. + + +More information +================ + - Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release + } +} + +module-whatis {Description: FlexiBLAS is a wrapper library that enables the exchange of the BLAS and LAPACK implementation +used by a program without recompiling or relinking it.} +module-whatis {Homepage: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} +module-whatis {URL: https://gitlab.mpi-magdeburg.mpg.de/software/flexiblas-release} + +set root /prefix/software/FlexiBLAS/3.3.1-GCC-12.3.0 + +conflict FlexiBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenBLAS/0.3.23-GCC-12.3.0 ] } { + module load OpenBLAS/0.3.23-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTFLEXIBLAS "$root" +setenv EBVERSIONFLEXIBLAS "3.3.1" +setenv EBDEVELFLEXIBLAS "$root/easybuild/FlexiBLAS-3.3.1-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCC/12.3.0 b/test/framework/modules/GCC/12.3.0 new file mode 100644 index 0000000000..5a21e5af1c --- /dev/null +++ b/test/framework/modules/GCC/12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCC/12.3.0 + +conflict GCC + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded binutils/2.40-GCCcore-12.3.0 ] } { + module load binutils/2.40-GCCcore-12.3.0 +} + +setenv EBROOTGCC "$root" +setenv EBVERSIONGCC "12.3.0" +setenv EBDEVELGCC "$root/easybuild/GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/GCCcore/12.3.0 b/test/framework/modules/GCCcore/12.3.0 new file mode 100644 index 0000000000..745b111007 --- /dev/null +++ b/test/framework/modules/GCCcore/12.3.0 @@ -0,0 +1,37 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...). + + +More information +================ + - Homepage: https://gcc.gnu.org/ + } +} + +module-whatis {Description: The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, + as well as libraries for these languages (libstdc++, libgcj,...).} +module-whatis {Homepage: https://gcc.gnu.org/} +module-whatis {URL: https://gcc.gnu.org/} + +set root /prefix/software/GCCcore/12.3.0 + +conflict GCCcore + +prepend-path CMAKE_LIBRARY_PATH $root/lib64 +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib64 +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTGCCCORE "$root" +setenv EBVERSIONGCCCORE "12.3.0" +setenv EBDEVELGCCCORE "$root/easybuild/GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 new file mode 100644 index 0000000000..89a72a65bc --- /dev/null +++ b/test/framework/modules/OpenBLAS/0.3.23-GCC-12.3.0 @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. + + +More information +================ + - Homepage: http://www.openblas.net/ + } +} + +module-whatis {Description: OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version.} +module-whatis {Homepage: http://www.openblas.net/} +module-whatis {URL: http://www.openblas.net/} + +set root /prefix/software/OpenBLAS/0.3.23-GCC-12.3.0 + +conflict OpenBLAS + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTOPENBLAS "$root" +setenv EBVERSIONOPENBLAS "0.3.23" +setenv EBDEVELOPENBLAS "$root/easybuild/OpenBLAS-0.3.23-GCC-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 new file mode 100644 index 0000000000..481f81e627 --- /dev/null +++ b/test/framework/modules/OpenMPI/4.1.5-GCC-12.3.0 @@ -0,0 +1,70 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The Open MPI Project is an open source MPI-3 implementation. + + +More information +================ + - Homepage: https://www.open-mpi.org/ + } +} + +module-whatis {Description: The Open MPI Project is an open source MPI-3 implementation.} +module-whatis {Homepage: https://www.open-mpi.org/} +module-whatis {URL: https://www.open-mpi.org/} + +set root /scratch/brussel/vo/000/bvo00005/vsc10009/ebtest/tclmodules/software/OpenMPI/4.1.5-GCC-12.3.0 + +conflict OpenMPI + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +if { ![ is-loaded hwloc/2.9.1-GCCcore-12.3.0 ] } { + module load hwloc/2.9.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libevent/2.1.12-GCCcore-12.3.0 ] } { + module load libevent/2.1.12-GCCcore-12.3.0 +} + +if { ![ is-loaded UCX/1.14.1-GCCcore-12.3.0 ] } { + module load UCX/1.14.1-GCCcore-12.3.0 +} + +if { ![ is-loaded libfabric/1.18.0-GCCcore-12.3.0 ] } { + module load libfabric/1.18.0-GCCcore-12.3.0 +} + +if { ![ is-loaded PMIx/4.2.4-GCCcore-12.3.0 ] } { + module load PMIx/4.2.4-GCCcore-12.3.0 +} + +if { ![ is-loaded UCC/1.2.0-GCCcore-12.3.0 ] } { + module load UCC/1.2.0-GCCcore-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTOPENMPI "$root" +setenv EBVERSIONOPENMPI "4.1.5" +setenv EBDEVELOPENMPI "$root/easybuild/OpenMPI-4.1.5-GCC-12.3.0-easybuild-devel" + +setenv SLURM_MPI_TYPE "pmix" +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/PMIx/4.2.4-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb new file mode 100644 index 0000000000..7118eebf4f --- /dev/null +++ b/test/framework/modules/ScaLAPACK/2.2.0-gompi-2023a-fb @@ -0,0 +1,43 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers. + + +More information +================ + - Homepage: https://www.netlib.org/scalapack/ + } +} + +module-whatis {Description: The ScaLAPACK (or Scalable LAPACK) library includes a subset of LAPACK routines + redesigned for distributed memory MIMD parallel computers.} +module-whatis {Homepage: https://www.netlib.org/scalapack/} +module-whatis {URL: https://www.netlib.org/scalapack/} + +set root /prefix/software/ScaLAPACK/2.2.0-gompi-2023a-fb + +conflict ScaLAPACK + +if { ![ is-loaded gompi/2023a ] } { + module load gompi/2023a +} + +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig + +setenv EBROOTSCALAPACK "$root" +setenv EBVERSIONSCALAPACK "2.2.0" +setenv EBDEVELSCALAPACK "$root/easybuild/ScaLAPACK-2.2.0-gompi-2023a-fb-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/UCC/1.2.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/UCX/1.14.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/binutils/2.40-GCCcore-12.3.0 b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 new file mode 100644 index 0000000000..7975adcb60 --- /dev/null +++ b/test/framework/modules/binutils/2.40-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +binutils: GNU binary utilities + + +More information +================ + - Homepage: https://directory.fsf.org/project/binutils/ + } +} + +module-whatis {Description: binutils: GNU binary utilities} +module-whatis {Homepage: https://directory.fsf.org/project/binutils/} +module-whatis {URL: https://directory.fsf.org/project/binutils/} + +set root /prefix/software/binutils/2.40-GCCcore-12.3.0 + +conflict binutils + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +if { ![ is-loaded zlib/1.2.13-GCCcore-12.3.0 ] } { + module load zlib/1.2.13-GCCcore-12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PATH $root/bin +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTBINUTILS "$root" +setenv EBVERSIONBINUTILS "2.40" +setenv EBDEVELBINUTILS "$root/easybuild/binutils-2.40-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/foss/2023a b/test/framework/modules/foss/2023a new file mode 100644 index 0000000000..448c5a77be --- /dev/null +++ b/test/framework/modules/foss/2023a @@ -0,0 +1,54 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK. + + +More information +================ + - Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, including + OpenMPI for MPI support, OpenBLAS (BLAS and LAPACK support), FFTW and ScaLAPACK.} +module-whatis {Homepage: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} +module-whatis {URL: https://easybuild.readthedocs.io/en/master/Common-toolchains.html#foss-toolchain} + +set root /prefix/software/foss/2023a + +conflict foss + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +if { ![ is-loaded FlexiBLAS/3.3.1-GCC-12.3.0 ] } { + module load FlexiBLAS/3.3.1-GCC-12.3.0 +} + +if { ![ is-loaded FFTW/3.3.10-GCC-12.3.0 ] } { + module load FFTW/3.3.10-GCC-12.3.0 +} + +if { ![ is-loaded FFTW.MPI/3.3.10-gompi-2023a ] } { + module load FFTW.MPI/3.3.10-gompi-2023a +} + +if { ![ is-loaded ScaLAPACK/2.2.0-gompi-2023a-fb ] } { + module load ScaLAPACK/2.2.0-gompi-2023a-fb +} + +setenv EBROOTFOSS "$root" +setenv EBVERSIONFOSS "2023a" +setenv EBDEVELFOSS "$root/easybuild/foss-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/gompi/2023a b/test/framework/modules/gompi/2023a new file mode 100644 index 0000000000..c81eeff8df --- /dev/null +++ b/test/framework/modules/gompi/2023a @@ -0,0 +1,38 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support. + + +More information +================ + - Homepage: (none) + } +} + +module-whatis {Description: GNU Compiler Collection (GCC) based compiler toolchain, + including OpenMPI for MPI support.} +module-whatis {Homepage: (none)} +module-whatis {URL: (none)} + +set root /prefix/software/gompi/2023a + +conflict gompi + +if { ![ is-loaded GCC/12.3.0 ] } { + module load GCC/12.3.0 +} + +if { ![ is-loaded OpenMPI/4.1.5-GCC-12.3.0 ] } { + module load OpenMPI/4.1.5-GCC-12.3.0 +} + +setenv EBROOTGOMPI "$root" +setenv EBVERSIONGOMPI "2023a" +setenv EBDEVELGOMPI "$root/easybuild/gompi-2023a-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/hwloc/2.9.1-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/libevent/2.1.12-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 new file mode 100644 index 0000000000..1c148cdd28 --- /dev/null +++ b/test/framework/modules/libfabric/1.18.0-GCCcore-12.3.0 @@ -0,0 +1 @@ +#%Module diff --git a/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 new file mode 100644 index 0000000000..a82945bd0c --- /dev/null +++ b/test/framework/modules/zlib/1.2.13-GCCcore-12.3.0 @@ -0,0 +1,44 @@ +#%Module +proc ModulesHelp { } { + puts stderr { + +Description +=========== +zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system. + + +More information +================ + - Homepage: https://www.zlib.net/ + } +} + +module-whatis {Description: zlib is designed to be a free, general-purpose, legally unencumbered -- that is, + not covered by any patents -- lossless data-compression library for use on virtually any + computer hardware and operating system.} +module-whatis {Homepage: https://www.zlib.net/} +module-whatis {URL: https://www.zlib.net/} + +set root /prefix/software/zlib/1.2.13-GCCcore-12.3.0 + +conflict zlib + +if { ![ is-loaded GCCcore/12.3.0 ] } { + module load GCCcore/12.3.0 +} + +prepend-path CMAKE_PREFIX_PATH $root +prepend-path CPATH $root/include +prepend-path LD_LIBRARY_PATH $root/lib +prepend-path LIBRARY_PATH $root/lib +prepend-path MANPATH $root/share/man +prepend-path PKG_CONFIG_PATH $root/lib/pkgconfig +prepend-path XDG_DATA_DIRS $root/share + +setenv EBROOTZLIB "$root" +setenv EBVERSIONZLIB "1.2.13" +setenv EBDEVELZLIB "$root/easybuild/zlib-1.2.13-GCCcore-12.3.0-easybuild-devel" + +# Built with EasyBuild version 4.9.0.dev0-rea8433dcf5e6edea3e72ad9bd9e23023ecc6b228 diff --git a/test/framework/modulestool.py b/test/framework/modulestool.py index 59c6872b93..f43a91e3b3 100644 --- a/test/framework/modulestool.py +++ b/test/framework/modulestool.py @@ -39,7 +39,7 @@ from easybuild.tools import modules, StrictVersion from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file, which, write_file -from easybuild.tools.modules import Lmod +from easybuild.tools.modules import EnvironmentModules, Lmod from test.framework.utilities import init_config @@ -192,6 +192,38 @@ def test_lmod_specific(self): # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty") + def test_environment_modules_specific(self): + """Environment Modules-specific test (skipped unless installed).""" + modulecmd_abspath = which(EnvironmentModules.COMMAND) + # only run this test if 'modulecmd.tcl' is installed + if modulecmd_abspath is not None: + # redefine 'module' and '_module_raw' function (deliberate mismatch with used module + # command in EnvironmentModules) + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/foo.tcl' bash $*`;\n}" + os.environ['module'] = "() { _module_raw \"$@\" 2>&1;\n}" + error_regex = ".*pattern .* not found in defined 'module' function" + self.assertErrorRegex(EasyBuildError, error_regex, EnvironmentModules, testing=True) + + # redefine '_module_raw' function with correct module command + os.environ['_module_raw'] = "() { eval `/usr/share/Modules/libexec/modulecmd.tcl' bash $*`;\n}" + mt = EnvironmentModules(testing=True) + self.assertIsInstance(mt.loaded_modules(), list) # dummy usage + + # initialize Environment Modules tool with non-official version number + # pass (fake) full path to 'modulecmd.tcl' via $MODULES_CMD + fake_path = os.path.join(self.test_installpath, 'libexec', 'modulecmd.tcl') + fake_modulecmd_txt = '\n'.join([ + 'puts stderr {Modules Release 5.3.1+unload-188-g14b6b59b (2023-10-21)}', + "puts {os.environ['FOO'] = 'foo'}", + ]) + write_file(fake_path, fake_modulecmd_txt) + os.chmod(fake_path, stat.S_IRUSR | stat.S_IXUSR) + os.environ['_module_raw'] = "() { eval `%s' bash $*`;\n}" % fake_path + os.environ['MODULES_CMD'] = fake_path + EnvironmentModules.COMMAND = fake_path + mt = EnvironmentModules(testing=True) + self.assertTrue(os.path.samefile(mt.cmd, fake_path), "%s - %s" % (mt.cmd, fake_path)) + def tearDown(self): """Testcase cleanup.""" super(ModulesToolTest, self).tearDown() diff --git a/test/framework/options.py b/test/framework/options.py index 57151c2679..5d87695661 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -64,7 +64,7 @@ from easybuild.tools.options import EasyBuildOptions, opts_dict_to_eb_opts, parse_external_modules_metadata from easybuild.tools.options import set_up_configuration, set_tmpdir, use_color from easybuild.tools.toolchain.utilities import TC_CONST_PREFIX -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import HAVE_ARCHSPEC from easybuild.tools.version import VERSION from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup, init_config @@ -577,6 +577,7 @@ def run_test(fmt=None): pattern_lines = [ r'^``%\(version_major\)s``\s+Major version\s*$', r'^``%\(cudaver\)s``\s+full version for CUDA\s*$', + r'^``%\(cudamajver\)s``\s+major version for CUDA\s*$', r'^``%\(pyshortver\)s``\s+short version for Python \(.\)\s*$', r'^\* ``%\(name\)s``$', r'^``%\(namelower\)s``\s+lower case of value of name\s*$', @@ -588,6 +589,7 @@ def run_test(fmt=None): pattern_lines = [ r'^\s+%\(version_major\)s: Major version$', r'^\s+%\(cudaver\)s: full version for CUDA$', + r'^\s+%\(cudamajver\)s: major version for CUDA$', r'^\s+%\(pyshortver\)s: short version for Python \(.\)$', r'^\s+%\(name\)s$', r'^\s+%\(namelower\)s: lower case of value of name$', @@ -734,6 +736,11 @@ def test_avail_hooks(self): " post_testcases_hook", " post_build_and_install_loop_hook", " end_hook", + " cancel_hook", + " crash_hook", + " fail_hook", + " pre_run_shell_cmd_hook", + " post_run_shell_cmd_hook", '', ]) self.assertEqual(stdout, expected) @@ -2223,7 +2230,7 @@ def test_ignore_osdeps(self): self.assertTrue(regex.search(outtxt), "OS dependencies are checked, outtxt: %s" % outtxt) msg = "One or more OS dependencies were not found: " msg += r"\[\('nosuchosdependency',\), \('nosuchdep_option1', 'nosuchdep_option2'\)\]" - regex = re.compile(r'%s' % msg, re.M) + regex = re.compile(msg, re.M) self.assertTrue(regex.search(outtxt), "OS dependencies are honored, outtxt: %s" % outtxt) # check whether OS dependencies are effectively ignored @@ -2461,6 +2468,22 @@ def test_try(self): with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True) + # Try changing only name or version of toolchain + args.pop(0) # Remove EC filename + foss_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-foss-2018a.eb') + copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0-gompi-2018a.eb'), foss_toy_ec) + write_file(foss_toy_ec, "toolchain['name'] = 'foss'", append=True) + + test_cases = [ + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-name=intel'], 'toy/0.0-iimpi-2018a'), + ([foss_toy_ec, '--try-toolchain-name=intel'], 'toy/0.0-intel-2018a'), + (['toy-0.0-gompi-2018a.eb', '--try-toolchain-version=2018b'], 'toy/0.0-gompi-2018b'), + ] + for extra_args, mod in test_cases: + outtxt = self.eb_main(args + extra_args, verbose=True, raise_error=True) + mod_regex = re.compile(r"\(module: %s\)$" % mod, re.M) + self.assertTrue(mod_regex.search(outtxt), "Pattern %s found in %s" % (mod_regex.pattern, outtxt)) + def test_try_with_copy(self): """Test whether --try options are taken into account.""" ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -3737,8 +3760,8 @@ def test_include_module_naming_schemes(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd("cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3753,8 +3776,9 @@ def test_include_module_naming_schemes(self): args = ['--avail-module-naming-schemes'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(mns_regex.search(res.output), + f"Unexpected pattern '{mns_regex.pattern}' found in: {res.output}") # include extra test MNS mns_txt = '\n'.join([ @@ -3770,8 +3794,9 @@ def test_include_module_naming_schemes(self): args.append('--include-module-naming-schemes=%s/*.py' % self.test_prefix) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(mns_regex.search(res.output), + f"Pattern '{mns_regex.pattern}' *not* found in: {res.output}") def test_use_included_module_naming_scheme(self): """Test using an included module naming scheme.""" @@ -3827,8 +3852,8 @@ def test_include_toolchains(self): # try and make sure top-level directory is in $PYTHONPATH if it isn't yet pythonpath = self.env_pythonpath with self.mocked_stdout_stderr(): - _, ec = run_cmd("cd %s; python -c 'import easybuild.framework'" % self.test_prefix, log_ok=False) - if ec > 0: + res = run_shell_cmd(f"cd {self.test_prefix}; python -c 'import easybuild.framework'", fail_on_error=False) + if res.exit_code != 0: pythonpath = '%s:%s' % (topdir, pythonpath) fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log') @@ -3846,8 +3871,9 @@ def test_include_toolchains(self): args = ['--list-toolchains'] test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertFalse(tc_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertFalse(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' *not* found in: {res.output}") # include extra test toolchain comp_txt = '\n'.join([ @@ -3868,8 +3894,9 @@ def test_include_toolchains(self): args.append('--include-toolchains=%s/*.py,%s/*/*.py' % (self.test_prefix, self.test_prefix)) test_cmd = self.mk_eb_test_cmd(args) with self.mocked_stdout_stderr(): - logtxt, _ = run_cmd(test_cmd, simple=False) - self.assertTrue(tc_regex.search(logtxt), "Pattern '%s' found in: %s" % (tc_regex.pattern, logtxt)) + res = run_shell_cmd(test_cmd) + self.assertTrue(tc_regex.search(res.output), + f"Pattern '{tc_regex.pattern}' found in: {res.output}") def test_cleanup_tmpdir(self): """Test --cleanup-tmpdir.""" @@ -3991,6 +4018,18 @@ def test_github_review_pr(self): self.mock_stderr(False) self.assertNotIn("2016.04", txt) + def test_set_multiple_pr_opts(self): + """Test that passing multiple PR options results in an error""" + test_cases = [ + ['--new-pr', 'dummy.eb', '--preview-pr'], + ['--new-pr', 'dummy.eb', '--update-pr', '42'], + ['--new-pr', 'dummy.eb', '--sync-pr-with-develop', '42'], + ['--new-pr', 'dummy.eb', '--new-pr-from-branch', 'mybranch'], + ] + for args in test_cases: + error_pattern = "The following options are set but incompatible.* " + args[0] + self.assertErrorRegex(EasyBuildError, error_pattern, self._run_mock_eb, args, raise_error=True) + def test_set_tmpdir(self): """Test set_tmpdir config function.""" self.purge_environment() @@ -4082,6 +4121,7 @@ def test_extended_dry_run(self): '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, '--debug', + '--disable-rpath', ] msg_regexs = [ @@ -4893,7 +4933,7 @@ def test_github_merge_pr(self): # --merge-pr also works on easyblocks (& framework) PRs args = [ '--merge-pr', - '2805', + '2995', '--pr-target-repo=easybuild-easyblocks', '-D', '--github-user=%s' % GITHUB_TEST_ACCOUNT, @@ -4901,12 +4941,12 @@ def test_github_merge_pr(self): stdout, stderr = self._run_mock_eb(args, do_build=True, raise_error=True, testing=False) self.assertEqual(stderr.strip(), '') expected_stdout = '\n'.join([ - "Checking eligibility of easybuilders/easybuild-easyblocks PR #2805 for merging...", + "Checking eligibility of easybuilders/easybuild-easyblocks PR #2995 for merging...", "* targets develop branch: OK", "* test suite passes: OK", "* no pending change requests: OK", - "* approved review: OK (by ocaisa)", - "* milestone is set: OK (4.6.2)", + "* approved review: OK (by boegel)", + "* milestone is set: OK (4.8.1)", "* mergeable state is clean: PR is already merged", '', "Review OK, merging pull request!", @@ -5141,13 +5181,13 @@ def test_dump_env_script(self): self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) with self.mocked_stdout_stderr(): - out, ec = run_cmd("function module { echo $@; } && source %s && echo FC: $FC" % env_script, simple=False) + res = run_shell_cmd(f"function module {{ echo $@; }} && source {env_script} && echo FC: $FC") expected_out = '\n'.join([ "load GCC/4.6.4", "load hwloc/1.11.8-GCC-4.6.4", "FC: gfortran", ]) - self.assertEqual(out.strip(), expected_out) + self.assertEqual(res.output.strip(), expected_out) def test_stop(self): """Test use of --stop.""" @@ -5174,19 +5214,22 @@ def test_fetch(self): lock_path = os.path.join(self.test_installpath, 'software', '.locks', lock_fn) mkdir(lock_path, parents=True) - args = ['toy-0.0.eb', '--fetch'] - stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) + # Run for a "regular" EC and one with an external module dependency + # which might trip up the dependency resolution (see #4298) + for ec in ('toy-0.0.eb', 'toy-0.0-deps.eb'): + args = [ec, '--fetch'] + stdout, stderr = self._run_mock_eb(args, raise_error=True, strip=True, testing=False) - patterns = [ - r"^== fetching files\.\.\.$", - r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", - ] - for pattern in patterns: - regex = re.compile(pattern, re.M) - self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) + patterns = [ + r"^== fetching files\.\.\.$", + r"^== COMPLETED: Installation STOPPED successfully \(took .* secs?\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' not found in: %s" % (regex.pattern, stdout)) - regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") - self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) + regex = re.compile(r"^== creating build dir, resetting environment\.\.\.$") + self.assertFalse(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) def test_parse_external_modules_metadata(self): """Test parse_external_modules_metadata function.""" @@ -5322,7 +5365,7 @@ def test_debug_lmod(self): init_config(build_options={'debug_lmod': True}) out = self.modtool.run_module('avail', return_output=True) - for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", "Master:avail"]: + for pattern in [r"^Lmod version", r"^lmod\(--terse -D avail\)\{", ":avail"]: regex = re.compile(pattern, re.M) self.assertTrue(regex.search(out), "Pattern '%s' found in: %s" % (regex.pattern, out)) else: @@ -6031,6 +6074,42 @@ def test_inject_checksums(self): ] self.assertEqual(ec['checksums'], expected_checksums) + # Also works for extensions (all 3 patch formats) + write_file(test_ec, textwrap.dedent(""" + exts_list = [ + ("bar", "0.0", { + 'sources': ['bar-0.0-local.tar.gz'], + 'patches': [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', # normal patch + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), # patch with patch level + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir'), + ], + }), + ] + """), append=True) + self._run_mock_eb(args, raise_error=True, strip=True) + ec = EasyConfigParser(test_ec).get_config_dict() + ext = ec['exts_list'][0] + self.assertEqual((ext[0], ext[1]), ("bar", "0.0")) + ext_opts = ext[2] + expected_patches = [ + 'bar-0.0_fix-silly-typo-in-printf-statement.patch', + ('bar-0.0_fix-very-silly-typo-in-printf-statement.patch', 0), + ('toy-0.0_fix-silly-typo-in-printf-statement.patch', 'toy_subdir') + ] + self.assertEqual(ext_opts['patches'], expected_patches) + expected_checksums = [ + {'bar-0.0-local.tar.gz': + 'f3676716b610545a4e8035087f5be0a0248adee0abb3930d3edb76d498ae91e7'}, + {'bar-0.0_fix-silly-typo-in-printf-statement.patch': + '84db53592e882b5af077976257f9c7537ed971cb2059003fd4faa05d02cae0ab'}, + {'bar-0.0_fix-very-silly-typo-in-printf-statement.patch': + 'd0bf102f9c5878445178c5f49b7cd7546e704c33fe2060c7354b7e473cfeb52b'}, + {'toy-0.0_fix-silly-typo-in-printf-statement.patch': + '81a3accc894592152f81814fbf133d39afad52885ab52c25018722c7bda92487'} + ] + self.assertEqual(ext_opts['checksums'], expected_checksums) + # passing easyconfig filename as argument to --inject-checksums results in error being reported, # because it's not a valid type of checksum args = ['--inject-checksums', test_ec] diff --git a/test/framework/package.py b/test/framework/package.py index 38f24242a7..7298d6ecc9 100644 --- a/test/framework/package.py +++ b/test/framework/package.py @@ -44,7 +44,7 @@ FPM_OUTPUT_FILE = 'fpm_mocked.out' -# purposely using non-bash script, to detect issues with shebang line being ignored (run_cmd with shell=False) +# purposely using non-bash script, to detect issues with shebang line being ignored (run_shell_cmd with use_bash=False) MOCKED_FPM = """#!/usr/bin/env python import os, sys diff --git a/test/framework/repository.py b/test/framework/repository.py index a10cfeebf8..acc2749c55 100644 --- a/test/framework/repository.py +++ b/test/framework/repository.py @@ -43,7 +43,7 @@ from easybuild.tools.repository.hgrepo import HgRepository from easybuild.tools.repository.svnrepo import SvnRepository from easybuild.tools.repository.repository import init_repository -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.version import VERSION @@ -95,10 +95,10 @@ def test_gitrepo(self): tmpdir = tempfile.mkdtemp() cmd = "cd %s && git clone --bare %s" % (tmpdir, test_repo_url) with self.mocked_stdout_stderr(): - _, ec = run_cmd(cmd, simple=False, log_all=False, log_ok=False) + res = run_shell_cmd(cmd, fail_on_error=False) # skip remainder of test if creating bare git repo didn't work - if ec == 0: + if res.exit_code == 0: repo = GitRepository(os.path.join(tmpdir, 'testrepository.git')) repo.init() toy_ec_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') diff --git a/test/framework/run.py b/test/framework/run.py index 4cd6821569..2f16e3978a 100644 --- a/test/framework/run.py +++ b/test/framework/run.py @@ -35,11 +35,14 @@ import os import re import signal +import string import stat import subprocess import sys import tempfile +import textwrap import time +from concurrent.futures import ThreadPoolExecutor from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config from unittest import TextTestRunner from easybuild.base.fancylogger import setLogLevelDebug @@ -47,9 +50,11 @@ import easybuild.tools.asyncprocess as asyncprocess import easybuild.tools.utilities from easybuild.tools.build_log import EasyBuildError, init_logging, stop_logging -from easybuild.tools.filetools import adjust_permissions, mkdir, read_file, write_file -from easybuild.tools.run import check_async_cmd, check_log_for_errors, complete_cmd, get_output_from_process -from easybuild.tools.run import RunResult, parse_log_for_error, run, run_cmd, run_cmd_qa, subprocess_terminate +from easybuild.tools.config import update_build_option +from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file +from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors +from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error +from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate from easybuild.tools.config import ERROR, IGNORE, WARN @@ -159,11 +164,11 @@ def test_run_cmd(self): self.assertTrue(out.startswith('foo ') and out.endswith(' bar')) self.assertEqual(type(out), str) - def test_run_basic(self): - """Basic test for run function.""" + def test_run_shell_cmd_basic(self): + """Basic test for run_shell_cmd function.""" with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") self.assertEqual(res.output, "hello\n") # no reason echo hello could fail self.assertEqual(res.cmd, "echo hello") @@ -183,13 +188,33 @@ def test_run_basic(self): cmd = "cat %s" % test_file with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 0) self.assertTrue(res.output.startswith('foo ') and res.output.endswith(' bar')) self.assertTrue(isinstance(res.output, str)) self.assertTrue(res.work_dir and isinstance(res.work_dir, str)) + def test_fileprefix_from_cmd(self): + """test simplifications from fileprefix_from_cmd.""" + cmds = { + 'abd123': 'abd123', + 'ab"a': 'aba', + 'a{:$:S@"a': 'aSa', + 'cmd-with-dash': 'cmd-with-dash', + 'cmd_with_underscore': 'cmd_with_underscore', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification) + + cmds = { + 'abd123': 'abd', + 'ab"a': 'aba', + '0a{:$:2@"a': 'aa', + } + for cmd, expected_simplification in cmds.items(): + self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification) + def test_run_cmd_log(self): """Test logging of executed commands.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') @@ -239,24 +264,25 @@ def test_run_cmd_log(self): self.assertTrue(logfiles[0].startswith("easybuild")) self.assertTrue(logfiles[0].endswith("log")) - def test_run_log(self): - """Test logging of executed commands with run function.""" + def test_run_shell_cmd_log(self): + """Test logging of executed commands with run_shell_cmd function.""" fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') os.close(fd) - regex_start_cmd = re.compile("Running command 'echo hello' in /") - regex_cmd_exit = re.compile("Command 'echo hello' exited with exit code [0-9]* and output:") + regex_start_cmd = re.compile("Running shell command 'echo hello' in /") + regex_cmd_exit = re.compile(r"Shell command completed successfully \(see output above\): echo hello") # command output is always logged init_logging(logfile, silent=True) with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") stop_logging(logfile) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') - self.assertEqual(len(regex_start_cmd.findall(read_file(logfile))), 1) - self.assertEqual(len(regex_cmd_exit.findall(read_file(logfile))), 1) + logtxt = read_file(logfile) + self.assertEqual(len(regex_start_cmd.findall(logtxt)), 1) + self.assertEqual(len(regex_cmd_exit.findall(logtxt)), 1) write_file(logfile, '') # with debugging enabled, exit code and output of command should only get logged once @@ -264,7 +290,7 @@ def test_run_log(self): init_logging(logfile, silent=True) with self.mocked_stdout_stderr(): - res = run("echo hello") + res = run_shell_cmd("echo hello") stop_logging(logfile) self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'hello\n') @@ -302,12 +328,15 @@ def handler(signum, _): signal.signal(signal.SIGALRM, orig_sigalrm_handler) signal.alarm(0) - def test_run_fail_cmd(self): - """Test run function with command that has negative exit code.""" + def test_run_shell_cmd_fail(self): + """Test run_shell_cmd function with command that has negative exit code.""" # define signal handler to call in case run takes too long def handler(signum, _): raise RuntimeError("Signal handler called with signal %s" % signum) + # disable trace output for this test (so stdout remains empty) + update_build_option('trace', False) + orig_sigalrm_handler = signal.getsignal(signal.SIGALRM) try: @@ -315,9 +344,87 @@ def handler(signum, _): signal.signal(signal.SIGALRM, handler) signal.alarm(3) - with self.mocked_stdout_stderr(): - res = run("kill -9 $$", fail_on_error=False) + # command to kill parent shell + cmd = "kill -9 $$" + + work_dir = os.path.realpath(self.test_prefix) + change_dir(work_dir) + + try: + run_shell_cmd(cmd) + self.assertFalse("This should never be reached, RunShellCmdError should occur!") + except RunShellCmdError as err: + self.assertEqual(str(err), "Shell command 'kill' failed!") + self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_name, 'kill') + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, None) + self.assertTrue(isinstance(err.caller_info, tuple)) + self.assertEqual(len(err.caller_info), 3) + self.assertEqual(err.caller_info[0], __file__) + self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site + self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') + + with self.mocked_stdout_stderr() as (_, stderr): + err.print() + + # check error reporting output + stderr = stderr.getvalue() + patterns = [ + r"^ERROR: Shell command failed!", + r"^\s+full command\s* -> kill -9 \$\$", + r"^\s+exit code\s* -> -9", + r"^\s+working directory\s* -> " + work_dir, + r"^\s+called from\s* -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"^\s+output \(stdout \+ stderr\)\s* -> .*/run-shell-cmd-output/kill-.*/out.txt", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr)) + + # check error reporting output when stdout/stderr are collected separately + try: + run_shell_cmd(cmd, split_stderr=True) + self.assertFalse("This should never be reached, RunShellCmdError should occur!") + except RunShellCmdError as err: + self.assertEqual(str(err), "Shell command 'kill' failed!") + self.assertEqual(err.cmd, "kill -9 $$") + self.assertEqual(err.cmd_name, 'kill') + self.assertEqual(err.exit_code, -9) + self.assertEqual(err.work_dir, work_dir) + self.assertEqual(err.output, '') + self.assertEqual(err.stderr, '') + self.assertTrue(isinstance(err.caller_info, tuple)) + self.assertEqual(len(err.caller_info), 3) + self.assertEqual(err.caller_info[0], __file__) + self.assertTrue(isinstance(err.caller_info[1], int)) # line number of calling site + self.assertEqual(err.caller_info[2], 'test_run_shell_cmd_fail') + + with self.mocked_stdout_stderr() as (_, stderr): + err.print() + + # check error reporting output + stderr = stderr.getvalue() + patterns = [ + r"^ERROR: Shell command failed!", + r"^\s+full command\s+ -> kill -9 \$\$", + r"^\s+exit code\s+ -> -9", + r"^\s+working directory\s+ -> " + work_dir, + r"^\s+called from\s+ -> 'test_run_shell_cmd_fail' function in .*/test/.*/run.py \(line [0-9]+\)", + r"^\s+output \(stdout\)\s+ -> .*/run-shell-cmd-output/kill-.*/out.txt", + r"^\s+error/warnings \(stderr\)\s+ -> .*/run-shell-cmd-output/kill-.*/err.txt", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stderr), "Pattern '%s' should be found in: %s" % (pattern, stderr)) + + # no error reporting when fail_on_error is disabled + with self.mocked_stdout_stderr() as (_, stderr): + res = run_shell_cmd(cmd, fail_on_error=False) self.assertEqual(res.exit_code, -9) + self.assertEqual(stderr.getvalue(), '') finally: # cleanup: disable the alarm + reset signal handler for SIGALRM @@ -333,15 +440,57 @@ def test_run_cmd_bis(self): self.assertEqual(len(out), len("hello\n" * 300)) self.assertEqual(ec, 0) - def test_run_bis(self): - """More 'complex' test for run function.""" + def test_run_shell_cmd_bis(self): + """More 'complex' test for run_shell_cmd function.""" # a more 'complex' command to run, make sure all required output is there with self.mocked_stdout_stderr(): - res = run("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done") + res = run_shell_cmd("for j in `seq 1 3`; do for i in `seq 1 100`; do echo hello; done; sleep 1.4; done") self.assertTrue(res.output.startswith('hello\nhello\n')) self.assertEqual(len(res.output), len("hello\n" * 300)) self.assertEqual(res.exit_code, 0) + def test_run_cmd_work_dir(self): + """ + Test running command in specific directory with run_cmd function. + """ + orig_wd = os.getcwd() + self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) + + test_dir = os.path.join(self.test_prefix, 'test') + for fn in ('foo.txt', 'bar.txt'): + write_file(os.path.join(test_dir, fn), 'test') + + with self.mocked_stdout_stderr(): + (out, ec) = run_cmd("ls | sort", path=test_dir) + + self.assertEqual(ec, 0) + self.assertEqual(out, 'bar.txt\nfoo.txt\n') + + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + + def test_run_shell_cmd_work_dir(self): + """ + Test running shell command in specific directory with run_shell_cmd function. + """ + orig_wd = os.getcwd() + self.assertFalse(os.path.samefile(orig_wd, self.test_prefix)) + + test_dir = os.path.join(self.test_prefix, 'test') + for fn in ('foo.txt', 'bar.txt'): + write_file(os.path.join(test_dir, fn), 'test') + + cmd = "ls | sort" + with self.mocked_stdout_stderr(): + res = run_shell_cmd(cmd, work_dir=test_dir) + + self.assertEqual(res.cmd, cmd) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'bar.txt\nfoo.txt\n') + self.assertEqual(res.stderr, None) + self.assertEqual(res.work_dir, test_dir) + + self.assertTrue(os.path.samefile(orig_wd, os.getcwd())) + def test_run_cmd_log_output(self): """Test run_cmd with log_output enabled""" with self.mocked_stdout_stderr(): @@ -374,8 +523,8 @@ def test_run_cmd_log_output(self): self.assertTrue(out.startswith('foo ') and out.endswith(' bar')) self.assertEqual(type(out), str) - def test_run_split_stderr(self): - """Test getting split stdout/stderr output from run function.""" + def test_run_shell_cmd_split_stderr(self): + """Test getting split stdout/stderr output from run_shell_cmd function.""" cmd = ';'.join([ "echo ok", "echo warning >&2", @@ -383,7 +532,7 @@ def test_run_split_stderr(self): # by default, output contains both stdout + stderr with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) self.assertEqual(res.exit_code, 0) output_lines = res.output.split('\n') self.assertTrue("ok" in output_lines) @@ -391,7 +540,7 @@ def test_run_split_stderr(self): self.assertEqual(res.stderr, None) with self.mocked_stdout_stderr(): - res = run(cmd, split_stderr=True) + res = run_shell_cmd(cmd, split_stderr=True) self.assertEqual(res.exit_code, 0) self.assertEqual(res.stderr, "warning\n") self.assertEqual(res.output, "ok\n") @@ -484,22 +633,22 @@ def test_run_cmd_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') - def test_run_trace(self): - """Test run in trace mode, and with tracing disabled.""" + def test_run_shell_cmd_trace(self): + """Test run_shell_cmd function in trace mode, and with tracing disabled.""" pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", - r"\techo hello", + r"\t\[output saved to .*\]", r" >> command completed: exit 0, ran in .*", ] # trace output is enabled by default (since EasyBuild v5.0) self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -514,7 +663,7 @@ def test_run_trace(self): self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -532,7 +681,7 @@ def test_run_trace(self): self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello", hidden=True) + res = run_shell_cmd("echo hello", hidden=True) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -542,23 +691,23 @@ def test_run_trace(self): self.assertEqual(stdout, '') self.assertEqual(stderr, '') - def test_run_trace_stdin(self): - """Test run under --trace + passing stdin input.""" + def test_run_shell_cmd_trace_stdin(self): + """Test run_shell_cmd function under --trace + passing stdin input.""" init_config(build_options={'trace': True}) pattern = [ - r"^ >> running command:", + r"^ >> running shell command:", + r"\techo hello", r"\t\[started at: [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", - r"\techo hello", + r"\t\[output saved to .*\]", r" >> command completed: exit 0, ran in .*", ] self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello") + res = run_shell_cmd("echo hello") stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -572,7 +721,7 @@ def test_run_trace_stdin(self): # also test with command that is fed input via stdin self.mock_stdout(True) self.mock_stderr(True) - res = run('cat', stdin='hello') + res = run_shell_cmd('cat', stdin='hello') stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -580,15 +729,15 @@ def test_run_trace_stdin(self): self.assertEqual(res.output, 'hello') self.assertEqual(res.exit_code, 0) self.assertEqual(stderr, '') - pattern.insert(3, r"\t\[input: hello\]") - pattern[-2] = "\tcat" + pattern.insert(4, r"\t\[input: hello\]") + pattern[1] = "\tcat" regex = re.compile('\n'.join(pattern)) self.assertTrue(regex.search(stdout), "Pattern '%s' found in: %s" % (regex.pattern, stdout)) # trace output can be disabled on a per-command basis by enabling 'hidden' self.mock_stdout(True) self.mock_stderr(True) - res = run("echo hello", hidden=True) + res = run_shell_cmd("echo hello", hidden=True) stdout = self.get_stdout() stderr = self.get_stderr() self.mock_stdout(False) @@ -764,26 +913,28 @@ def test_run_cmd_cache(self): run_cmd.clear_cache() - def test_run_cache(self): - """Test caching for run""" + def test_run_shell_cmd_cache(self): + """Test caching for run_shell_cmd function""" cmd = "ulimit -u" with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) first_out = res.output self.assertEqual(res.exit_code, 0) with self.mocked_stdout_stderr(): - res = run(cmd) + res = run_shell_cmd(cmd) cached_out = res.output self.assertEqual(res.exit_code, 0) self.assertEqual(first_out, cached_out) # inject value into cache to check whether executing command again really returns cached value with self.mocked_stdout_stderr(): - cached_res = RunResult(cmd=cmd, output="123456", exit_code=123, stderr=None, work_dir='/test_ulimit') - run.update_cache({(cmd, None): cached_res}) - res = run(cmd) + cached_res = RunShellCmdResult(cmd=cmd, output="123456", exit_code=123, stderr=None, + work_dir='/test_ulimit', out_file='/tmp/foo.out', err_file=None, + thread_id=None, task_id=None) + run_shell_cmd.update_cache({(cmd, None): cached_res}) + res = run_shell_cmd(cmd) self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 123) self.assertEqual(res.output, "123456") @@ -793,22 +944,24 @@ def test_run_cache(self): # also test with command that uses stdin cmd = "cat" with self.mocked_stdout_stderr(): - res = run(cmd, stdin='foo') + res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.exit_code, 0) self.assertEqual(res.output, 'foo') # inject different output for cat with 'foo' as stdin to check whether cached value is used with self.mocked_stdout_stderr(): - cached_res = RunResult(cmd=cmd, output="bar", exit_code=123, stderr=None, work_dir='/test_cat') - run.update_cache({(cmd, 'foo'): cached_res}) - res = run(cmd, stdin='foo') + cached_res = RunShellCmdResult(cmd=cmd, output="bar", exit_code=123, stderr=None, + work_dir='/test_cat', out_file='/tmp/cat.out', err_file=None, + thread_id=None, task_id=None) + run_shell_cmd.update_cache({(cmd, 'foo'): cached_res}) + res = run_shell_cmd(cmd, stdin='foo') self.assertEqual(res.cmd, cmd) self.assertEqual(res.exit_code, 123) self.assertEqual(res.output, 'bar') self.assertEqual(res.stderr, None) self.assertEqual(res.work_dir, '/test_cat') - run.clear_cache() + run_shell_cmd.clear_cache() def test_parse_log_error(self): """Test basic parse_log_for_error functionality.""" @@ -858,8 +1011,8 @@ def test_run_cmd_dry_run(self): expected = """ running interactive command "some_qa_cmd"\n""" self.assertIn(expected, stdout) - def test_run_dry_run(self): - """Test use of run function under (extended) dry run.""" + def test_run_shell_cmd_dry_run(self): + """Test use of run_shell_cmd function under (extended) dry run.""" build_options = { 'extended_dry_run': True, 'silent': False, @@ -869,7 +1022,7 @@ def test_run_dry_run(self): cmd = "somecommand foo 123 bar" self.mock_stdout(True) - res = run(cmd) + res = run_shell_cmd(cmd) stdout = self.get_stdout() self.mock_stdout(False) # fake output/exit code is returned for commands not actually run in dry run mode @@ -877,12 +1030,12 @@ def test_run_dry_run(self): self.assertEqual(res.output, '') self.assertEqual(res.stderr, None) # check dry run output - expected = """ running command "somecommand foo 123 bar"\n""" + expected = """ running shell command "somecommand foo 123 bar"\n""" self.assertIn(expected, stdout) # check enabling 'hidden' self.mock_stdout(True) - res = run(cmd, hidden=True) + res = run_shell_cmd(cmd, hidden=True) stdout = self.get_stdout() self.mock_stdout(False) # fake output/exit code is returned for commands not actually run in dry run mode @@ -896,10 +1049,11 @@ def test_run_dry_run(self): outfile = os.path.join(self.test_prefix, 'cmd.out') self.assertNotExists(outfile) self.mock_stdout(True) - res = run("echo 'This is always echoed' > %s; echo done; false" % outfile, fail_on_error=False, in_dry_run=True) + res = run_shell_cmd("echo 'This is always echoed' > %s; echo done; false" % outfile, + fail_on_error=False, in_dry_run=True) stdout = self.get_stdout() self.mock_stdout(False) - self.assertNotIn('running command "', stdout) + self.assertNotIn('running shell command "', stdout) self.assertNotEqual(res.exit_code, 0) self.assertEqual(res.output, 'done\n') self.assertEqual(res.stderr, None) @@ -936,6 +1090,25 @@ def test_run_cmd_script(self): self.assertEqual(ec, 0) self.assertEqual(out, "hello\n") + def test_run_shell_cmd_no_bash(self): + """Testing use of run_shell_cmd with use_bash=False to call external scripts""" + py_test_script = os.path.join(self.test_prefix, 'test.py') + write_file(py_test_script, '\n'.join([ + '#!%s' % sys.executable, + 'print("hello")', + ])) + adjust_permissions(py_test_script, stat.S_IXUSR) + + with self.mocked_stdout_stderr(): + res = run_shell_cmd(py_test_script) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "hello\n") + + with self.mocked_stdout_stderr(): + res = run_shell_cmd([py_test_script], use_bash=False) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, "hello\n") + def test_run_cmd_stream(self): """Test use of run_cmd with streaming output.""" self.mock_stdout(True) @@ -958,6 +1131,40 @@ def test_run_cmd_stream(self): for line in expected: self.assertIn(line, stdout) + def test_run_shell_cmd_stream(self): + """Test use of run_shell_cmd with streaming output.""" + self.mock_stdout(True) + self.mock_stderr(True) + cmd = '; '.join([ + "echo hello there", + "sleep 1", + "echo testing command that produces a fair amount of output", + "sleep 1", + "echo more than 128 bytes which means a whole bunch of characters...", + "sleep 1", + "echo more than 128 characters in fact, which is quite a bit when you think of it", + ]) + res = run_shell_cmd(cmd, stream_output=True) + stdout = self.get_stdout() + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + + expected_output = '\n'.join([ + "hello there", + "testing command that produces a fair amount of output", + "more than 128 bytes which means a whole bunch of characters...", + "more than 128 characters in fact, which is quite a bit when you think of it", + '', + ]) + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, expected_output) + + self.assertEqual(stderr, '') + expected = ("== (streaming) output for command 'echo hello" + '\n' + expected_output).split('\n') + for line in expected: + self.assertIn(line, stdout) + def test_run_cmd_async(self): """Test asynchronously running of a shell command via run_cmd + complete_cmd.""" @@ -1024,7 +1231,7 @@ def test_run_cmd_async(self): "for i in $(seq 1 50)", "do sleep 0.1", "for j in $(seq 1000)", - "do echo foo", + "do echo foo${i}${j}", "done", "done", "echo done", @@ -1074,8 +1281,68 @@ def test_run_cmd_async(self): res = check_async_cmd(*cmd_info, output=res['output']) self.assertEqual(res['done'], True) self.assertEqual(res['exit_code'], 0) - self.assertTrue(res['output'].startswith('start\n')) - self.assertTrue(res['output'].endswith('\ndone\n')) + self.assertEqual(len(res['output']), 435661) + self.assertTrue(res['output'].startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res['output']) + self.assertTrue(res['output'].endswith('\nfoo501000\ndone\n')) + + def test_run_shell_cmd_async(self): + """Test asynchronously running of a shell command via run_shell_cmd """ + + thread_pool = ThreadPoolExecutor() + + os.environ['TEST'] = 'test123' + env = os.environ.copy() + + test_cmd = "echo 'sleeping...'; sleep 2; echo $TEST" + task = thread_pool.submit(run_shell_cmd, test_cmd, hidden=True, asynchronous=True, env=env) + + # change value of $TEST to check that command is completed with correct environment + os.environ['TEST'] = 'some_other_value' + + # initial poll should result in None, since it takes a while for the command to complete + self.assertEqual(task.done(), False) + + # wait until command is done + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(res.output, 'sleeping...\ntest123\n') + + # check asynchronous running of failing command + error_test_cmd = "echo 'FAIL!' >&2; exit 123" + task = thread_pool.submit(run_shell_cmd, error_test_cmd, hidden=True, fail_on_error=False, asynchronous=True) + time.sleep(1) + res = task.result() + self.assertEqual(res.exit_code, 123) + self.assertEqual(res.output, "FAIL!\n") + self.assertTrue(res.thread_id) + + # also test with a command that produces a lot of output, + # since that tends to lock up things unless we frequently grab some output... + verbose_test_cmd = ';'.join([ + "echo start", + "for i in $(seq 1 50)", + "do sleep 0.1", + "for j in $(seq 1000)", + "do echo foo${i}${j}", + "done", + "done", + "echo done", + ]) + task = thread_pool.submit(run_shell_cmd, verbose_test_cmd, hidden=True, asynchronous=True) + + while not task.done(): + time.sleep(1) + res = task.result() + + self.assertEqual(res.exit_code, 0) + self.assertEqual(len(res.output), 435661) + self.assertTrue(res.output.startswith('start\nfoo11\nfoo12\n')) + self.assertTrue('\nfoo49999\nfoo491000\nfoo501\n' in res.output) + self.assertTrue(res.output.endswith('\nfoo501000\ndone\n')) def test_check_log_for_errors(self): fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-') @@ -1094,8 +1361,9 @@ def test_check_log_for_errors(self): "enabling -Werror", "the process crashed with 0" ]) - expected_msg = r"Found 2 error\(s\) in command output "\ - r"\(output: error found\n\tthe process crashed with 0\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\tthe process crashed with 0" # String promoted to list self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, @@ -1107,14 +1375,17 @@ def test_check_log_for_errors(self): self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [(r"\b(error|crashed)\b", ERROR)]) - expected_msg = "Found 2 potential error(s) in command output " \ - "(output: error found\n\tthe process crashed with 0)" + expected_msg = "Found 2 potential error(s) in command output:\n"\ + "\terror found\n"\ + "\tthe process crashed with 0" init_logging(logfile, silent=True) check_log_for_errors(input_text, [(r"\b(error|crashed)\b", WARN)]) stop_logging(logfile) self.assertIn(expected_msg, read_file(logfile)) - expected_msg = r"Found 2 error\(s\) in command output \(output: error found\n\ttest failed\)" + expected_msg = r"Found 2 error\(s\) in command output:\n"\ + r"\terror found\n"\ + r"\ttest failed" write_file(logfile, '') init_logging(logfile, silent=True) self.assertErrorRegex(EasyBuildError, expected_msg, check_log_for_errors, input_text, [ @@ -1124,9 +1395,114 @@ def test_check_log_for_errors(self): "fail" ]) stop_logging(logfile) - expected_msg = "Found 1 potential error(s) in command output (output: the process crashed with 0)" + expected_msg = "Found 1 potential error(s) in command output:\n\tthe process crashed with 0" self.assertIn(expected_msg, read_file(logfile)) + def test_run_cmd_with_hooks(self): + """ + Test running command with run_cmd with pre/post run_shell_cmd hooks in place. + """ + cwd = os.getcwd() + + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + print("pre-run hook interactive '%s' in %s" % (cmd, work_dir)) + else: + print("pre-run hook '%s' in %s" % (cmd, work_dir)) + if not cmd.startswith('echo'): + cmds = cmd.split(';') + return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()]) + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs.get('exit_code') + output = kwargs.get('output') + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + msg = "post-run hook interactive '%s'" % cmd + else: + msg = "post-run hook '%s'" % cmd + msg += " (exit code: %s, output: '%s')" % (exit_code, output) + print(msg) + """) + write_file(hooks_file, hooks_file_txt) + update_build_option('hooks', hooks_file) + + # disable trace output to make checking of generated output produced by hooks easier + update_build_option('trace', False) + + with self.mocked_stdout_stderr(): + run_cmd("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + "post-run hook 'echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + + with self.mocked_stdout_stderr(): + run_cmd_qa("sleep 2; make", qa={}) + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook interactive 'sleep 2; make' in %s" % cwd, + "post-run hook interactive 'sleep 2; echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + + def test_run_shell_cmd_with_hooks(self): + """ + Test running shell command with run_shell_cmd function with pre/post run_shell_cmd hooks in place. + """ + cwd = os.getcwd() + + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + print("pre-run hook interactive '||%s||' in %s" % (cmd, work_dir)) + else: + print("pre-run hook '%s' in %s" % (cmd, work_dir)) + import sys + sys.stderr.write('pre-run hook done\\n') + if not cmd.startswith('echo'): + cmds = cmd.split(';') + return '; '.join(cmds[:-1] + ["echo " + cmds[-1].lstrip()]) + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs.get('exit_code') + output = kwargs.get('output') + work_dir = kwargs['work_dir'] + if kwargs.get('interactive'): + msg = "post-run hook interactive '%s'" % cmd + else: + msg = "post-run hook '%s'" % cmd + msg += " (exit code: %s, output: '%s')" % (exit_code, output) + print(msg) + """) + write_file(hooks_file, hooks_file_txt) + update_build_option('hooks', hooks_file) + + # disable trace output to make checking of generated output produced by hooks easier + update_build_option('trace', False) + + with self.mocked_stdout_stderr(): + run_shell_cmd("make") + stdout = self.get_stdout() + + expected_stdout = '\n'.join([ + "pre-run hook 'make' in %s" % cwd, + "post-run hook 'echo make' (exit code: 0, output: 'make\n')", + '', + ]) + self.assertEqual(stdout, expected_stdout) + def suite(): """ returns all the testcases in this module """ diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index 9c700cf779..15fe5773aa 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -27,12 +27,13 @@ @author: Kenneth Hoste (Ghent University) """ +import os from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.easyblocks.toy import EB_toy, compose_toy_build_cmd from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd class Toy_Extension(ExtensionEasyBlock): @@ -67,7 +68,7 @@ def run(self, *args, **kwargs): EB_toy.build_step(self.master, name=self.name, cfg=self.cfg) if self.cfg['toy_ext_param']: - run_cmd(self.cfg['toy_ext_param']) + run_shell_cmd(self.cfg['toy_ext_param']) return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name) @@ -81,22 +82,24 @@ def prerun(self): super(Toy_Extension, self).run(unpack_src=True) EB_toy.configure_step(self.master, name=self.name, cfg=self.cfg) - def run_async(self): + def run_async(self, thread_pool): """ Install toy extension asynchronously. """ + task_id = f'ext_{self.name}_{self.version}' if self.src: cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) else: - self.async_cmd_info = False + cmd = f"echo 'no sources for {self.name}'" + + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ Wrap up installation of toy extension. """ super(Toy_Extension, self).postrun() - EB_toy.install_step(self.master, name=self.name) def sanity_check_step(self, *args, **kwargs): diff --git a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py index 50b573649a..2c0ba5a9e5 100644 --- a/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py +++ b/test/framework/sandbox/easybuild/easyblocks/l/libtoy.py @@ -30,7 +30,7 @@ import os from easybuild.framework.easyblock import EasyBlock -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext SHLIB_EXT = get_shared_lib_ext() @@ -40,7 +40,7 @@ class EB_libtoy(EasyBlock): """Support for building/installing libtoy.""" def banned_linked_shared_libs(self): - default = '/thiswillnotbethere,libtoytoytoy.%s,toytoytoy' % SHLIB_EXT + default = f'/thiswillnotbethere,libtoytoytoy.{SHLIB_EXT},toytoytoy' return os.getenv('EB_LIBTOY_BANNED_SHARED_LIBS', default).split(',') def required_linked_shared_libs(self): @@ -53,8 +53,8 @@ def configure_step(self, name=None): def build_step(self, name=None, buildopts=None): """Build libtoy.""" - run_cmd('make') + run_shell_cmd('make') def install_step(self, name=None): """Install libtoy.""" - run_cmd('make install PREFIX="%s"' % self.installdir) + run_shell_cmd(f'make install PREFIX="{self.installdir}"') diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy.py index 88ff864454..fb842bfe06 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy.py @@ -34,11 +34,11 @@ from easybuild.framework.easyconfig import CUSTOM from easybuild.framework.extensioneasyblock import ExtensionEasyBlock -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.environment import setvar from easybuild.tools.filetools import mkdir, write_file from easybuild.tools.modules import get_software_root, get_software_version -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd def compose_toy_build_cmd(cfg, name, prebuildopts, buildopts): @@ -108,7 +108,7 @@ def configure_step(self, name=None, cfg=None): 'echo "Configured"', cfg['configopts'] ]) - run_cmd(cmd) + run_shell_cmd(cmd) if os.path.exists("%s.source" % name): os.rename('%s.source' % name, '%s.c' % name) @@ -122,7 +122,11 @@ def build_step(self, name=None, cfg=None): name = self.name cmd = compose_toy_build_cmd(self.cfg, name, cfg['prebuildopts'], cfg['buildopts']) - run_cmd(cmd) + # purposely run build command without checking exit code; + # we rely on this in test_toy_build_hooks + res = run_shell_cmd(cmd, fail_on_error=False) + if res.exit_code: + print_warning("Command '%s' failed, but we'll ignore it..." % cmd) def install_step(self, name=None): """Install toy.""" @@ -159,12 +163,14 @@ def run(self): """ self.build_step() - def run_async(self): + def run_async(self, thread_pool): """ Asynchronous installation of toy as extension. """ cmd = compose_toy_build_cmd(self.cfg, self.name, self.cfg['prebuildopts'], self.cfg['buildopts']) - self.async_cmd_start(cmd) + task_id = f'ext_{self.name}_{self.version}' + return thread_pool.submit(run_shell_cmd, cmd, asynchronous=True, env=os.environ.copy(), + fail_on_error=False, task_id=task_id) def postrun(self): """ diff --git a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py index 3695744630..43d7433265 100644 --- a/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py +++ b/test/framework/sandbox/easybuild/easyblocks/t/toy_buggy.py @@ -41,7 +41,7 @@ def configure_step(self): def build_step(self): """Build toy.""" # note: import is (purposely) missing, so this will go down hard - run_cmd('gcc toy.c -o toy') # noqa + run_shell_cmd('gcc toy.c -o toy') # noqa def install_step(self): """Install toy.""" diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py index 5eb5f66682..5d8dec26c8 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme.py @@ -66,4 +66,4 @@ def is_short_modname_for(self, modname, name): """ Determine whether the specified (short) module name is a module for software with the specified name. """ - return modname.find('%s' % name) != -1 + return modname.find(name) != -1 diff --git a/test/framework/systemtools.py b/test/framework/systemtools.py index 82f1315dc5..6d8395a5fc 100644 --- a/test/framework/systemtools.py +++ b/test/framework/systemtools.py @@ -40,7 +40,7 @@ import easybuild.tools.systemtools as st from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import adjust_permissions, read_file, symlink, which, write_file -from easybuild.tools.run import RunResult, run +from easybuild.tools.run import RunShellCmdResult, run_shell_cmd from easybuild.tools.systemtools import CPU_ARCHITECTURES, AARCH32, AARCH64, POWER, X86_64 from easybuild.tools.systemtools import CPU_FAMILIES, POWER_LE, DARWIN, LINUX, UNKNOWN from easybuild.tools.systemtools import CPU_VENDORS, AMD, APM, ARM, CAVIUM, IBM, INTEL @@ -320,8 +320,8 @@ def mocked_is_readable(mocked_fp, fp): return fp == mocked_fp -def mocked_run(cmd, **kwargs): - """Mocked version of run, with specified output for known commands.""" +def mocked_run_shell_cmd(cmd, **kwargs): + """Mocked version of run_shell_cmd, with specified output for known commands.""" known_cmds = { "gcc --version": "gcc (GCC) 5.1.1 20150618 (Red Hat 5.1.1-4)", "ldd --version": "ldd (GNU libc) 2.12; ", @@ -340,9 +340,10 @@ def mocked_run(cmd, **kwargs): "ulimit -u": '40', } if cmd in known_cmds: - return RunResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd()) + return RunShellCmdResult(cmd=cmd, exit_code=0, output=known_cmds[cmd], stderr=None, work_dir=os.getcwd(), + out_file=None, err_file=None, thread_id=None, task_id=None) else: - return run(cmd, **kwargs) + return run_shell_cmd(cmd, **kwargs) def mocked_uname(): @@ -361,7 +362,7 @@ def setUp(self): self.orig_get_os_type = st.get_os_type self.orig_is_readable = st.is_readable self.orig_read_file = st.read_file - self.orig_run = st.run + self.orig_run_shell_cmd = st.run_shell_cmd self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None self.orig_platform_uname = st.platform.uname self.orig_get_tool_version = st.get_tool_version @@ -369,10 +370,7 @@ def setUp(self): self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC self.orig_HAVE_DISTRO = st.HAVE_DISTRO self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE - if hasattr(st, 'archspec_cpu_host'): - self.orig_archspec_cpu_host = st.archspec_cpu_host - else: - self.orig_archspec_cpu_host = None + self.orig_archspec_cpu_host = getattr(st, 'archspec_cpu_host', None) def tearDown(self): """Cleanup after systemtools test.""" @@ -381,7 +379,7 @@ def tearDown(self): st.get_cpu_architecture = self.orig_get_cpu_architecture st.get_os_name = self.orig_get_os_name st.get_os_type = self.orig_get_os_type - st.run = self.orig_run + st.run_shell_cmd = self.orig_run_shell_cmd if self.orig_platform_dist is not None: st.platform.dist = self.orig_platform_dist st.platform.uname = self.orig_platform_uname @@ -412,7 +410,7 @@ def test_avail_core_count_linux(self): def test_avail_core_count_darwin(self): """Test getting core count (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_avail_core_count(), 10) def test_cpu_model_native(self): @@ -450,7 +448,7 @@ def test_cpu_model_linux(self): def test_cpu_model_darwin(self): """Test getting CPU model (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_model(), "Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz") def test_cpu_speed_native(self): @@ -485,7 +483,7 @@ def test_cpu_speed_linux(self): def test_cpu_speed_darwin(self): """Test getting CPU speed (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_speed(), 2400.0) def test_cpu_features_native(self): @@ -540,7 +538,7 @@ def test_cpu_features_linux(self): def test_cpu_features_darwin(self): """Test getting CPU features (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd expected = ['1gbpage', 'acpi', 'aes', 'apic', 'avx1.0', 'avx2', 'bmi1', 'bmi2', 'clfsh', 'cmov', 'cx16', 'cx8', 'de', 'ds', 'dscpl', 'dtes64', 'em64t', 'erms', 'est', 'f16c', 'fma', 'fpu', 'fpu_csds', 'fxsr', 'htt', 'invpcid', 'lahf', 'lzcnt', 'mca', 'mce', 'mmx', 'mon', 'movbe', 'msr', 'mtrr', @@ -634,12 +632,12 @@ def test_cpu_vendor_linux(self): def test_cpu_vendor_darwin(self): """Test getting CPU vendor (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_cpu_vendor(), INTEL) def test_cpu_family_native(self): """Test get_cpu_family function.""" - run.clear_cache() + run_shell_cmd.clear_cache() cpu_family = get_cpu_family() self.assertTrue(cpu_family in CPU_FAMILIES or cpu_family == UNKNOWN) @@ -684,8 +682,8 @@ def test_cpu_family_linux(self): def test_cpu_family_darwin(self): """Test get_cpu_family function (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run - run.clear_cache() + st.run_shell_cmd = mocked_run_shell_cmd + run_shell_cmd.clear_cache() self.assertEqual(get_cpu_family(), INTEL) def test_os_type(self): @@ -767,15 +765,17 @@ def test_gcc_version_native(self): def test_gcc_version_linux(self): """Test getting gcc version (mocked for Linux).""" st.get_os_type = lambda: st.LINUX - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_gcc_version(), '5.1.1') def test_gcc_version_darwin(self): """Test getting gcc version (mocked for Darwin).""" st.get_os_type = lambda: st.DARWIN out = "Apple LLVM version 7.0.0 (clang-700.1.76)" - mocked_run_res = RunResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=os.getcwd()) - st.run = lambda *args, **kwargs: mocked_run_res + cwd = os.getcwd() + mocked_run_res = RunShellCmdResult(cmd="gcc --version", exit_code=0, output=out, stderr=None, work_dir=cwd, + out_file=None, err_file=None, thread_id=None, task_id=None) + st.run_shell_cmd = lambda *args, **kwargs: mocked_run_res self.assertEqual(get_gcc_version(), None) def test_glibc_version_native(self): @@ -786,7 +786,7 @@ def test_glibc_version_native(self): def test_glibc_version_linux(self): """Test getting glibc version (mocked for Linux).""" st.get_os_type = lambda: st.LINUX - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_glibc_version(), '2.12') def test_glibc_version_linux_gentoo(self): @@ -817,7 +817,7 @@ def test_get_total_memory_linux(self): def test_get_total_memory_darwin(self): """Test the function that gets the total memory.""" st.get_os_type = lambda: st.DARWIN - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertEqual(get_total_memory(), 8192) def test_get_total_memory_native(self): @@ -833,9 +833,9 @@ def test_system_info(self): def test_det_parallelism_native(self): """Test det_parallelism function (native calls).""" self.assertTrue(det_parallelism() > 0) - # specified parallellism + # specified parallelism self.assertEqual(det_parallelism(par=5), 5) - # max parallellism caps + # max parallelism caps self.assertEqual(det_parallelism(maxpar=1), 1) self.assertEqual(det_parallelism(16, 1), 1) self.assertEqual(det_parallelism(par=5, maxpar=2), 2) @@ -849,7 +849,7 @@ def test_det_parallelism_mocked(self): st.get_avail_core_count = lambda: 8 self.assertTrue(det_parallelism(), 8) # make 'ulimit -u' return '40', which should result in default (max) parallelism of 4 ((40-15)/6) - st.run = mocked_run + st.run_shell_cmd = mocked_run_shell_cmd self.assertTrue(det_parallelism(), 4) self.assertTrue(det_parallelism(par=6), 4) self.assertTrue(det_parallelism(maxpar=2), 2) @@ -1038,10 +1038,10 @@ def test_check_linked_shared_libs(self): os_type = get_os_type() if os_type == LINUX: with self.mocked_stdout_stderr(): - res = run("ldd %s" % bin_ls_path) + res = run_shell_cmd("ldd %s" % bin_ls_path) elif os_type == DARWIN: with self.mocked_stdout_stderr(): - res = run("otool -L %s" % bin_ls_path) + res = run_shell_cmd("otool -L %s" % bin_ls_path) else: raise EasyBuildError("Unknown OS type: %s" % os_type) diff --git a/test/framework/toolchain.py b/test/framework/toolchain.py index 042d366acd..918b344dcf 100644 --- a/test/framework/toolchain.py +++ b/test/framework/toolchain.py @@ -50,7 +50,7 @@ from easybuild.tools.environment import setvar from easybuild.tools.filetools import adjust_permissions, copy_dir, find_eb_script, mkdir from easybuild.tools.filetools import read_file, symlink, write_file, which -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.toolchain.mpi import get_mpi_cmd_template from easybuild.tools.toolchain.toolchain import env_vars_external_module @@ -1942,6 +1942,58 @@ def test_old_new_iccifort(self): self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], scalapack_mt_static_libs_fosscuda) self.modtool.purge() + tc = self.get_toolchain('foss', version='2023a') + tc.prepare() + self.assertEqual(os.environ['BLAS_SHARED_LIBS'], blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_STATIC_LIBS'], blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS'], libblas_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBBLAS_MT'], libblas_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['LAPACK_SHARED_LIBS'], lapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_STATIC_LIBS'], lapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_SHARED_LIBS'], + lapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LAPACK_MT_STATIC_LIBS'], + lapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK'], liblapack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBLAPACK_MT'], liblapack_mt_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['BLAS_LAPACK_SHARED_LIBS'], + blas_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_STATIC_LIBS'], + blas_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_SHARED_LIBS'], + blas_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['BLAS_LAPACK_MT_STATIC_LIBS'], + blas_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + + self.assertEqual(os.environ['FFT_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFT_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFT_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS'], fft_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS'], fft_static_libs_fosscuda) + self.assertEqual(os.environ['FFTW_SHARED_LIBS_MT'], fft_mt_shared_libs_fosscuda) + self.assertEqual(os.environ['FFTW_STATIC_LIBS_MT'], fft_mt_static_libs_fosscuda) + self.assertEqual(os.environ['LIBFFT'], libfft_fosscuda) + self.assertEqual(os.environ['LIBFFT_MT'], libfft_mt_fosscuda) + + self.assertEqual(os.environ['LIBSCALAPACK'], libscalack_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['LIBSCALAPACK_MT'], libscalack_mt_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_SHARED_LIBS'], + scalapack_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_STATIC_LIBS'], + scalapack_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_SHARED_LIBS'], + scalapack_mt_shared_libs_fosscuda.replace('openblas', 'flexiblas')) + self.assertEqual(os.environ['SCALAPACK_MT_STATIC_LIBS'], + scalapack_mt_static_libs_fosscuda.replace('openblas', 'flexiblas')) + self.modtool.purge() + tc = self.get_toolchain('intel', version='2018a') with self.mocked_stdout_stderr(): tc.prepare() @@ -2110,7 +2162,7 @@ def test_independence(self): 'CrayIntel': "-O2 -ftz -fp-speculation=safe -fp-model source -fopenmp -craype-verbose", 'GCC': "-O2 -ftree-vectorize -test -fno-math-errno -fopenmp", 'iccifort': "-O2 -test -ftz -fp-speculation=safe -fp-model source -fopenmp", - 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -fiopenmp", + 'intel-compilers': "-O2 -test -ftz -fp-speculation=safe -fp-model precise -qopenmp", } toolchains = [ @@ -2217,6 +2269,7 @@ def test_compiler_cache(self): "--force", "--debug", "--disable-cleanup-tmpdir", + "--disable-rpath", ] ccache = which('ccache') @@ -2314,8 +2367,8 @@ def test_rpath_args_script(self): # simplest possible compiler command with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -c foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -c foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2326,12 +2379,12 @@ def test_rpath_args_script(self): "'-c'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # linker command, --enable-new-dtags should be replaced with --disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld '' '%s' --enable-new-dtags foo.o" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld '' '{rpath_inc}' --enable-new-dtags foo.o") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2342,12 +2395,12 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "'foo.o'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # compiler command, -Wl,--enable-new-dtags should be replaced with -Wl,--disable-new-dtags with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' -Wl,--enable-new-dtags foo.c" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' -Wl,--enable-new-dtags foo.c") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2358,12 +2411,12 @@ def test_rpath_args_script(self): "'-Wl,--disable-new-dtags'", "'foo.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing no arguments with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s'" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}'") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2372,12 +2425,12 @@ def test_rpath_args_script(self): "'-Wl,-rpath=$ORIGIN/../lib64'", "'-Wl,--disable-new-dtags'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test passing a single empty argument with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s ld.gold '' '%s' ''" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} ld.gold '' '{rpath_inc}' ''") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2387,13 +2440,13 @@ def test_rpath_args_script(self): "'--disable-new-dtags'", "''", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, but non-existing path => not used in RPATH, but -L option is retained - cmd = "%s gcc '' '%s' foo.c -L%s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L{self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2405,13 +2458,13 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument again, with existing path mkdir(os.path.join(self.test_prefix, 'foo')) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2424,12 +2477,12 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # relative paths passed to -L are *not* RPATH'ed in with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s gcc '' '%s' foo.c -L../lib -lfoo" % (script, rpath_inc), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} gcc '' '{rpath_inc}' foo.c -L../lib -lfoo") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2441,13 +2494,13 @@ def test_rpath_args_script(self): "'-L../lib'", "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # single -L argument, with value separated by a space - cmd = "%s gcc '' '%s' foo.c -L %s/foo -lfoo" % (script, rpath_inc, self.test_prefix) + cmd = f"{script} gcc '' '{rpath_inc}' foo.c -L {self.test_prefix}/foo -lfoo" with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2460,7 +2513,7 @@ def test_rpath_args_script(self): "'-L%s/foo'" % self.test_prefix, "'-lfoo'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) mkdir(os.path.join(self.test_prefix, 'bar')) mkdir(os.path.join(self.test_prefix, 'lib64')) @@ -2482,8 +2535,8 @@ def test_rpath_args_script(self): '-L%s/bar' % self.test_prefix, ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2504,7 +2557,7 @@ def test_rpath_args_script(self): "'-L/usr/lib'", "'-L%s/bar'" % self.test_prefix, ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # test specifying of custom rpath filter cmd = ' '.join([ @@ -2520,8 +2573,8 @@ def test_rpath_args_script(self): '-lbar', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-rpath=%s/lib'" % self.test_prefix, "'-rpath=%s/lib64'" % self.test_prefix, @@ -2537,7 +2590,7 @@ def test_rpath_args_script(self): "'-L/bar'", "'-lbar'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # slightly trimmed down real-life example (compilation of XZ) for subdir in ['icc/lib/intel64', 'imkl/lib', 'imkl/mkl/lib/intel64', 'gettext/lib']: @@ -2560,8 +2613,8 @@ def test_rpath_args_script(self): '-Wl,/example/software/XZ/5.2.2-intel-2016b/lib', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd("%s icc '' '%s' %s" % (script, rpath_inc, args), simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(f"{script} icc '' '{rpath_inc}' {args}") + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, "'-Wl,-rpath=%s/lib64'" % self.test_prefix, @@ -2588,7 +2641,7 @@ def test_rpath_args_script(self): "'-Wl,-rpath'", "'-Wl,/example/software/XZ/5.2.2-intel-2016b/lib'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # trimmed down real-life example involving quotes and escaped quotes (compilation of GCC) args = [ @@ -2605,8 +2658,8 @@ def test_rpath_args_script(self): ] cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, ' '.join(args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = [ "'-Wl,-rpath=%s/lib'" % self.test_prefix, @@ -2626,15 +2679,16 @@ def test_rpath_args_script(self): "'-o' 'build/version.o'", "'../../gcc/version.c'", ] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # verify that no -rpath arguments are injected when command is run in 'version check' mode for extra_args in ["-v", "-V", "--version", "-dumpversion", "-v -L/test/lib"]: cmd = "%s g++ '' '%s' %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(["'%s'" % x for x in extra_args.split(' ')])) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) + cmd_args = ' '.join(["'%s'" % x for x in extra_args.split(' ')]) + self.assertEqual(res.output.strip(), f"CMD_ARGS=({cmd_args})") # if a compiler command includes "-x c++-header" or "-x c-header" (which imply no linking is done), # we should *not* inject -Wl,-rpath options, since those enable linking as a side-effect; @@ -2647,10 +2701,10 @@ def test_rpath_args_script(self): for extra_args in test_cases: cmd = "%s g++ '' '%s' foo.c -O2 %s" % (script, rpath_inc, extra_args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) cmd_args = ["'foo.c'", "'-O2'"] + ["'%s'" % x for x in extra_args.split(' ')] - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # check whether $LIBRARY_PATH is taken into account test_cmd_gcc = "%s gcc '' '%s' -c foo.c" % (script, rpath_inc) @@ -2716,16 +2770,16 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % x for x in library_path if x] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) # paths already listed via -L don't get included again as RPATH option new_lib64 = os.path.join(self.test_prefix, 'new', 'lib64') @@ -2744,19 +2798,19 @@ def test_rpath_args_script(self): os.environ['LIBRARY_PATH'] = ':'.join(library_path) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_gcc, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_gcc) + self.assertEqual(res.exit_code, 0) # no -L options in GCC command, so all $LIBRARY_PATH entries are retained except for last one (lib symlink) cmd_args = pre_cmd_args_gcc + ["'-Wl,-rpath=%s'" % x for x in library_path[:-1] if x] + post_cmd_args_gcc - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) with self.mocked_stdout_stderr(): - out, ec = run_cmd(test_cmd_ld, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(test_cmd_ld) + self.assertEqual(res.exit_code, 0) # only new path from $LIBRARY_PATH is included as -rpath option, # since others are already included via corresponding -L flag cmd_args = pre_cmd_args_ld + ["'-rpath=%s'" % new_lib64] + post_cmd_args_ld - self.assertEqual(out.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) + self.assertEqual(res.output.strip(), "CMD_ARGS=(%s)" % ' '.join(cmd_args)) def test_toolchain_prepare_rpath(self): """Test toolchain.prepare under --rpath""" @@ -2874,8 +2928,8 @@ def test_toolchain_prepare_rpath(self): '-DX="\\"\\""', ]) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', '-Wl,-rpath=%s/foo' % self.test_prefix, @@ -2885,7 +2939,7 @@ def test_toolchain_prepare_rpath(self): '$FOO', '-DX=""', ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # check whether 'stubs' library directory are correctly filtered out paths = [ @@ -2911,8 +2965,8 @@ def test_toolchain_prepare_rpath(self): cmd = "g++ ${USER}.c %s" % ' '.join(args) with self.mocked_stdout_stderr(): - out, ec = run_cmd(cmd, simple=False) - self.assertEqual(ec, 0) + res = run_shell_cmd(cmd) + self.assertEqual(res.exit_code, 0) expected = ' '.join([ '-Wl,--disable-new-dtags', @@ -2937,7 +2991,7 @@ def test_toolchain_prepare_rpath(self): '-L%s/prefix/software/bleh/0/lib/stubs' % self.test_prefix, '-L%s/prefix/software/foobar/4.5/stubsbutnotreally' % self.test_prefix, ]) - self.assertEqual(out.strip(), expected % {'user': os.getenv('USER')}) + self.assertEqual(res.output.strip(), expected % {'user': os.getenv('USER')}) # calling prepare() again should *not* result in wrapping the existing RPATH wrappers # this can happen when building extensions @@ -3050,6 +3104,21 @@ def test_env_vars_external_module(self): expected = {} self.assertEqual(res, expected) + def test_get_flag(self): + """Test get_flag function""" + tc = self.get_toolchain('gompi', version='2018a') + + checks = { + '-a': 'a', + '-openmp': 'openmp', + '-foo': ['foo'], + '-foo -bar': ['foo', 'bar'], + } + + for flagstring, flags in checks.items(): + tc.options.options_map['openmp'] = flags + self.assertEqual(tc.get_flag('openmp'), flagstring) + def suite(): """ return all the tests""" diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 2192b0589c..61a9cca2e5 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -42,7 +42,7 @@ import textwrap from easybuild.tools import LooseVersion from importlib import reload -from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered +from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, cleanup from test.framework.package import mock_fpm from unittest import TextTestRunner @@ -50,6 +50,7 @@ import easybuild.tools.module_naming_scheme # required to dynamically load test module naming scheme(s) from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.framework.easyconfig.parser import EasyConfigParser +from easybuild.main import main_with_hooks from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax, get_repositorypath from easybuild.tools.environment import modify_env @@ -57,7 +58,8 @@ from easybuild.tools.filetools import read_file, remove_dir, remove_file, which, write_file from easybuild.tools.module_generator import ModuleGeneratorTcl from easybuild.tools.modules import Lmod -from easybuild.tools.run import run_cmd +from easybuild.tools.run import run_shell_cmd +from easybuild.tools.utilities import nub from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.version import VERSION as EASYBUILD_VERSION @@ -100,6 +102,9 @@ def tearDown(self): del sys.modules['easybuild.easyblocks.toytoy'] del sys.modules['easybuild.easyblocks.generic.toy_extension'] + # reset cached hooks + easybuild.tools.hooks._cached_hooks.clear() + super(ToyBuildTest, self).tearDown() # remove logs @@ -154,7 +159,7 @@ def check_toy(self, installpath, outtxt, name='toy', version='0.0', versionprefi def _test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=True, fails=False, verbose=True, raise_error=False, test_report=None, name='toy', versionsuffix='', testing=True, - raise_systemexit=False, force=True, test_report_regexs=None): + raise_systemexit=False, force=True, test_report_regexs=None, debug=True): """Perform a toy build.""" if extra_args is None: extra_args = [] @@ -168,10 +173,11 @@ def _test_toy_build(self, extra_args=None, ec_file=None, tmpdir=None, verify=Tru '--sourcepath=%s' % self.test_sourcepath, '--buildpath=%s' % self.test_buildpath, '--installpath=%s' % self.test_installpath, - '--debug', '--unittest-file=%s' % self.logfile, '--robot=%s' % os.pathsep.join([self.test_buildpath, os.path.dirname(__file__)]), ] + if debug: + args.append('--debug') if force: args.append('--force') if tmpdir is not None: @@ -292,6 +298,7 @@ def test_toy_tweaked(self): ec_extra = '\n'.join([ "versionsuffix = '-tweaked'", "modextrapaths = {'SOMEPATH': ['foo/bar', 'baz', '']}", + "modextrapaths_append = {'SOMEPATH_APPEND': ['qux/fred', 'thud', '']}", "modextravars = {'FOO': 'bar'}", "modloadmsg = '%s'" % modloadmsg, "modtclfooter = 'puts stderr \"oh hai!\"'", # ignored when module syntax is Lua @@ -326,6 +333,9 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/foo/bar$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root/baz$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend-path\s*SOMEPATH\s*\$root$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/qux/fred$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root/thud$', toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append-path\s*SOMEPATH_APPEND\s*\$root$', toy_module_txt, re.M)) mod_load_msg = r'module-info mode load.*\n\s*puts stderr\s*.*%s$' % modloadmsg_regex_tcl self.assertTrue(re.search(mod_load_msg, toy_module_txt, re.M)) self.assertTrue(re.search(r'^puts stderr "oh hai!"$', toy_module_txt, re.M)) @@ -333,6 +343,11 @@ def test_toy_tweaked(self): self.assertTrue(re.search(r'^setenv\("FOO", "bar"\)', toy_module_txt, re.M)) pattern = r'^prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)$' self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + pattern = r'^append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)$' + self.assertTrue(re.search(pattern, toy_module_txt, re.M)) + self.assertTrue(re.search(r'^append_path\("SOMEPATH_APPEND", root\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)$', toy_module_txt, re.M)) self.assertTrue(re.search(r'^prepend_path\("SOMEPATH", root\)$', toy_module_txt, re.M)) mod_load_msg = r'^if mode\(\) == "load" then\n\s*io.stderr:write\(%s\)$' % modloadmsg_regex_lua @@ -368,8 +383,8 @@ def test_toy_buggy_easyblock(self): 'verify': False, 'verbose': False, } - err_regex = r"Traceback[\S\s]*toy_buggy.py.*build_step[\S\s]*name 'run_cmd' is not defined" - self.assertErrorRegex(EasyBuildError, err_regex, self.run_test_toy_build_with_output, **kwargs) + err_regex = r"name 'run_shell_cmd' is not defined" + self.assertErrorRegex(NameError, err_regex, self.run_test_toy_build_with_output, **kwargs) def test_toy_build_formatv2(self): """Perform a toy build (format v2).""" @@ -736,9 +751,9 @@ def test_toy_group_check(self): # figure out a group that we're a member of to use in the test with self.mocked_stdout_stderr(): - out, ec = run_cmd('groups', simple=False) - self.assertEqual(ec, 0, "Failed to select group to use in test") - group_name = out.split(' ')[0].strip() + res = run_shell_cmd('groups') + self.assertEqual(res.exit_code, 0, "Failed to select group to use in test") + group_name = res.output.split(' ')[0].strip() toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') @@ -1278,8 +1293,8 @@ def test_toy_extension_patches_postinstallcmds(self): # make sure that patches were actually applied (without them the message producded by 'bar' is different) bar_bin = os.path.join(installdir, 'bin', 'bar') with self.mocked_stdout_stderr(): - out, _ = run_cmd(bar_bin) - self.assertEqual(out, "I'm a bar, and very very proud of it.\n") + res = run_shell_cmd(bar_bin) + self.assertEqual(res.output, "I'm a bar, and very very proud of it.\n") # verify that post-install command for 'bar' extension was executed fn = 'created-via-postinstallcmds.txt' @@ -1429,10 +1444,12 @@ def test_toy_extension_extract_cmd(self): ]) write_file(test_ec, test_ec_txt) - error_pattern = "unzip .*/bar-0.0.tar.gz.* exited with exit code [1-9]" + error_pattern = r"shell command 'unzip \.\.\.' failed in extensions step for test.eb" with self.mocked_stdout_stderr(): - self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec, - raise_error=True, verbose=False) + # for now, we expect subprocess.CalledProcessError, but eventually 'run' function will + # do proper error reporting + self.assertErrorRegex(EasyBuildError, error_pattern, + self._test_toy_build, ec_file=test_ec, raise_error=True, verbose=False) def test_toy_extension_sources_git_config(self): """Test install toy that includes extensions with 'sources' spec including 'git_config'.""" @@ -1528,7 +1545,7 @@ def test_toy_module_fulltxt(self): mod_txt_regex_pattern = '\n'.join([ r'help\(\[==\[', r'', - r'%s' % help_txt, + help_txt, r'\]==\]\)', r'', r'whatis\(\[==\[Description: Toy C program, 100% toy.\]==\]\)', @@ -1551,6 +1568,9 @@ def test_toy_module_fulltxt(self): r'prepend_path\("SOMEPATH", pathJoin\(root, "foo/bar"\)\)', r'prepend_path\("SOMEPATH", pathJoin\(root, "baz"\)\)', r'prepend_path\("SOMEPATH", root\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "qux/fred"\)\)', + r'append_path\("SOMEPATH_APPEND", pathJoin\(root, "thud"\)\)', + r'append_path\("SOMEPATH_APPEND", root\)', r'', r'if mode\(\) == "load" then', ] + modloadmsg_lua + [ @@ -1565,7 +1585,7 @@ def test_toy_module_fulltxt(self): r'proc ModulesHelp { } {', r' puts stderr {', r'', - r'%s' % help_txt, + help_txt, r' }', r'}', r'', @@ -1589,6 +1609,9 @@ def test_toy_module_fulltxt(self): r'prepend-path SOMEPATH \$root/foo/bar', r'prepend-path SOMEPATH \$root/baz', r'prepend-path SOMEPATH \$root', + r'append-path SOMEPATH_APPEND \$root/qux/fred', + r'append-path SOMEPATH_APPEND \$root/thud', + r'append-path SOMEPATH_APPEND \$root', r'', r'if { \[ module-info mode load \] } {', ] + modloadmsg_tcl + [ @@ -1894,7 +1917,7 @@ def test_toy_exts_parallel(self): write_file(test_ec, test_ec_txt) args = ['--parallel-extensions-install', '--experimental', '--force', '--parallel=3'] - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, @@ -1913,7 +1936,7 @@ def test_toy_exts_parallel(self): # also test skipping of extensions in parallel args.append('--skip') - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # order in which these patterns occur is not fixed, so check them one by one @@ -1939,7 +1962,7 @@ def test_toy_exts_parallel(self): write_file(toy_ext_eb, toy_ext_eb_txt) args[-1] = '--include-easyblocks=%s' % toy_ext_eb - stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args) + stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') # take into account that each of these lines may appear multiple times, # in case no progress was made between checks @@ -2406,7 +2429,14 @@ def test_sanity_check_paths_lib64(self): # modify test easyconfig: move lib/libtoy.a to lib64/libtoy.a ectxt = re.sub(r"\s*'files'.*", "'files': ['bin/toy', ('lib/libtoy.a', 'lib/libfoo.a')],", ectxt) - postinstallcmd = "mkdir %(installdir)s/lib64 && mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a" + postinstallcmd = ' && '.join([ + # remove lib64 symlink (if it's there) + "rm -f %(installdir)s/lib64", + # create empty lib64 dir + "mkdir %(installdir)s/lib64", + # move libtoy.a + "mv %(installdir)s/lib/libtoy.a %(installdir)s/lib64/libtoy.a", + ]) ectxt = re.sub("postinstallcmds.*", "postinstallcmds = ['%s']" % postinstallcmd, ectxt) test_ec = os.path.join(self.test_prefix, 'toy-0.0.eb') @@ -2634,7 +2664,7 @@ def test_toy_build_enhanced_sanity_check(self): test_ec_txt = test_ec_txt + '\nenhance_sanity_check = False' write_file(test_ec, test_ec_txt) - error_pattern = r" Missing mandatory key 'dirs' in sanity_check_paths." + error_pattern = r"Missing mandatory key 'dirs' in sanity_check_paths." with self.mocked_stdout_stderr(): self.assertErrorRegex(EasyBuildError, error_pattern, self._test_toy_build, ec_file=test_ec, extra_args=eb_args, raise_error=True, verbose=False) @@ -2811,17 +2841,19 @@ def test_toy_filter_rpath_sanity_libs(self): libtoy_libdir = os.path.join(self.test_installpath, 'software', 'libtoy', '0.0', 'lib') toyapp_bin = os.path.join(self.test_installpath, 'software', 'toy-app', '0.0', 'bin', 'toy-app') - rpath_regex = re.compile(r"RPATH.*%s" % libtoy_libdir, re.M) + rpath_regex = re.compile(r"RPATH.*" + libtoy_libdir, re.M) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertTrue(rpath_regex.search(out), "Pattern '%s' should be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertTrue(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"ldd {toyapp_bin}") + out = res.output libtoy_regex = re.compile(r"libtoy.so => /.*/libtoy.so", re.M) notfound = re.compile(r"libtoy\.so\s*=>\s*not found", re.M) - self.assertTrue(libtoy_regex.search(out), "Pattern '%s' should be found in: %s" % (libtoy_regex.pattern, out)) - self.assertFalse(notfound.search(out), "Pattern '%s' should not be found in: %s" % (notfound.pattern, out)) + self.assertTrue(libtoy_regex.search(out), f"Pattern '{libtoy_regex.pattern}' should be found in: {out}") + self.assertFalse(notfound.search(out), f"Pattern '{notfound.pattern}' should not be found in: {out}") # test sanity error when --rpath-filter is used to filter a required library # In this test, libtoy.so will be linked, but not RPATH-ed due to the --rpath-filter @@ -2840,16 +2872,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) - self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") + self.assertFalse(rpath_regex.search(res.output), + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") # test again with list of library names passed to --filter-rpath-sanity-libs args = ['--rpath', '--rpath-filter=.*libtoy.*', '--filter-rpath-sanity-libs=libfoo.so,libtoy.so,libbar.so'] @@ -2857,16 +2889,16 @@ def test_toy_filter_rpath_sanity_libs(self): self._test_toy_build(ec_file=toy_ec, name='toy-app', extra_args=args, raise_error=True) with self.mocked_stdout_stderr(): - out, ec = run_cmd("readelf -d %s" % toyapp_bin, simple=False) + res = run_shell_cmd(f"readelf -d {toyapp_bin}") self.assertFalse(rpath_regex.search(out), - "Pattern '%s' should not be found in: %s" % (rpath_regex.pattern, out)) + f"Pattern '{rpath_regex.pattern}' should not be found in: {res.output}") with self.mocked_stdout_stderr(): - out, ec = run_cmd("ldd %s" % toyapp_bin, simple=False) - self.assertFalse(libtoy_regex.search(out), - "Pattern '%s' should not be found in: %s" % (libtoy_regex.pattern, out)) - self.assertTrue(notfound.search(out), - "Pattern '%s' should be found in: %s" % (notfound.pattern, out)) + res = run_shell_cmd(f"ldd {toyapp_bin}") + self.assertFalse(libtoy_regex.search(res.output), + f"Pattern '{libtoy_regex.pattern}' should not be found in: {res.output}") + self.assertTrue(notfound.search(res.output), + f"Pattern '{notfound.pattern}' should be found in: {res.output}") def test_toy_modaltsoftname(self): """Build two dependent toys as in test_toy_toy but using modaltsoftname""" @@ -2950,11 +2982,11 @@ def test_toy_build_trace(self): r"^== fetching files\.\.\.\n >> sources:\n >> .*/toy-0\.0\.tar\.gz \[SHA256: 44332000.*\]$", r"^ >> applying patch toy-0\.0_fix-silly-typo-in-printf-statement\.patch$", r'\n'.join([ - r"^ >> running command:", + r"^ >> running shell command:", + r"\tgcc toy.c -o toy\n" r"\t\[started at: .*\]", r"\t\[working dir: .*\]", - r"\t\[output logged in .*\]", - r"\tgcc toy.c -o toy\n" + r"\t\[output saved to .*\]", r'', ]), r" >> command completed: exit 0, ran in .*", @@ -2982,6 +3014,9 @@ def test_toy_build_hooks(self): hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') hooks_file_txt = textwrap.dedent(""" import os + from easybuild.tools.filetools import change_dir, copy_file + + TOY_COMP_CMD = "gcc toy.c -o toy" def start_hook(): print('start hook triggered') @@ -3005,6 +3040,9 @@ def post_install_hook(self): print('in post-install hook for %s v%s' % (self.name, self.version)) print(', '.join(sorted(os.listdir(self.installdir)))) + copy_of_toy = os.path.join(self.start_dir, 'copy_of_toy') + copy_file(copy_of_toy, os.path.join(self.installdir, 'bin')) + def module_write_hook(self, module_path, module_txt): print('in module-write hook hook for %s' % os.path.basename(module_path)) return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore') @@ -3017,12 +3055,36 @@ def pre_sanitycheck_hook(self): def end_hook(): print('end hook triggered, all done!') + + def pre_run_shell_cmd_hook(cmd, *args, **kwargs): + if isinstance(cmd, str) and cmd.strip() == TOY_COMP_CMD: + print("pre_run_shell_cmd_hook triggered for '%s'" % cmd) + # 'copy_toy_file' command doesn't exist, but don't worry, + # this problem will be fixed in post_run_shell_cmd_hook + cmd += " && copy_toy_file toy copy_of_toy" + return cmd + + def post_run_shell_cmd_hook(cmd, *args, **kwargs): + exit_code = kwargs['exit_code'] + output = kwargs['output'] + work_dir = kwargs['work_dir'] + if isinstance(cmd, str) and cmd.strip().startswith(TOY_COMP_CMD) and exit_code: + cwd = change_dir(work_dir) + copy_file('toy', 'copy_of_toy') + change_dir(cwd) + print("'%s' command failed (exit code %s), but I fixed it!" % (cmd, exit_code)) """) write_file(hooks_file, hooks_file_txt) + extra_args = [ + '--hooks=%s' % hooks_file, + # disable trace output to make checking of generated output produced by hooks easier + '--disable-trace', + ] + self.mock_stderr(True) self.mock_stdout(True) - self._test_toy_build(ec_file=test_ec, extra_args=['--hooks=%s' % hooks_file], raise_error=True) + self._test_toy_build(ec_file=test_ec, extra_args=extra_args, raise_error=True, debug=False) stderr = self.get_stderr() stdout = self.get_stdout() self.mock_stderr(False) @@ -3032,35 +3094,55 @@ def end_hook(): toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0') if get_module_syntax() == 'Lua': toy_mod_file += '.lua' - - self.assertEqual(stderr, '') mod_name = os.path.basename(toy_mod_file) + + warnings = nub([x for x in stderr.strip().splitlines() if x]) + self.assertEqual(warnings, ["WARNING: Command ' gcc toy.c -o toy ' failed, but we'll ignore it..."]) + # parse hook is triggered 3 times: once for main install, and then again for each extension; # module write hook is triggered 5 times: # - before installing extensions # - for fake module file being created during sanity check (triggered twice, for main + toy install) # - for final module file # - for devel module file - expected_output_lines = [ - "== Running start hook...\nstart hook triggered\n", - "== Running parse hook for test.eb...\ntoy 0.0\n['%(name)s-%(version)s.tar.gz']\necho toy\n", - "== Running pre-configure hook...\npre-configure: toy.source: True\n", - "== Running post-configure hook...\npost-configure: toy.source: False\n", - "== Running post-install hook...\nin post-install hook for toy v0.0\nbin, lib\n", - "== Running parse hook...\ntoy 0.0\n['%(name)s-%(version)s.tar.gz']\necho toy\n" * 2, # twice - "== Running post-single_extension hook...\ninstalling of extension bar is done!\n", - "== Running post-single_extension hook...\ninstalling of extension toy is done!\n", - "== Running pre-sanitycheck hook...\npre_sanity_check_hook\n", - "== Running end hook...\nend hook triggered, all done!", - ] - for line in expected_output_lines: - self.assertIn(line, stdout.strip()) - txt = f"== Running module_write hook...\nin module-write hook hook for {mod_name}\n" - self.assertEqual(5, stdout.strip().count(txt)) + expected_output = textwrap.dedent(f""" + start hook triggered + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + pre-configure: toy.source: True + post-configure: toy.source: False + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! + in post-install hook for toy v0.0 + bin, lib + in module-write hook hook for {mod_name} + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + toy 0.0 + ['%(name)s-%(version)s.tar.gz'] + echo toy + installing of extension bar is done! + pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' + ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! + installing of extension toy is done! + pre_sanity_check_hook + in module-write hook hook for {mod_name} + in module-write hook hook for {mod_name} + in module-write hook hook for {mod_name} + in module-write hook hook for {mod_name} + end hook triggered, all done! + """).strip().format(mod_name=os.path.basename(toy_mod_file)) + self.assertEqual(stdout.strip(), expected_output) toy_mod = read_file(toy_mod_file) self.assertIn('Not a toy anymore', toy_mod) + toy_bin_dir = os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin') + toy_bins = sorted(os.listdir(toy_bin_dir)) + self.assertEqual(toy_bins, ['bar', 'copy_of_toy', 'toy']) + def test_toy_multi_deps(self): """Test installation of toy easyconfig that uses multi_deps.""" test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') @@ -3596,7 +3678,7 @@ def __exit__(self, type, value, traceback): wait_matches = wait_regex.findall(stdout) # we can't rely on an exact number of 'waiting' messages, so let's go with a range... - self.assertIn(len(wait_matches), range(2, 5)) + self.assertIn(len(wait_matches), range(1, 5)) self.assertTrue(ok_regex.search(stdout), "Pattern '%s' found in: %s" % (ok_regex.pattern, stdout)) @@ -3769,7 +3851,6 @@ def test_toy_build_lib_lib64_symlink(self): toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') test_ec_txt = read_file(toy_ec) - test_ec_txt += "\npostinstallcmds += ['mv %(installdir)s/lib %(installdir)s/lib64']" test_ec = os.path.join(self.test_prefix, 'test.eb') write_file(test_ec, test_ec_txt) @@ -3782,30 +3863,30 @@ def test_toy_build_lib_lib64_symlink(self): lib_path = os.path.join(toy_installdir, 'lib') lib64_path = os.path.join(toy_installdir, 'lib64') - # lib64 subdir exists, is not a symlink - self.assertExists(lib64_path) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) - - # lib subdir is a symlink to lib64 subdir + # lib subdir exists, is not a symlink self.assertExists(lib_path) self.assertTrue(os.path.isdir(lib_path)) - self.assertTrue(os.path.islink(lib_path)) - self.assertTrue(os.path.samefile(lib_path, lib64_path)) + self.assertFalse(os.path.islink(lib_path)) + + # lib64 subdir is a symlink to lib subdir + self.assertExists(lib64_path) + self.assertTrue(os.path.isdir(lib64_path)) + self.assertTrue(os.path.islink(lib64_path)) + self.assertTrue(os.path.samefile(lib64_path, lib_path)) - # lib symlink should point to a relative path - self.assertFalse(os.path.isabs(os.readlink(lib_path))) + # lib64 symlink should point to a relative path + self.assertFalse(os.path.isabs(os.readlink(lib64_path))) # cleanup and try again with --disable-lib-lib64-symlink remove_dir(self.test_installpath) with self.mocked_stdout_stderr(): - self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib-lib64-symlink']) + self._test_toy_build(ec_file=test_ec, extra_args=['--disable-lib64-lib-symlink']) - self.assertExists(lib64_path) - self.assertNotExists(lib_path) - self.assertNotIn('lib', os.listdir(toy_installdir)) - self.assertTrue(os.path.isdir(lib64_path)) - self.assertFalse(os.path.islink(lib64_path)) + self.assertExists(lib_path) + self.assertNotExists(lib64_path) + self.assertNotIn('lib64', os.listdir(toy_installdir)) + self.assertTrue(os.path.isdir(lib_path)) + self.assertFalse(os.path.islink(lib_path)) def test_toy_build_sanity_check_linked_libs(self): """Test sanity checks for banned/requires libraries.""" @@ -3897,6 +3978,44 @@ def test_toy_build_sanity_check_linked_libs(self): self._test_toy_build(ec_file=test_ec, extra_args=args, force=False, raise_error=True, verbose=False, verify=False) + def test_toy_mod_files(self): + """Check detection of .mod files""" + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + test_ec_txt = read_file(toy_ec) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + test_ec_txt += "\npostinstallcmds += ['touch %(installdir)s/lib/file.mod']" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec) + + args = ['--try-toolchain=GCCcore,6.2.0', '--disable-map-toolchains'] + self.mock_stdout(True) + self.mock_stderr(True) + self._test_toy_build(ec_file=test_ec, extra_args=args) + stderr = self.get_stderr() + self.mock_stdout(False) + self.mock_stderr(False) + pattern = r"WARNING: One or more \.mod files found in .*/software/toy/0.0-GCCcore-6.2.0: .*/lib64/file.mod" + self.assertRegex(stderr.strip(), pattern) + + args += ['--fail-on-mod-files-gcccore'] + pattern = r"Sanity check failed: One or more \.mod files found in .*/toy/0.0-GCCcore-6.2.0: .*/lib/file.mod" + self.assertErrorRegex(EasyBuildError, pattern, self.run_test_toy_build_with_output, ec_file=test_ec, + extra_args=args, verify=False, fails=True, verbose=False, raise_error=True) + + test_ec_txt += "\nskip_mod_files_sanity_check = True" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, extra_args=args) + def test_toy_ignore_test_failure(self): """Check whether use of --ignore-test-failure is mentioned in build output.""" args = ['--ignore-test-failure'] @@ -3987,6 +4106,104 @@ def test_toy_post_install_messages(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + def test_toy_build_info_msg(self): + """ + Test use of build info message + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nbuild_info_msg = "Are you sure you want to install this toy software?"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr(): + self._test_toy_build(ec_file=test_ec, testing=False, verify=False, raise_error=True) + stdout = self.get_stdout() + + pattern = '\n'.join([ + r"== This easyconfig provides the following build information:", + r'', + r"Are you sure you want to install this toy software\?", + ]) + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(stdout), "Pattern '%s' should be found in: %s" % (regex.pattern, stdout)) + + def test_toy_failing_test_step(self): + """ + Test behaviour when test step fails, using toy easyconfig. + """ + test_ecs = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs') + toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb') + + test_ec_txt = read_file(toy_ec) + test_ec_txt += '\nruntest = "false"' + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ec_txt) + + error_pattern = r"shell command 'false \.\.\.' failed in test step" + self.assertErrorRegex(EasyBuildError, error_pattern, self.run_test_toy_build_with_output, + ec_file=test_ec, raise_error=True) + + def test_eb_crash(self): + """ + Test behaviour when EasyBuild crashes, for example due to a buggy hook + """ + hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') + hooks_file_txt = textwrap.dedent(""" + def pre_configure_hook(self, *args, **kwargs): + no_such_thing + """) + write_file(hooks_file, hooks_file_txt) + + topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + toy_eb = os.path.join(topdir, 'test', 'framework', 'sandbox', 'easybuild', 'easyblocks', 't', 'toy.py') + toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + args = [ + toy_ec, + f'--hooks={hooks_file}', + '--force', + f'--installpath={self.test_prefix}', + f'--include-easyblocks={toy_eb}', + ] + + with self.mocked_stdout_stderr() as (_, stderr): + cleanup() + try: + main_with_hooks(args=args) + self.assertFalse("This should never be reached, main function should have crashed!") + except NameError as err: + self.assertEqual(str(err), "name 'no_such_thing' is not defined") + + regex = re.compile(r"EasyBuild crashed! Please consider reporting a bug, this should not happen") + stderr = stderr.getvalue() + self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + + def test_eb_error(self): + """ + Test whether main function as run by 'eb' command print error messages to stderr. + """ + topdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + toy_ec = os.path.join(topdir, 'test', 'framework', 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') + + test_ec = os.path.join(self.test_prefix, 'test.eb') + test_ec_txt = read_file(toy_ec) + test_ec_txt += "\ndependencies = [('nosuchdep', '1.0')]" + write_file(test_ec, test_ec_txt) + + with self.mocked_stdout_stderr() as (_, stderr): + cleanup() + try: + main_with_hooks(args=[test_ec, '--robot', '--force']) + except SystemExit: + pass + + regex = re.compile("^ERROR: Missing dependencies", re.M) + stderr = stderr.getvalue() + self.assertTrue(regex.search(stderr), f"Pattern '{regex.pattern}' should be found in {stderr}") + def suite(): """ return all the tests in this file """ diff --git a/test/framework/utilities_test.py b/test/framework/utilities_test.py index 0aa3a9956e..ba4766f302 100644 --- a/test/framework/utilities_test.py +++ b/test/framework/utilities_test.py @@ -123,6 +123,15 @@ def test_LooseVersion(self): self.assertLess(LooseVersion('1.02'), '2.01') self.assertLessEqual('1.02', LooseVersion('2.01')) self.assertLessEqual(LooseVersion('1.02'), '2.01') + # Negation of all ops, i.e. verify each op can return False + self.assertFalse(LooseVersion('2.02') != '2.02') + self.assertFalse(LooseVersion('2.02') <= '2.01') + self.assertFalse(LooseVersion('2.02') < '2.01') + self.assertFalse(LooseVersion('2.02') < '2.02') + self.assertFalse(LooseVersion('2.02') == '2.03') + self.assertFalse(LooseVersion('2.02') >= '2.03') + self.assertFalse(LooseVersion('2.02') > '2.03') + self.assertFalse(LooseVersion('2.02') > '2.02') # Some comparisons we might do: Full version on left hand side, shorter on right self.assertGreater(LooseVersion('2.1.5'), LooseVersion('2.1')) @@ -135,7 +144,7 @@ def test_LooseVersion(self): self.assertGreater(LooseVersion('1.0'), LooseVersion('1')) self.assertLess(LooseVersion('1'), LooseVersion('1.0')) - # The following test is taken from Python disutils tests + # The following test is taken from Python distutils tests # licensed under the Python Software Foundation License Version 2 versions = (('1.5.1', '1.5.2b2', -1), ('161', '3.10a', 1),