diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 3805c1a240..fb00471b90 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -38,6 +38,7 @@ jobs: - TEMPLATE: "template_skip_nf_core_configs.yml" runner: ubuntu-latest profile: "docker" + fail-fast: false steps: - name: go to working directory diff --git a/CHANGELOG.md b/CHANGELOG.md index ee6f8aabf2..707dc8635f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - switch to new image syntax in readme ([#2645](https://github.com/nf-core/tools/pull/2645)) - Add conda channel order to nextflow.config ([#2094](https://github.com/nf-core/tools/pull/2094)) - Fix tyop in pipeline nextflow.config ([#2664](https://github.com/nf-core/tools/pull/2664)) +- Remove `nfcore_external_java_deps.jar` from lib directory in pipeline template ([#2675](https://github.com/nf-core/tools/pull/2675)) - Add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678)) - Add new pipeline error message pointing to docs when 'requirement exceeds available memory' error message ([#2680](https://github.com/nf-core/tools/pull/2680)) @@ -42,6 +43,7 @@ - Update actions/cache action to v4 ([#2666](https://github.com/nf-core/tools/pull/2666)) - Handle api redirects from the old site ([#2672](https://github.com/nf-core/tools/pull/2672)) - Remove redundanct v in pipeline version for emails ([#2667](https://github.com/nf-core/tools/pull/2667)) +- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678)) # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20] diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 117704d1f1..707f98a53a 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -1,5 +1,6 @@ import logging -import os +from pathlib import Path +from typing import Union log = logging.getLogger(__name__) @@ -51,7 +52,6 @@ def files_exist(self): docs/output.md docs/README.md docs/usage.md - lib/nfcore_external_java_deps.jar lib/NfcoreTemplate.groovy lib/Utils.groovy lib/WorkflowMain.groovy @@ -98,6 +98,12 @@ def files_exist(self): .travis.yml + Files that *must not* be present if a certain entry is present in ``nextflow.config``: + + .. code-block:: bash + + lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -132,48 +138,46 @@ def files_exist(self): ["CHANGELOG.md"], ["CITATIONS.md"], ["CODE_OF_CONDUCT.md"], - ["CODE_OF_CONDUCT.md"], ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling ["nextflow_schema.json"], ["nextflow.config"], ["README.md"], - [os.path.join(".github", ".dockstore.yml")], - [os.path.join(".github", "CONTRIBUTING.md")], - [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")], - [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], - [os.path.join(".github", "workflows", "branch.yml")], - [os.path.join(".github", "workflows", "ci.yml")], - [os.path.join(".github", "workflows", "linting_comment.yml")], - [os.path.join(".github", "workflows", "linting.yml")], - [os.path.join("assets", "email_template.html")], - [os.path.join("assets", "email_template.txt")], - [os.path.join("assets", "sendmail_template.txt")], - [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("conf", "modules.config")], - [os.path.join("conf", "test.config")], - [os.path.join("conf", "test_full.config")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")], - [os.path.join("docs", "output.md")], - [os.path.join("docs", "README.md")], - [os.path.join("docs", "README.md")], - [os.path.join("docs", "usage.md")], - [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreTemplate.groovy")], - [os.path.join("lib", "Utils.groovy")], - [os.path.join("lib", "WorkflowMain.groovy")], + [Path(".github", ".dockstore.yml")], + [Path(".github", "CONTRIBUTING.md")], + [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")], + [Path(".github", "ISSUE_TEMPLATE", "config.yml")], + [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")], + [Path(".github", "PULL_REQUEST_TEMPLATE.md")], + [Path(".github", "workflows", "branch.yml")], + [Path(".github", "workflows", "ci.yml")], + [Path(".github", "workflows", "linting_comment.yml")], + [Path(".github", "workflows", "linting.yml")], + [Path("assets", "email_template.html")], + [Path("assets", "email_template.txt")], + [Path("assets", "sendmail_template.txt")], + [Path("assets", f"nf-core-{short_name}_logo_light.png")], + [Path("conf", "modules.config")], + [Path("conf", "test.config")], + [Path("conf", "test_full.config")], + [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")], + [Path("docs", "output.md")], + [Path("docs", "README.md")], + [Path("docs", "README.md")], + [Path("docs", "usage.md")], + [Path("lib", "NfcoreTemplate.groovy")], + [Path("lib", "Utils.groovy")], + [Path("lib", "WorkflowMain.groovy")], ] files_warn = [ ["main.nf"], - [os.path.join("assets", "multiqc_config.yml")], - [os.path.join("conf", "base.config")], - [os.path.join("conf", "igenomes.config")], - [os.path.join(".github", "workflows", "awstest.yml")], - [os.path.join(".github", "workflows", "awsfulltest.yml")], - [os.path.join("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], + [Path("assets", "multiqc_config.yml")], + [Path("conf", "base.config")], + [Path("conf", "igenomes.config")], + [Path(".github", "workflows", "awstest.yml")], + [Path(".github", "workflows", "awsfulltest.yml")], + [Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")], ["modules.json"], ["pyproject.toml"], ] @@ -184,45 +188,48 @@ def files_exist(self): "parameters.settings.json", "pipeline_template.yml", # saving information in .nf-core.yml ".nf-core.yaml", # yml not yaml - os.path.join("bin", "markdown_to_html.r"), - os.path.join("conf", "aws.config"), - os.path.join(".github", "workflows", "push_dockerhub.yml"), - os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md"), - os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md"), - os.path.join("docs", "images", f"nf-core-{short_name}_logo.png"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path(".github", "workflows", "push_dockerhub.yml"), + Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), + Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), ".markdownlint.yml", ".yamllint.yml", - os.path.join("lib", "Checks.groovy"), - os.path.join("lib", "Completion.groovy"), - os.path.join("lib", "Workflow.groovy"), + Path("lib", "Checks.groovy"), + Path("lib", "Completion.groovy"), + Path("lib", "Workflow.groovy"), ] files_warn_ifexists = [".travis.yml"] + files_fail_ifinconfig = [[Path("lib", "nfcore_external_java_deps.jar"), "nf-validation"]] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) + log.info(f"Files to ignore: {ignore_files}") - def pf(file_path): - return os.path.join(self.wf_path, file_path) + def pf(file_path: Union[str, Path]) -> Path: + return Path(self.wf_path, file_path) # First - critical files. Check that this is actually a Nextflow pipeline - if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")): + if not pf("nextflow.config").is_file() and not pf("main.nf").is_file(): failed.append("File not found: nextflow.config or main.nf") raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?") # Files that cause an error if they don't exist for files in files_fail: - if any([f in ignore_files for f in files]): + print(files) + if any([str(f) in ignore_files for f in files]): continue - if any([os.path.isfile(pf(f)) for f in files]): + if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: failed.append(f"File not found: {self._wrap_quotes(files)}") # Files that cause a warning if they don't exist for files in files_warn: - if any([f in ignore_files for f in files]): + if any([str(f) in ignore_files for f in files]): continue - if any([os.path.isfile(pf(f)) for f in files]): + if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: warned.append(f"File not found: {self._wrap_quotes(files)}") @@ -231,16 +238,32 @@ def pf(file_path): for file in files_fail_ifexists: if file in ignore_files: continue - if os.path.isfile(pf(file)): + if pf(file).is_file(): failed.append(f"File must be removed: {self._wrap_quotes(file)}") else: passed.append(f"File not found check: {self._wrap_quotes(file)}") - + # Files that cause an error if they exists together with a certain entry in nextflow.config + for file in files_fail_ifinconfig: + if str(file[0]) in ignore_files: + continue + nextflow_config = pf("nextflow.config") + in_config = False + with open(nextflow_config) as f: + if file[1] in f.read(): + in_config = True + if pf(file[0]).is_file() and in_config: + failed.append(f"File must be removed: {self._wrap_quotes(file[0])}") + elif pf(file[0]).is_file() and not in_config: + passed.append(f"File found check: {self._wrap_quotes(file[0])}") + elif not pf(file[0]).is_file() and not in_config: + failed.append(f"File not found check: {self._wrap_quotes(file[0])}") + elif not pf(file[0]).is_file() and in_config: + passed.append(f"File not found check: {self._wrap_quotes(file[0])}") # Files that cause a warning if they exist for file in files_warn_ifexists: if file in ignore_files: continue - if os.path.isfile(pf(file)): + if pf(file).is_file(): warned.append(f"File should be removed: {self._wrap_quotes(file)}") else: passed.append(f"File not found check: {self._wrap_quotes(file)}") diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 82b286fb44..176b0e9e65 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -1,8 +1,9 @@ import filecmp import logging -import os import shutil import tempfile +from pathlib import Path +from typing import Union import yaml @@ -39,7 +40,6 @@ def files_unchanged(self): docs/images/nf-core-PIPELINE_logo_light.png docs/images/nf-core-PIPELINE_logo_dark.png docs/README.md' - lib/nfcore_external_java_deps.jar lib/NfcoreTemplate.groovy ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling @@ -49,6 +49,10 @@ def files_unchanged(self): .prettierignore pyproject.toml + Files that need to be there or not based on a entry in nextflow config:: + + lib/nfcore_external_java_deps.jar # if config doesn't mention nf-validation + .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example: @@ -87,28 +91,30 @@ def files_unchanged(self): [".prettierrc.yml"], ["CODE_OF_CONDUCT.md"], ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"], # NB: British / American spelling - [os.path.join(".github", ".dockstore.yml")], - [os.path.join(".github", "CONTRIBUTING.md")], - [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")], - [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")], - [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")], - [os.path.join(".github", "workflows", "branch.yml")], - [os.path.join(".github", "workflows", "linting_comment.yml")], - [os.path.join(".github", "workflows", "linting.yml")], - [os.path.join("assets", "email_template.html")], - [os.path.join("assets", "email_template.txt")], - [os.path.join("assets", "sendmail_template.txt")], - [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")], - [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")], - [os.path.join("docs", "README.md")], - [os.path.join("lib", "nfcore_external_java_deps.jar")], - [os.path.join("lib", "NfcoreTemplate.groovy")], + [Path(".github", ".dockstore.yml")], + [Path(".github", "CONTRIBUTING.md")], + [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")], + [Path(".github", "ISSUE_TEMPLATE", "config.yml")], + [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")], + [Path(".github", "PULL_REQUEST_TEMPLATE.md")], + [Path(".github", "workflows", "branch.yml")], + [Path(".github", "workflows", "linting_comment.yml")], + [Path(".github", "workflows", "linting.yml")], + [Path("assets", "email_template.html")], + [Path("assets", "email_template.txt")], + [Path("assets", "sendmail_template.txt")], + [Path("assets", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")], + [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")], + [Path("docs", "README.md")], + [Path("lib", "NfcoreTemplate.groovy")], ] files_partial = [ [".gitignore", ".prettierignore", "pyproject.toml"], ] + files_conditional = [ + [Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf_validation"}], + ] # Only show error messages from pipeline creation logging.getLogger("nf_core.create").setLevel(logging.ERROR) @@ -124,24 +130,24 @@ def files_unchanged(self): "prefix": prefix, } - template_yaml_path = os.path.join(tmp_dir, "template.yaml") + template_yaml_path = Path(tmp_dir, "template.yaml") with open(template_yaml_path, "w") as fh: yaml.dump(template_yaml, fh, default_flow_style=False) - test_pipeline_dir = os.path.join(tmp_dir, f"{prefix}-{short_name}") + test_pipeline_dir = Path(tmp_dir, f"{prefix}-{short_name}") create_obj = nf_core.create.PipelineCreate( None, None, None, no_git=True, outdir=test_pipeline_dir, template_yaml_path=template_yaml_path ) create_obj.init_pipeline() # Helper functions for file paths - def _pf(file_path): + def _pf(file_path: Union[str, Path]) -> Path: """Helper function - get file path for pipeline file""" - return os.path.join(self.wf_path, file_path) + return Path(self.wf_path, file_path) - def _tf(file_path): + def _tf(file_path: Union[str, Path]) -> Path: """Helper function - get file path for template file""" - return os.path.join(test_pipeline_dir, file_path) + return Path(test_pipeline_dir, file_path) # Files that must be completely unchanged from template for files in files_exact: @@ -151,7 +157,7 @@ def _tf(file_path): ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file - elif not any([os.path.isfile(_pf(f)) for f in files]): + elif not any([_pf(f).is_file() for f in files]): ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file has an identical match @@ -180,7 +186,7 @@ def _tf(file_path): ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") # Ignore if we can't find the file - elif not any([os.path.isfile(_pf(f)) for f in files]): + elif not any([_pf(f).is_file() for f in files]): ignored.append(f"File does not exist: {self._wrap_quotes(files)}") # Check that the file contains the template file contents @@ -208,6 +214,39 @@ def _tf(file_path): except FileNotFoundError: pass + # Files that should be there only if an entry in nextflow config is not set + for files in files_conditional: + # Ignore if file specified in linting config + ignore_files = self.lint_config.get("files_unchanged", []) + if files[0] in ignore_files: + ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}") + + # Ignore if we can't find the file + elif _pf(files[0]).is_file(): + ignored.append(f"File does not exist: {self._wrap_quotes(files[0])}") + + # Check that the file has an identical match + else: + config_key, config_value = list(files[1].items())[0] + if config_key in self.nf_config and self.nf_config[config_key] == config_value: + # Ignore if the config key is set to the expected value + ignored.append(f"File ignored due to config: {self._wrap_quotes(files)}") + else: + try: + if filecmp.cmp(_pf(files[0]), _tf(files[0]), shallow=True): + passed.append(f"`{files[0]}` matches the template") + else: + if "files_unchanged" in self.fix: + # Try to fix the problem by overwriting the pipeline file + shutil.copy(_tf(files[0]), _pf(files[0])) + passed.append(f"`{files[0]}` matches the template") + fixed.append(f"`{files[0]}` overwritten with template file") + else: + failed.append(f"`{files[0]}` does not match the template") + could_fix = True + except FileNotFoundError: + pass + # cleaning up temporary dir shutil.rmtree(tmp_dir) diff --git a/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar b/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar deleted file mode 100644 index 805c8bb5e4..0000000000 Binary files a/nf_core/pipeline-template/lib/nfcore_external_java_deps.jar and /dev/null differ diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml index 984c091091..7d08e1c8ef 100644 --- a/nf_core/pipeline-template/pyproject.toml +++ b/nf_core/pipeline-template/pyproject.toml @@ -1,4 +1,4 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Ruff. +# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. [tool.ruff] line-length = 120 diff --git a/tests/lint/files_exist.py b/tests/lint/files_exist.py index 4e5e4d3c2b..5ba26d77a0 100644 --- a/tests/lint/files_exist.py +++ b/tests/lint/files_exist.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import nf_core.lint @@ -7,7 +8,7 @@ def test_files_exist_missing_config(self): """Lint test: critical files missing FAIL""" new_pipeline = self._make_pipeline_copy() - os.remove(os.path.join(new_pipeline, "CHANGELOG.md")) + Path(new_pipeline, "CHANGELOG.md").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() @@ -21,7 +22,7 @@ def test_files_exist_missing_main(self): """Check if missing main issues warning""" new_pipeline = self._make_pipeline_copy() - os.remove(os.path.join(new_pipeline, "main.nf")) + Path(new_pipeline, "main.nf").unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) lint_obj._load() @@ -34,7 +35,7 @@ def test_files_exist_depreciated_file(self): """Check whether depreciated file issues warning""" new_pipeline = self._make_pipeline_copy() - nf = os.path.join(new_pipeline, "parameters.settings.json") + nf = Path(new_pipeline, "parameters.settings.json") os.system(f"touch {nf}") lint_obj = nf_core.lint.PipelineLint(new_pipeline)