Merge pull request #2675 from nf-core/rm_nfcore_external_java_deps

Remove nfcore_external_java_deps.jar from lib directory in pipeline template

.github/workflows/create-test-lint-wf-template.yml

@@ -38,6 +38,7 @@ jobs:
           - TEMPLATE: "template_skip_nf_core_configs.yml"
             runner: ubuntu-latest
             profile: "docker"
+      fail-fast: false
 
     steps:
       - name: go to working directory

CHANGELOG.md

@@ -10,6 +10,7 @@
 - switch to new image syntax in readme ([#2645](https://github.com/nf-core/tools/pull/2645))
 - Add conda channel order to nextflow.config ([#2094](https://github.com/nf-core/tools/pull/2094))
 - Fix tyop in pipeline nextflow.config ([#2664](https://github.com/nf-core/tools/pull/2664))
+- Remove `nfcore_external_java_deps.jar` from lib directory in pipeline template ([#2675](https://github.com/nf-core/tools/pull/2675))
 - Add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678))
 - Add new pipeline error message pointing to docs when 'requirement exceeds available memory' error message ([#2680](https://github.com/nf-core/tools/pull/2680))
 
@@ -42,6 +43,7 @@
 - Update actions/cache action to v4 ([#2666](https://github.com/nf-core/tools/pull/2666))
 - Handle api redirects from the old site ([#2672](https://github.com/nf-core/tools/pull/2672))
 - Remove redundanct v in pipeline version for emails ([#2667](https://github.com/nf-core/tools/pull/2667))
+- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678))
 
 # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]
 

nf_core/lint/files_exist.py

@@ -1,5 +1,6 @@
 import logging
-import os
+from pathlib import Path
+from typing import Union
 
 log = logging.getLogger(__name__)
 
@@ -51,7 +52,6 @@ def files_exist(self):
         docs/output.md
         docs/README.md
         docs/usage.md
-        lib/nfcore_external_java_deps.jar
         lib/NfcoreTemplate.groovy
         lib/Utils.groovy
         lib/WorkflowMain.groovy
@@ -98,6 +98,12 @@ def files_exist(self):
 
         .travis.yml
 
+    Files that *must not* be present if a certain entry is present in ``nextflow.config``:
+
+    .. code-block:: bash
+
+        lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config
+
     .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting
             the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example:
 
@@ -132,48 +138,46 @@ def files_exist(self):
         ["CHANGELOG.md"],
         ["CITATIONS.md"],
         ["CODE_OF_CONDUCT.md"],
-        ["CODE_OF_CONDUCT.md"],
         ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"],  # NB: British / American spelling
         ["nextflow_schema.json"],
         ["nextflow.config"],
         ["README.md"],
-        [os.path.join(".github", ".dockstore.yml")],
-        [os.path.join(".github", "CONTRIBUTING.md")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
-        [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")],
-        [os.path.join(".github", "workflows", "branch.yml")],
-        [os.path.join(".github", "workflows", "ci.yml")],
-        [os.path.join(".github", "workflows", "linting_comment.yml")],
-        [os.path.join(".github", "workflows", "linting.yml")],
-        [os.path.join("assets", "email_template.html")],
-        [os.path.join("assets", "email_template.txt")],
-        [os.path.join("assets", "sendmail_template.txt")],
-        [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("conf", "modules.config")],
-        [os.path.join("conf", "test.config")],
-        [os.path.join("conf", "test_full.config")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
-        [os.path.join("docs", "output.md")],
-        [os.path.join("docs", "README.md")],
-        [os.path.join("docs", "README.md")],
-        [os.path.join("docs", "usage.md")],
-        [os.path.join("lib", "nfcore_external_java_deps.jar")],
-        [os.path.join("lib", "NfcoreTemplate.groovy")],
-        [os.path.join("lib", "Utils.groovy")],
-        [os.path.join("lib", "WorkflowMain.groovy")],
+        [Path(".github", ".dockstore.yml")],
+        [Path(".github", "CONTRIBUTING.md")],
+        [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "config.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
+        [Path(".github", "PULL_REQUEST_TEMPLATE.md")],
+        [Path(".github", "workflows", "branch.yml")],
+        [Path(".github", "workflows", "ci.yml")],
+        [Path(".github", "workflows", "linting_comment.yml")],
+        [Path(".github", "workflows", "linting.yml")],
+        [Path("assets", "email_template.html")],
+        [Path("assets", "email_template.txt")],
+        [Path("assets", "sendmail_template.txt")],
+        [Path("assets", f"nf-core-{short_name}_logo_light.png")],
+        [Path("conf", "modules.config")],
+        [Path("conf", "test.config")],
+        [Path("conf", "test_full.config")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
+        [Path("docs", "output.md")],
+        [Path("docs", "README.md")],
+        [Path("docs", "README.md")],
+        [Path("docs", "usage.md")],
+        [Path("lib", "NfcoreTemplate.groovy")],
+        [Path("lib", "Utils.groovy")],
+        [Path("lib", "WorkflowMain.groovy")],
     ]
 
     files_warn = [
         ["main.nf"],
-        [os.path.join("assets", "multiqc_config.yml")],
-        [os.path.join("conf", "base.config")],
-        [os.path.join("conf", "igenomes.config")],
-        [os.path.join(".github", "workflows", "awstest.yml")],
-        [os.path.join(".github", "workflows", "awsfulltest.yml")],
-        [os.path.join("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")],
+        [Path("assets", "multiqc_config.yml")],
+        [Path("conf", "base.config")],
+        [Path("conf", "igenomes.config")],
+        [Path(".github", "workflows", "awstest.yml")],
+        [Path(".github", "workflows", "awsfulltest.yml")],
+        [Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")],
         ["modules.json"],
         ["pyproject.toml"],
     ]
@@ -184,45 +188,48 @@ def files_exist(self):
         "parameters.settings.json",
         "pipeline_template.yml",  # saving information in .nf-core.yml
         ".nf-core.yaml",  # yml not yaml
-        os.path.join("bin", "markdown_to_html.r"),
-        os.path.join("conf", "aws.config"),
-        os.path.join(".github", "workflows", "push_dockerhub.yml"),
-        os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md"),
-        os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md"),
-        os.path.join("docs", "images", f"nf-core-{short_name}_logo.png"),
+        Path("bin", "markdown_to_html.r"),
+        Path("conf", "aws.config"),
+        Path(".github", "workflows", "push_dockerhub.yml"),
+        Path(".github", "ISSUE_TEMPLATE", "bug_report.md"),
+        Path(".github", "ISSUE_TEMPLATE", "feature_request.md"),
+        Path("docs", "images", f"nf-core-{short_name}_logo.png"),
         ".markdownlint.yml",
         ".yamllint.yml",
-        os.path.join("lib", "Checks.groovy"),
-        os.path.join("lib", "Completion.groovy"),
-        os.path.join("lib", "Workflow.groovy"),
+        Path("lib", "Checks.groovy"),
+        Path("lib", "Completion.groovy"),
+        Path("lib", "Workflow.groovy"),
     ]
     files_warn_ifexists = [".travis.yml"]
+    files_fail_ifinconfig = [[Path("lib", "nfcore_external_java_deps.jar"), "nf-validation"]]
 
     # Remove files that should be ignored according to the linting config
     ignore_files = self.lint_config.get("files_exist", [])
+    log.info(f"Files to ignore: {ignore_files}")
 
-    def pf(file_path):
-        return os.path.join(self.wf_path, file_path)
+    def pf(file_path: Union[str, Path]) -> Path:
+        return Path(self.wf_path, file_path)
 
     # First - critical files. Check that this is actually a Nextflow pipeline
-    if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")):
+    if not pf("nextflow.config").is_file() and not pf("main.nf").is_file():
         failed.append("File not found: nextflow.config or main.nf")
         raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?")
 
     # Files that cause an error if they don't exist
     for files in files_fail:
-        if any([f in ignore_files for f in files]):
+        print(files)
+        if any([str(f) in ignore_files for f in files]):
             continue
-        if any([os.path.isfile(pf(f)) for f in files]):
+        if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
             failed.append(f"File not found: {self._wrap_quotes(files)}")
 
     # Files that cause a warning if they don't exist
     for files in files_warn:
-        if any([f in ignore_files for f in files]):
+        if any([str(f) in ignore_files for f in files]):
             continue
-        if any([os.path.isfile(pf(f)) for f in files]):
+        if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
             warned.append(f"File not found: {self._wrap_quotes(files)}")
@@ -231,16 +238,32 @@ def pf(file_path):
     for file in files_fail_ifexists:
         if file in ignore_files:
             continue
-        if os.path.isfile(pf(file)):
+        if pf(file).is_file():
             failed.append(f"File must be removed: {self._wrap_quotes(file)}")
         else:
             passed.append(f"File not found check: {self._wrap_quotes(file)}")
-
+    # Files that cause an error if they exists together with a certain entry in nextflow.config
+    for file in files_fail_ifinconfig:
+        if str(file[0]) in ignore_files:
+            continue
+        nextflow_config = pf("nextflow.config")
+        in_config = False
+        with open(nextflow_config) as f:
+            if file[1] in f.read():
+                in_config = True
+        if pf(file[0]).is_file() and in_config:
+            failed.append(f"File must be removed: {self._wrap_quotes(file[0])}")
+        elif pf(file[0]).is_file() and not in_config:
+            passed.append(f"File found check: {self._wrap_quotes(file[0])}")
+        elif not pf(file[0]).is_file() and not in_config:
+            failed.append(f"File not found check: {self._wrap_quotes(file[0])}")
+        elif not pf(file[0]).is_file() and in_config:
+            passed.append(f"File not found check: {self._wrap_quotes(file[0])}")
     # Files that cause a warning if they exist
     for file in files_warn_ifexists:
         if file in ignore_files:
             continue
-        if os.path.isfile(pf(file)):
+        if pf(file).is_file():
             warned.append(f"File should be removed: {self._wrap_quotes(file)}")
         else:
             passed.append(f"File not found check: {self._wrap_quotes(file)}")