Remove nfcore_external_java_deps.jar from lib directory in pipeline template

CHANGELOG.md

@@ -10,7 +10,7 @@
 - switch to new image syntax in readme ([#2645](https://github.com/nf-core/tools/pull/2645))
 - Add conda channel order to nextflow.config ([#2094](https://github.com/nf-core/tools/pull/2094))
 - Fix tyop in pipeline nextflow.config ([#2664](https://github.com/nf-core/tools/pull/2664))
-- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678))
+- Remove `nfcore_external_java_deps.jar` from lib directory in pipeline template ([#2675](https://github.com/nf-core/tools/pull/2675))
 
 ### Download
 
@@ -40,6 +40,7 @@
 - Add new subcommand: `nf-core logo-create` to output an nf-core logo for a pipeline (instead of going through the website) ([#2662](https://github.com/nf-core/tools/pull/2662))
 - Update actions/cache action to v4 ([#2666](https://github.com/nf-core/tools/pull/2666))
 - Remove redundanct v in pipeline version for emails ([#2667](https://github.com/nf-core/tools/pull/2667))
+- add function to check `-profile` is well formatted ([#2678](https://github.com/nf-core/tools/pull/2678))
 
 # [v2.11.1 - Magnesium Dragon Patch](https://github.com/nf-core/tools/releases/tag/2.11) - [2023-12-20]
 

nf_core/lint/files_exist.py

@@ -1,5 +1,6 @@
 import logging
-import os
+from pathlib import Path
+from typing import Union
 
 log = logging.getLogger(__name__)
 
@@ -51,7 +52,6 @@ def files_exist(self):
         docs/output.md
         docs/README.md
         docs/usage.md
-        lib/nfcore_external_java_deps.jar
         lib/NfcoreTemplate.groovy
         lib/Utils.groovy
         lib/WorkflowMain.groovy
@@ -98,6 +98,12 @@ def files_exist(self):
 
         .travis.yml
 
+    Files that *must not* be present if a certain entry is present in ``nextflow.config``:
+
+    .. code-block:: bash
+
+        lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config
+
     .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting
             the ``files_exist`` key as follows in your ``.nf-core.yml`` config file. For example:
 
@@ -137,43 +143,42 @@ def files_exist(self):
         ["nextflow_schema.json"],
         ["nextflow.config"],
         ["README.md"],
-        [os.path.join(".github", ".dockstore.yml")],
-        [os.path.join(".github", "CONTRIBUTING.md")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
-        [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")],
-        [os.path.join(".github", "workflows", "branch.yml")],
-        [os.path.join(".github", "workflows", "ci.yml")],
-        [os.path.join(".github", "workflows", "linting_comment.yml")],
-        [os.path.join(".github", "workflows", "linting.yml")],
-        [os.path.join("assets", "email_template.html")],
-        [os.path.join("assets", "email_template.txt")],
-        [os.path.join("assets", "sendmail_template.txt")],
-        [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("conf", "modules.config")],
-        [os.path.join("conf", "test.config")],
-        [os.path.join("conf", "test_full.config")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
-        [os.path.join("docs", "output.md")],
-        [os.path.join("docs", "README.md")],
-        [os.path.join("docs", "README.md")],
-        [os.path.join("docs", "usage.md")],
-        [os.path.join("lib", "nfcore_external_java_deps.jar")],
-        [os.path.join("lib", "NfcoreTemplate.groovy")],
-        [os.path.join("lib", "Utils.groovy")],
-        [os.path.join("lib", "WorkflowMain.groovy")],
+        [Path(".github", ".dockstore.yml")],
+        [Path(".github", "CONTRIBUTING.md")],
+        [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "config.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
+        [Path(".github", "PULL_REQUEST_TEMPLATE.md")],
+        [Path(".github", "workflows", "branch.yml")],
+        [Path(".github", "workflows", "ci.yml")],
+        [Path(".github", "workflows", "linting_comment.yml")],
+        [Path(".github", "workflows", "linting.yml")],
+        [Path("assets", "email_template.html")],
+        [Path("assets", "email_template.txt")],
+        [Path("assets", "sendmail_template.txt")],
+        [Path("assets", f"nf-core-{short_name}_logo_light.png")],
+        [Path("conf", "modules.config")],
+        [Path("conf", "test.config")],
+        [Path("conf", "test_full.config")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
+        [Path("docs", "output.md")],
+        [Path("docs", "README.md")],
+        [Path("docs", "README.md")],
+        [Path("docs", "usage.md")],
+        [Path("lib", "NfcoreTemplate.groovy")],
+        [Path("lib", "Utils.groovy")],
+        [Path("lib", "WorkflowMain.groovy")],
     ]
 
     files_warn = [
         ["main.nf"],
-        [os.path.join("assets", "multiqc_config.yml")],
-        [os.path.join("conf", "base.config")],
-        [os.path.join("conf", "igenomes.config")],
-        [os.path.join(".github", "workflows", "awstest.yml")],
-        [os.path.join(".github", "workflows", "awsfulltest.yml")],
-        [os.path.join("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")],
+        [Path("assets", "multiqc_config.yml")],
+        [Path("conf", "base.config")],
+        [Path("conf", "igenomes.config")],
+        [Path(".github", "workflows", "awstest.yml")],
+        [Path(".github", "workflows", "awsfulltest.yml")],
+        [Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy")],
         ["modules.json"],
         ["pyproject.toml"],
     ]
@@ -184,36 +189,37 @@ def files_exist(self):
         "parameters.settings.json",
         "pipeline_template.yml",  # saving information in .nf-core.yml
         ".nf-core.yaml",  # yml not yaml
-        os.path.join("bin", "markdown_to_html.r"),
-        os.path.join("conf", "aws.config"),
-        os.path.join(".github", "workflows", "push_dockerhub.yml"),
-        os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.md"),
-        os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.md"),
-        os.path.join("docs", "images", f"nf-core-{short_name}_logo.png"),
+        Path("bin", "markdown_to_html.r"),
+        Path("conf", "aws.config"),
+        Path(".github", "workflows", "push_dockerhub.yml"),
+        Path(".github", "ISSUE_TEMPLATE", "bug_report.md"),
+        Path(".github", "ISSUE_TEMPLATE", "feature_request.md"),
+        Path("docs", "images", f"nf-core-{short_name}_logo.png"),
         ".markdownlint.yml",
         ".yamllint.yml",
-        os.path.join("lib", "Checks.groovy"),
-        os.path.join("lib", "Completion.groovy"),
-        os.path.join("lib", "Workflow.groovy"),
+        Path("lib", "Checks.groovy"),
+        Path("lib", "Completion.groovy"),
+        Path("lib", "Workflow.groovy"),
     ]
     files_warn_ifexists = [".travis.yml"]
+    files_fail_ifinconfig = [[Path("lib", "nfcore_external_java_deps.jar"), "nf-validation"]]
 
     # Remove files that should be ignored according to the linting config
     ignore_files = self.lint_config.get("files_exist", [])
 
-    def pf(file_path):
-        return os.path.join(self.wf_path, file_path)
+    def pf(file_path: Union[str, Path]) -> Path:
+        return Path(self.wf_path, file_path)
 
     # First - critical files. Check that this is actually a Nextflow pipeline
-    if not os.path.isfile(pf("nextflow.config")) and not os.path.isfile(pf("main.nf")):
+    if not pf("nextflow.config").is_file() and not pf("main.nf").is_file():
         failed.append("File not found: nextflow.config or main.nf")
         raise AssertionError("Neither nextflow.config or main.nf found! Is this a Nextflow pipeline?")
 
     # Files that cause an error if they don't exist
     for files in files_fail:
         if any([f in ignore_files for f in files]):
             continue
-        if any([os.path.isfile(pf(f)) for f in files]):
+        if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
             failed.append(f"File not found: {self._wrap_quotes(files)}")
@@ -222,7 +228,7 @@ def pf(file_path):
     for files in files_warn:
         if any([f in ignore_files for f in files]):
             continue
-        if any([os.path.isfile(pf(f)) for f in files]):
+        if any([pf(f).is_file() for f in files]):
             passed.append(f"File found: {self._wrap_quotes(files)}")
         else:
             warned.append(f"File not found: {self._wrap_quotes(files)}")
@@ -231,16 +237,32 @@ def pf(file_path):
     for file in files_fail_ifexists:
         if file in ignore_files:
             continue
-        if os.path.isfile(pf(file)):
+        if pf(file).is_file():
             failed.append(f"File must be removed: {self._wrap_quotes(file)}")
         else:
             passed.append(f"File not found check: {self._wrap_quotes(file)}")
-
+    # Files that cause an error if they exists together with a certain entry in nextflow.config
+    for file in files_fail_ifinconfig:
+        if file[0] in ignore_files:
+            continue
+        nextflow_config = pf("nextflow.config")
+        in_config = False
+        with open(nextflow_config) as f:
+            if file[1] in f.read():
+                in_config = True
+        if pf(file[0]).is_file() and in_config:
+            failed.append(f"File must be removed: {self._wrap_quotes(file[0])}")
+        elif pf(file[0]).is_file() and not in_config:
+            passed.append(f"File not found check: {self._wrap_quotes(file[0])}")
+        elif not pf(file[0]).is_file() and not in_config:
+            failed.append(f"File not found check: {self._wrap_quotes(file[0])}")
+        elif not pf(file[0]).is_file() and in_config:
+            passed.append(f"File not found check: {self._wrap_quotes(file[0])}")
     # Files that cause a warning if they exist
     for file in files_warn_ifexists:
         if file in ignore_files:
             continue
-        if os.path.isfile(pf(file)):
+        if pf(file).is_file():
             warned.append(f"File should be removed: {self._wrap_quotes(file)}")
         else:
             passed.append(f"File not found check: {self._wrap_quotes(file)}")

nf_core/lint/files_unchanged.py

@@ -1,8 +1,9 @@
 import filecmp
 import logging
-import os
 import shutil
 import tempfile
+from pathlib import Path
+from typing import Union
 
 import yaml
 
@@ -39,7 +40,6 @@ def files_unchanged(self):
         docs/images/nf-core-PIPELINE_logo_light.png
         docs/images/nf-core-PIPELINE_logo_dark.png
         docs/README.md'
-        lib/nfcore_external_java_deps.jar
         lib/NfcoreTemplate.groovy
         ['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling
 
@@ -49,6 +49,10 @@ def files_unchanged(self):
         .prettierignore
         pyproject.toml
 
+    Files that need to be there or not based on a entry in nextflow config::
+
+        lib/nfcore_external_java_deps.jar # if config doesn't mention nf-validation
+
     .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting
              the ``files_unchanged`` key as follows in your ``.nf-core.yml`` config file. For example:
 
@@ -87,28 +91,30 @@ def files_unchanged(self):
         [".prettierrc.yml"],
         ["CODE_OF_CONDUCT.md"],
         ["LICENSE", "LICENSE.md", "LICENCE", "LICENCE.md"],  # NB: British / American spelling
-        [os.path.join(".github", ".dockstore.yml")],
-        [os.path.join(".github", "CONTRIBUTING.md")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "config.yml")],
-        [os.path.join(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
-        [os.path.join(".github", "PULL_REQUEST_TEMPLATE.md")],
-        [os.path.join(".github", "workflows", "branch.yml")],
-        [os.path.join(".github", "workflows", "linting_comment.yml")],
-        [os.path.join(".github", "workflows", "linting.yml")],
-        [os.path.join("assets", "email_template.html")],
-        [os.path.join("assets", "email_template.txt")],
-        [os.path.join("assets", "sendmail_template.txt")],
-        [os.path.join("assets", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_light.png")],
-        [os.path.join("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
-        [os.path.join("docs", "README.md")],
-        [os.path.join("lib", "nfcore_external_java_deps.jar")],
-        [os.path.join("lib", "NfcoreTemplate.groovy")],
+        [Path(".github", ".dockstore.yml")],
+        [Path(".github", "CONTRIBUTING.md")],
+        [Path(".github", "ISSUE_TEMPLATE", "bug_report.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "config.yml")],
+        [Path(".github", "ISSUE_TEMPLATE", "feature_request.yml")],
+        [Path(".github", "PULL_REQUEST_TEMPLATE.md")],
+        [Path(".github", "workflows", "branch.yml")],
+        [Path(".github", "workflows", "linting_comment.yml")],
+        [Path(".github", "workflows", "linting.yml")],
+        [Path("assets", "email_template.html")],
+        [Path("assets", "email_template.txt")],
+        [Path("assets", "sendmail_template.txt")],
+        [Path("assets", f"nf-core-{short_name}_logo_light.png")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_light.png")],
+        [Path("docs", "images", f"nf-core-{short_name}_logo_dark.png")],
+        [Path("docs", "README.md")],
+        [Path("lib", "NfcoreTemplate.groovy")],
     ]
     files_partial = [
         [".gitignore", ".prettierignore", "pyproject.toml"],
     ]
+    files_conditional = [
+        [Path("lib", "nfcore_external_java_deps.jar"), {"plugins": "nf_validation"}],
+    ]
 
     # Only show error messages from pipeline creation
     logging.getLogger("nf_core.create").setLevel(logging.ERROR)
@@ -124,24 +130,24 @@ def files_unchanged(self):
         "prefix": prefix,
     }
 
-    template_yaml_path = os.path.join(tmp_dir, "template.yaml")
+    template_yaml_path = Path(tmp_dir, "template.yaml")
     with open(template_yaml_path, "w") as fh:
         yaml.dump(template_yaml, fh, default_flow_style=False)
 
-    test_pipeline_dir = os.path.join(tmp_dir, f"{prefix}-{short_name}")
+    test_pipeline_dir = Path(tmp_dir, f"{prefix}-{short_name}")
     create_obj = nf_core.create.PipelineCreate(
         None, None, None, no_git=True, outdir=test_pipeline_dir, template_yaml_path=template_yaml_path
     )
     create_obj.init_pipeline()
 
     # Helper functions for file paths
-    def _pf(file_path):
+    def _pf(file_path: Union[str, Path]) -> Path:
         """Helper function - get file path for pipeline file"""
-        return os.path.join(self.wf_path, file_path)
+        return Path(self.wf_path, file_path)
 
-    def _tf(file_path):
+    def _tf(file_path: Union[str, Path]) -> Path:
         """Helper function - get file path for template file"""
-        return os.path.join(test_pipeline_dir, file_path)
+        return Path(test_pipeline_dir, file_path)
 
     # Files that must be completely unchanged from template
     for files in files_exact:
@@ -151,7 +157,7 @@ def _tf(file_path):
             ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}")
 
         # Ignore if we can't find the file
-        elif not any([os.path.isfile(_pf(f)) for f in files]):
+        elif not any([_pf(f).is_file() for f in files]):
             ignored.append(f"File does not exist: {self._wrap_quotes(files)}")
 
         # Check that the file has an identical match
@@ -180,7 +186,7 @@ def _tf(file_path):
             ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}")
 
         # Ignore if we can't find the file
-        elif not any([os.path.isfile(_pf(f)) for f in files]):
+        elif not any([_pf(f).is_file() for f in files]):
             ignored.append(f"File does not exist: {self._wrap_quotes(files)}")
 
         # Check that the file contains the template file contents
@@ -208,6 +214,39 @@ def _tf(file_path):
                 except FileNotFoundError:
                     pass
 
+    # Files that should be there only if an entry in nextflow config is not set
+    for files in files_conditional:
+        # Ignore if file specified in linting config
+        ignore_files = self.lint_config.get("files_unchanged", [])
+        if files[0] in ignore_files:
+            ignored.append(f"File ignored due to lint config: {self._wrap_quotes(files)}")
+
+        # Ignore if we can't find the file
+        elif _pf(files[0]).is_file():
+            ignored.append(f"File does not exist: {self._wrap_quotes(files[0])}")
+
+        # Check that the file has an identical match
+        else:
+            config_key, config_value = list(files[1].items())[0]
+            if config_key in self.nf_config and self.nf_config[config_key] == config_value:
+                # Ignore if the config key is set to the expected value
+                ignored.append(f"File ignored due to config: {self._wrap_quotes(files)}")
+            else:
+                try:
+                    if filecmp.cmp(_pf(files[0]), _tf(files[0]), shallow=True):
+                        passed.append(f"`{files[0]}` matches the template")
+                    else:
+                        if "files_unchanged" in self.fix:
+                            # Try to fix the problem by overwriting the pipeline file
+                            shutil.copy(_tf(files[0]), _pf(files[0]))
+                            passed.append(f"`{files[0]}` matches the template")
+                            fixed.append(f"`{files[0]}` overwritten with template file")
+                        else:
+                            failed.append(f"`{files[0]}` does not match the template")
+                            could_fix = True
+                except FileNotFoundError:
+                    pass
+
     # cleaning up temporary dir
     shutil.rmtree(tmp_dir)
 

nf_core/pipeline-template/pyproject.toml

@@ -1,4 +1,4 @@
-# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Ruff.
+# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff.
 # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation.
 [tool.ruff]
 line-length = 120