Merge pull request #3128 from mirpedrol/template-customisation-nfcore…

…modules

Template: add option to exclude nf-core components from pipeline template

nf_core/commands_pipelines.py

@@ -144,9 +144,12 @@ def pipelines_lint(
             ctx.obj["hide_progress"],
         )
         swf_failed = 0
+        module_failed = 0
         if subworkflow_lint_obj is not None:
             swf_failed = len(subworkflow_lint_obj.failed)
-        if len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed > 0:
+        if module_lint_obj is not None:
+            module_failed = len(module_lint_obj.failed)
+        if len(lint_obj.failed) + module_failed + swf_failed > 0:
             sys.exit(1)
     except AssertionError as e:
         log.critical(e)

nf_core/pipeline-template/.editorconfig

@@ -11,6 +11,7 @@ indent_style = space
 [*.{md,yml,yaml,html,css,scss,js}]
 indent_size = 2
 
+{%- if modules %}
 # These files are edited and tested upstream in nf-core/modules
 [/modules/nf-core/**]
 charset = unset
@@ -24,6 +25,7 @@ end_of_line = unset
 insert_final_newline = unset
 trim_trailing_whitespace = unset
 indent_style = unset
+{%- endif %}
 
 {%- if email %}
 [/assets/email*]

nf_core/pipeline-template/main.nf

@@ -18,6 +18,7 @@
 */
 
 include { {{ short_name|upper }}  } from './workflows/{{ short_name }}'
+{%- if modules %}
 include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'
 include { PIPELINE_COMPLETION     } from './subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'
 {%- if igenomes %}
@@ -33,7 +34,7 @@ include { getGenomeAttribute      } from './subworkflows/local/utils_nfcore_{{ s
 //   This is an example of how to use getGenomeAttribute() to fetch parameters
 //   from igenomes.config using `--genome`
 params.fasta = getGenomeAttribute('fasta')
-{% endif %}
+{% endif %}{% endif %}
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     NAMED WORKFLOWS FOR PIPELINE
@@ -56,10 +57,10 @@ workflow {{ prefix_nodash|upper }}_{{ short_name|upper }} {
     {{ short_name|upper }} (
         samplesheet
     )
-{%- if multiqc %}
+{%- if multiqc %}{%- if modules %}
     emit:
     multiqc_report = {{ short_name|upper }}.out.multiqc_report // channel: /path/to/multiqc_report.html
-{%- endif %}
+{%- endif %}{%- endif %}
 }
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -71,6 +72,7 @@ workflow {
 
     main:
 
+    {%- if modules %}
     //
     // SUBWORKFLOW: Run initialisation tasks
     //
@@ -82,14 +84,19 @@ workflow {
         params.outdir,
         params.input
     )
-
+    {% endif %}
     //
     // WORKFLOW: Run main workflow
     //
     {{ prefix_nodash|upper }}_{{ short_name|upper }} (
+        {%- if modules %}
         PIPELINE_INITIALISATION.out.samplesheet
+        {%- else %}
+        params.input
+        {%- endif %}
     )
 
+    {%- if modules %}
     //
     // SUBWORKFLOW: Run completion tasks
     //
@@ -104,6 +111,7 @@ workflow {
         {% if adaptivecard or slackreport %}params.hook_url,{% endif %}
         {% if multiqc %}{{ prefix_nodash|upper }}_{{ short_name|upper }}.out.multiqc_report{% endif %}
     )
+    {%- endif %}
 }
 
 /*

nf_core/pipeline-template/nextflow.config

@@ -66,10 +66,21 @@ params {
     validate_params            = true
 
 }
-
+{% if modules %}
 // Load base.config by default for all pipelines
 includeConfig 'conf/base.config'
+{%- else %}
+process {
+    // TODO nf-core: Check the defaults for all processes
+    cpus   = { check_max( 1    * task.attempt, 'cpus'   ) }
+    memory = { check_max( 6.GB * task.attempt, 'memory' ) }
+    time   = { check_max( 4.h  * task.attempt, 'time'   ) }
 
+    errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
+    maxRetries    = 1
+    maxErrors     = '-1'
+}
+{% endif %}
 {% if nf_core_configs -%}
 // Load nf-core custom profiles from different Institutions
 try {
@@ -289,7 +300,7 @@ validation {
 -\033[2m----------------------------------------------------\033[0m-
 """
         afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { "  https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""}
-* The nf-core framework 
+* The nf-core framework
   https://doi.org/10.1038/s41587-020-0439-x
 
 * Software dependencies
@@ -302,10 +313,10 @@ validation {
     }{% endif %}
 }
 {% endif -%}
-
+{%- if modules %}
 // Load modules.config for DSL2 module specific options
 includeConfig 'conf/modules.config'
-
+{% endif %}
 // Function to ensure that resource requirements don't go beyond
 // a maximum limit
 def check_max(obj, type) {

nf_core/pipeline-template/nextflow_schema.json

@@ -56,6 +56,7 @@
                     "fa_icon": "fas fa-book",
                     "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details."
                 },
+                {%- if modules %}
                 "fasta": {
                     "type": "string",
                     "format": "file-path",
@@ -66,6 +67,7 @@
                     "help_text": "This parameter is *mandatory* if `--genome` is not specified. If you don't have a BWA index available this will be generated for you automatically. Combine with `--save_reference` to save BWA index for future runs.",
                     "fa_icon": "far fa-file-code"
                 },
+                {%- endif %}
                 "igenomes_ignore": {
                     "type": "boolean",
                     "description": "Do not load the iGenomes reference config.",

nf_core/pipeline-template/workflows/pipeline.nf

@@ -4,12 +4,14 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 */
 
+{%- if modules %}
 {% if fastqc %}include { FASTQC                 } from '../modules/nf-core/fastqc/main'{% endif %}
 {% if multiqc %}include { MULTIQC                } from '../modules/nf-core/multiqc/main'{% endif %}
 {% if nf_schema %}include { paramsSummaryMap       } from 'plugin/nf-schema'{% endif %}
 {% if multiqc %}include { paramsSummaryMultiqc   } from '../subworkflows/nf-core/utils_nfcore_pipeline'{% endif %}
 include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline'
 {% if citations or multiqc %}include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_{{ short_name }}_pipeline'{% endif %}
+{%- endif %}
 
 /*
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -22,6 +24,7 @@ workflow {{ short_name|upper }} {
     take:
     ch_samplesheet // channel: samplesheet read in from --input
 
+    {%- if modules %}
     main:
 
     ch_versions = Channel.empty()
@@ -98,6 +101,7 @@ workflow {{ short_name|upper }} {
     emit:
     {%- if multiqc %}multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html{% endif %}
     versions       = ch_versions                 // channel: [ path(versions.yml) ]
+{% endif %}
 }
 
 /*

nf_core/pipelines/create/templatefeatures.yml

@@ -242,6 +242,29 @@ fastqc:
     The pipeline will include the FastQC module.
   nfcore_pipelines: True
   custom_pipelines: True
+modules:
+  skippable_paths:
+    - "conf/base.config"
+    - "conf/modules.config"
+    - "modules.json"
+    - "modules"
+    - "subworkflows"
+  short_description: "Use nf-core components"
+  description: "Include all required files to use nf-core modules and subworkflows"
+  help_text: |
+    It is *recommended* to use this feature if you want to use modules and subworkflows in your pipeline.
+    This will add all required files to use nf-core components or any compatible components from private repos by using `nf-core modules` and `nf-core subworkflows` commands.
+  linting:
+    nfcore_components: False
+    modules_json: False
+    base_config: False
+    modules_config: False
+    files_exist:
+      - "conf/base.config"
+      - "conf/modules.config"
+      - "modules.json"
+  nfcore_pipelines: False
+  custom_pipelines: True
 changelog:
   skippable_paths:
     - "CHANGELOG.md"

nf_core/pipelines/lint/__init__.py

@@ -9,7 +9,7 @@
 import logging
 import os
 from pathlib import Path
-from typing import List, Tuple, Union
+from typing import List, Optional, Tuple, Union
 
 import git
 import rich
@@ -177,7 +177,8 @@ def _load_lint_config(self) -> bool:
         # Check if we have any keys that don't match lint test names
         if self.lint_config is not None:
             for k in self.lint_config:
-                if k not in self.lint_tests:
+                if k != "nfcore_components" and k not in self.lint_tests:
+                    # nfcore_components is an exception to allow custom pipelines without nf-core components
                     log.warning(f"Found unrecognised test name '{k}' in pipeline lint config")
                     is_correct = False
 
@@ -546,7 +547,7 @@ def run_linting(
     md_fn=None,
     json_fn=None,
     hide_progress: bool = False,
-) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]:
+) -> Tuple[PipelineLint, Optional[ComponentLint], Optional[ComponentLint]]:
     """Runs all nf-core linting checks on a given Nextflow pipeline project
     in either `release` mode or `normal` mode (default). Returns an object
     of type :class:`PipelineLint` after finished.
@@ -591,41 +592,45 @@ def run_linting(
     lint_obj._load_lint_config()
     lint_obj.load_pipeline_config()
 
-    # Create the modules lint object
-    module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress)
-    # Create the subworkflows lint object
-    try:
-        subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress)
-    except LookupError:
+    if "nfcore_components" in lint_obj.lint_config and not lint_obj.lint_config["nfcore_components"]:
+        module_lint_obj = None
         subworkflow_lint_obj = None
-
-    # Verify that the pipeline is correctly configured and has  a modules.json file
-    module_lint_obj.has_valid_directory()
-    module_lint_obj.has_modules_file()
-    # Run only the tests we want
-    if key:
-        # Select only the module lint tests
-        module_lint_tests = list(
-            set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)))
-        )
-        # Select only the subworkflow lint tests
-        subworkflow_lint_tests = list(
-            set(key).intersection(
-                set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True))
-            )
-        )
     else:
-        # If no key is supplied, run the default modules tests
-        module_lint_tests = list(("module_changes", "module_version"))
-        subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version"))
-    module_lint_obj.filter_tests_by_key(module_lint_tests)
-    if subworkflow_lint_obj is not None:
-        subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests)
-
-    # Set up files for component linting test
-    module_lint_obj.set_up_pipeline_files()
-    if subworkflow_lint_obj is not None:
-        subworkflow_lint_obj.set_up_pipeline_files()
+        # Create the modules lint object
+        module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress)
+        # Create the subworkflows lint object
+        try:
+            subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress)
+        except LookupError:
+            subworkflow_lint_obj = None
+
+        # Verify that the pipeline is correctly configured and has  a modules.json file
+        module_lint_obj.has_valid_directory()
+        module_lint_obj.has_modules_file()
+        # Run only the tests we want
+        if key:
+            # Select only the module lint tests
+            module_lint_tests = list(
+                set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)))
+            )
+            # Select only the subworkflow lint tests
+            subworkflow_lint_tests = list(
+                set(key).intersection(
+                    set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True))
+                )
+            )
+        else:
+            # If no key is supplied, run the default modules tests
+            module_lint_tests = list(("module_changes", "module_version"))
+            subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version"))
+        module_lint_obj.filter_tests_by_key(module_lint_tests)
+        if subworkflow_lint_obj is not None:
+            subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests)
+
+        # Set up files for component linting test
+        module_lint_obj.set_up_pipeline_files()
+        if subworkflow_lint_obj is not None:
+            subworkflow_lint_obj.set_up_pipeline_files()
 
     # Run the pipeline linting tests
     try:
@@ -635,21 +640,23 @@ def run_linting(
         log.info("Stopping tests...")
         return lint_obj, module_lint_obj, subworkflow_lint_obj
 
-    # Run the module lint tests
-    if len(module_lint_obj.all_local_components) > 0:
-        module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True)
-    if len(module_lint_obj.all_remote_components) > 0:
-        module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False)
-    # Run the subworkflows lint tests
+    if module_lint_obj is not None:
+        # Run the module lint tests
+        if len(module_lint_obj.all_local_components) > 0:
+            module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True)
+        if len(module_lint_obj.all_remote_components) > 0:
+            module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False)
     if subworkflow_lint_obj is not None:
+        # Run the subworkflows lint tests
         if len(subworkflow_lint_obj.all_local_components) > 0:
             subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True)
         if len(subworkflow_lint_obj.all_remote_components) > 0:
             subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False)
 
     # Print the results
     lint_obj._print_results(show_passed)
-    module_lint_obj._print_results(show_passed, sort_by=sort_by)
+    if module_lint_obj is not None:
+        module_lint_obj._print_results(show_passed, sort_by=sort_by)
     if subworkflow_lint_obj is not None:
         subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by)
     nf_core.pipelines.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj)

nf_core/pipelines/lint_utils.py

@@ -22,15 +22,22 @@ def print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj):
     swf_passed = 0
     swf_warned = 0
     swf_failed = 0
+    module_passed = 0
+    module_warned = 0
+    module_failed = 0
     if subworkflow_lint_obj is not None:
         swf_passed = len(subworkflow_lint_obj.passed)
         swf_warned = len(subworkflow_lint_obj.warned)
         swf_failed = len(subworkflow_lint_obj.failed)
-    nbr_passed = len(lint_obj.passed) + len(module_lint_obj.passed) + swf_passed
+    if module_lint_obj is not None:
+        module_passed = len(module_lint_obj.passed)
+        module_warned = len(module_lint_obj.warned)
+        module_failed = len(module_lint_obj.failed)
+    nbr_passed = len(lint_obj.passed) + module_passed + swf_passed
     nbr_ignored = len(lint_obj.ignored)
     nbr_fixed = len(lint_obj.fixed)
-    nbr_warned = len(lint_obj.warned) + len(module_lint_obj.warned) + swf_warned
-    nbr_failed = len(lint_obj.failed) + len(module_lint_obj.failed) + swf_failed
+    nbr_warned = len(lint_obj.warned) + module_warned + swf_warned
+    nbr_failed = len(lint_obj.failed) + module_failed + swf_failed
 
     summary_colour = "red" if nbr_failed > 0 else "green"
     table = Table(box=rich.box.ROUNDED, style=summary_colour)