Merge pull request #3116 from nvnieuwk/feat/linting-nf-schema

nf-schema support
nf-core · Aug 27, 2024 · 7d23535 · 7d23535
2 parents ee85104 + e816faf
commit 7d23535
Show file tree

Hide file tree

Showing 42 changed files with 1,083 additions and 972 deletions.
diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml
@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir

diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml
@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: go to working directory

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,8 @@
 - add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104))
 - add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125))
 - add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126))
+- Use nf-schema instead of nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116))
+- add option to exclude nf-schema from the template ([#3116](https://github.com/nf-core/tools/pull/3116))
 - add option to exclude fastqc from pipeline template ([#3129](https://github.com/nf-core/tools/pull/3129))
 - add option to exclude documentation from pipeline template ([#3130](https://github.com/nf-core/tools/pull/3130))
 - add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133))
@@ -42,6 +44,10 @@
 - Remove defaults from conda `environment.yml` file. ([#3029](https://github.com/nf-core/tools/pull/3029))
 - Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070))
 - Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086))
+- Linting will now fail when an unpinned plugin is used ([#3116](https://github.com/nf-core/tools/pull/3116))
+- Linting will now check if the schema is correct for the used validation plugin ([#3116])(https://github.com/nf-core/tools/pull/3116)
+- Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116)
+- Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116))
 
 ### Pipeline create command
 

diff --git a/docs/api/_src/pipeline_lint_tests/plugin_includes.md b/docs/api/_src/pipeline_lint_tests/plugin_includes.md
@@ -0,0 +1,5 @@
+# plugin_includes
+
+```{eval-rst}
+.. automethod:: nf_core.pipelines.lint.PipelineLint.plugin_includes
+```
diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml
@@ -24,7 +24,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: Check out pipeline code

diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md
@@ -16,7 +16,7 @@
 [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX)
 [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com)
 
-[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/)
+[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/)
 [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/)
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)

diff --git a/nf_core/pipeline-template/assets/schema_input.json b/nf_core/pipeline-template/assets/schema_input.json
@@ -1,5 +1,5 @@
 {
-    "$schema": "http://json-schema.org/draft-07/schema",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
     "$id": "https://github.com/{{ name }}/master/assets/schema_input.json",
     "title": "{{ name }} pipeline - params.input schema",
     "description": "Schema for the file provided with params.input",

diff --git a/nf_core/pipeline-template/main.nf b/nf_core/pipeline-template/main.nf
@@ -76,7 +76,6 @@ workflow {
     //
     PIPELINE_INITIALISATION (
         params.version,
-        params.help,
         params.validate_params,
         params.monochrome_logs,
         args,

diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json
@@ -23,19 +23,19 @@
                 "nf-core": {
                     "utils_nextflow_pipeline": {
                         "branch": "master",
-                        "git_sha": "20c03aede5a80ff520a905cea1f8ca121b5bb661",
+                        "git_sha": "d20fb2a9cc3e2835e9d067d1046a63252eb17352",
                         "installed_by": ["subworkflows"]
                     },
                     "utils_nfcore_pipeline": {
                         "branch": "master",
-                        "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3",
+                        "git_sha": "2fdce49d30c0254f76bc0f13c55c17455c1251ab",
                         "installed_by": ["subworkflows"]
-                    },
-                    "utils_nfvalidation_plugin": {
+                    }{% if nf_schema %},
+                    "utils_nfschema_plugin": {
                         "branch": "master",
-                        "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa",
+                        "git_sha": "bbd5a41f4535a8defafe6080e00ea74c45f4f96c",
                         "installed_by": ["subworkflows"]
-                    }
+                    }{% endif %}
                 }
             }
         }

diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config
@@ -39,7 +39,9 @@ params {
     {%- endif %}
     monochrome_logs              = false
     hook_url                     = null
-    help                         = false
+    {% if nf_schema %}help                         = false
+    help_full                    = false
+    show_hidden                  = false{% endif %}
     version                      = false
     {% if test_config %}pipelines_testdata_base_path = 'https://github.com/nf-core/test-datasets/'{% endif %}
 
@@ -61,11 +63,7 @@ params {
     max_time                   = '240.h'
 
     // Schema validation default options
-    validationFailUnrecognisedParams = false
-    validationLenientMode            = false
-    validationSchemaIgnoreParams     = 'genomes,igenomes_base'
-    validationShowHiddenParams       = false
-    validate_params                  = true
+    validate_params            = true
 
 }
 
@@ -204,11 +202,6 @@ podman.registry       = 'quay.io'
 singularity.registry  = 'quay.io'
 charliecloud.registry = 'quay.io'
 
-// Nextflow plugins
-plugins {
-    id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet
-}
-
 {% if igenomes -%}
 // Load igenomes.config if required
 if (!params.igenomes_ignore) {
@@ -266,11 +259,50 @@ manifest {
     homePage        = 'https://github.com/{{ name }}'
     description     = """{{ description }}"""
     mainScript      = 'main.nf'
-    nextflowVersion = '!>=23.04.0'
+    nextflowVersion = '!>=23.10.0'
     version         = '{{ version }}'
     doi             = ''
 }
 
+{% if nf_schema -%}
+// Nextflow plugins
+plugins {
+    id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet
+}
+
+validation {
+    defaultIgnoreParams = ["genomes", "helpFull", "showHidden", "help-full", "show-hidden"] // The last 4 parameters are here because of a bug in nf-schema. This will be fixed in a later version
+    help {
+        enabled = true
+        command = "nextflow run $manifest.name -profile <docker/singularity/.../institute> --input samplesheet.csv --outdir <OUTDIR>"
+        fullParameter = "help_full"
+        showHiddenParameter = "show_hidden"
+        {%- if is_nfcore %}
+        beforeText = """
+-\033[2m----------------------------------------------------\033[0m-
+                                        \033[0;32m,--.\033[0;30m/\033[0;32m,-.\033[0m
+\033[0;34m        ___     __   __   __   ___     \033[0;32m/,-._.--~\'\033[0m
+\033[0;34m  |\\ | |__  __ /  ` /  \\ |__) |__         \033[0;33m}  {\033[0m
+\033[0;34m  | \\| |       \\__, \\__/ |  \\ |___     \033[0;32m\\`-._,-`-,\033[0m
+                                        \033[0;32m`._,._,\'\033[0m
+\033[0;35m  ${manifest.name} ${manifest.version}\033[0m
+-\033[2m----------------------------------------------------\033[0m-
+"""
+        afterText = """${manifest.doi ? "* The pipeline\n" : ""}${manifest.doi.tokenize(",").collect { "  https://doi.org/${it.trim().replace('https://doi.org/','')}"}.join("\n")}${manifest.doi ? "\n" : ""}
+* The nf-core framework 
+  https://doi.org/10.1038/s41587-020-0439-x
+
+* Software dependencies
+  https://github.com/${manifest.name}/blob/master/CITATIONS.md
+"""{% endif %}
+    }{% if is_nfcore %}
+    summary {
+        beforeText = validation.help.beforeText
+        afterText = validation.help.afterText
+    }{% endif %}
+}
+{% endif -%}
+
 // Load modules.config for DSL2 module specific options
 includeConfig 'conf/modules.config'
 

diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json
@@ -1,10 +1,10 @@
 {
-    "$schema": "http://json-schema.org/draft-07/schema",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
     "$id": "https://github.com/{{ name }}/master/nextflow_schema.json",
     "title": "{{ name }} pipeline parameters",
     "description": "{{ description }}",
     "type": "object",
-    "definitions": {
+    "$defs": {
         "input_output_options": {
             "title": "Input/output options",
             "type": "object",
@@ -40,8 +40,7 @@
                     "type": "string",
                     "description": "MultiQC report title. Printed as page header, used for filename if not otherwise specified.",
                     "fa_icon": "fas fa-file-signature"
-                }
-                {% endif %}
+                }{% endif %}
             }
         },
         {%- if igenomes %}
@@ -73,6 +72,14 @@
                     "fa_icon": "fas fa-ban",
                     "hidden": true,
                     "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`."
+                },
+                "igenomes_base": {
+                    "type": "string",
+                    "format": "directory-path",
+                    "description": "The base path to the igenomes reference files",
+                    "fa_icon": "fas fa-ban",
+                    "hidden": true,
+                    "default": "s3://ngi-igenomes/igenomes/"
                 }
             }
         },
@@ -169,12 +176,6 @@
             "description": "Less common options for the pipeline, typically set in a config file.",
             "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.",
             "properties": {
-                "help": {
-                    "type": "boolean",
-                    "description": "Display help text.",
-                    "fa_icon": "fas fa-question-circle",
-                    "hidden": true
-                },
                 "version": {
                     "type": "boolean",
                     "description": "Display version and exit.",
@@ -252,27 +253,6 @@
                     "default": true,
                     "fa_icon": "fas fa-check-square",
                     "hidden": true
-                },
-                "validationShowHiddenParams": {
-                    "type": "boolean",
-                    "fa_icon": "far fa-eye-slash",
-                    "description": "Show all params when using `--help`",
-                    "hidden": true,
-                    "help_text": "By default, parameters set as _hidden_ in the schema are not shown on the command line when a user runs with `--help`. Specifying this option will tell the pipeline to show all parameters."
-                },
-                "validationFailUnrecognisedParams": {
-                    "type": "boolean",
-                    "fa_icon": "far fa-check-circle",
-                    "description": "Validation of parameters fails when an unrecognised parameter is found.",
-                    "hidden": true,
-                    "help_text": "By default, when an unrecognised parameter is found, it returns a warinig."
-                },
-                "validationLenientMode": {
-                    "type": "boolean",
-                    "fa_icon": "far fa-check-circle",
-                    "description": "Validation of parameters in lenient more.",
-                    "hidden": true,
-                    "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)."
                 }{% if test_config %},
                 "pipelines_testdata_base_path": {
                     "type": "string",
@@ -286,19 +266,19 @@
     },
     "allOf": [
         {
-            "$ref": "#/definitions/input_output_options"
+            "$ref": "#/$defs/input_output_options"
         },
         {% if igenomes %}{
-            "$ref": "#/definitions/reference_genome_options"
+            "$ref": "#/$defs/reference_genome_options"
         },{% endif %}
         {% if nf_core_configs %}{
-            "$ref": "#/definitions/institutional_config_options"
+            "$ref": "#/$defs/institutional_config_options"
         },{% endif %}
         {
-            "$ref": "#/definitions/max_job_request_options"
+            "$ref": "#/$defs/max_job_request_options"
         },
         {
-            "$ref": "#/definitions/generic_options"
+            "$ref": "#/$defs/generic_options"
         }
     ]
 }