Merge pull request #2306 from adamrtalbot/fix_parsing_container_spec

fix: Container directive parsing when it is on a single line

CHANGELOG.md

@@ -14,6 +14,7 @@
 - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298))
 - Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301))
 - Remove schema validation from `lib` folder and use Nextflow [nf-validation plugin](https://nextflow-io.github.io/nf-validation/) instead ([#1771](https://github.com/nf-core/tools/pull/1771/))
+- Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306))
 
 ### Download
 

nf_core/modules/lint/main_nf.py

@@ -223,8 +223,8 @@ def check_process_section(self, lines, fix_version, progress_bar):
     self.passed.append(("process_exist", "Process definition exists", self.main_nf))
 
     # Checks that build numbers of bioconda, singularity and docker container are matching
-    singularity_tag = "singularity"
-    docker_tag = "docker"
+    singularity_tag = None
+    docker_tag = None
     bioconda_packages = []
 
     # Process name should be all capital letters
@@ -240,7 +240,12 @@ def check_process_section(self, lines, fix_version, progress_bar):
     # Deprecated enable_conda
     for i, l in enumerate(lines):
         url = None
-        l = l.strip(" '\"")
+        l = l.strip(" \n'\"}:")
+
+        # Catch preceeding "container "
+        if l.startswith("container"):
+            l = l.replace("container", "").strip(" \n'\"}:")
+
         if _container_type(l) == "conda":
             bioconda_packages = [b for b in l.split() if "bioconda::" in b]
             match = re.search(r"params\.enable_conda", l)
@@ -261,9 +266,10 @@ def check_process_section(self, lines, fix_version, progress_bar):
                     )
                 )
         if _container_type(l) == "singularity":
-            # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1
-            # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0
-            match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?'", l)
+            # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1
+            # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0
+            # Please god let's find a better way to do this than regex
+            match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l)
             if match is not None:
                 singularity_tag = match.group(1)
                 self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf))
@@ -273,15 +279,15 @@ def check_process_section(self, lines, fix_version, progress_bar):
             url = urlparse(l.split("'")[0])
 
         if _container_type(l) == "docker":
-            # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
-            # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
-            match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l)
+            # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5 -> 2.7.1--pl526_5
+            # e.g. "biocontainers/biocontainers:v1.2.0_cv1 -> v1.2.0_cv1
+            match = re.search(r":([A-Za-z\d\-_.]+)$", l)
             if match is not None:
                 docker_tag = match.group(1)
                 self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf))
             else:
                 self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
-                docker_tag = NoneD
+                docker_tag = None
             if l.startswith("quay.io/"):
                 l_stripped = re.sub(r"\W+$", "", l)
                 self.failed.append(
@@ -412,7 +418,11 @@ def check_process_section(self, lines, fix_version, progress_bar):
             else:
                 self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf))
 
-    return docker_tag == singularity_tag
+    # Check if a tag exists at all. If not, return None.
+    if singularity_tag is None or docker_tag is None:
+        return None
+    else:
+        return docker_tag == singularity_tag
 
 
 def check_process_labels(self, lines):
@@ -591,7 +601,7 @@ def _container_type(line):
     """Returns the container type of a build."""
     if line.startswith("conda"):
         return "conda"
-    if line.startswith("https://containers") or line.startswith("https://depot"):
+    if line.startswith("https://") or line.startswith("https://depot"):
         # Look for a http download URL.
         # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980
         url_regex = (
@@ -601,5 +611,5 @@ def _container_type(line):
         if url_match:
             return "singularity"
         return None
-    if line.count("/") >= 1 and line.count(":") == 1:
+    if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0 and "https://" not in line:
         return "docker"

nf_core/pipeline-template/.github/workflows/awsfulltest.yml

@@ -26,7 +26,7 @@ jobs:
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %}
           parameters: |
             {
-              "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
+              "hook_url": "{% raw %}${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}{% endraw %}",
               "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}"
             }
           profiles: test_full