Merge branch 'dev' into fix-linting-multiple-remotes

README.md

@@ -214,6 +214,22 @@ pip install --upgrade nf-core
 
 Please refer to the respective documentation for further details to manage packages, as for example [conda](https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-pkgs.html#updating-packages) or [pip](https://packaging.python.org/en/latest/tutorials/installing-packages/#upgrading-packages).
 
+### Activate shell completions for nf-core/tools
+
+Auto-completion for the `nf-core` command is available for bash, zsh and fish. To activate it, add the following lines to the respective shell config files.
+shell | shell config file | command
+--- | --- | ---
+bash | ~/.bashrc | `eval "$(_NF_CORE_COMPLETE=bash_source nf-core)"`
+Zsh | ~/.zshrc | `eval "$(_NF_CORE_COMPLETE=zsh_source nf-core)"`
+fish | ~/.config/fish/completions/nf-core.fish | `eval (env _NF_CORE_COMPLETE=fish_source nf-core)`
+
+After a restart of the shell session you should have auto-completion for the `nf-core` command and all its sub-commands and options.
+
+> **NB:** The added line will run the command `nf-core` (which will also slow down startup time of your shell). You should therefore either have the nf-core/tools installed globally.
+> You can also wrap it inside `if type nf-core > /dev/null; then ` \<YOUR EVAL CODE LINE\> `fi` for bash and zsh or `if command -v nf-core &> /dev/null eval (env _NF_CORE_COMPLETE=fish_source nf-core) end` for fish. You need to then source the config in your environment for the completions to be activated.
+
+> **NB:** If you see the error `command not found compdef` , be sure that your config file contains the line `autoload -Uz compinit && compinit` before the eval line.
+
 ## Listing pipelines
 
 The command `nf-core list` shows all available nf-core pipelines along with their latest version, when that was published and how recently the pipeline code was pulled to your local system (if at all).

nf_core/__main__.py

@@ -79,27 +79,28 @@
 
 
 def run_nf_core():
-    # Print nf-core header
-    stderr.print(f"\n[green]{' ' * 42},--.[grey39]/[green],-.", highlight=False)
-    stderr.print("[blue]          ___     __   __   __   ___     [green]/,-._.--~\\", highlight=False)
-    stderr.print(r"[blue]    |\ | |__  __ /  ` /  \ |__) |__      [yellow]   }  {", highlight=False)
-    stderr.print(r"[blue]    | \| |       \__, \__/ |  \ |___     [green]\`-._,-`-,", highlight=False)
-    stderr.print("[green]                                          `._,._,'\n", highlight=False)
-    stderr.print(
-        f"[grey39]    nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]",
-        highlight=False,
-    )
-    try:
-        is_outdated, _, remote_vers = nf_core.utils.check_if_outdated()
-        if is_outdated:
-            stderr.print(
-                f"[bold bright_yellow]    There is a new version of nf-core/tools available! ({remote_vers})",
-                highlight=False,
-            )
-    except Exception as e:
-        log.debug(f"Could not check latest version: {e}")
-    stderr.print("\n")
-
+    # print nf-core header if environment variable is not set
+    if os.environ.get("_NF_CORE_COMPLETE") is None:
+        # Print nf-core header
+        stderr.print(f"\n[green]{' ' * 42},--.[grey39]/[green],-.", highlight=False)
+        stderr.print("[blue]          ___     __   __   __   ___     [green]/,-._.--~\\", highlight=False)
+        stderr.print(r"[blue]    |\ | |__  __ /  ` /  \ |__) |__      [yellow]   }  {", highlight=False)
+        stderr.print(r"[blue]    | \| |       \__, \__/ |  \ |___     [green]\`-._,-`-,", highlight=False)
+        stderr.print("[green]                                          `._,._,'\n", highlight=False)
+        stderr.print(
+            f"[grey39]    nf-core/tools version {nf_core.__version__} - [link=https://nf-co.re]https://nf-co.re[/]",
+            highlight=False,
+        )
+        try:
+            is_outdated, _, remote_vers = nf_core.utils.check_if_outdated()
+            if is_outdated:
+                stderr.print(
+                    f"[bold bright_yellow]    There is a new version of nf-core/tools available! ({remote_vers})",
+                    highlight=False,
+                )
+        except Exception as e:
+            log.debug(f"Could not check latest version: {e}")
+        stderr.print("\n")
     # Launch the click cli
     nf_core_cli(auto_envvar_prefix="NFCORE")
 

nf_core/components/components_utils.py

@@ -69,7 +69,7 @@ def get_repo_info(directory, use_prompt=True):
             ).unsafe_ask()
             log.info("To avoid this prompt in the future, add the 'org_path' key to a root '%s' file.", config_fn.name)
             if rich.prompt.Confirm.ask("[bold][blue]?[/] Would you like me to add this config now?", default=True):
-                with open(os.path.join(dir, ".nf-core.yml"), "a+") as fh:
+                with open(config_fn, "a+") as fh:
                     fh.write(f"org_path: {org}\n")
                     log.info(f"Config added to '{config_fn.name}'")
 

nf_core/components/update.py

@@ -9,7 +9,12 @@
 import nf_core.modules.modules_utils
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
-from nf_core.components.components_utils import prompt_component_version_sha
+from nf_core.components.components_utils import (
+    get_components_to_install,
+    prompt_component_version_sha,
+)
+from nf_core.components.install import ComponentInstall
+from nf_core.components.remove import ComponentRemove
 from nf_core.modules.modules_differ import ModulesDiffer
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
@@ -254,6 +259,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                         self.update_linked_components(
                             modules_to_update, subworkflows_to_update, updated, check_diff_exist=False
                         )
+                        self.manage_changes_in_linked_components(component, modules_to_update, subworkflows_to_update)
 
                 elif self.show_diff:
                     ModulesDiffer.print_diff(
@@ -278,7 +284,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 # Clear the component directory and move the installed files there
                 self.move_files_from_tmp_dir(component, install_tmp_dir, modules_repo.repo_path, version)
                 # Update modules.json with newly installed component
-                self.modules_json.update(self.component_type, modules_repo, component, version, self.component_type)
+                self.modules_json.update(self.component_type, modules_repo, component, version, installed_by=None)
                 updated.append(component)
                 recursive_update = True
                 modules_to_update, subworkflows_to_update = self.get_components_to_update(component)
@@ -299,10 +305,16 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
                 if recursive_update and len(modules_to_update + subworkflows_to_update) > 0:
                     # Update linked components
                     self.update_linked_components(modules_to_update, subworkflows_to_update, updated)
+                    self.manage_changes_in_linked_components(component, modules_to_update, subworkflows_to_update)
             else:
                 # Don't save to a file, just iteratively update the variable
                 self.modules_json.update(
-                    self.component_type, modules_repo, component, version, self.component_type, write_file=False
+                    self.component_type,
+                    modules_repo,
+                    component,
+                    version,
+                    installed_by=None,
+                    write_file=False,
                 )
 
         if self.save_diff_fn:
@@ -869,6 +881,34 @@ def update_linked_components(self, modules_to_update, subworkflows_to_update, up
             self.update(m_update, silent=True, updated=updated, check_diff_exist=check_diff_exist)
             self._reset_component_type(original_component_type, original_update_all)
 
+    def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update):
+        """Check for linked components added or removed in the new subworkflow version"""
+        if self.component_type == "subworkflows":
+            subworkflow_directory = Path(self.dir, self.component_type, self.modules_repo.repo_path, component)
+            included_modules, included_subworkflows = get_components_to_install(subworkflow_directory)
+            # If a new module/subworkflow is included in the subworklfow and wasn't included before
+            for module in included_modules:
+                if module not in modules_to_update:
+                    log.info(f"Installing newly included module '{module}' for '{component}'")
+                    install_module_object = ComponentInstall(self.dir, "modules", installed_by=component)
+                    install_module_object.install(module, silent=True)
+            for subworkflow in included_subworkflows:
+                if subworkflow not in subworkflows_to_update:
+                    log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'")
+                    install_subworkflow_object = ComponentInstall(self.dir, "subworkflows", installed_by=component)
+                    install_subworkflow_object.install(subworkflow, silent=True)
+            # If a module/subworkflow has been removed from the subworkflow
+            for module in modules_to_update:
+                if module not in included_modules:
+                    log.info(f"Removing module '{module}' which is not included in '{component}' anymore.")
+                    remove_module_object = ComponentRemove("modules", self.dir)
+                    remove_module_object.remove(module, removed_by=component)
+            for subworkflow in subworkflows_to_update:
+                if subworkflow not in included_subworkflows:
+                    log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.")
+                    remove_subworkflow_object = ComponentRemove("subworkflows", self.dir)
+                    remove_subworkflow_object.remove(subworkflow, removed_by=component)
+
     def _change_component_type(self, new_component_type):
         original_component_type = self.component_type
         self.component_type = new_component_type

nf_core/modules/lint/main_nf.py

@@ -254,14 +254,14 @@ def check_process_section(self, lines, fix_version, progress_bar):
                 self.passed.append(("process_standard_label", "Correct process label", self.main_nf))
     else:
         self.warned.append(("process_standard_label", "Process label unspecified", self.main_nf))
-    for l in lines:
+    for i, l in enumerate(lines):
         if _container_type(l) == "bioconda":
             bioconda_packages = [b for b in l.split() if "bioconda::" in b]
         l = l.strip(" '\"")
         if _container_type(l) == "singularity":
             # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1
             # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0
-            match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?['\"]", l)
+            match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?'", l)
             if match is not None:
                 singularity_tag = match.group(1)
                 self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf))
@@ -271,13 +271,19 @@ def check_process_section(self, lines, fix_version, progress_bar):
         if _container_type(l) == "docker":
             # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
             # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
-            match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)['\"]", l)
+            match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l)
             if match is not None:
                 docker_tag = match.group(1)
                 self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf))
             else:
                 self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
                 docker_tag = None
+        if l.startswith("container"):
+            container_section = l + lines[i + 1] + lines[i + 2]
+            if container_section.count('"') > 2:
+                self.failed.append(
+                    ("container_links", "Too many double quotes found when specifying containers", self.main_nf)
+                )
 
     # Check that all bioconda packages have build numbers
     # Also check for newer versions

nf_core/modules/modules_json.py

@@ -656,7 +656,7 @@ def update(
         repo_component_entry[component_name]["git_sha"] = component_version
         repo_component_entry[component_name]["branch"] = branch
         try:
-            if installed_by not in repo_component_entry[component_name]["installed_by"]:
+            if installed_by not in repo_component_entry[component_name]["installed_by"] and installed_by is not None:
                 repo_component_entry[component_name]["installed_by"].append(installed_by)
         except KeyError:
             repo_component_entry[component_name]["installed_by"] = [installed_by]

tests/subworkflows/update.py

@@ -296,6 +296,33 @@ def test_update_all_subworkflows_from_module(self):
     )
 
 
+def test_update_change_of_included_modules(self):
+    """Update a subworkflow which has a module change in the new version."""
+    # Install an old version of vcf_annotate_ensemblvep with tabix/bgziptabix and without tabix/tabix
+    self.subworkflow_install_module_change.install("vcf_annotate_ensemblvep")
+    old_mod_json = ModulesJson(self.pipeline_dir).get_modules_json()
+
+    # Check that tabix/bgziptabix is there
+    assert "tabix/bgziptabix" in old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/bgziptabix").is_dir()
+    # Check that tabix/tabix is not there
+    assert "tabix/tabix" not in old_mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert not Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir()
+
+    # Update vcf_annotate_ensemblvep without tabix/bgziptabix and with tabix/tabix
+    update_obj = SubworkflowUpdate(self.pipeline_dir, update_deps=True, show_diff=False)
+    assert update_obj.update("vcf_annotate_ensemblvep") is True
+
+    mod_json = ModulesJson(self.pipeline_dir).get_modules_json()
+
+    # Check that tabix/bgziptabix is not there
+    assert "tabix/bgziptabix" not in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert not Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/bgziptabix").is_dir()
+    # Check that tabix/tabix is there
+    assert "tabix/tabix" in mod_json["repos"][NF_CORE_MODULES_REMOTE]["modules"][NF_CORE_MODULES_NAME]
+    assert Path(self.pipeline_dir, "modules", NF_CORE_MODULES_NAME, "tabix/tabix").is_dir()
+
+
 def cmp_component(dir1, dir2):
     """Compare two versions of the same component"""
     files = ["main.nf", "meta.yml"]

tests/test_subworkflows.py

@@ -67,6 +67,12 @@ def setUp(self):
             force=False,
             sha=OLD_SUBWORKFLOWS_SHA,
         )
+        self.subworkflow_install_module_change = nf_core.subworkflows.SubworkflowInstall(
+            self.pipeline_dir,
+            prompt=False,
+            force=False,
+            sha="8c343b3c8a0925949783dc547666007c245c235b",
+        )
         self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True)
 
         # Set up remove objects
@@ -126,6 +132,7 @@ def tearDown(self):
         test_update_all,
         test_update_all_linked_components_from_subworkflow,
         test_update_all_subworkflows_from_module,
+        test_update_change_of_included_modules,
         test_update_with_config_dont_update,
         test_update_with_config_fix_all,
         test_update_with_config_fixed_version,