From da6f860e83d6a32959aa9acc4924cf7c2522201b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 24 Jan 2023 19:32:57 +0100 Subject: [PATCH 1/4] Make jobs automatically resubmit for a much wider range of exit codes now `130..145` --- CHANGELOG.md | 1 + nf_core/pipeline-template/conf/base.config | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 856bc9a604..0e8e84504b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) +- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) ### Linting diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index c5c691057d..59622840f2 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From eeba92b6f4c9c774876a2cc19e00696fa09dfaeb Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 30 Jan 2023 10:46:21 +0100 Subject: [PATCH 2/4] Add 104 to retry error codes --- nf_core/pipeline-template/conf/base.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index 59622840f2..f73c5afaa4 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -15,7 +15,7 @@ process { memory = { check_max( 6.GB * task.attempt, 'memory' ) } time = { check_max( 4.h * task.attempt, 'time' ) } - errorStrategy = { task.exitStatus in 130..145 ? 'retry' : 'finish' } + errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' From c0a6ff56bcf5a599dd86928f1652ae816dc54390 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 1 Feb 2023 01:33:58 +0100 Subject: [PATCH 3/4] Update CHANGELOG.md Co-authored-by: Matthieu Muffato --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e8e84504b..691a3a0b66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Template - Turn on automatic clean up of intermediate files in `work/` on successful pipeline completion in full-test config ([#2163](https://github.com/nf-core/tools/pull/2163)) -- Make jobs automatically resubmit for a much wider range of exit codes (now `130..145`) +- Make jobs automatically resubmit for a much wider range of exit codes (now `104` and `130..145`) ### Linting From 6ff70af488b642d43a27a3d89008fc1cac7ed0e4 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 4 Feb 2023 19:33:35 +0100 Subject: [PATCH 4/4] Run new version of Black --- nf_core/bump_version.py | 2 -- nf_core/create.py | 1 - nf_core/download.py | 1 - nf_core/launch.py | 3 --- nf_core/lint/__init__.py | 1 - nf_core/lint/files_unchanged.py | 2 -- nf_core/lint/template_strings.py | 1 - nf_core/list.py | 1 - nf_core/modules/bump_versions.py | 1 - nf_core/utils.py | 2 -- tests/test_download.py | 1 - tests/test_test_utils.py | 1 - 12 files changed, 17 deletions(-) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index 1f38775bb6..129016fa38 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -133,12 +133,10 @@ def update_file_version(filename, pipeline_obj, patterns): replacements = [] for pattern in patterns: - found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/create.py b/nf_core/create.py index 045c35d1b4..74c9df1b87 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -283,7 +283,6 @@ def render_template(self): # Set the paths to skip according to customization for template_fn_path_obj in template_files: - template_fn_path = str(template_fn_path_obj) # Skip files that are in the self.skip_paths list diff --git a/nf_core/download.py b/nf_core/download.py index 2f964b3afd..9d430e3352 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -506,7 +506,6 @@ def get_singularity_images(self): containers_download = [] containers_pull = [] for container in self.containers: - # Fetch the output and cached filenames for this container out_path, cache_path = self.singularity_image_filenames(container) diff --git a/nf_core/launch.py b/nf_core/launch.py index 87150172f7..648c8775f8 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -98,7 +98,6 @@ def __init__( self.cli_launch = True def launch_pipeline(self): - # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: launch_type = questionary.select( @@ -461,7 +460,6 @@ def prompt_group(self, group_id, group_obj): answers = {} error_msgs = [] while not while_break: - if len(error_msgs) == 0: self.print_param_header(group_id, group_obj, True) @@ -698,7 +696,6 @@ def build_command(self): # Pipeline parameters if len(self.schema_obj.input_params) > 0: - # Write the user selection to a file and run nextflow with that if self.use_params_file: dump_json_with_prettier(self.params_out, self.schema_obj.input_params) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index b46399eb97..e014a933ea 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -434,7 +434,6 @@ def format_result(test_results): ) def _print_summary(self): - # Summary table summary_colour = "red" if len(self.failed) > 0 else "green" table = Table(box=rich.box.ROUNDED, style=summary_colour) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index cadced5483..c0be64d0d7 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -147,7 +147,6 @@ def _tf(file_path): # Files that must be completely unchanged from template for files in files_exact: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): @@ -177,7 +176,6 @@ def _tf(file_path): # Files that can be added to, but that must contain the template contents for files in files_partial: - # Ignore if file specified in linting config ignore_files = self.lint_config.get("files_unchanged", []) if any([f in ignore_files for f in files]): diff --git a/nf_core/lint/template_strings.py b/nf_core/lint/template_strings.py index 436abe7b2b..fb1f0f32e5 100644 --- a/nf_core/lint/template_strings.py +++ b/nf_core/lint/template_strings.py @@ -24,7 +24,6 @@ def template_strings(self): # Loop through files, searching for string num_matches = 0 for fn in self.files: - # Skip binary files binary_ftypes = ["image", "application/java-archive"] (ftype, encoding) = mimetypes.guess_type(fn) diff --git a/nf_core/list.py b/nf_core/list.py index 53307ac9bd..77a9ac3919 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -328,7 +328,6 @@ def get_local_nf_workflow_details(self): """Get full details about a local cached workflow""" if self.local_path is None: - # Try to guess the local cache directory if len(os.environ.get("NXF_ASSETS", "")) > 0: nf_wfdir = os.path.join(os.environ.get("NXF_ASSETS"), self.full_name) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 7b9702622e..6d61d4c750 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -187,7 +187,6 @@ def bump_module_version(self, module): found_match = False newcontent = [] for line in content.splitlines(): - # Match the pattern matches_pattern = re.findall(rf"^.*{pattern[0]}.*$", line) if matches_pattern: diff --git a/nf_core/utils.py b/nf_core/utils.py index b60f61fff0..55703ab450 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -872,7 +872,6 @@ def get_repo_releases_branches(pipeline, wfs): # Repo is a nf-core pipeline for wf in wfs.remote_workflows: if wf.full_name == pipeline or wf.name == pipeline: - # Set to full name just in case it didn't have the nf-core/ prefix pipeline = wf.full_name @@ -883,7 +882,6 @@ def get_repo_releases_branches(pipeline, wfs): # Arbitrary GitHub repo else: if pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo log.debug( f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" diff --git a/tests/test_download.py b/tests/test_download.py index 4577a83992..e2ae882394 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,7 +18,6 @@ class DownloadTest(unittest.TestCase): - # # Tests for 'get_release_hash' # diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index ddf88ef74a..c4e3d49ae0 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -33,7 +33,6 @@ def test_tmp_folder_does_not_exist_after(): def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: with set_wd(tmpdirname): context_wd = Path().resolve()