From c07d8aa0cde0a1677c7e9ed70a0a36cc08d139c2 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 12:47:13 +0200 Subject: [PATCH 01/11] better testing --- .travis.yml | 10 ++-------- scripts/test.sh | 24 +++--------------------- 2 files changed, 5 insertions(+), 29 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e18eb1d1d..2ab8e7780f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,19 +11,13 @@ env: global: - NXF_VER=0.31.0 SGT_VER=2.5.1 matrix: - - CE=singularity TEST=TOOLS - - CE=singularity TEST=MANTA - - CE=docker TEST=MANTA - - CE=docker TEST=TOOLS + - CE=singularity TEST=SOMATIC + - CE=docker TEST=SOMATIC - CE=docker TEST=ANNOTATEVEP - CE=singularity TEST=ANNOTATESNPEFF - - CE=singularity TEST=STEP - CE=singularity TEST=GERMLINE - - CE=singularity TEST=DIR - CE=docker TEST=ANNOTATESNPEFF - - CE=docker TEST=STEP - CE=docker TEST=GERMLINE - - CE=docker TEST=DIR install: # Install Nextflow (and Singularity if needed) diff --git a/scripts/test.sh b/scripts/test.sh index bcd017d1f1..13492a34f6 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -94,38 +94,20 @@ then fi fi -if [[ ALL,DIR =~ $TEST ]] -then - run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal - clean_repo -fi - -if [[ ALL,STEP =~ $TEST ]] -then - run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal - run_wrapper --germline --step recalibrate --noReports - clean_repo -fi - if [[ ALL,GERMLINE =~ $TEST ]] then run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal --variantCalling --tools HaplotypeCaller + run_wrapper --germline --step recalibrate --noReports clean_repo fi -if [[ ALL,TOOLS =~ $TEST ]] +if [[ ALL,SOMATIC =~ $TEST ]] then - run_wrapper --somatic --sample $SAMPLE --variantCalling --tools FreeBayes,HaplotypeCaller,Mutect2 -fi - -if [[ ALL,MANTA =~ $TEST ]] -then - run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta --noReports + run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools FreeBayes,HaplotypeCaller,Manta,Mutect2 --noReports run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta,Strelka --noReports --strelkaBP clean_repo fi - if [[ ALL,ANNOTATEALL,ANNOTATESNPEFF,ANNOTATEVEP =~ $TEST ]] then if [[ $TEST = ANNOTATESNPEFF ]] From f11106eb18804c784e8a38c06e321600f143a8b6 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 13:56:01 +0200 Subject: [PATCH 02/11] sort tests by amount of time needed --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ab8e7780f..3d1dd30eb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,8 @@ env: - CE=docker TEST=SOMATIC - CE=docker TEST=ANNOTATEVEP - CE=singularity TEST=ANNOTATESNPEFF - - CE=singularity TEST=GERMLINE - CE=docker TEST=ANNOTATESNPEFF + - CE=singularity TEST=GERMLINE - CE=docker TEST=GERMLINE install: From 3fe164fce49322416d0af03a511b31f314bb5dde Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 13:56:51 +0200 Subject: [PATCH 03/11] sort exports --- scripts/test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 13492a34f6..37bf133dde 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -13,10 +13,8 @@ TMPDIR=`pwd`/tmp mkdir -p $TMPDIR export NXF_SINGULARITY_CACHEDIR=$TMPDIR export NXF_TEMP=$TMPDIR - -export SINGULARITY_TMPDIR=$TMPDIR export SINGULARITY_CACHEDIR=$TMPDIR - +export SINGULARITY_TMPDIR=$TMPDIR # remove Reference directory rm -rf References From 46e187731f80aeb50e8ceea856be31d8695bf376 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 13:57:26 +0200 Subject: [PATCH 04/11] update depreciated syntax for singularity --- buildContainers.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildContainers.nf b/buildContainers.nf index 8aa546c657..845c54f9ff 100644 --- a/buildContainers.nf +++ b/buildContainers.nf @@ -98,7 +98,7 @@ process PullSingularityContainers { script: """ - singularity pull --name ${container}-${params.tag}.img docker://${params.repository}/${container}:${params.tag} + singularity build --name ${container}-${params.tag}.img docker://${params.repository}/${container}:${params.tag} """ } From b432dd7eb8b52fa6b15edc581c49b761d29b3c10 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 13:57:54 +0200 Subject: [PATCH 05/11] add pulling other big containers --- scripts/containers.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/containers.sh b/scripts/containers.sh index fc5dc12a31..f7d41565b8 100755 --- a/scripts/containers.sh +++ b/scripts/containers.sh @@ -25,7 +25,19 @@ do esac done -if [[ $TEST = ANNOTATEVEP ]] && [[ $PROFILE = docker ]] && [[ $TRAVIS == true ]] +if [[ $PROFILE = docker ]] && [[ $TRAVIS == true ]] then - docker pull maxulysse/vepgrch37:latest + if [[ $TEST = ANNOTATEVEP ]] + then + docker pull maxulysse/vepgrch37:latest + else + docker pull maxulysse/snpeffgrch37:latest + fi +fi + +if [[ $TEST = ANNOTATESNPEFF ]] && [[ $PROFILE = singularity ]] && [[ $TRAVIS == true ]] +then + cd tmp + singularity build --name maxulysse-snpeffgrch37-latest.img docker://maxulysse/snpeffgrch37:latest + cd .. fi From ce93c115fbb184e8acb779463750700f40b13500 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 13:58:04 +0200 Subject: [PATCH 06/11] update docs --- docs/TESTS.md | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/TESTS.md b/docs/TESTS.md index 458ffbad82..3e059cc5eb 100644 --- a/docs/TESTS.md +++ b/docs/TESTS.md @@ -24,11 +24,6 @@ nextflow run main.nf --sampleDir Sarek-data/testdata/manta/normal \ --step mapping --genome smallGRCh37 --genome_base References/smallGRCh37 \ --tag latest -profile singularity -# Testing to restart from `realign` -nextflow run main.nf --step realign \ - --genome smallGRCh37 --genome_base References/smallGRCh37 \ - --tag latest -profile singularity - # Testing to restart from `recalibrate` nextflow run main.nf --step recalibrate \ --genome smallGRCh37 --genome_base References/smallGRCh37 \ @@ -93,11 +88,8 @@ Four optional arguments are supported: - `-s` || `--sample`: Use to change the test sample (default=`Sarek-data/testdata/tsv/tiny.tsv`) - `-t` || `--test`: - - `DIR`: test `mapping` with an input directory - - `STEP`: test `mapping`, `realign` and `recalibrate` - - `GERMLINE`: test `mapping` and Variant Calling with `HaplotypeCaller` - - `TOOLS`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect1`, `MuTect2`, `Strelka` - - `MANTA`: test `mapping` and Variant Calling with `Manta` + - `GERMLINE`: test `mapping`, `recalibrate` and Variant Calling with `HaplotypeCaller` + - `SOMATIC`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect1`, `MuTect2`, `Strelka` and `Manta` - `ANNOTATESNPEFF`: test annotation using `snpEFF` - `ANNOTATEVEP`: test annotation using `VEP` - `BUILDCONTAINERS`: test building all containers except `snpeffgrch37`, `snpeffgrch38`, `vepgrch37` and `vepgrch38` From d910c134b35434039d3cf7e5c180a1f54f9d3a92 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 14:02:52 +0200 Subject: [PATCH 07/11] fix path to TMPDIR --- scripts/containers.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/containers.sh b/scripts/containers.sh index f7d41565b8..3f9f5c5343 100755 --- a/scripts/containers.sh +++ b/scripts/containers.sh @@ -5,6 +5,9 @@ PROFILE=singularity TEST=ALL TRAVIS=${TRAVIS:-false} +TMPDIR=`pwd`/tmp +mkdir -p $TMPDIR + while [[ $# -gt 0 ]] do key=$1 @@ -37,7 +40,7 @@ fi if [[ $TEST = ANNOTATESNPEFF ]] && [[ $PROFILE = singularity ]] && [[ $TRAVIS == true ]] then - cd tmp + cd $TMPDIR singularity build --name maxulysse-snpeffgrch37-latest.img docker://maxulysse/snpeffgrch37:latest cd .. fi From 365876d2e40ec6c513afee74ca1843d1184de4bb Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 14:09:01 +0200 Subject: [PATCH 08/11] update with .simg extension for singularity image + remove --name option --- buildContainers.nf | 2 +- conf/singularity-path.config | 72 ++++++++++++++++++------------------ scripts/containers.sh | 4 +- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/buildContainers.nf b/buildContainers.nf index 845c54f9ff..69e869d646 100644 --- a/buildContainers.nf +++ b/buildContainers.nf @@ -98,7 +98,7 @@ process PullSingularityContainers { script: """ - singularity build --name ${container}-${params.tag}.img docker://${params.repository}/${container}:${params.tag} + singularity build ${container}-${params.tag}.simg docker://${params.repository}/${container}:${params.tag} """ } diff --git a/conf/singularity-path.config b/conf/singularity-path.config index 938829dc77..448e7d1432 100644 --- a/conf/singularity-path.config +++ b/conf/singularity-path.config @@ -14,111 +14,111 @@ singularity { process { withName:BuildBWAindexes { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:BuildReferenceIndex { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:BuildSAMToolsIndex { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:BuildVCFIndex { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:CompressVCF { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:ConcatVCF { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:CreateRecalibrationTable { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:GetVersionAll { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:GetVersionAlleleCount { - container = "${params.containerPath}/runallelecount-${params.tag}.img" + container = "${params.containerPath}/runallelecount-${params.tag}.simg" } withName:GetVersionASCAT { - container = "${params.containerPath}/r-base-${params.tag}.img" + container = "${params.containerPath}/r-base-${params.tag}.simg" } withName:GetVersionSnpeff { - container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"} + container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.simg" : "${params.containerPath}/snpeffgrch37-${params.tag}.simg"} } withName:GetVersionVEP { - container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"} + container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.simg" : "${params.containerPath}/vepgrch37-${params.tag}.simg"} } withName:MapReads { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:MarkDuplicates { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:MergeBams { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RecalibrateBam { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunAlleleCount { - container = "${params.containerPath}/runallelecount-${params.tag}.img" + container = "${params.containerPath}/runallelecount-${params.tag}.simg" } withName:RunAscat { - container = "${params.containerPath}/r-base-${params.tag}.img" + container = "${params.containerPath}/r-base-${params.tag}.simg" } withName:RunBamQC { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunBcftoolsStats { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunConvertAlleleCounts { - container = "${params.containerPath}/r-base-${params.tag}.img" + container = "${params.containerPath}/r-base-${params.tag}.simg" } withName:RunFastQC { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunFreeBayes { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunGenotypeGVCFs { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunHaplotypecaller { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunManta { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunMultiQC { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunMutect2 { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunSamtoolsStats { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunSingleManta { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunSingleStrelka { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunSnpeff { - container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"} + container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.simg" : "${params.containerPath}/snpeffgrch37-${params.tag}.simg"} } withName:RunStrelka { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunStrelkaBP { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunVcftools { - container = "${params.containerPath}/sarek-${params.tag}.img" + container = "${params.containerPath}/sarek-${params.tag}.simg" } withName:RunVEP { - container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"} + container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.simg" : "${params.containerPath}/vepgrch37-${params.tag}.simg"} } } diff --git a/scripts/containers.sh b/scripts/containers.sh index 3f9f5c5343..da6c2fd1ba 100755 --- a/scripts/containers.sh +++ b/scripts/containers.sh @@ -40,7 +40,5 @@ fi if [[ $TEST = ANNOTATESNPEFF ]] && [[ $PROFILE = singularity ]] && [[ $TRAVIS == true ]] then - cd $TMPDIR - singularity build --name maxulysse-snpeffgrch37-latest.img docker://maxulysse/snpeffgrch37:latest - cd .. + singularity build $TMPDIR/maxulysse-snpeffgrch37-latest.simg docker://maxulysse/snpeffgrch37:latest fi From f88e4c3499b772d5bb6b3aac01c58a3f284496c9 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 14:33:33 +0200 Subject: [PATCH 09/11] clean up test.sh script --- scripts/test.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/test.sh b/scripts/test.sh index 37bf133dde..a170eb1803 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -82,14 +82,6 @@ then echo "$(tput setaf 1)Building references$(tput sgr0)" nextflow run buildReferences.nf --refDir Sarek-data/reference --outDir References/$GENOME -profile $PROFILE --genome $GENOME --verbose fi - # Remove images only on TRAVIS - if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]] - then - docker rmi -f maxulysse/igvtools:latest - elif [[ $PROFILE == singularity ]] && [[ $TRAVIS == true ]] - then - rm -rf work/singularity/igvtools-latest.img - fi fi if [[ ALL,GERMLINE =~ $TEST ]] @@ -118,15 +110,6 @@ then then ANNOTATOR=merge,snpEFF,VEP fi - if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]] - then - docker rmi -f maxulysse/sarek:latest - docker rmi -f maxulysse/picard:latest - elif [[ $PROFILE == singularity ]] && [[ $TRAVIS == true ]] - then - rm -rf work/singularity/sarek-latest.img - rm -rf work/singularity/picard-latest.img - fi run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz --noReports run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz,Sarek-data/testdata/vcf/Strelka_9876T_variants.vcf.gz clean_repo From 1558b0707a1d31f41e95e8a65b8306821b099061 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 14:55:33 +0200 Subject: [PATCH 10/11] update CHANGELOG --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4115f195c..9b7750e652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - [#608](https://github.com/SciLifeLab/Sarek/pull/608) - Update Nextflow required version - [#616](https://github.com/SciLifeLab/Sarek/pull/616) - Update CHANGELOG - [#615](https://github.com/SciLifeLab/Sarek/pull/615) - Use `splitCsv` instead of `readlines` -- [#621](https://github.com/SciLifeLab/Sarek/pull/621) - Improve install script -- [#621](https://github.com/SciLifeLab/Sarek/pull/621) - Simplify tests +- [#621](https://github.com/SciLifeLab/Sarek/pull/621), [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Improve install script +- [#621](https://github.com/SciLifeLab/Sarek/pull/621), [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Simplify tests - [#627](https://github.com/SciLifeLab/Sarek/pull/627), [#629](https://github.com/SciLifeLab/Sarek/pull/629), [#637](https://github.com/SciLifeLab/Sarek/pull/637) - Refactor docs - [#629](https://github.com/SciLifeLab/Sarek/pull/629) - Refactor config - [#632](https://github.com/SciLifeLab/Sarek/pull/632) - Use 2 threads and 2 cpus FastQC processes - [#637](https://github.com/SciLifeLab/Sarek/pull/637) - Update tool version gathering +- [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Use correct `.simg` extension for Singularity images ### `Removed` - [#616](https://github.com/SciLifeLab/Sarek/pull/616) - Remove old Issue Template From ab70d0e8b79134a37ff1b9393ab392c797daee27 Mon Sep 17 00:00:00 2001 From: Maxime Garcia Date: Tue, 11 Sep 2018 15:36:43 +0200 Subject: [PATCH 11/11] forgot one MuTecT1 [skip ci] --- docs/TESTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/TESTS.md b/docs/TESTS.md index 3e059cc5eb..1e160bd9c9 100644 --- a/docs/TESTS.md +++ b/docs/TESTS.md @@ -89,7 +89,7 @@ Four optional arguments are supported: Use to change the test sample (default=`Sarek-data/testdata/tsv/tiny.tsv`) - `-t` || `--test`: - `GERMLINE`: test `mapping`, `recalibrate` and Variant Calling with `HaplotypeCaller` - - `SOMATIC`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect1`, `MuTect2`, `Strelka` and `Manta` + - `SOMATIC`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect2`, `Strelka` and `Manta` - `ANNOTATESNPEFF`: test annotation using `snpEFF` - `ANNOTATEVEP`: test annotation using `VEP` - `BUILDCONTAINERS`: test building all containers except `snpeffgrch37`, `snpeffgrch38`, `vepgrch37` and `vepgrch38`