SciLifeLab · szilvajuhos · Sep 11, 2018 · Sep 11, 2018 · Sep 11, 2018 · Sep 11, 2018
diff --git a/.travis.yml b/.travis.yml
@@ -11,19 +11,13 @@ env:
   global:
     - NXF_VER=0.31.0 SGT_VER=2.5.1
   matrix:
-    - CE=singularity TEST=TOOLS
-    - CE=singularity TEST=MANTA
-    - CE=docker      TEST=MANTA
-    - CE=docker      TEST=TOOLS
+    - CE=singularity TEST=SOMATIC
+    - CE=docker      TEST=SOMATIC
     - CE=docker      TEST=ANNOTATEVEP
     - CE=singularity TEST=ANNOTATESNPEFF
-    - CE=singularity TEST=STEP
-    - CE=singularity TEST=GERMLINE
-    - CE=singularity TEST=DIR
     - CE=docker      TEST=ANNOTATESNPEFF
-    - CE=docker      TEST=STEP
+    - CE=singularity TEST=GERMLINE
     - CE=docker      TEST=GERMLINE
-    - CE=docker      TEST=DIR
 
 install:
   # Install Nextflow (and Singularity if needed)

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,12 +17,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - [#608](https://github.com/SciLifeLab/Sarek/pull/608) - Update Nextflow required version
 - [#616](https://github.com/SciLifeLab/Sarek/pull/616) - Update CHANGELOG
 - [#615](https://github.com/SciLifeLab/Sarek/pull/615) - Use `splitCsv` instead of `readlines`
-- [#621](https://github.com/SciLifeLab/Sarek/pull/621) - Improve install script
-- [#621](https://github.com/SciLifeLab/Sarek/pull/621) - Simplify tests
+- [#621](https://github.com/SciLifeLab/Sarek/pull/621), [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Improve install script
+- [#621](https://github.com/SciLifeLab/Sarek/pull/621), [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Simplify tests
 - [#627](https://github.com/SciLifeLab/Sarek/pull/627), [#629](https://github.com/SciLifeLab/Sarek/pull/629), [#637](https://github.com/SciLifeLab/Sarek/pull/637) - Refactor docs
 - [#629](https://github.com/SciLifeLab/Sarek/pull/629) - Refactor config
 - [#632](https://github.com/SciLifeLab/Sarek/pull/632) - Use 2 threads and 2 cpus FastQC processes
 - [#637](https://github.com/SciLifeLab/Sarek/pull/637) - Update tool version gathering
+- [#638](https://github.com/SciLifeLab/Sarek/pull/638) - Use correct `.simg` extension for Singularity images
 
 ### `Removed`
 - [#616](https://github.com/SciLifeLab/Sarek/pull/616) - Remove old Issue Template

diff --git a/buildContainers.nf b/buildContainers.nf
@@ -98,7 +98,7 @@ process PullSingularityContainers {
 
   script:
   """
-  singularity pull --name ${container}-${params.tag}.img docker://${params.repository}/${container}:${params.tag}
+  singularity build ${container}-${params.tag}.simg docker://${params.repository}/${container}:${params.tag}
   """
 }
 

diff --git a/conf/singularity-path.config b/conf/singularity-path.config
@@ -14,111 +14,111 @@ singularity {
 
 process {
   withName:BuildBWAindexes {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:BuildReferenceIndex {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:BuildSAMToolsIndex {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:BuildVCFIndex {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:CompressVCF {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:ConcatVCF {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:CreateRecalibrationTable {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:GetVersionAll {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:GetVersionAlleleCount {
-    container = "${params.containerPath}/runallelecount-${params.tag}.img"
+    container = "${params.containerPath}/runallelecount-${params.tag}.simg"
   }
   withName:GetVersionASCAT {
-    container = "${params.containerPath}/r-base-${params.tag}.img"
+    container = "${params.containerPath}/r-base-${params.tag}.simg"
   }
   withName:GetVersionSnpeff {
-    container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"}
+    container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.simg" : "${params.containerPath}/snpeffgrch37-${params.tag}.simg"}
   }
   withName:GetVersionVEP {
-    container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"}
+    container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.simg" : "${params.containerPath}/vepgrch37-${params.tag}.simg"}
   }
   withName:MapReads {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:MarkDuplicates {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:MergeBams {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RecalibrateBam {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunAlleleCount {
-    container = "${params.containerPath}/runallelecount-${params.tag}.img"
+    container = "${params.containerPath}/runallelecount-${params.tag}.simg"
   }
   withName:RunAscat {
-    container = "${params.containerPath}/r-base-${params.tag}.img"
+    container = "${params.containerPath}/r-base-${params.tag}.simg"
   }
   withName:RunBamQC {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunBcftoolsStats {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunConvertAlleleCounts {
-    container = "${params.containerPath}/r-base-${params.tag}.img"
+    container = "${params.containerPath}/r-base-${params.tag}.simg"
   }
   withName:RunFastQC {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunFreeBayes {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunGenotypeGVCFs {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunHaplotypecaller {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunManta {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunMultiQC {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunMutect2 {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunSamtoolsStats {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunSingleManta {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunSingleStrelka {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunSnpeff {
-    container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.img" : "${params.containerPath}/snpeffgrch37-${params.tag}.img"}
+    container = {params.genome == 'GRCh38' ? "${params.containerPath}/snpeffgrch38-${params.tag}.simg" : "${params.containerPath}/snpeffgrch37-${params.tag}.simg"}
   }
   withName:RunStrelka {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunStrelkaBP {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunVcftools {
-    container = "${params.containerPath}/sarek-${params.tag}.img"
+    container = "${params.containerPath}/sarek-${params.tag}.simg"
   }
   withName:RunVEP {
-    container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.img" : "${params.containerPath}/vepgrch37-${params.tag}.img"}
+    container = {params.genome == 'GRCh38' ? "${params.containerPath}/vepgrch38-${params.tag}.simg" : "${params.containerPath}/vepgrch37-${params.tag}.simg"}
   }
 }
diff --git a/docs/TESTS.md b/docs/TESTS.md
@@ -24,11 +24,6 @@ nextflow run main.nf --sampleDir Sarek-data/testdata/manta/normal \
   --step mapping --genome smallGRCh37 --genome_base References/smallGRCh37 \
   --tag latest -profile singularity
 
-# Testing to restart from `realign`
-nextflow run main.nf --step realign \
-  --genome smallGRCh37 --genome_base References/smallGRCh37 \
-  --tag latest -profile singularity
-
 # Testing to restart from `recalibrate`
 nextflow run main.nf --step recalibrate \
   --genome smallGRCh37 --genome_base References/smallGRCh37 \
@@ -93,11 +88,8 @@ Four optional arguments are supported:
 - `-s` || `--sample`:
   Use to change the test sample (default=`Sarek-data/testdata/tsv/tiny.tsv`)
 - `-t` || `--test`:
- - `DIR`: test `mapping` with an input directory
- - `STEP`: test `mapping`, `realign` and `recalibrate`
- - `GERMLINE`: test `mapping` and Variant Calling with `HaplotypeCaller`
- - `TOOLS`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect1`, `MuTect2`, `Strelka`
- - `MANTA`: test `mapping` and Variant Calling with `Manta`
+ - `GERMLINE`: test `mapping`, `recalibrate` and Variant Calling with `HaplotypeCaller`
+ - `SOMATIC`: test `mapping` and Variant Calling with `FreeBayes`, `HaplotypeCaller`, `MuTect2`, `Strelka` and `Manta`
  - `ANNOTATESNPEFF`: test annotation using `snpEFF`
  - `ANNOTATEVEP`: test annotation using `VEP`
  - `BUILDCONTAINERS`: test building all containers except `snpeffgrch37`, `snpeffgrch38`, `vepgrch37` and `vepgrch38`

diff --git a/scripts/containers.sh b/scripts/containers.sh
@@ -5,6 +5,9 @@ PROFILE=singularity
 TEST=ALL
 TRAVIS=${TRAVIS:-false}
 
+TMPDIR=`pwd`/tmp
+mkdir -p $TMPDIR
+
 while [[ $# -gt 0 ]]
 do
   key=$1
@@ -25,7 +28,17 @@ do
   esac
 done
 
-if [[ $TEST = ANNOTATEVEP ]] && [[ $PROFILE = docker ]] && [[ $TRAVIS == true ]]
+if [[ $PROFILE = docker ]] && [[ $TRAVIS == true ]]
+then
+  if [[ $TEST = ANNOTATEVEP ]]
+  then
+    docker pull maxulysse/vepgrch37:latest
+  else
+    docker pull maxulysse/snpeffgrch37:latest
+  fi
+fi
+
+if [[ $TEST = ANNOTATESNPEFF ]] && [[ $PROFILE = singularity ]] && [[ $TRAVIS == true ]]
 then
-  docker pull maxulysse/vepgrch37:latest
+  singularity build $TMPDIR/maxulysse-snpeffgrch37-latest.simg docker://maxulysse/snpeffgrch37:latest
 fi
diff --git a/scripts/test.sh b/scripts/test.sh
@@ -13,10 +13,8 @@ TMPDIR=`pwd`/tmp
 mkdir -p $TMPDIR
 export NXF_SINGULARITY_CACHEDIR=$TMPDIR
 export NXF_TEMP=$TMPDIR
-
-export SINGULARITY_TMPDIR=$TMPDIR
 export SINGULARITY_CACHEDIR=$TMPDIR
-
+export SINGULARITY_TMPDIR=$TMPDIR
 
 # remove Reference directory
 rm -rf References
@@ -84,48 +82,22 @@ then
     echo "$(tput setaf 1)Building references$(tput sgr0)"
     nextflow run buildReferences.nf --refDir Sarek-data/reference --outDir References/$GENOME -profile $PROFILE --genome $GENOME --verbose
   fi
-  # Remove images only on TRAVIS
-  if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]]
-  then
-    docker rmi -f maxulysse/igvtools:latest
-  elif [[ $PROFILE == singularity ]] && [[ $TRAVIS == true ]]
-  then
-    rm -rf work/singularity/igvtools-latest.img
-  fi
-fi
-
-if [[ ALL,DIR =~ $TEST ]]
-then
-  run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal
-  clean_repo
-fi
-
-if [[ ALL,STEP =~ $TEST ]]
-then
-  run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal
-  run_wrapper --germline --step recalibrate --noReports
-  clean_repo
 fi
 
 if [[ ALL,GERMLINE =~ $TEST ]]
 then
   run_wrapper --germline --sampleDir Sarek-data/testdata/tiny/normal --variantCalling --tools HaplotypeCaller
+  run_wrapper --germline --step recalibrate --noReports
   clean_repo
 fi
 
-if [[ ALL,TOOLS =~ $TEST ]]
-then
-  run_wrapper --somatic --sample $SAMPLE --variantCalling  --tools FreeBayes,HaplotypeCaller,Mutect2
-fi
-
-if [[ ALL,MANTA =~ $TEST ]]
+if [[ ALL,SOMATIC =~ $TEST ]]
 then
-  run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta --noReports
+  run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools FreeBayes,HaplotypeCaller,Manta,Mutect2 --noReports
   run_wrapper --somatic --sample Sarek-data/testdata/tsv/tiny-manta.tsv --variantCalling --tools Manta,Strelka --noReports --strelkaBP
   clean_repo
 fi
 
-
 if [[ ALL,ANNOTATEALL,ANNOTATESNPEFF,ANNOTATEVEP =~ $TEST ]]
 then
   if [[ $TEST = ANNOTATESNPEFF ]]
@@ -138,15 +110,6 @@ then
   then
     ANNOTATOR=merge,snpEFF,VEP
   fi
-  if [[ $PROFILE == docker ]] && [[ $TRAVIS == true ]]
-  then
-    docker rmi -f maxulysse/sarek:latest
-    docker rmi -f maxulysse/picard:latest
-  elif [[ $PROFILE == singularity ]] && [[ $TRAVIS == true ]]
-  then
-    rm -rf work/singularity/sarek-latest.img
-    rm -rf work/singularity/picard-latest.img
-  fi
   run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz --noReports
   run_wrapper --annotate --tools ${ANNOTATOR} --annotateVCF Sarek-data/testdata/vcf/Strelka_1234N_variants.vcf.gz,Sarek-data/testdata/vcf/Strelka_9876T_variants.vcf.gz
   clean_repo