From a8eb1c05cf8e005cc71029b1e94f9ddbe071599d Mon Sep 17 00:00:00 2001 From: Sebastian Schoenherr Date: Tue, 4 Apr 2023 16:03:28 +0200 Subject: [PATCH 1/3] Remove url-based uploads (#104) * Remove error messages * Remove url-based uploads * Remove http and sftp tests * Write correct context messages when URL-based uploads are used --------- Co-authored-by: Lukas Forer --- pom.xml | 7 - .../steps/InputValidation.java | 67 +------ .../steps/ImputationTest.java | 175 +----------------- .../imputationserver/util/TestSFTPServer.java | 65 ------- 4 files changed, 4 insertions(+), 310 deletions(-) delete mode 100644 src/test/java/genepi/imputationserver/util/TestSFTPServer.java diff --git a/pom.xml b/pom.xml index 7919f5aa..cac2cd92 100644 --- a/pom.xml +++ b/pom.xml @@ -284,13 +284,6 @@ - - org.apache.sshd - sshd-sftp - 0.9.0 - test - - org.codehaus.groovy groovy diff --git a/src/main/java/genepi/imputationserver/steps/InputValidation.java b/src/main/java/genepi/imputationserver/steps/InputValidation.java index 10603ebc..c70389ea 100644 --- a/src/main/java/genepi/imputationserver/steps/InputValidation.java +++ b/src/main/java/genepi/imputationserver/steps/InputValidation.java @@ -8,7 +8,6 @@ import cloudgene.sdk.internal.WorkflowContext; import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.importer.IImporter; import genepi.hadoop.importer.ImporterFactory; import genepi.imputationserver.steps.imputation.ImputationPipeline; import genepi.imputationserver.steps.vcf.VcfFile; @@ -327,70 +326,10 @@ private boolean importVcfFiles(WorkflowContext context) { if (ImporterFactory.needsImport(context.get(input))) { - context.beginTask("Importing files..."); - - String[] urlList = context.get(input).split(";")[0].split("\\s+"); - - String username = ""; - if (context.get(input).split(";").length > 1) { - username = context.get(input).split(";")[1]; - } - - String password = ""; - if (context.get(input).split(";").length > 2) { - password = context.get(input).split(";")[2]; - } - - for (String url2 : urlList) { - - String url = url2 + ";" + username + ";" + password; - String target = FileUtil.path(context.getLocalTemp(), "importer", input); - FileUtil.createDirectory(target); - context.println("Import to local workspace " + target + "..."); - - try { - - context.updateTask("Import " + url2 + "...", WorkflowContext.RUNNING); - context.log("Import " + url2 + "..."); - IImporter importer = ImporterFactory.createImporter(url, target); - - if (importer != null) { - - boolean successful = importer.importFiles("vcf.gz"); - - if (successful) { - - context.setInput(input, target); - - } else { - - context.updateTask("Import " + url2 + " failed: " + importer.getErrorMessage(), - WorkflowContext.ERROR); - - return false; - - } - - } else { - - context.updateTask("Import " + url2 + " failed: Protocol not supported", - WorkflowContext.ERROR); - - return false; - - } - - } catch (Exception e) { - context.updateTask("Import File(s) " + url2 + " failed: " + e.toString(), - WorkflowContext.ERROR); - - return false; - } - - } - - context.updateTask("File Import successful. ", WorkflowContext.OK); + context.log("URL-based uploads are no longer supported. Please use direct file uploads instead."); + context.error("URL-based uploads are no longer supported. Please use direct file uploads instead."); + return false; } } diff --git a/src/test/java/genepi/imputationserver/steps/ImputationTest.java b/src/test/java/genepi/imputationserver/steps/ImputationTest.java index 45d0b600..30972b90 100644 --- a/src/test/java/genepi/imputationserver/steps/ImputationTest.java +++ b/src/test/java/genepi/imputationserver/steps/ImputationTest.java @@ -274,179 +274,6 @@ public void testPipelineWidthInvalidFileFormat() throws IOException, ZipExceptio } - @Test - public void testPipelineWithHttpUrl() throws IOException, ZipException { - - String configFolder = "test-data/configs/hapmap-chr1"; - String inputFolder = "https://imputationserver.sph.umich.edu/static/downloads/hapmap300.chr1.recode.vcf.gz"; - - // create workflow context - WorkflowTestContext context = buildContext(inputFolder, "hapmap2"); - - // create step instance - InputValidation inputValidation = new InputValidationMock(configFolder); - - // run and test - boolean result = run(context, inputValidation); - - // check if step is failed - assertEquals(true, result); - - // run qc to create chunkfile - QcStatisticsMock qcStats = new QcStatisticsMock(configFolder); - result = run(context, qcStats); - - // add panel to hdfs - importRefPanel(FileUtil.path(configFolder, "ref-panels")); - // importMinimacMap("test-data/B38_MAP_FILE.map"); - importBinaries("files/bin"); - - // run imputation - ImputationMinimac3Mock imputation = new ImputationMinimac3Mock(configFolder); - result = run(context, imputation); - assertTrue(result); - - // run export - CompressionEncryptionMock export = new CompressionEncryptionMock("files"); - result = run(context, export); - assertTrue(result); - - ZipFile zipFile = new ZipFile("test-data/tmp/local/chr_1.zip", PASSWORD.toCharArray()); - zipFile.extractAll("test-data/tmp"); - - VcfFile file = VcfFileUtil.load("test-data/tmp/chr1.dose.vcf.gz", 100000000, false); - - assertEquals("1", file.getChromosome()); - assertEquals(60, file.getNoSamples()); - assertEquals(true, file.isPhased()); - - FileUtil.deleteDirectory("test-data/tmp"); - - } - - /* - * @Test public void testPipelineWithS3() throws IOException, ZipException { - * - * String configFolder = "test-data/configs/hapmap-chr1"; String inputFolder = - * "s3://imputationserver-aws-testdata/test-s3/hapmap300.chr1.recode.vcf.gz"; - * - * // create workflow context WorkflowTestContext context = - * buildContext(inputFolder, "hapmap2"); - * - * // create step instance InputValidation inputValidation = new - * InputValidationMock(configFolder); - * - * // run and test boolean result = run(context, inputValidation); - * - * // check if step is failed assertEquals(true, result); - * - * // run qc to create chunkfile QcStatisticsMock qcStats = new - * QcStatisticsMock(configFolder); result = run(context, qcStats); - * - * // add panel to hdfs importRefPanel(FileUtil.path(configFolder, - * "ref-panels")); // importMinimacMap("test-data/B38_MAP_FILE.map"); - * importBinaries("files/bin"); - * - * // run imputation ImputationMinimac3Mock imputation = new - * ImputationMinimac3Mock(configFolder); result = run(context, imputation); - * assertTrue(result); - * - * // run export CompressionEncryptionMock export = new - * CompressionEncryptionMock("files"); result = run(context, export); - * assertTrue(result); - * - * ZipFile zipFile = new ZipFile("test-data/tmp/local/chr_1.zip"); if - * (zipFile.isEncrypted()) { zipFile.setPassword(PASSWORD); } - * zipFile.extractAll("test-data/tmp"); - * - * VcfFile file = VcfFileUtil.load("test-data/tmp/chr1.dose.vcf.gz", 100000000, - * false); - * - * assertEquals("1", file.getChromosome()); assertEquals(60, - * file.getNoSamples()); assertEquals(true, file.isPhased()); - * - * FileUtil.deleteDirectory("test-data/tmp"); - * - * } - */ - - /* - * @Test public void testPipelineWithSFTP() throws IOException, ZipException, - * InterruptedException { - * - * TestSFTPServer server = new TestSFTPServer("test-data/data"); - * - * String configFolder = "test-data/configs/hapmap-chr20"; String inputFolder = - * "sftp://localhost:8001/" + new - * File("test-data/data/chr20-phased").getAbsolutePath() + ";" + - * TestSFTPServer.USERNAME + ";" + TestSFTPServer.PASSWORD; - * - * // create workflow context WorkflowTestContext context = - * buildContext(inputFolder, "hapmap2"); - * - * // create step instance InputValidation inputValidation = new - * InputValidationMock(configFolder); - * - * // run and test boolean result = run(context, inputValidation); - * - * // check if step is failed assertEquals(true, result); - * - * // run qc to create chunkfile QcStatisticsMock qcStats = new - * QcStatisticsMock(configFolder); result = run(context, qcStats); - * - * // add panel to hdfs importRefPanel(FileUtil.path(configFolder, - * "ref-panels")); // importMinimacMap("test-data/B38_MAP_FILE.map"); - * importBinaries("files/bin"); - * - * // run imputation ImputationMinimac3Mock imputation = new - * ImputationMinimac3Mock(configFolder); result = run(context, imputation); - * assertTrue(result); - * - * // run export CompressionEncryptionMock export = new - * CompressionEncryptionMock("files"); result = run(context, export); - * assertTrue(result); - * - * ZipFile zipFile = new ZipFile("test-data/tmp/local/chr_20.zip"); if - * (zipFile.isEncrypted()) { zipFile.setPassword(PASSWORD); } - * zipFile.extractAll("test-data/tmp"); - * - * VcfFile file = VcfFileUtil.load("test-data/tmp/chr20.dose.vcf.gz", 100000000, - * false); - * - * assertEquals("20", file.getChromosome()); assertEquals(51, - * file.getNoSamples()); assertEquals(true, file.isPhased()); - * assertEquals(TOTAL_REFPANEL_CHR20_B37 + ONLY_IN_INPUT, file.getNoSnps()); - * - * FileUtil.deleteDirectory("test-data/tmp"); - * - * server.stop(); - * - * } - * - * @Test public void testPipelineWithWrongSFTPCredentials() throws IOException, - * ZipException, InterruptedException { - * - * TestSFTPServer server = new TestSFTPServer("test-data/data"); - * - * String configFolder = "test-data/configs/hapmap-chr20"; String inputFolder = - * "sftp://localhost:8001/" + new File("data/chr20-phased").getAbsolutePath() + - * ";" + "WRONG_USERNAME" + ";" + TestSFTPServer.PASSWORD; - * - * // create workflow context WorkflowTestContext context = - * buildContext(inputFolder, "hapmap2"); - * - * // create step instance InputValidation inputValidation = new - * InputValidationMock(configFolder); - * - * // run and test boolean result = run(context, inputValidation); - * - * // check if step is failed assertEquals(false, result); - * - * server.stop(); - * - * } - */ - @Test public void testPipelineWithEagle() throws IOException, ZipException { @@ -1571,7 +1398,7 @@ protected WorkflowTestContext buildContext(String folder, String refpanel) { context.setOutput("pgs_output", file.getAbsolutePath() + "/pgs_output"); FileUtil.createDirectory(file.getAbsolutePath() + "/pgs_output"); - + context.setOutput("logfile", file.getAbsolutePath() + "/logfile"); FileUtil.createDirectory(file.getAbsolutePath() + "/logfile"); diff --git a/src/test/java/genepi/imputationserver/util/TestSFTPServer.java b/src/test/java/genepi/imputationserver/util/TestSFTPServer.java deleted file mode 100644 index 859d5a36..00000000 --- a/src/test/java/genepi/imputationserver/util/TestSFTPServer.java +++ /dev/null @@ -1,65 +0,0 @@ -package genepi.imputationserver.util; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - -import org.apache.commons.lang.StringUtils; -import org.apache.sshd.SshServer; -import org.apache.sshd.common.NamedFactory; -import org.apache.sshd.common.Session; -import org.apache.sshd.common.file.FileSystemView; -import org.apache.sshd.common.file.nativefs.NativeFileSystemFactory; -import org.apache.sshd.common.file.nativefs.NativeFileSystemView; -import org.apache.sshd.server.Command; -import org.apache.sshd.server.PasswordAuthenticator; -import org.apache.sshd.server.command.ScpCommandFactory; -import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; -import org.apache.sshd.server.session.ServerSession; -import org.apache.sshd.sftp.subsystem.SftpSubsystem; - -public class TestSFTPServer { - - private SshServer sshd; - - public static final String USERNAME = "username"; - - public static final String PASSWORD = "password"; - - public TestSFTPServer(final String rootFolder) throws IOException { - - sshd = SshServer.setUpDefaultServer(); - sshd.setFileSystemFactory(new NativeFileSystemFactory() { - @Override - public FileSystemView createFileSystemView(final Session session) { - return new NativeFileSystemView(session.getUsername(), false) { - @Override - public String getVirtualUserDir() { - System.out.println("Virtual Root: " + new File(rootFolder).getAbsolutePath()); - return new File(rootFolder).getAbsolutePath(); - } - - - }; - }; - }); - sshd.setPort(8001); - sshd.setSubsystemFactories(Arrays.>asList(new SftpSubsystem.Factory())); - sshd.setCommandFactory(new ScpCommandFactory()); - sshd.setKeyPairProvider( - new SimpleGeneratorHostKeyProvider(new File(rootFolder + "/hostkey.ser").getAbsolutePath())); - sshd.setPasswordAuthenticator(new PasswordAuthenticator() { - @Override - public boolean authenticate(final String username, final String password, final ServerSession session) { - return StringUtils.equals(username, USERNAME) && StringUtils.equals(password, PASSWORD); - } - }); - sshd.start(); - - } - - public void stop() throws InterruptedException{ - sshd.stop(); - } - -} From 675a1cb80f4bf0e9eeff8e883e195432c4d00a17 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 4 Apr 2023 16:07:51 +0200 Subject: [PATCH 2/3] Update groovy to 3.0.17 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cac2cd92..713d26d7 100644 --- a/pom.xml +++ b/pom.xml @@ -293,7 +293,7 @@ org.codehaus.groovy groovy-templates - 3.0.9 + 3.0.17 From 780f92ede93da4457281cd1b5ffc18e6fed6af2b Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 4 Apr 2023 21:09:48 +0200 Subject: [PATCH 3/3] Prepare release 1.7.2 --- files/imputationserver-beagle.yaml | 2 +- files/imputationserver-hla.yaml | 2 +- files/imputationserver-pgs.yaml | 2 +- files/minimac4.yaml | 2 +- pom.xml | 2 +- .../imputationserver/steps/imputation/ImputationPipeline.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/files/imputationserver-beagle.yaml b/files/imputationserver-beagle.yaml index 7da06e7d..e56dde47 100644 --- a/files/imputationserver-beagle.yaml +++ b/files/imputationserver-beagle.yaml @@ -1,7 +1,7 @@ id: imputationserver-beagle name: Genotype Imputation supporting Beagle (Minimac4) description: This is the new Michigan Imputation Server Pipeline using Minimac4. Documentation can be found here.

If your input data is GRCh37/hg19 please ensure chromosomes are encoded without prefix (e.g. 20).
If your input data is GRCh38hg38 please ensure chromosomes are encoded with prefix 'chr' (e.g. chr20). -version: 1.7.1 +version: 1.7.2 website: https://imputationserver.readthedocs.io category: diff --git a/files/imputationserver-hla.yaml b/files/imputationserver-hla.yaml index f110d695..ebfff5ac 100644 --- a/files/imputationserver-hla.yaml +++ b/files/imputationserver-hla.yaml @@ -1,7 +1,7 @@ id: imputationserver-hla name: Genotype Imputation HLA (Minimac4) description: This is the new Michigan Imputation Server Pipeline using Minimac4. Documentation can be found here.

If your input data is GRCh37/hg19 please ensure chromosomes are encoded without prefix (e.g. 20).
If your input data is GRCh38hg38 please ensure chromosomes are encoded with prefix 'chr' (e.g. chr20). -version: 1.7.1 +version: 1.7.2 website: https://imputationserver.readthedocs.io category: diff --git a/files/imputationserver-pgs.yaml b/files/imputationserver-pgs.yaml index 6088edf4..1de5bc75 100644 --- a/files/imputationserver-pgs.yaml +++ b/files/imputationserver-pgs.yaml @@ -1,7 +1,7 @@ id: imputationserver-pgs name: Genotype Imputation (PGS Calc Integration) description: This is the new Michigan Imputation Server Pipeline using Minimac4. Documentation can be found here.

If your input data is GRCh37/hg19 please ensure chromosomes are encoded without prefix (e.g. 20).
If your input data is GRCh38hg38 please ensure chromosomes are encoded with prefix 'chr' (e.g. chr20). -version: 1.7.1 +version: 1.7.2 website: https://imputationserver.readthedocs.io category: diff --git a/files/minimac4.yaml b/files/minimac4.yaml index e16bace7..5f5e91e1 100644 --- a/files/minimac4.yaml +++ b/files/minimac4.yaml @@ -1,7 +1,7 @@ id: imputationserver name: Genotype Imputation (Minimac4) description: This is the new Michigan Imputation Server Pipeline using Minimac4. Documentation can be found here.

If your input data is GRCh37/hg19 please ensure chromosomes are encoded without prefix (e.g. 20).
If your input data is GRCh38hg38 please ensure chromosomes are encoded with prefix 'chr' (e.g. chr20). -version: 1.7.1 +version: 1.7.2 website: https://imputationserver.readthedocs.io category: diff --git a/pom.xml b/pom.xml index 713d26d7..39c3b75f 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ genepi imputationserver - 1.7.1 + 1.7.2 jar diff --git a/src/main/java/genepi/imputationserver/steps/imputation/ImputationPipeline.java b/src/main/java/genepi/imputationserver/steps/imputation/ImputationPipeline.java index 39e84f9f..30c9a576 100644 --- a/src/main/java/genepi/imputationserver/steps/imputation/ImputationPipeline.java +++ b/src/main/java/genepi/imputationserver/steps/imputation/ImputationPipeline.java @@ -24,7 +24,7 @@ public class ImputationPipeline { - public static final String PIPELINE_VERSION = "michigan-imputationserver-1.7.1"; + public static final String PIPELINE_VERSION = "michigan-imputationserver-1.7.2"; public static final String IMPUTATION_VERSION = "minimac4-1.0.2";