diff --git a/README.md b/README.md index f79f89b..d8292fa 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ Just make sure you have proper internet access. ```groovy plugins { - id 'nf-float@0.3.2' + id 'nf-float@0.4.0' } ``` @@ -66,9 +66,9 @@ Go to the folder where you just install the `nextflow` command line. Let's call this folder the Nextflow home directory. Create the float plugin folder with: ```bash -mkdir -p .nextflow/plugins/nf-float-0.3.2 +mkdir -p .nextflow/plugins/nf-float-0.4.0 ``` -where `0.3.2` is the version of the float plugin. This version number should +where `0.4.0` is the version of the float plugin. This version number should align with the version in of your plugin and the property in your configuration file. (check the configuration section) @@ -76,7 +76,7 @@ Retrieve your plugin zip file and unzip it in this folder. If everything goes right, you should be able to see two sub-folders: ```bash -$ ll .nextflow/plugins/nf-float-0.3.2/ +$ ll .nextflow/plugins/nf-float-0.4.0/ total 48 drwxr-xr-x 4 ec2-user ec2-user 51 Jan 5 07:17 classes drwxr-xr-x 2 ec2-user ec2-user 25 Jan 5 07:17 META-INF @@ -89,7 +89,7 @@ file with the command line option `-c`. Here is a sample of the configuration. ```groovy plugins { - id 'nf-float@0.3.2' + id 'nf-float@0.4.0' } workDir = '/mnt/memverge/shared' @@ -124,6 +124,10 @@ Available `float` config options: for the list of available options. * `timeFactor`: a float number. default to 1. An extra factor to multiply based on the time supplied by the task. Use it to resolve some task timeouts. +* `maxCpuFactor`: a float number. default to 4. The maximum CPU cores of the instance is set + to `maxCpuFactor` * `cpus` of the task. +* `maxMemoryFactor`: a float number. default to 4. The maximum memory of the instance is set + to `maxMemoryFactor` * `memory` of the task. * `commonExtra`: allows the user to specify other submit CLI options. This parameter will be appended to every float submit command. @@ -167,7 +171,7 @@ Unknown config secret 'MMC_USERNAME' To enable s3 as work directory, user need to set work directory to a s3 bucket. ```groovy plugins { - id 'nf-float@0.3.2' + id 'nf-float@0.4.0' } workDir = 's3://bucket/path' diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/FloatBin.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/FloatBin.groovy index b87fcc2..ef0898b 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/FloatBin.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/FloatBin.groovy @@ -12,7 +12,20 @@ import java.util.regex.Pattern class FloatBin { private static final binName = 'float' - static Path get(String opCenterAddr) { + /** + * This function checks if the float binary is available in the plugin + * directory or the $PATH environment variable. + * + * If it's not available, download the proper version from the op-center. + * If it's available, use the sync command to update. + * + * @param opCenterAddr address of the op-center + * @param targetDir the directory to check or download binary, default to + * the plugin directory if set to null. + * + * @return location of the float binary + */ + static Path get(String opCenterAddr, Path targetDir = null) { def ret = getFloatBinPath() if (ret == null) { if (!opCenterAddr) { @@ -20,8 +33,10 @@ class FloatBin { return Paths.get(binName) } final URL src = getDownloadUrl(opCenterAddr) - final Path pluginsDir = Global.getPluginsDir() - ret = pluginsDir.resolve(binName) + if (targetDir == null) { + targetDir = Global.getPluginsDir() + } + ret = targetDir.resolve(binName) try { log.info "try downloading $src to $ret" Global.download(src, ret) diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy index 8f77604..b50df5d 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy @@ -37,7 +37,10 @@ class FloatConf { static final String NF_RUN_NAME = 'nextflow-io-run-name' static final String NF_SESSION_ID = 'nextflow-io-session-id' static final String NF_TASK_NAME = 'nextflow-io-task-name' - static final String NF_INPUT_SIZE = 'input-size' + static final String FLOAT_INPUT_SIZE = 'input-size' + static final String FLOAT_JOB_KIND = 'job-kind' + static final int DFT_MAX_CPU_FACTOR = 4 + static final int DFT_MAX_MEM_FACTOR = 4 /** credentials for op center */ String username @@ -58,8 +61,8 @@ class FloatConf { float cpuFactor = 1 float memoryFactory = 1 - float maxCpuFactor = 2 - float maxMemoryFactor = 2 + float maxCpuFactor = DFT_MAX_CPU_FACTOR + float maxMemoryFactor = DFT_MAX_MEM_FACTOR /** * Create a FloatConf instance and initialize the content from the diff --git a/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy b/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy index 03f1c92..297920c 100644 --- a/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy +++ b/plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy @@ -282,9 +282,12 @@ class FloatGridExecutor extends AbstractGridExecutor { result[FloatConf.NF_JOB_ID] = floatJobs.getNfJobID(task.id) result[FloatConf.NF_SESSION_ID] = "uuid-${session.uniqueId}".toString() result[FloatConf.NF_TASK_NAME] = task.name - result[FloatConf.NF_INPUT_SIZE] = getInputFileSize(task).toString() - if (task.processor.name) { - result[FloatConf.NF_PROCESS_NAME] = task.processor.name + result[FloatConf.FLOAT_INPUT_SIZE] = getInputFileSize(task).toString() + + final processName = task.processor.name + if (processName) { + result[FloatConf.NF_PROCESS_NAME] = processName + result[FloatConf.FLOAT_JOB_KIND] = getJobKind(processName) } if (session.runName) { result[FloatConf.NF_RUN_NAME] = session.runName @@ -295,6 +298,14 @@ class FloatGridExecutor extends AbstractGridExecutor { return result } + private String getJobKind(String processName) { + def fsPrefix = workDir.getScheme() + if (isFusionEnabled()) { + fsPrefix += 'fu' + } + return fsPrefix + '-' + processName + } + @Override List getSubmitCommandLine(TaskRun task, Path scriptFile) { return getSubmitCommandLine(new FloatTaskHandler(task, this), scriptFile) diff --git a/plugins/nf-float/src/resources/META-INF/MANIFEST.MF b/plugins/nf-float/src/resources/META-INF/MANIFEST.MF index 3b65bea..0f010a9 100644 --- a/plugins/nf-float/src/resources/META-INF/MANIFEST.MF +++ b/plugins/nf-float/src/resources/META-INF/MANIFEST.MF @@ -1,6 +1,6 @@ Manifest-Version: 1.0 Plugin-Class: com.memverge.nextflow.FloatPlugin Plugin-Id: nf-float -Plugin-Version: 0.3.2 +Plugin-Version: 0.4.0 Plugin-Provider: MemVerge Corp. Plugin-Requires: >=23.04.0 diff --git a/plugins/nf-float/src/test/com/memverge/nextflow/FloatBaseTest.groovy b/plugins/nf-float/src/test/com/memverge/nextflow/FloatBaseTest.groovy index 2f84d04..289e7c8 100644 --- a/plugins/nf-float/src/test/com/memverge/nextflow/FloatBaseTest.groovy +++ b/plugins/nf-float/src/test/com/memverge/nextflow/FloatBaseTest.groovy @@ -108,13 +108,13 @@ class FloatBaseTest extends BaseTest { 'submit', '--dataVolume', param.nfs ?: nfs + ':' + workDir, '--image', param.image ?: image, - '--cpu', realCpu + ':' + realCpu * 2, - '--mem', realMem + ':' + realMem * 2, + '--cpu', realCpu + ':' + realCpu * FloatConf.DFT_MAX_CPU_FACTOR, + '--mem', realMem + ':' + realMem * FloatConf.DFT_MAX_MEM_FACTOR, '--job', script, '--customTag', jobID(taskID), '--customTag', "${FloatConf.NF_SESSION_ID}:uuid-$uuid", '--customTag', "${FloatConf.NF_TASK_NAME}:foo-$taskIndex", - '--customTag', "${FloatConf.NF_INPUT_SIZE}:0", + '--customTag', "${FloatConf.FLOAT_INPUT_SIZE}:0", '--customTag', "${FloatConf.NF_RUN_NAME}:test-run"] } }