[FLOAT-4502] Avoid using plain credential.

Avoid using plain credential in the command line which can be exposed in
logs or GUI.  Set the credentials in float secret and retrieve them
in the user script.

plugins/nf-float/src/main/com/memverge/nextflow/FloatConf.groovy

@@ -230,11 +230,19 @@ class FloatConf {
         return ret
     }
 
+    List<List<String>> getCliPrefixes() {
+        return this.addresses.collect{ getCliPrefixOf(it)}
+    }
+
     List<String> getCliPrefix(TaskId id) {
         if (id == null) {
             id = new TaskId(0)
         }
         final address = addresses[id.intValue() % (addresses.size())]
+        return getCliPrefixOf(address)
+    }
+
+    private List<String> getCliPrefixOf(String address) {
         validate()
 
         def bin = FloatBin.get(address)

plugins/nf-float/src/main/com/memverge/nextflow/FloatGridExecutor.groovy

@@ -62,6 +62,35 @@ class FloatGridExecutor extends AbstractGridExecutor {
         super.register()
         uploadBinDir()
         syncFloatBin()
+        setSecrets()
+    }
+
+    void shutdown() {
+        deleteSecrets()
+        super.shutdown()
+    }
+
+    private void setSecrets() {
+        floatConf.getCliPrefixes().each { prefix ->
+            for (def entry : floatConf.s3cred.getRunSecretsMap(getRunName())) {
+                def cmd = prefix.collect(it -> it)
+                cmd << "secret" << "set" << entry.key << entry.value
+                def res = Global.execute(cmd)
+                log.info "[FLOAT] set secret ${entry.key}, $res"
+            }
+        }
+
+    }
+
+    private void deleteSecrets() {
+        floatConf.getCliPrefixes().each { prefix ->
+            for (def entry : floatConf.s3cred.getRunSecretsMap(getRunName())) {
+                def cmd = prefix.collect(it -> it)
+                cmd << "secret" << "unset" << entry.key
+                def res = Global.execute(cmd)
+                log.info "[FLOAT] delete secret ${entry.key}, $res"
+            }
+        }
     }
 
     @Override
@@ -96,6 +125,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
 
         if (floatConf.s3cred.isValid()) {
             // if we have s3 credential, make sure aws cli is available in path
+            result += floatConf.s3cred.getExportS3CredScript(getRunName())
             result += "export PATH=\$PATH:/opt/aws/dist\n"
             result += "export LD_LIBRARY_PATH=\$LIBRARY_PATH:/opt/aws/dist\n"
         }
@@ -270,7 +300,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
         def ret = isFusionEnabled()
                 ? handler.fusionLauncher().fusionEnv()
                 : [:]
-        return floatConf.s3cred.updateEnvMap(ret)
+        return ret
     }
 
     String getRunName() {
@@ -413,7 +443,7 @@ class FloatGridExecutor extends AbstractGridExecutor {
             size = Math.max(size, FUSION_MIN_VOL_SIZE)
         }
         if (task.scratch) {
-            double inputSizeGB =  (double)(getInputFileSize(task)) / 1024 / 1024 / 1024
+            double inputSizeGB = (double) (getInputFileSize(task)) / 1024 / 1024 / 1024
             long minDiskSizeBasedOnInput = Math.round(inputSizeGB * DISK_INPUT_FACTOR)
             size = Math.max(size, minDiskSizeBasedOnInput)
         }

plugins/nf-float/src/main/com/memverge/nextflow/Global.groovy

@@ -64,6 +64,31 @@ class AWSCred {
         return opts.join(",")
     }
 
+    private static String getS3UserSecretKey(String runName) {
+        return "AWS_ACCESS_KEY_ID_${runName.toUpperCase()}"
+    }
+
+    private static String getS3PassSecretKey(String runName) {
+        return "AWS_SECRET_ACCESS_KEY_${runName.toUpperCase()}"
+    }
+
+    private static String getS3tokenSecretKey(String runName) {
+        return "AWS_SESSION_TOKEN_${runName.toUpperCase()}"
+    }
+
+    Map<String, String> getRunSecretsMap(String runName) {
+        def ret = new HashMap<String, String>()
+        if (isValid()) {
+            ret.put(getS3UserSecretKey(runName), accessKey)
+            ret.put(getS3PassSecretKey(runName), secretKey)
+            if (token) {
+                ret.put(getS3tokenSecretKey(runName), token)
+            }
+        }
+        return ret
+    }
+
+
     def updateMap(Map map) {
         if (!isValid()) {
             return map
@@ -79,21 +104,37 @@ class AWSCred {
         return map
     }
 
-    Map<String, String> updateEnvMap(Map map) {
+    Map<String, String> updateEnvMap(Map map, String runName) {
         if (!isValid()) {
             return map
         }
         if (hasAllKeyCaseInsensitive(map, ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"])) {
             return map
         }
-        map.put("AWS_ACCESS_KEY_ID", accessKey)
-        map.put("AWS_SECRET_ACCESS_KEY", secretKey)
+        map.put("AWS_ACCESS_KEY_ID", "{secret:${getS3UserSecretKey(runName)}}")
+        map.put("AWS_SECRET_ACCESS_KEY", "{secret:${getS3PassSecretKey(runName)}}")
         if (token) {
-            map.put("AWS_SESSION_TOKEN", token)
+            map.put("AWS_SESSION_TOKEN", "{secret:${getS3tokenSecretKey(runName)}}")
         }
         return map
     }
 
+    private static String getExportCmd(String secretName) {
+        return "\$(/opt/memverge/bin/float secret get $secretName -a \$FLOAT_ADDR)"
+    }
+
+    String getExportS3CredScript(String runName) {
+        def ret = ""
+        if (isValid()) {
+            ret = "export AWS_ACCESS_KEY_ID=${getExportCmd(getS3UserSecretKey(runName))}\n"
+            ret += "export AWS_SECRET_ACCESS_KEY=${getExportCmd(getS3PassSecretKey(runName))}\n"
+            if (token) {
+                ret += "export AWS_SESSION_TOKEN=${getExportCmd(getS3tokenSecretKey(runName))}\n"
+            }
+        }
+        return ret
+    }
+
     List<String> getOpts() {
         def ret = ["accesskey=${accessKey}", "secret=${secretKey}"]
         if (token) {
@@ -120,7 +161,6 @@ class AWSCred {
 }
 
 
-
 /**
  * The Global class is copied from the same class in NextFlow.
  * We copy some implementation because they are updated in different releases.

plugins/nf-float/src/test/com/memverge/nextflow/GlobalTest.groovy

@@ -138,13 +138,13 @@ class GlobalTest extends FloatBaseTest {
 
         when:
         def res = Global.getAwsCredentials(config)
-        def envMap = res.updateEnvMap([:])
+        def envMap = res.updateEnvMap([:], "banana")
 
         then:
         res.isValid() == true
-        envMap['AWS_ACCESS_KEY_ID'] == "A"
-        envMap['AWS_SECRET_ACCESS_KEY'] == "B"
-        envMap['AWS_SESSION_TOKEN'] == "C"
+        envMap['AWS_ACCESS_KEY_ID'] == "{secret:AWS_ACCESS_KEY_ID_BANANA}"
+        envMap['AWS_SECRET_ACCESS_KEY'] == "{secret:AWS_SECRET_ACCESS_KEY_BANANA}"
+        envMap['AWS_SESSION_TOKEN'] == "{secret:AWS_SESSION_TOKEN_BANANA}"
 
         cleanup:
         clearAwsEnvs()