From 6d5e1b8c0d61b02f23bd4c3ecf2299fc5a73609c Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:24:38 -0800 Subject: [PATCH] [Backport 2.x] Include previous node substitutions in nested lists and maps (#373) Include previous node substitutions in nested lists and maps (#371) (cherry picked from commit 27991070134f1895941e92ec78b3e1724f90d9ed) Signed-off-by: Daniel Widdis Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../flowframework/util/ParseUtils.java | 34 +++++++++++++++---- .../flowframework/util/ParseUtilsTests.java | 23 +++++++++++-- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/opensearch/flowframework/util/ParseUtils.java b/src/main/java/org/opensearch/flowframework/util/ParseUtils.java index 88caf8f5a..b452a33a9 100644 --- a/src/main/java/org/opensearch/flowframework/util/ParseUtils.java +++ b/src/main/java/org/opensearch/flowframework/util/ParseUtils.java @@ -30,12 +30,14 @@ import java.time.Instant; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; import static org.opensearch.flowframework.common.CommonValue.PARAMETERS_FIELD; @@ -277,15 +279,20 @@ public static Map getInputsFromPreviousSteps( value = matchedValue.get(); } } - // Check for substitution if (value != null) { - Matcher m = SUBSTITUTION_PATTERN.matcher(value.toString()); - if (m.matches()) { - WorkflowData data = outputs.get(m.group(1)); - if (data != null && data.getContent().containsKey(m.group(2))) { - value = data.getContent().get(m.group(2)); - } + // Check for any substitution(s) in value, list, or map + if (value instanceof Map) { + @SuppressWarnings("unchecked") + Map valueMap = (Map) value; + value = valueMap.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> conditionallySubstitute(e.getValue(), outputs))); + } else if (value instanceof List) { + value = ((List) value).stream().map(v -> conditionallySubstitute(v, outputs)).collect(Collectors.toList()); + } else { + value = conditionallySubstitute(value, outputs); } + // Add value to inputs and mark that a required key was present inputs.put(key, value); requiredKeys.remove(key); } @@ -306,4 +313,17 @@ public static Map getInputsFromPreviousSteps( // Finally return the map return inputs; } + + private static Object conditionallySubstitute(Object value, Map outputs) { + if (value instanceof String) { + Matcher m = SUBSTITUTION_PATTERN.matcher((String) value); + if (m.matches()) { + WorkflowData data = outputs.get(m.group(1)); + if (data != null && data.getContent().containsKey(m.group(2))) { + return data.getContent().get(m.group(2)); + } + } + } + return value; + } } diff --git a/src/test/java/org/opensearch/flowframework/util/ParseUtilsTests.java b/src/test/java/org/opensearch/flowframework/util/ParseUtilsTests.java index 02222b9aa..1ed556445 100644 --- a/src/test/java/org/opensearch/flowframework/util/ParseUtilsTests.java +++ b/src/test/java/org/opensearch/flowframework/util/ParseUtilsTests.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; import java.util.Set; @@ -72,7 +73,13 @@ public void testBuildAndParseStringToStringMap() throws IOException { public void testGetInputsFromPreviousSteps() { WorkflowData currentNodeInputs = new WorkflowData( - Map.ofEntries(Map.entry("content1", 1), Map.entry("param1", 2), Map.entry("content3", "${{step1.output1}}")), + Map.ofEntries( + Map.entry("content1", 1), + Map.entry("param1", 2), + Map.entry("content3", "${{step1.output1}}"), + Map.entry("nestedMap", Map.of("content4", "${{step3.output3}}")), + Map.entry("nestedList", List.of("${{step4.output4}}")) + ), Map.of("param1", "value1"), "workflowId", "nodeId" @@ -86,11 +93,13 @@ public void testGetInputsFromPreviousSteps() { "step1" ) ), - Map.entry("step2", new WorkflowData(Map.of("output2", "step2outputvalue2"), "workflowId", "step2")) + Map.entry("step2", new WorkflowData(Map.of("output2", "step2outputvalue2"), "workflowId", "step2")), + Map.entry("step3", new WorkflowData(Map.of("output3", "step3outputvalue3"), "workflowId", "step3")), + Map.entry("step4", new WorkflowData(Map.of("output4", "step4outputvalue4"), "workflowId", "step4")) ); Map previousNodeInputs = Map.of("step2", "output2"); Set requiredKeys = Set.of("param1", "content1"); - Set optionalKeys = Set.of("output1", "output2", "content3", "no-output"); + Set optionalKeys = Set.of("output1", "output2", "content3", "nestedMap", "nestedList", "no-output"); Map inputs = ParseUtils.getInputsFromPreviousSteps( requiredKeys, @@ -104,7 +113,15 @@ public void testGetInputsFromPreviousSteps() { assertEquals(1, inputs.get("content1")); assertEquals("outputvalue1", inputs.get("output1")); assertEquals("step2outputvalue2", inputs.get("output2")); + + // Substitutions assertEquals("outputvalue1", inputs.get("content3")); + @SuppressWarnings("unchecked") + Map nestedMap = (Map) inputs.get("nestedMap"); + assertEquals("step3outputvalue3", nestedMap.get("content4")); + @SuppressWarnings("unchecked") + List nestedList = (List) inputs.get("nestedList"); + assertEquals(List.of("step4outputvalue4"), nestedList); assertNull(inputs.get("no-output")); Set missingRequiredKeys = Set.of("not-here");