From 68cdf97dc186be3e24f9d938d12d4191f7043146 Mon Sep 17 00:00:00 2001 From: DanCardin Date: Thu, 18 Jan 2024 13:57:11 -0500 Subject: [PATCH 1/3] fix: Handle multiline strings in yaml serialization. Signed-off-by: DanCardin --- src/hera/_yaml.py | 31 +++++++++++++++++++++++++++++++ src/hera/workflows/workflow.py | 15 +-------------- tests/test_yaml.py | 26 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 src/hera/_yaml.py create mode 100644 tests/test_yaml.py diff --git a/src/hera/_yaml.py b/src/hera/_yaml.py new file mode 100644 index 000000000..1583319c4 --- /dev/null +++ b/src/hera/_yaml.py @@ -0,0 +1,31 @@ +from types import ModuleType +from typing import Optional + +_yaml: Optional[ModuleType] = None +try: + import yaml + + _yaml = yaml +except ImportError: + _yaml = None +else: + + def str_presenter(dumper, data): + """Configures yaml for dumping multiline strings. + + Ref: https://github.com/yaml/pyyaml/issues/240 + """ + if data.count("\n") > 0: # check for multiline string + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + _yaml.add_representer(str, str_presenter) + _yaml.representer.SafeRepresenter.add_representer(str, str_presenter) + + +def dump(*args, default_flow_style: bool = False, sort_keys: bool = False, **kwargs) -> str: + """Builds the Workflow as an Argo schema Workflow object and returns it as yaml string.""" + if not _yaml: + raise ImportError("`PyYAML` is not installed. Install `hera[yaml]` to bring in the extra dependency") + + return _yaml.dump(*args, default_flow_style=default_flow_style, sort_keys=sort_keys, **kwargs) diff --git a/src/hera/workflows/workflow.py b/src/hera/workflows/workflow.py index 1fa1a395f..bb6c699ab 100644 --- a/src/hera/workflows/workflow.py +++ b/src/hera/workflows/workflow.py @@ -5,7 +5,6 @@ """ import time from pathlib import Path -from types import ModuleType from typing import Any, Dict, List, Optional, Type, Union try: @@ -13,6 +12,7 @@ except ImportError: from typing_extensions import Annotated, get_args # type: ignore +from hera import _yaml from hera.shared import global_config from hera.shared._pydantic import BaseModel, validator from hera.workflows._mixins import ( @@ -64,14 +64,6 @@ from hera.workflows.service import WorkflowsService from hera.workflows.workflow_status import WorkflowStatus -_yaml: Optional[ModuleType] = None -try: - import yaml - - _yaml = yaml -except ImportError: - _yaml = None - ImagePullSecretsT = Optional[Union[LocalObjectReference, List[LocalObjectReference], str, List[str]]] NAME_LIMIT = 63 @@ -358,11 +350,6 @@ def __eq__(self, other) -> bool: def to_yaml(self, *args, **kwargs) -> str: """Builds the Workflow as an Argo schema Workflow object and returns it as yaml string.""" - if not _yaml: - raise ImportError("`PyYAML` is not installed. Install `hera[yaml]` to bring in the extra dependency") - # Set some default options if not provided by the user - kwargs.setdefault("default_flow_style", False) - kwargs.setdefault("sort_keys", False) return _yaml.dump(self.to_dict(), *args, **kwargs) def create(self, wait: bool = False, poll_interval: int = 5) -> TWorkflow: diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 000000000..3caa9a86b --- /dev/null +++ b/tests/test_yaml.py @@ -0,0 +1,26 @@ +from textwrap import dedent +from unittest.mock import patch + +import pytest + +from hera import _yaml + + +def test_dump_multiline_string(): + result = _yaml.dump({"args": '\'payload={\n "text": "example"\n}\''}) + assert result == dedent( + """\ + args: |- + 'payload={ + "text": "example" + }' + """ + ) + + +def test_yaml_missing(): + with patch("hera._yaml._yaml", new=None): + with pytest.raises(ImportError) as e: + _yaml.dump({}) + + assert "Install `hera[yaml]`" in str(e.value) From 7b29e30edd09a5e32a3f3a6494f8b1883c7cc03e Mon Sep 17 00:00:00 2001 From: DanCardin Date: Thu, 18 Jan 2024 14:39:21 -0500 Subject: [PATCH 2/3] docs: Update docs and examples with regenerated yaml. Signed-off-by: DanCardin --- docs/examples/workflows/artifacts/artifact.md | 16 ++- .../artifacts/artifact_with_fanout.md | 41 ++++---- docs/examples/workflows/daemon.md | 49 +++++----- .../workflows/dags/any_success_all_fail.md | 46 +++++---- docs/examples/workflows/dags/callable_dag.md | 15 +-- docs/examples/workflows/dags/complex_deps.md | 14 ++- docs/examples/workflows/dags/conditional.md | 27 ++--- .../dags/dag_conditional_on_task_status.md | 25 ++--- .../dags/dag_conditional_parameters.md | 6 +- .../dag_diamond_with_callable_decorators.md | 12 +-- .../dags/dag_diamond_with_callable_script.md | 12 +-- .../dag_with_script_output_param_passing.md | 23 +++-- .../dags/dag_with_script_param_passing.md | 23 ++--- docs/examples/workflows/dags/on_exit.md | 23 ++--- docs/examples/workflows/dynamic_resources.md | 60 +++++------- docs/examples/workflows/dynamic_volumes.md | 9 +- docs/examples/workflows/global_config.md | 8 +- docs/examples/workflows/hello_world.md | 15 +-- .../workflows/loops/dynamic_fanout.md | 25 ++--- .../loops/dynamic_fanout_extra_kwargs.md | 40 +++----- .../workflows/loops/dynamic_fanout_fanin.md | 41 ++++---- .../loops/dynamic_fanout_json_payload.md | 40 +++----- .../workflows/loops/script_loops_maps.md | 21 ++-- .../examples/workflows/loops/with_sequence.md | 23 ++--- docs/examples/workflows/multi_env.md | 15 +-- docs/examples/workflows/resource_flags.md | 22 ++++- docs/examples/workflows/scripts/coinflip.md | 28 ++---- .../scripts/default_param_overwrite.md | 23 ++--- .../scripts/script_artifact_passing.md | 17 +++- .../workflows/scripts/script_auto_infer.md | 21 +++- .../workflows/scripts/script_variations.md | 30 ++---- .../scripts/script_with_default_params.md | 29 ++---- .../scripts/script_with_image_pull_policy.md | 8 +- .../scripts/script_with_resources.md | 8 +- .../workflows/steps/callable_steps.md | 15 +-- .../upstream/artifact_gc_workflow.md | 6 +- .../upstream/artifact_path_placeholders.md | 9 +- .../workflows/upstream/ci_workflowtemplate.md | 34 +++---- docs/examples/workflows/upstream/coinflip.md | 9 +- .../workflows/upstream/coinflip_recursive.md | 9 +- .../workflows/upstream/colored_logs.md | 13 +-- .../upstream/conditional_artifacts.md | 14 ++- ...r_set_template__outputs_result_workflow.md | 5 +- .../upstream/dag_conditional_parameters.md | 6 +- .../upstream/exit_handler_with_artifacts.md | 4 +- .../workflows/upstream/input_artifact_raw.md | 7 +- .../upstream/k8s_json_patch_workflow.md | 5 +- .../upstream/k8s_resource_log_selector.md | 45 +++++---- .../upstream/k8s_set_owner_reference.md | 9 +- .../loops_arbitrary_sequential_steps.md | 14 ++- .../workflows/upstream/loops_param_result.md | 7 +- .../workflows/upstream/parallelism_nested.md | 10 +- .../upstream/pod_spec_patch_wf_tmpl.md | 8 +- .../upstream/resource_delete_with_flags.md | 11 ++- .../workflows/upstream/retry_script.md | 8 +- .../upstream/workflow_of_workflows.md | 26 +++-- docs/examples/workflows/use-cases/dask.md | 39 ++------ .../workflows/use-cases/map_reduce.md | 57 ++++++++--- .../use-cases/spacy_inference_pipeline.md | 98 ++++++++++++------- docs/examples/workflows/use-cases/spark.md | 49 +++------- .../use-cases/workflow_of_workflows.md | 38 +++++-- docs/examples/workflows/user_container.md | 8 +- docs/examples/workflows/volume_mounts.md | 13 +-- docs/examples/workflows/volume_mounts_nfs.md | 13 +-- docs/examples/workflows/volume_mounts_wt.md | 13 +-- docs/examples/workflows/workflow_on_exit.md | 15 +-- .../workflows/workflow_with_global_params.md | 15 +-- .../artifacts/artifact-with-fanout.yaml | 41 ++++---- examples/workflows/artifacts/artifact.yaml | 16 ++- examples/workflows/daemon.yaml | 49 +++++----- .../workflows/dags/any-success-all-fail.yaml | 46 +++++---- examples/workflows/dags/callable-dag.yaml | 15 +-- examples/workflows/dags/complex-deps.yaml | 14 ++- examples/workflows/dags/conditional.yaml | 27 ++--- .../dags/dag-conditional-on-task-status.yaml | 25 ++--- .../dags/dag-conditional-parameters.yaml | 6 +- .../dag-diamond-with-callable-decorators.yaml | 12 +-- .../dag-diamond-with-callable-script.yaml | 12 +-- .../dag-with-script-output-param-passing.yaml | 23 +++-- .../dags/dag-with-script-param-passing.yaml | 23 ++--- examples/workflows/dags/on-exit.yaml | 23 ++--- examples/workflows/dynamic-resources.yaml | 60 +++++------- examples/workflows/dynamic-volumes.yaml | 9 +- examples/workflows/global-config.yaml | 8 +- examples/workflows/hello-world.yaml | 15 +-- .../loops/dynamic-fanout-extra-kwargs.yaml | 40 +++----- .../workflows/loops/dynamic-fanout-fanin.yaml | 41 ++++---- .../loops/dynamic-fanout-json-payload.yaml | 40 +++----- examples/workflows/loops/dynamic-fanout.yaml | 25 ++--- .../workflows/loops/script-loops-maps.yaml | 21 ++-- examples/workflows/loops/with-sequence.yaml | 23 ++--- examples/workflows/multi-env.yaml | 15 +-- examples/workflows/resource-flags.yaml | 22 ++++- examples/workflows/scripts/coinflip.yaml | 28 ++---- .../scripts/default-param-overwrite.yaml | 23 ++--- .../scripts/script-artifact-passing.yaml | 17 +++- .../workflows/scripts/script-auto-infer.yaml | 21 +++- .../workflows/scripts/script-variations.yaml | 30 ++---- .../scripts/script-with-default-params.yaml | 27 ++--- .../script-with-image-pull-policy.yaml | 8 +- .../scripts/script-with-resources.yaml | 8 +- examples/workflows/steps/callable-steps.yaml | 15 +-- .../upstream/artifact-gc-workflow.yaml | 6 +- .../upstream/artifact-path-placeholders.yaml | 9 +- .../upstream/ci-workflowtemplate.yaml | 34 +++---- .../upstream/coinflip-recursive.yaml | 9 +- examples/workflows/upstream/coinflip.yaml | 9 +- examples/workflows/upstream/colored-logs.yaml | 13 +-- .../upstream/conditional-artifacts.yaml | 14 ++- ...set-template--outputs-result-workflow.yaml | 5 +- .../upstream/dag-conditional-parameters.yaml | 6 +- .../upstream/exit-handler-with-artifacts.yaml | 4 +- .../upstream/input-artifact-raw.yaml | 7 +- .../upstream/k8s-json-patch-workflow.yaml | 5 +- .../upstream/k8s-resource-log-selector.yaml | 45 +++++---- .../upstream/k8s-set-owner-reference.yaml | 9 +- .../loops-arbitrary-sequential-steps.yaml | 14 ++- .../upstream/loops-param-result.yaml | 7 +- .../upstream/parallelism-nested.yaml | 10 +- .../upstream/pod-spec-patch-wf-tmpl.yaml | 8 +- .../upstream/resource-delete-with-flags.yaml | 11 ++- examples/workflows/upstream/retry-script.yaml | 8 +- .../upstream/workflow-of-workflows.yaml | 26 +++-- examples/workflows/use_cases/dask.yaml | 39 ++------ examples/workflows/use_cases/map-reduce.yaml | 57 ++++++++--- .../use_cases/spacy-inference-pipeline.yaml | 98 ++++++++++++------- examples/workflows/use_cases/spark.yaml | 49 +++------- .../use_cases/workflow-of-workflows.yaml | 38 +++++-- examples/workflows/user-container.yaml | 8 +- examples/workflows/volume-mounts-nfs.yaml | 13 +-- examples/workflows/volume-mounts-wt.yaml | 13 +-- examples/workflows/volume-mounts.yaml | 13 +-- examples/workflows/workflow-on-exit.yaml | 15 +-- .../workflow-with-global-params.yaml | 15 +-- 134 files changed, 1385 insertions(+), 1507 deletions(-) diff --git a/docs/examples/workflows/artifacts/artifact.md b/docs/examples/workflows/artifacts/artifact.md index d63791beb..a71105612 100644 --- a/docs/examples/workflows/artifacts/artifact.md +++ b/docs/examples/workflows/artifacts/artifact.md @@ -65,8 +65,12 @@ task, consumer, takes this artifact, places it at its own `/file` path, and prin command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/file',\ - \ 'w+') as f:\n f.write('Hello, world!')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/file', 'w+') as f: + f.write('Hello, world!') - inputs: artifacts: - name: in-art @@ -76,7 +80,11 @@ task, consumer, takes this artifact, places it at its own `/file` path, and prin command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/file',\ - \ 'r') as f:\n print(f.readlines())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/file', 'r') as f: + print(f.readlines()) ``` diff --git a/docs/examples/workflows/artifacts/artifact_with_fanout.md b/docs/examples/workflows/artifacts/artifact_with_fanout.md index a3d12b884..bd25caa2f 100644 --- a/docs/examples/workflows/artifacts/artifact_with_fanout.md +++ b/docs/examples/workflows/artifacts/artifact_with_fanout.md @@ -86,9 +86,14 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - with open('/tmp/file', 'w+') as f:\n for i in range(10):\n f.write(json.dumps(i)\ - \ + '\\n')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + with open('/tmp/file', 'w+') as f: + for i in range(10): + f.write(json.dumps(i) + '\n') - inputs: artifacts: - name: in-art @@ -98,10 +103,17 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import sys\nindices = []\nwith open('/tmp/file', 'r') as f:\n for line\ - \ in f.readlines():\n indices.append(line.strip())\njson.dump(indices,\ - \ sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import sys + indices = [] + with open('/tmp/file', 'r') as f: + for line in f.readlines(): + indices.append(line.strip()) + json.dump(indices, sys.stdout) - inputs: parameters: - name: i @@ -110,19 +122,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: i = json.loads(r'''{{inputs.parameters.i}}''') + except: i = r'''{{inputs.parameters.i}}''' - try: i = json.loads(r''''''{{inputs.parameters.i}}'''''') - - except: i = r''''''{{inputs.parameters.i}}'''''' - - - print(i)' + print(i) ``` diff --git a/docs/examples/workflows/daemon.md b/docs/examples/workflows/daemon.md index 0c5b91d4b..7be5190b2 100644 --- a/docs/examples/workflows/daemon.md +++ b/docs/examples/workflows/daemon.md @@ -77,12 +77,21 @@ http requests to the server. command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nfrom http.server\ - \ import BaseHTTPRequestHandler, HTTPServer\n\nclass MyServer(BaseHTTPRequestHandler):\n\ - \n def do_GET(self):\n self.send_response(200)\n self.send_header('Content-type',\ - \ 'application/json')\n self.end_headers()\n self.wfile.write(bytes(\"\ - {'name':'John'}\", 'utf-8'))\nwebServer = HTTPServer(('0.0.0.0', 8080), MyServer)\n\ - webServer.serve_forever()" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + from http.server import BaseHTTPRequestHandler, HTTPServer + + class MyServer(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(bytes("{'name':'John'}", 'utf-8')) + webServer = HTTPServer(('0.0.0.0', 8080), MyServer) + webServer.serve_forever() - inputs: parameters: - name: ip @@ -91,33 +100,21 @@ http requests to the server. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: ip = json.loads(r''''''{{inputs.parameters.ip}}'''''') - - except: ip = r''''''{{inputs.parameters.ip}}'''''' - + try: ip = json.loads(r'''{{inputs.parameters.ip}}''') + except: ip = r'''{{inputs.parameters.ip}}''' import http.client - import os - print(os.environ) - - server_ip = ip.replace(''"'', '''') - - connection = http.client.HTTPConnection(''{server_ip}:8080''.format(server_ip=server_ip)) - - connection.request(''GET'', ''/'') - + server_ip = ip.replace('"', '') + connection = http.client.HTTPConnection('{server_ip}:8080'.format(server_ip=server_ip)) + connection.request('GET', '/') response = connection.getresponse() - - print(response.read())' + print(response.read()) ``` diff --git a/docs/examples/workflows/dags/any_success_all_fail.md b/docs/examples/workflows/dags/any_success_all_fail.md index 1f41fb539..12b5a7052 100644 --- a/docs/examples/workflows/dags/any_success_all_fail.md +++ b/docs/examples/workflows/dags/any_success_all_fail.md @@ -84,10 +84,18 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: a = json.loads(r'''{{inputs.parameters.a}}''')\nexcept: a = r'''{{inputs.parameters.a}}'''\n\ - \nimport random\nrandom.seed(a)\nif random.random() < 0.5:\n raise Exception('Oh,\ - \ no!')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + + import random + random.seed(a) + if random.random() < 0.5: + raise Exception('Oh, no!') - inputs: parameters: - name: a @@ -96,20 +104,15 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - raise Exception(a)' + raise Exception(a) - inputs: parameters: - name: a @@ -118,19 +121,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) ``` diff --git a/docs/examples/workflows/dags/callable_dag.md b/docs/examples/workflows/dags/callable_dag.md index 7ebeb00fe..de3684415 100644 --- a/docs/examples/workflows/dags/callable_dag.md +++ b/docs/examples/workflows/dags/callable_dag.md @@ -66,20 +66,15 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: name = json.loads(r'''{{inputs.parameters.name}}''') + except: name = r'''{{inputs.parameters.name}}''' - try: name = json.loads(r''''''{{inputs.parameters.name}}'''''') - - except: name = r''''''{{inputs.parameters.name}}'''''' - - - print(''Hello, {name}!''.format(name=name))' + print('Hello, {name}!'.format(name=name)) - dag: tasks: - arguments: diff --git a/docs/examples/workflows/dags/complex_deps.md b/docs/examples/workflows/dags/complex_deps.md index 57a599778..4934622b4 100644 --- a/docs/examples/workflows/dags/complex_deps.md +++ b/docs/examples/workflows/dags/complex_deps.md @@ -76,8 +76,16 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: p = json.loads(r'''{{inputs.parameters.p}}''')\nexcept: p = r'''{{inputs.parameters.p}}'''\n\ - \nif p < 0.5:\n raise Exception(p)\nprint(42)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: p = json.loads(r'''{{inputs.parameters.p}}''') + except: p = r'''{{inputs.parameters.p}}''' + + if p < 0.5: + raise Exception(p) + print(42) ``` diff --git a/docs/examples/workflows/dags/conditional.md b/docs/examples/workflows/dags/conditional.md index b821d3289..2d4281ef6 100644 --- a/docs/examples/workflows/dags/conditional.md +++ b/docs/examples/workflows/dags/conditional.md @@ -67,31 +67,34 @@ This example showcases conditional execution on success, failure, and error command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport random\n\ - p = random.random()\nif p <= 0.5:\n raise Exception('failure')\nprint('success')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import random + p = random.random() + if p <= 0.5: + raise Exception('failure') + print('success') - name: success script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''success'')' + print('success') - name: failure script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''failure'')' + print('failure') ``` diff --git a/docs/examples/workflows/dags/dag_conditional_on_task_status.md b/docs/examples/workflows/dags/dag_conditional_on_task_status.md index 73723f073..1597f140c 100644 --- a/docs/examples/workflows/dags/dag_conditional_on_task_status.md +++ b/docs/examples/workflows/dags/dag_conditional_on_task_status.md @@ -63,31 +63,32 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport random\n\ - if random.randint(0, 1) == 0:\n raise ValueError" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import random + if random.randint(0, 1) == 0: + raise ValueError - name: when-failed script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''It was a failure'')' + print('It was a failure') - name: when-succeeded script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''It was a success'')' + print('It was a success') ``` diff --git a/docs/examples/workflows/dags/dag_conditional_parameters.md b/docs/examples/workflows/dags/dag_conditional_parameters.md index b08e87dad..c6bbc3a40 100644 --- a/docs/examples/workflows/dags/dag_conditional_parameters.md +++ b/docs/examples/workflows/dags/dag_conditional_parameters.md @@ -77,9 +77,9 @@ command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - name: heads script: command: diff --git a/docs/examples/workflows/dags/dag_diamond_with_callable_decorators.md b/docs/examples/workflows/dags/dag_diamond_with_callable_decorators.md index 0bb2c0d02..9652696c2 100644 --- a/docs/examples/workflows/dags/dag_diamond_with_callable_decorators.md +++ b/docs/examples/workflows/dags/dag_diamond_with_callable_decorators.md @@ -77,13 +77,11 @@ command: - python image: python:alpine3.6 - source: 'import json + source: |- + import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) ``` diff --git a/docs/examples/workflows/dags/dag_diamond_with_callable_script.md b/docs/examples/workflows/dags/dag_diamond_with_callable_script.md index 3d39712e4..89b4944c5 100644 --- a/docs/examples/workflows/dags/dag_diamond_with_callable_script.md +++ b/docs/examples/workflows/dags/dag_diamond_with_callable_script.md @@ -63,14 +63,12 @@ command: - python image: python:alpine3.6 - source: 'import json + source: |- + import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) - dag: tasks: - arguments: diff --git a/docs/examples/workflows/dags/dag_with_script_output_param_passing.md b/docs/examples/workflows/dags/dag_with_script_output_param_passing.md index b12b2c7c5..7237fa00e 100644 --- a/docs/examples/workflows/dags/dag_with_script_output_param_passing.md +++ b/docs/examples/workflows/dags/dag_with_script_output_param_passing.md @@ -68,8 +68,12 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/test',\ - \ 'w') as f_out:\n f_out.write('test')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/test', 'w') as f_out: + f_out.write('test') - inputs: parameters: - name: a @@ -78,19 +82,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) ``` diff --git a/docs/examples/workflows/dags/dag_with_script_param_passing.md b/docs/examples/workflows/dags/dag_with_script_param_passing.md index f0f9ba5df..5f52b4d9e 100644 --- a/docs/examples/workflows/dags/dag_with_script_param_passing.md +++ b/docs/examples/workflows/dags/dag_with_script_param_passing.md @@ -55,13 +55,11 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(42)' + print(42) - inputs: parameters: - name: a @@ -70,19 +68,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) ``` diff --git a/docs/examples/workflows/dags/on_exit.md b/docs/examples/workflows/dags/on_exit.md index 5ddc9601c..cae26be6d 100644 --- a/docs/examples/workflows/dags/on_exit.md +++ b/docs/examples/workflows/dags/on_exit.md @@ -45,13 +45,11 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Bye Hera'')' + print('Bye Hera') - dag: tasks: - arguments: @@ -77,19 +75,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(''Hello Hera, {s}''.format(s=s))' + print('Hello Hera, {s}'.format(s=s)) ``` diff --git a/docs/examples/workflows/dynamic_resources.md b/docs/examples/workflows/dynamic_resources.md index ac8c8c0ba..f1d81775e 100644 --- a/docs/examples/workflows/dynamic_resources.md +++ b/docs/examples/workflows/dynamic_resources.md @@ -129,11 +129,17 @@ you can compute the resources dynamically based on the amount of data you need t command: - python image: python:3.10 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\n\"\"\"Computes\ - \ the resources necessary by the following job, which could be anything.\"\ - \"\"\nimport json\nimport sys\nresources = []\nfor i in range(1, 4):\n \ - \ resources.append({'cpu': i, 'mem': '{v}Mi'.format(v=i * 100)})\njson.dump(resources,\ - \ sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + """Computes the resources necessary by the following job, which could be anything.""" + import json + import sys + resources = [] + for i in range(1, 4): + resources.append({'cpu': i, 'mem': '{v}Mi'.format(v=i * 100)}) + json.dump(resources, sys.stdout) - inputs: parameters: - name: cpu @@ -146,26 +152,18 @@ you can compute the resources dynamically based on the amount of data you need t command: - python image: python:3.10 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: cpu = json.loads(r''''''{{inputs.parameters.cpu}}'''''') - - except: cpu = r''''''{{inputs.parameters.cpu}}'''''' - - try: mem = json.loads(r''''''{{inputs.parameters.mem}}'''''') - - except: mem = r''''''{{inputs.parameters.mem}}'''''' - + try: cpu = json.loads(r'''{{inputs.parameters.cpu}}''') + except: cpu = r'''{{inputs.parameters.cpu}}''' + try: mem = json.loads(r'''{{inputs.parameters.mem}}''') + except: mem = r'''{{inputs.parameters.mem}}''' """Perform some computation.""" - - print(''received cpu {cpu} and mem {mem}''.format(cpu=cpu, mem=mem))' + print('received cpu {cpu} and mem {mem}'.format(cpu=cpu, mem=mem)) - inputs: parameters: - default: '1' @@ -180,25 +178,17 @@ you can compute the resources dynamically based on the amount of data you need t command: - python image: python:3.10 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: cpu = json.loads(r''''''{{inputs.parameters.cpu}}'''''') - - except: cpu = r''''''{{inputs.parameters.cpu}}'''''' - - try: mem = json.loads(r''''''{{inputs.parameters.mem}}'''''') - - except: mem = r''''''{{inputs.parameters.mem}}'''''' - + try: cpu = json.loads(r'''{{inputs.parameters.cpu}}''') + except: cpu = r'''{{inputs.parameters.cpu}}''' + try: mem = json.loads(r'''{{inputs.parameters.mem}}''') + except: mem = r'''{{inputs.parameters.mem}}''' """Perform some computation.""" - - print(''received cpu {cpu} and mem {mem}''.format(cpu=cpu, mem=mem))' + print('received cpu {cpu} and mem {mem}'.format(cpu=cpu, mem=mem)) ``` diff --git a/docs/examples/workflows/dynamic_volumes.md b/docs/examples/workflows/dynamic_volumes.md index 17e35c2f9..3b2b65776 100644 --- a/docs/examples/workflows/dynamic_volumes.md +++ b/docs/examples/workflows/dynamic_volumes.md @@ -43,15 +43,12 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import subprocess - - print(subprocess.run(''cd && /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(subprocess.run('cd && /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: v diff --git a/docs/examples/workflows/global_config.md b/docs/examples/workflows/global_config.md index 7db61374e..90946f09e 100644 --- a/docs/examples/workflows/global_config.md +++ b/docs/examples/workflows/global_config.md @@ -52,12 +52,10 @@ command: - python3 image: image-say - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''hello'')' + print('hello') ``` diff --git a/docs/examples/workflows/hello_world.md b/docs/examples/workflows/hello_world.md index 52c691304..10d77d5b0 100644 --- a/docs/examples/workflows/hello_world.md +++ b/docs/examples/workflows/hello_world.md @@ -46,19 +46,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(''Hello, {s}!''.format(s=s))' + print('Hello, {s}!'.format(s=s)) ``` diff --git a/docs/examples/workflows/loops/dynamic_fanout.md b/docs/examples/workflows/loops/dynamic_fanout.md index 547b52aea..a39906f44 100644 --- a/docs/examples/workflows/loops/dynamic_fanout.md +++ b/docs/examples/workflows/loops/dynamic_fanout.md @@ -64,17 +64,13 @@ they may need to process. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([i for i in range(10)], sys.stdout)' + json.dump([i for i in range(10)], sys.stdout) - inputs: parameters: - name: value @@ -83,19 +79,14 @@ they may need to process. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: value = json.loads(r'''{{inputs.parameters.value}}''') + except: value = r'''{{inputs.parameters.value}}''' - try: value = json.loads(r''''''{{inputs.parameters.value}}'''''') - - except: value = r''''''{{inputs.parameters.value}}'''''' - - - print(''Received value: {value}!''.format(value=value))' + print('Received value: {value}!'.format(value=value)) ``` diff --git a/docs/examples/workflows/loops/dynamic_fanout_extra_kwargs.md b/docs/examples/workflows/loops/dynamic_fanout_extra_kwargs.md index 66dbe37c1..889bd502e 100644 --- a/docs/examples/workflows/loops/dynamic_fanout_extra_kwargs.md +++ b/docs/examples/workflows/loops/dynamic_fanout_extra_kwargs.md @@ -96,17 +96,13 @@ the job to dictate what the fanout should execute over. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([i for i in range(10)], sys.stdout)' + json.dump([i for i in range(10)], sys.stdout) - inputs: parameters: - name: value @@ -118,28 +114,18 @@ the job to dictate what the fanout should execute over. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: extra_param1 = json.loads(r''''''{{inputs.parameters.extra_param1}}'''''') - - except: extra_param1 = r''''''{{inputs.parameters.extra_param1}}'''''' - - try: extra_param2 = json.loads(r''''''{{inputs.parameters.extra_param2}}'''''') - - except: extra_param2 = r''''''{{inputs.parameters.extra_param2}}'''''' - - try: value = json.loads(r''''''{{inputs.parameters.value}}'''''') - - except: value = r''''''{{inputs.parameters.value}}'''''' - - - print(''Received value={value}, extra_param1={extra_param1}, extra_param2={extra_param2}!''.format(value=value, - extra_param1=extra_param1, extra_param2=extra_param2))' + try: extra_param1 = json.loads(r'''{{inputs.parameters.extra_param1}}''') + except: extra_param1 = r'''{{inputs.parameters.extra_param1}}''' + try: extra_param2 = json.loads(r'''{{inputs.parameters.extra_param2}}''') + except: extra_param2 = r'''{{inputs.parameters.extra_param2}}''' + try: value = json.loads(r'''{{inputs.parameters.value}}''') + except: value = r'''{{inputs.parameters.value}}''' + + print('Received value={value}, extra_param1={extra_param1}, extra_param2={extra_param2}!'.format(value=value, extra_param1=extra_param1, extra_param2=extra_param2)) ``` diff --git a/docs/examples/workflows/loops/dynamic_fanout_fanin.md b/docs/examples/workflows/loops/dynamic_fanout_fanin.md index 9351257e9..730545153 100644 --- a/docs/examples/workflows/loops/dynamic_fanout_fanin.md +++ b/docs/examples/workflows/loops/dynamic_fanout_fanin.md @@ -78,17 +78,13 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([{''value'': i} for i in range(10)], sys.stdout)' + json.dump([{'value': i} for i in range(10)], sys.stdout) - inputs: parameters: - name: object @@ -102,10 +98,18 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: object = json.loads(r'''{{inputs.parameters.object}}''')\nexcept: object\ - \ = r'''{{inputs.parameters.object}}'''\n\nprint('Received object: {object}!'.format(object=object))\n\ - value = object['value']\nwith open('/tmp/value', 'w') as f:\n f.write(str(value))" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: object = json.loads(r'''{{inputs.parameters.object}}''') + except: object = r'''{{inputs.parameters.object}}''' + + print('Received object: {object}!'.format(object=object)) + value = object['value'] + with open('/tmp/value', 'w') as f: + f.write(str(value)) - inputs: parameters: - name: values @@ -114,19 +118,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: values = json.loads(r'''{{inputs.parameters.values}}''') + except: values = r'''{{inputs.parameters.values}}''' - try: values = json.loads(r''''''{{inputs.parameters.values}}'''''') - - except: values = r''''''{{inputs.parameters.values}}'''''' - - - print(''Received values: {values}!''.format(values=values))' + print('Received values: {values}!'.format(values=values)) ``` diff --git a/docs/examples/workflows/loops/dynamic_fanout_json_payload.md b/docs/examples/workflows/loops/dynamic_fanout_json_payload.md index 2636ab545..0349278ee 100644 --- a/docs/examples/workflows/loops/dynamic_fanout_json_payload.md +++ b/docs/examples/workflows/loops/dynamic_fanout_json_payload.md @@ -68,18 +68,13 @@ they may need to process. The fanout occurs over independent JSON payloads comin command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([{''p1'': i + 1, ''p2'': i + 2, ''p3'': i + 3} for i in range(10)], - sys.stdout)' + json.dump([{'p1': i + 1, 'p2': i + 2, 'p3': i + 3} for i in range(10)], sys.stdout) - inputs: parameters: - name: p1 @@ -90,27 +85,18 @@ they may need to process. The fanout occurs over independent JSON payloads comin command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: p1 = json.loads(r''''''{{inputs.parameters.p1}}'''''') - - except: p1 = r''''''{{inputs.parameters.p1}}'''''' - - try: p2 = json.loads(r''''''{{inputs.parameters.p2}}'''''') - - except: p2 = r''''''{{inputs.parameters.p2}}'''''' - - try: p3 = json.loads(r''''''{{inputs.parameters.p3}}'''''') - - except: p3 = r''''''{{inputs.parameters.p3}}'''''' - - - print(''Received p1={p1}, p2={p2}, p3={p3}''.format(p1=p1, p2=p2, p3=p3))' + try: p1 = json.loads(r'''{{inputs.parameters.p1}}''') + except: p1 = r'''{{inputs.parameters.p1}}''' + try: p2 = json.loads(r'''{{inputs.parameters.p2}}''') + except: p2 = r'''{{inputs.parameters.p2}}''' + try: p3 = json.loads(r'''{{inputs.parameters.p3}}''') + except: p3 = r'''{{inputs.parameters.p3}}''' + + print('Received p1={p1}, p2={p2}, p3={p3}'.format(p1=p1, p2=p2, p3=p3)) ``` diff --git a/docs/examples/workflows/loops/script_loops_maps.md b/docs/examples/workflows/loops/script_loops_maps.md index 4a52d9f91..b7630d41c 100644 --- a/docs/examples/workflows/loops/script_loops_maps.md +++ b/docs/examples/workflows/loops/script_loops_maps.md @@ -69,23 +69,16 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: key_1 = json.loads(r'''{{inputs.parameters.key_1}}''') + except: key_1 = r'''{{inputs.parameters.key_1}}''' + try: key_2 = json.loads(r'''{{inputs.parameters.key_2}}''') + except: key_2 = r'''{{inputs.parameters.key_2}}''' - try: key_1 = json.loads(r''''''{{inputs.parameters.key_1}}'''''') - - except: key_1 = r''''''{{inputs.parameters.key_1}}'''''' - - try: key_2 = json.loads(r''''''{{inputs.parameters.key_2}}'''''') - - except: key_2 = r''''''{{inputs.parameters.key_2}}'''''' - - - print(''{key_1}, {key_2}''.format(key_1=key_1, key_2=key_2))' + print('{key_1}, {key_2}'.format(key_1=key_1, key_2=key_2)) ``` diff --git a/docs/examples/workflows/loops/with_sequence.md b/docs/examples/workflows/loops/with_sequence.md index ddb04e875..1486ae32c 100644 --- a/docs/examples/workflows/loops/with_sequence.md +++ b/docs/examples/workflows/loops/with_sequence.md @@ -75,13 +75,11 @@ This example showcases how to generate and parallelize generated sequences command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(3)' + print(3) - inputs: parameters: - name: message @@ -90,19 +88,14 @@ This example showcases how to generate and parallelize generated sequences command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) ``` diff --git a/docs/examples/workflows/multi_env.md b/docs/examples/workflows/multi_env.md index 06f590676..00deb2b2d 100644 --- a/docs/examples/workflows/multi_env.md +++ b/docs/examples/workflows/multi_env.md @@ -53,18 +53,13 @@ - name: c value: '3' image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - - assert os.environ[''a''] == ''1'', os.environ[''a''] - - assert os.environ[''b''] == ''2'', os.environ[''b''] - - assert os.environ[''c''] == ''3'', os.environ[''c'']' + assert os.environ['a'] == '1', os.environ['a'] + assert os.environ['b'] == '2', os.environ['b'] + assert os.environ['c'] == '3', os.environ['c'] ``` diff --git a/docs/examples/workflows/resource_flags.md b/docs/examples/workflows/resource_flags.md index c9eb7be43..1ac01bd80 100644 --- a/docs/examples/workflows/resource_flags.md +++ b/docs/examples/workflows/resource_flags.md @@ -59,15 +59,29 @@ - name: create-route resource: action: create - manifest: "apiVersion: route.openshift.io/v1\nkind: Route\nmetadata:\n name:\ - \ host-route\nspec:\n to:\n kind: Service\n name: service-name\n" + manifest: | + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: host-route + spec: + to: + kind: Service + name: service-name - name: create-route-without-validation resource: action: create flags: - --validate=false - manifest: "apiVersion: route.openshift.io/v1\nkind: Route\nmetadata:\n name:\ - \ host-route\nspec:\n to:\n kind: Service\n name: service-name\n" + manifest: | + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: host-route + spec: + to: + kind: Service + name: service-name - name: resource-validate-example steps: - - continueOn: diff --git a/docs/examples/workflows/scripts/coinflip.md b/docs/examples/workflows/scripts/coinflip.md index 618ea28f7..abf7ef731 100644 --- a/docs/examples/workflows/scripts/coinflip.md +++ b/docs/examples/workflows/scripts/coinflip.md @@ -64,40 +64,32 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) - name: heads script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''it was heads'')' + print('it was heads') - name: tails script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''it was tails'')' + print('it was tails') ``` diff --git a/docs/examples/workflows/scripts/default_param_overwrite.md b/docs/examples/workflows/scripts/default_param_overwrite.md index da127c8fb..54da50004 100644 --- a/docs/examples/workflows/scripts/default_param_overwrite.md +++ b/docs/examples/workflows/scripts/default_param_overwrite.md @@ -60,13 +60,11 @@ conditionally. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Another message for the world!'')' + print('Another message for the world!') - inputs: parameters: - default: Hello, world! @@ -76,19 +74,14 @@ conditionally. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) ``` diff --git a/docs/examples/workflows/scripts/script_artifact_passing.md b/docs/examples/workflows/scripts/script_artifact_passing.md index bdd9ffd99..352c1e605 100644 --- a/docs/examples/workflows/scripts/script_artifact_passing.md +++ b/docs/examples/workflows/scripts/script_artifact_passing.md @@ -62,8 +62,12 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/hello_world.txt',\ - \ 'w') as f:\n f.write('hello world')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/hello_world.txt', 'w') as f: + f.write('hello world') - inputs: artifacts: - name: message @@ -73,7 +77,12 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/message',\ - \ 'r') as f:\n message = f.readline()\nprint(message)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/message', 'r') as f: + message = f.readline() + print(message) ``` diff --git a/docs/examples/workflows/scripts/script_auto_infer.md b/docs/examples/workflows/scripts/script_auto_infer.md index fd53f1727..229eb9867 100644 --- a/docs/examples/workflows/scripts/script_auto_infer.md +++ b/docs/examples/workflows/scripts/script_auto_infer.md @@ -67,9 +67,14 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - result = 'foo testing'\nwith open('/tmp/result', 'wb') as f:\n pickle.dump(result,\ - \ f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import pickle + result = 'foo testing' + with open('/tmp/result', 'wb') as f: + pickle.dump(result, f) - inputs: artifacts: - name: i @@ -79,7 +84,13 @@ command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - with open('/tmp/i', 'rb') as f:\n i = pickle.load(f)\nprint(i)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import pickle + with open('/tmp/i', 'rb') as f: + i = pickle.load(f) + print(i) ``` diff --git a/docs/examples/workflows/scripts/script_variations.md b/docs/examples/workflows/scripts/script_variations.md index 1c5b58f05..1e4d68505 100644 --- a/docs/examples/workflows/scripts/script_variations.md +++ b/docs/examples/workflows/scripts/script_variations.md @@ -56,13 +56,11 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Hello World!'')' + print('Hello World!') - inputs: parameters: - name: test @@ -72,25 +70,17 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: another_test = json.loads(r''''''{{inputs.parameters.another_test}}'''''') - - except: another_test = r''''''{{inputs.parameters.another_test}}'''''' - - try: test = json.loads(r''''''{{inputs.parameters.test}}'''''') - - except: test = r''''''{{inputs.parameters.test}}'''''' - + try: another_test = json.loads(r'''{{inputs.parameters.another_test}}''') + except: another_test = r'''{{inputs.parameters.another_test}}''' + try: test = json.loads(r'''{{inputs.parameters.test}}''') + except: test = r'''{{inputs.parameters.test}}''' print(test) - - print(another_test)' + print(another_test) ``` diff --git a/docs/examples/workflows/scripts/script_with_default_params.md b/docs/examples/workflows/scripts/script_with_default_params.md index 5c0805ccf..85be90ebb 100644 --- a/docs/examples/workflows/scripts/script_with_default_params.md +++ b/docs/examples/workflows/scripts/script_with_default_params.md @@ -81,27 +81,18 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - try: b = json.loads(r''''''{{inputs.parameters.b}}'''''') - - except: b = r''''''{{inputs.parameters.b}}'''''' - - try: c = json.loads(r''''''{{inputs.parameters.c}}'''''') - - except: c = r''''''{{inputs.parameters.c}}'''''' - - - print(a, b, c)' + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + try: b = json.loads(r'''{{inputs.parameters.b}}''') + except: b = r'''{{inputs.parameters.b}}''' + try: c = json.loads(r'''{{inputs.parameters.c}}''') + except: c = r'''{{inputs.parameters.c}}''' + + print(a, b, c) ``` diff --git a/docs/examples/workflows/scripts/script_with_image_pull_policy.md b/docs/examples/workflows/scripts/script_with_image_pull_policy.md index a97fc0a10..1273fc93c 100644 --- a/docs/examples/workflows/scripts/script_with_image_pull_policy.md +++ b/docs/examples/workflows/scripts/script_with_image_pull_policy.md @@ -37,12 +37,10 @@ - python image: python:3.8 imagePullPolicy: Always - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''ok'')' + print('ok') ``` diff --git a/docs/examples/workflows/scripts/script_with_resources.md b/docs/examples/workflows/scripts/script_with_resources.md index 421865cb1..62ac268f5 100644 --- a/docs/examples/workflows/scripts/script_with_resources.md +++ b/docs/examples/workflows/scripts/script_with_resources.md @@ -38,12 +38,10 @@ resources: requests: memory: 5Gi - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''ok'')' + print('ok') ``` diff --git a/docs/examples/workflows/steps/callable_steps.md b/docs/examples/workflows/steps/callable_steps.md index 18a012fea..c13cc4f56 100644 --- a/docs/examples/workflows/steps/callable_steps.md +++ b/docs/examples/workflows/steps/callable_steps.md @@ -64,20 +64,15 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: name = json.loads(r'''{{inputs.parameters.name}}''') + except: name = r'''{{inputs.parameters.name}}''' - try: name = json.loads(r''''''{{inputs.parameters.name}}'''''') - - except: name = r''''''{{inputs.parameters.name}}'''''' - - - print(''Hello, {name}!''.format(name=name))' + print('Hello, {name}!'.format(name=name)) - name: calling-steps steps: - - arguments: diff --git a/docs/examples/workflows/upstream/artifact_gc_workflow.md b/docs/examples/workflows/upstream/artifact_gc_workflow.md index 92324c9f5..eb21427e6 100644 --- a/docs/examples/workflows/upstream/artifact_gc_workflow.md +++ b/docs/examples/workflows/upstream/artifact_gc_workflow.md @@ -56,11 +56,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl templates: - container: args: - - 'echo "hello world" > /tmp/on-completion.txt - + - | + echo "hello world" > /tmp/on-completion.txt echo "hello world" > /tmp/on-deletion.txt - - ' command: - sh - -c diff --git a/docs/examples/workflows/upstream/artifact_path_placeholders.md b/docs/examples/workflows/upstream/artifact_path_placeholders.md index c9dd1f680..394464849 100644 --- a/docs/examples/workflows/upstream/artifact_path_placeholders.md +++ b/docs/examples/workflows/upstream/artifact_path_placeholders.md @@ -59,17 +59,12 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl artifacts: - name: text raw: - data: '1 - + data: | + 1 2 - 3 - 4 - 5 - - ' parameters: - name: lines-count value: '3' diff --git a/docs/examples/workflows/upstream/ci_workflowtemplate.md b/docs/examples/workflows/upstream/ci_workflowtemplate.md index e987111a8..bfe2f6ec0 100644 --- a/docs/examples/workflows/upstream/ci_workflowtemplate.md +++ b/docs/examples/workflows/upstream/ci_workflowtemplate.md @@ -190,15 +190,11 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl templates: - container: args: - - 'mkdir -p $(go env GOMODCACHE) - + - | + mkdir -p $(go env GOMODCACHE) [ -e /mnt/GOMODCACHE ] && cp -Rf /mnt/GOMODCACHE $(go env GOMODCACHE) - mkdir -p $(go env GOCACHE) - [ -e /mnt/GOCACHE ] && cp -Rf /mnt/GOCACHE $(go env GOCACHE) - - ' command: - sh - -euxc @@ -226,10 +222,8 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl name: cache-restore - container: args: - - 'git clone -v -b "{{workflow.parameters.branch}}" --single-branch --depth - 1 https://github.com/golang/example.git . - - ' + - | + git clone -v -b "{{workflow.parameters.branch}}" --single-branch --depth 1 https://github.com/golang/example.git . command: - sh - -euxc @@ -248,9 +242,8 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl name: clone - container: args: - - 'go mod download -x - - ' + - | + go mod download -x command: - sh - -xuce @@ -269,9 +262,8 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl name: deps - container: args: - - 'go build ./... - - ' + - | + go build ./... command: - sh - -xuce @@ -290,17 +282,13 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl name: build - container: args: - - 'go install github.com/jstemmer/go-junit-report@latest - + - | + go install github.com/jstemmer/go-junit-report@latest go install github.com/alexec/junit2html@v0.0.2 - - trap ''cat test.out | go-junit-report | junit2html > test-report.html'' EXIT - + trap 'cat test.out | go-junit-report | junit2html > test-report.html' EXIT go test -v ./... 2>&1 > test.out - - ' command: - sh - -euxc diff --git a/docs/examples/workflows/upstream/coinflip.md b/docs/examples/workflows/upstream/coinflip.md index 35bb0142a..33c664ca4 100644 --- a/docs/examples/workflows/upstream/coinflip.md +++ b/docs/examples/workflows/upstream/coinflip.md @@ -96,10 +96,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + source: |- + import random + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) ``` diff --git a/docs/examples/workflows/upstream/coinflip_recursive.md b/docs/examples/workflows/upstream/coinflip_recursive.md index 85b2570a6..4d21a1865 100644 --- a/docs/examples/workflows/upstream/coinflip_recursive.md +++ b/docs/examples/workflows/upstream/coinflip_recursive.md @@ -74,10 +74,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + source: |- + import random + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) ``` diff --git a/docs/examples/workflows/upstream/colored_logs.md b/docs/examples/workflows/upstream/colored_logs.md index 5784e634e..7d63e9ac7 100644 --- a/docs/examples/workflows/upstream/colored_logs.md +++ b/docs/examples/workflows/upstream/colored_logs.md @@ -56,11 +56,12 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - name: PYTHONUNBUFFERED value: '1' image: python:3.7 - source: "import time\nimport random\nmessages = ['No Color', '\\x1b[30m%s\\\ - x1b[0m' % 'FG Black', '\\x1b[32m%s\\x1b[0m' % 'FG Green', '\\x1b[34m%s\\x1b[0m'\ - \ % 'FG Blue', '\\x1b[36m%s\\x1b[0m' % 'FG Cyan', '\\x1b[41m%s\\x1b[0m' %\ - \ 'BG Red', '\\x1b[43m%s\\x1b[0m' % 'BG Yellow', '\\x1b[45m%s\\x1b[0m' % 'BG\ - \ Magenta']\nfor i in range(1, 100):\n print(random.choice(messages))\n\ - \ time.sleep(1)" + source: |- + import time + import random + messages = ['No Color', '\x1b[30m%s\x1b[0m' % 'FG Black', '\x1b[32m%s\x1b[0m' % 'FG Green', '\x1b[34m%s\x1b[0m' % 'FG Blue', '\x1b[36m%s\x1b[0m' % 'FG Cyan', '\x1b[41m%s\x1b[0m' % 'BG Red', '\x1b[43m%s\x1b[0m' % 'BG Yellow', '\x1b[45m%s\x1b[0m' % 'BG Magenta'] + for i in range(1, 100): + print(random.choice(messages)) + time.sleep(1) ``` diff --git a/docs/examples/workflows/upstream/conditional_artifacts.md b/docs/examples/workflows/upstream/conditional_artifacts.md index ae7f00491..f064ad438 100644 --- a/docs/examples/workflows/upstream/conditional_artifacts.md +++ b/docs/examples/workflows/upstream/conditional_artifacts.md @@ -108,9 +108,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - name: heads outputs: artifacts: @@ -120,7 +120,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('it was heads')" + source: |- + with open('result.txt', 'w') as f: + f.write('it was heads') - name: tails outputs: artifacts: @@ -130,7 +132,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('it was tails')" + source: |- + with open('result.txt', 'w') as f: + f.write('it was tails') - name: main outputs: artifacts: diff --git a/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md b/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md index 104840877..7a56b4a0f 100644 --- a/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md +++ b/docs/examples/workflows/upstream/container_set_template__outputs_result_workflow.md @@ -66,9 +66,8 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - containerSet: containers: - args: - - 'print("hi") - - ' + - | + print("hi") command: - python - -c diff --git a/docs/examples/workflows/upstream/dag_conditional_parameters.md b/docs/examples/workflows/upstream/dag_conditional_parameters.md index bb62153e0..d02e82a90 100644 --- a/docs/examples/workflows/upstream/dag_conditional_parameters.md +++ b/docs/examples/workflows/upstream/dag_conditional_parameters.md @@ -85,9 +85,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - dag: tasks: - name: flip-coin diff --git a/docs/examples/workflows/upstream/exit_handler_with_artifacts.md b/docs/examples/workflows/upstream/exit_handler_with_artifacts.md index 0c6e6b933..8b876ca50 100644 --- a/docs/examples/workflows/upstream/exit_handler_with_artifacts.md +++ b/docs/examples/workflows/upstream/exit_handler_with_artifacts.md @@ -142,7 +142,9 @@ spec: command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('Welcome')" + source: |- + with open('result.txt', 'w') as f: + f.write('Welcome') - container: args: - cat /tmp/message diff --git a/docs/examples/workflows/upstream/input_artifact_raw.md b/docs/examples/workflows/upstream/input_artifact_raw.md index c38692db2..e05131c4f 100644 --- a/docs/examples/workflows/upstream/input_artifact_raw.md +++ b/docs/examples/workflows/upstream/input_artifact_raw.md @@ -51,13 +51,10 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - name: myfile path: /tmp/file raw: - data: 'this is - + data: | + this is the raw file - contents - - ' name: raw-contents ``` diff --git a/docs/examples/workflows/upstream/k8s_json_patch_workflow.md b/docs/examples/workflows/upstream/k8s_json_patch_workflow.md index 350c017ad..bafa8ee21 100644 --- a/docs/examples/workflows/upstream/k8s_json_patch_workflow.md +++ b/docs/examples/workflows/upstream/k8s_json_patch_workflow.md @@ -47,7 +47,10 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl flags: - workflow - '{{workflow.name}}' - manifest: "- op: add\n path: /metadata/labels/foo\n value: bar\n" + manifest: | + - op: add + path: /metadata/labels/foo + value: bar mergeStrategy: json ``` diff --git a/docs/examples/workflows/upstream/k8s_resource_log_selector.md b/docs/examples/workflows/upstream/k8s_resource_log_selector.md index 255d4c1b5..851bf5d86 100644 --- a/docs/examples/workflows/upstream/k8s_resource_log_selector.md +++ b/docs/examples/workflows/upstream/k8s_resource_log_selector.md @@ -63,24 +63,33 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl resource: action: create failureCondition: status.replicaStatuses.Worker.failed > 0 - manifest: "apiVersion: kubeflow.org/v1\nkind: TFJob\nmetadata:\n name: tfjob-examples\n\ - spec:\n tfReplicaSpecs:\n Worker:\n replicas: 2\n restartPolicy:\ - \ Never\n template:\n metadata:\n # We add this label\ - \ to the pods created by TFJob custom resource to inform Argo Workflows\n\ - \ # that we want to include the logs from the created pods. Once\ - \ the pods are created with this\n # label, you can then use `argo\ - \ logs -c tensorflow` to the logs from this particular container.\n \ - \ # Note that `workflow.name` is a supported global variable provided\ - \ by Argo Workflows.\n #\n # The Kubeflow training controller\ - \ will take this CRD and automatically created worker pods with\n \ - \ # labels, such as `job-role` and `replica-index`. If you'd like to query\ - \ logs for pods with\n # specific labels, you can specify the label\ - \ selector explicitly via `argo logs -l `.\n \ - \ # For example, you can use `argo logs -c tensorflow -l replica-index=0`\ - \ to see the first worker pod's logs.\n labels:\n workflows.argoproj.io/workflow:\ - \ {{workflow.name}}\n spec:\n containers:\n \ - \ - name: tensorflow\n image: \"Placeholder for TensorFlow distributed\ - \ training image\"\n" + manifest: | + apiVersion: kubeflow.org/v1 + kind: TFJob + metadata: + name: tfjob-examples + spec: + tfReplicaSpecs: + Worker: + replicas: 2 + restartPolicy: Never + template: + metadata: + # We add this label to the pods created by TFJob custom resource to inform Argo Workflows + # that we want to include the logs from the created pods. Once the pods are created with this + # label, you can then use `argo logs -c tensorflow` to the logs from this particular container. + # Note that `workflow.name` is a supported global variable provided by Argo Workflows. + # + # The Kubeflow training controller will take this CRD and automatically created worker pods with + # labels, such as `job-role` and `replica-index`. If you'd like to query logs for pods with + # specific labels, you can specify the label selector explicitly via `argo logs -l `. + # For example, you can use `argo logs -c tensorflow -l replica-index=0` to see the first worker pod's logs. + labels: + workflows.argoproj.io/workflow: {{workflow.name}} + spec: + containers: + - name: tensorflow + image: "Placeholder for TensorFlow distributed training image" successCondition: status.replicaStatuses.Worker.succeeded = 2 ``` diff --git a/docs/examples/workflows/upstream/k8s_set_owner_reference.md b/docs/examples/workflows/upstream/k8s_set_owner_reference.md index 6bb72ca4a..500414867 100644 --- a/docs/examples/workflows/upstream/k8s_set_owner_reference.md +++ b/docs/examples/workflows/upstream/k8s_set_owner_reference.md @@ -45,8 +45,13 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - name: k8s-set-owner-reference resource: action: create - manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n generateName: owned-eg-\n\ - data:\n some: value\n" + manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + generateName: owned-eg- + data: + some: value setOwnerReference: true ``` diff --git a/docs/examples/workflows/upstream/loops_arbitrary_sequential_steps.md b/docs/examples/workflows/upstream/loops_arbitrary_sequential_steps.md index 462508371..a137222cf 100644 --- a/docs/examples/workflows/upstream/loops_arbitrary_sequential_steps.md +++ b/docs/examples/workflows/upstream/loops_arbitrary_sequential_steps.md @@ -67,11 +67,15 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl arguments: parameters: - name: step_params - value: "[\n { \"exit_code\": 0, \"message\": \"succeeds 1\" },\n { \"exit_code\"\ - : 0, \"message\": \"succeeds 2\" },\n { \"exit_code\": 0, \"message\": \"\ - succeeds 3\" },\n { \"exit_code\": 1, \"message\": \"will fail and stop here\"\ - \ },\n { \"exit_code\": 0, \"message\": \"will not run\" },\n { \"exit_code\"\ - : 0, \"message\": \"will not run\" }\n]\n" + value: | + [ + { "exit_code": 0, "message": "succeeds 1" }, + { "exit_code": 0, "message": "succeeds 2" }, + { "exit_code": 0, "message": "succeeds 3" }, + { "exit_code": 1, "message": "will fail and stop here" }, + { "exit_code": 0, "message": "will not run" }, + { "exit_code": 0, "message": "will not run" } + ] entrypoint: loop-arbitrary-sequential-steps-example templates: - container: diff --git a/docs/examples/workflows/upstream/loops_param_result.md b/docs/examples/workflows/upstream/loops_param_result.md index 6b5e24ecb..9d83e24f4 100644 --- a/docs/examples/workflows/upstream/loops_param_result.md +++ b/docs/examples/workflows/upstream/loops_param_result.md @@ -83,10 +83,9 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import json - + source: |- + import json import sys - - json.dump([i for i in range(20, 31)], sys.stdout)' + json.dump([i for i in range(20, 31)], sys.stdout) ``` diff --git a/docs/examples/workflows/upstream/parallelism_nested.md b/docs/examples/workflows/upstream/parallelism_nested.md index 910983a85..935ec0912 100644 --- a/docs/examples/workflows/upstream/parallelism_nested.md +++ b/docs/examples/workflows/upstream/parallelism_nested.md @@ -67,13 +67,11 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl arguments: parameters: - name: seq-list - value: '["a","b","c","d"] - - ' + value: | + ["a","b","c","d"] - name: parallel-list - value: '[1,2,3,4] - - ' + value: | + [1,2,3,4] entrypoint: parallel-worker templates: - container: diff --git a/docs/examples/workflows/upstream/pod_spec_patch_wf_tmpl.md b/docs/examples/workflows/upstream/pod_spec_patch_wf_tmpl.md index dcf250f30..7988384f6 100644 --- a/docs/examples/workflows/upstream/pod_spec_patch_wf_tmpl.md +++ b/docs/examples/workflows/upstream/pod_spec_patch_wf_tmpl.md @@ -50,8 +50,12 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - name: mem-limit value: 100Mi entrypoint: whalesay - podSpecPatch: "containers:\n - name: main\n resources:\n limits:\n \ - \ memory: \"{{workflow.parameters.mem-limit}}\"\n" + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "{{workflow.parameters.mem-limit}}" templates: - container: args: diff --git a/docs/examples/workflows/upstream/resource_delete_with_flags.md b/docs/examples/workflows/upstream/resource_delete_with_flags.md index 39cd2ff0c..ff0fba3ee 100644 --- a/docs/examples/workflows/upstream/resource_delete_with_flags.md +++ b/docs/examples/workflows/upstream/resource_delete_with_flags.md @@ -71,8 +71,15 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl - name: create-configmap resource: action: create - manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: resource-delete-with-flags\n\ - \ labels:\n cleanup: \"true\"\ndata:\n key: value\n" + manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: resource-delete-with-flags + labels: + cleanup: "true" + data: + key: value - inputs: parameters: - name: selector diff --git a/docs/examples/workflows/upstream/retry_script.md b/docs/examples/workflows/upstream/retry_script.md index 17e3afdf0..0c7c651cc 100644 --- a/docs/examples/workflows/upstream/retry_script.md +++ b/docs/examples/workflows/upstream/retry_script.md @@ -44,12 +44,10 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl command: - python image: python:alpine3.6 - source: 'import random - + source: |- + import random import sys - exit_code = random.choice([0, 1, 1]) - - sys.exit(exit_code)' + sys.exit(exit_code) ``` diff --git a/docs/examples/workflows/upstream/workflow_of_workflows.md b/docs/examples/workflows/upstream/workflow_of_workflows.md index 7e743139f..f382139e7 100644 --- a/docs/examples/workflows/upstream/workflow_of_workflows.md +++ b/docs/examples/workflows/upstream/workflow_of_workflows.md @@ -81,8 +81,14 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ workflow-of-workflows-1-\nspec:\n workflowTemplateRef:\n name: {{inputs.parameters.workflowtemplate}}\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: workflow-of-workflows-1- + spec: + workflowTemplateRef: + name: {{inputs.parameters.workflowtemplate}} successCondition: status.phase == Succeeded - inputs: parameters: @@ -92,10 +98,18 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ workflow-of-workflows-2-\nspec:\n arguments:\n parameters:\n - name:\ - \ message\n value: {{inputs.parameters.message}}\n workflowTemplateRef:\n\ - \ name: {{inputs.parameters.workflowtemplate}}\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: workflow-of-workflows-2- + spec: + arguments: + parameters: + - name: message + value: {{inputs.parameters.message}} + workflowTemplateRef: + name: {{inputs.parameters.workflowtemplate}} successCondition: status.phase == Succeeded - name: main steps: diff --git a/docs/examples/workflows/use-cases/dask.md b/docs/examples/workflows/use-cases/dask.md index 21c209b56..31b3b74ac 100644 --- a/docs/examples/workflows/use-cases/dask.md +++ b/docs/examples/workflows/use-cases/dask.md @@ -78,44 +78,25 @@ This example showcases how to run Dask within a Hera submitted Argo workflow. command: - python image: ghcr.io/dask/dask:latest - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: n_workers = json.loads(r''''''{{inputs.parameters.n_workers}}'''''') - - except: n_workers = r''''''{{inputs.parameters.n_workers}}'''''' - - try: namespace = json.loads(r''''''{{inputs.parameters.namespace}}'''''') - - except: namespace = r''''''{{inputs.parameters.namespace}}'''''' - + try: n_workers = json.loads(r'''{{inputs.parameters.n_workers}}''') + except: n_workers = r'''{{inputs.parameters.n_workers}}''' + try: namespace = json.loads(r'''{{inputs.parameters.namespace}}''') + except: namespace = r'''{{inputs.parameters.namespace}}''' import subprocess - - subprocess.run([''pip'', ''install'', ''dask-kubernetes'', ''dask[distributed]''], - stdout=subprocess.PIPE, universal_newlines=True) - + subprocess.run(['pip', 'install', 'dask-kubernetes', 'dask[distributed]'], stdout=subprocess.PIPE, universal_newlines=True) import dask.array as da - from dask.distributed import Client - from dask_kubernetes.classic import KubeCluster, make_pod_spec - - cluster = KubeCluster(pod_template=make_pod_spec(image=''ghcr.io/dask/dask:latest'', - memory_limit=''4G'', memory_request=''2G'', cpu_limit=1, cpu_request=1), namespace=namespace, - n_workers=n_workers) - + cluster = KubeCluster(pod_template=make_pod_spec(image='ghcr.io/dask/dask:latest', memory_limit='4G', memory_request='2G', cpu_limit=1, cpu_request=1), namespace=namespace, n_workers=n_workers) client = Client(cluster) - array = da.ones((1000, 1000, 1000)) - - print(''Array mean = {array_mean}, expected = 1.0''.format(array_mean=array.mean().compute())) - - client.close()' + print('Array mean = {array_mean}, expected = 1.0'.format(array_mean=array.mean().compute())) + client.close() ``` diff --git a/docs/examples/workflows/use-cases/map_reduce.md b/docs/examples/workflows/use-cases/map_reduce.md index ef22a1c4a..a3e18914c 100644 --- a/docs/examples/workflows/use-cases/map_reduce.md +++ b/docs/examples/workflows/use-cases/map_reduce.md @@ -135,13 +135,23 @@ See the upstream example [here](https://github.com/argoproj/argo-workflows/blob/ command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: num_parts = json.loads(r'''{{inputs.parameters.num_parts}}''')\nexcept:\ - \ num_parts = r'''{{inputs.parameters.num_parts}}'''\n\nimport json\nimport\ - \ os\nimport sys\nos.mkdir('/mnt/out')\npart_ids = list(map_(lambda x: str(x),\ - \ range(num_parts)))\nfor (i, part_id) in enumerate(part_ids, start=1):\n\ - \ with open('/mnt/out/' + part_id + '.json', 'w') as f:\n json.dump({'foo':\ - \ i}, f)\njson.dump(part_ids, sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: num_parts = json.loads(r'''{{inputs.parameters.num_parts}}''') + except: num_parts = r'''{{inputs.parameters.num_parts}}''' + + import json + import os + import sys + os.mkdir('/mnt/out') + part_ids = list(map_(lambda x: str(x), range(num_parts))) + for (i, part_id) in enumerate(part_ids, start=1): + with open('/mnt/out/' + part_id + '.json', 'w') as f: + json.dump({'foo': i}, f) + json.dump(part_ids, sys.stdout) - inputs: artifacts: - name: part @@ -159,10 +169,17 @@ See the upstream example [here](https://github.com/argoproj/argo-workflows/blob/ command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import os\nos.mkdir('/mnt/out')\nwith open('/mnt/in/part.json') as f:\n \ - \ part = json.load(f)\nwith open('/mnt/out/part.json', 'w') as f:\n json.dump({'bar':\ - \ part['foo'] * 2}, f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import os + os.mkdir('/mnt/out') + with open('/mnt/in/part.json') as f: + part = json.load(f) + with open('/mnt/out/part.json', 'w') as f: + json.dump({'bar': part['foo'] * 2}, f) - inputs: artifacts: - name: results @@ -182,10 +199,18 @@ See the upstream example [here](https://github.com/argoproj/argo-workflows/blob/ command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import os\nos.mkdir('/mnt/out')\ntotal = 0\nfor f in list(map_(lambda x: open('/mnt/in/'\ - \ + x), os.listdir('/mnt/in'))):\n result = json.load(f)\n total = total\ - \ + result['bar']\nwith open('/mnt/out/total.json', 'w') as f:\n json.dump({'total':\ - \ total}, f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import os + os.mkdir('/mnt/out') + total = 0 + for f in list(map_(lambda x: open('/mnt/in/' + x), os.listdir('/mnt/in'))): + result = json.load(f) + total = total + result['bar'] + with open('/mnt/out/total.json', 'w') as f: + json.dump({'total': total}, f) ``` diff --git a/docs/examples/workflows/use-cases/spacy_inference_pipeline.md b/docs/examples/workflows/use-cases/spacy_inference_pipeline.md index cb8d09366..e2d786990 100644 --- a/docs/examples/workflows/use-cases/spacy_inference_pipeline.md +++ b/docs/examples/workflows/use-cases/spacy_inference_pipeline.md @@ -149,15 +149,21 @@ Step 2: Performs inference on the dataset in the volume path /mnt/data using Spa requests: cpu: '0.5' memory: 1Gi - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import subprocess\nfrom spacy.lang.en.examples import sentences\nprint(subprocess.run('cd\ - \ /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode())\n\ - ' the used image does not have `spacy` installed, so we need to install it\ - \ first! '\nsubprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE,\ - \ universal_newlines=True)\n' dumping spacy example sentences data into a\ - \ file\\n replace this with real dataset '\nwith open('/mnt/data/input_data.json',\ - \ 'w') as json_file:\n json.dump(sentences, json_file)\nprint('Data preparation\ - \ completed')\nprint(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import subprocess + from spacy.lang.en.examples import sentences + print(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode()) + ' the used image does not have `spacy` installed, so we need to install it first! ' + subprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE, universal_newlines=True) + ' dumping spacy example sentences data into a file\n replace this with real dataset ' + with open('/mnt/data/input_data.json', 'w') as json_file: + json.dump(sentences, json_file) + print('Data preparation completed') + print(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/data name: data-dir @@ -170,31 +176,55 @@ Step 2: Performs inference on the dataset in the volume path /mnt/data using Spa requests: cpu: '0.5' memory: 1Gi - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport subprocess\n\ - ' the used image does not have `spacy` installed, so we need to install it\ - \ first! '\nsubprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE,\ - \ universal_newlines=True)\nprint(subprocess.run('cd /mnt/data && ls -l ',\ - \ shell=True, capture_output=True).stdout.decode())\nimport json\nfrom typing\ - \ import List\nimport pydantic\nimport spacy\nfrom pydantic import BaseModel\n\ - from spacy.cli import download\n' download and load spacy model https://spacy.io/models/en#en_core_web_lg\ - \ '\nspacy_model_name = 'en_core_web_lg'\ndownload(spacy_model_name)\nnlp\ - \ = spacy.load(spacy_model_name)\n' build pydantic model '\nprint(pydantic.version.version_info())\n\ - \nclass NEROutput(BaseModel):\n input_text: str\n ner_entities: List[str]\ - \ = []\nner_output_list: List[NEROutput] = []\n' read data prepared from previous\ - \ step data_prep '\nwith open('/mnt/data/input_data.json', 'r') as json_file:\n\ - \ input_data = json.load(json_file)\n print(input_data)\n ' iterate\ - \ each sentence in the data and perform NER '\n for sentence in input_data:\n\ - \ print('input text: ' + sentence)\n doc = nlp(sentence)\n \ - \ print('output NER:')\n ner_entities: List[str] = []\n \ - \ for entity in doc.ents:\n ' Print the entity text and its NER\ - \ label '\n ner_entity = entity.text + ' is ' + entity.label_\n\ - \ print(ner_entity)\n ner_entities.append(ner_entity)\n\ - \ print('ner_entities = + ' + ner_entities)\n ner_output = NEROutput(input_text=sentence,\ - \ ner_entities=ner_entities)\n ner_output_list.append(dict(ner_output))\n\ - \ print('ner_output_list = ' + ner_output_list)\nprint('Inference completed')\n\ - ' save output in a file '\nwith open('/mnt/data/output_data.json', 'w') as\ - \ json_file:\n json.dump(ner_output_list, json_file)\nprint(subprocess.run('cd\ - \ /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + ' the used image does not have `spacy` installed, so we need to install it first! ' + subprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE, universal_newlines=True) + print(subprocess.run('cd /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode()) + import json + from typing import List + import pydantic + import spacy + from pydantic import BaseModel + from spacy.cli import download + ' download and load spacy model https://spacy.io/models/en#en_core_web_lg ' + spacy_model_name = 'en_core_web_lg' + download(spacy_model_name) + nlp = spacy.load(spacy_model_name) + ' build pydantic model ' + print(pydantic.version.version_info()) + + class NEROutput(BaseModel): + input_text: str + ner_entities: List[str] = [] + ner_output_list: List[NEROutput] = [] + ' read data prepared from previous step data_prep ' + with open('/mnt/data/input_data.json', 'r') as json_file: + input_data = json.load(json_file) + print(input_data) + ' iterate each sentence in the data and perform NER ' + for sentence in input_data: + print('input text: ' + sentence) + doc = nlp(sentence) + print('output NER:') + ner_entities: List[str] = [] + for entity in doc.ents: + ' Print the entity text and its NER label ' + ner_entity = entity.text + ' is ' + entity.label_ + print(ner_entity) + ner_entities.append(ner_entity) + print('ner_entities = + ' + ner_entities) + ner_output = NEROutput(input_text=sentence, ner_entities=ner_entities) + ner_output_list.append(dict(ner_output)) + print('ner_output_list = ' + ner_output_list) + print('Inference completed') + ' save output in a file ' + with open('/mnt/data/output_data.json', 'w') as json_file: + json.dump(ner_output_list, json_file) + print(subprocess.run('cd /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/data name: data-dir diff --git a/docs/examples/workflows/use-cases/spark.md b/docs/examples/workflows/use-cases/spark.md index cb6d548d5..b2803fef9 100644 --- a/docs/examples/workflows/use-cases/spark.md +++ b/docs/examples/workflows/use-cases/spark.md @@ -123,62 +123,35 @@ compares a regular Pandas dataframe with a Spark dataframe. Inspired by: https:/ requests: cpu: '4' memory: 8Gi - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: n = json.loads(r''''''{{inputs.parameters.n}}'''''') - - except: n = r''''''{{inputs.parameters.n}}'''''' - + try: n = json.loads(r'''{{inputs.parameters.n}}''') + except: n = r'''{{inputs.parameters.n}}''' import random - import subprocess - import time - - subprocess.run([''pip'', ''install'', ''pyspark'', ''pandas''], stdout=subprocess.PIPE, - universal_newlines=True) - + subprocess.run(['pip', 'install', 'pyspark', 'pandas'], stdout=subprocess.PIPE, universal_newlines=True) import pandas as pd - from pyspark.sql import SparkSession - - spark = SparkSession.builder.master(''local[1]'').appName(''my-spark-example-running-in-hera.com'').getOrCreate() - - (data, columns) = ([random.randint(0, n) for _ in range(n)], [''value'']) - + spark = SparkSession.builder.master('local[1]').appName('my-spark-example-running-in-hera.com').getOrCreate() + (data, columns) = ([random.randint(0, n) for _ in range(n)], ['value']) pandas_df = pd.DataFrame(data=data, columns=columns) - start = time.time() - pandas_result = pandas_df.describe() - pandas_elapsed = time.time() - start - - print(''Pandas dataframe: '') - + print('Pandas dataframe: ') print(pandas_result) - - print(''Pandas dataframe took {pandas_elapsed} seconds to compute''.format(pandas_elapsed=pandas_elapsed)) - + print('Pandas dataframe took {pandas_elapsed} seconds to compute'.format(pandas_elapsed=pandas_elapsed)) spark_df = spark.createDataFrame(data=pandas_df, schema=columns) - start = time.time() - spark_result = spark_df.describe() - spark_elapsed = time.time() - start - - print(''Spark dataframe: '') - + print('Spark dataframe: ') print(spark_result) - - print(''Spark dataframe took {spark_elapsed} seconds to compute''.format(spark_elapsed=spark_elapsed))' + print('Spark dataframe took {spark_elapsed} seconds to compute'.format(spark_elapsed=spark_elapsed)) ``` diff --git a/docs/examples/workflows/use-cases/workflow_of_workflows.md b/docs/examples/workflows/use-cases/workflow_of_workflows.md index 6ca0571b6..035de96ee 100644 --- a/docs/examples/workflows/use-cases/workflow_of_workflows.md +++ b/docs/examples/workflows/use-cases/workflow_of_workflows.md @@ -68,19 +68,41 @@ resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ sub-workflow-1-\nspec:\n entrypoint: echo\n templates:\n - container:\n\ - \ args:\n - I'm workflow 1\n command:\n - cowsay\n \ - \ image: docker/whalesay:latest\n name: echo\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: sub-workflow-1- + spec: + entrypoint: echo + templates: + - container: + args: + - I'm workflow 1 + command: + - cowsay + image: docker/whalesay:latest + name: echo successCondition: status.phase == Succeeded - name: w2-resource resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ sub-workflow-2-\nspec:\n entrypoint: echo\n templates:\n - container:\n\ - \ args:\n - I'm workflow 2\n command:\n - cowsay\n \ - \ image: docker/whalesay:latest\n name: echo\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: sub-workflow-2- + spec: + entrypoint: echo + templates: + - container: + args: + - I'm workflow 2 + command: + - cowsay + image: docker/whalesay:latest + name: echo successCondition: status.phase == Succeeded - name: main steps: diff --git a/docs/examples/workflows/user_container.md b/docs/examples/workflows/user_container.md index 09ddbd055..1d0c5e77c 100644 --- a/docs/examples/workflows/user_container.md +++ b/docs/examples/workflows/user_container.md @@ -51,13 +51,11 @@ This example showcases the user of a user container with a volume mount. command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''hi'')' + print('hi') sidecars: - name: sidecar-name volumeMounts: diff --git a/docs/examples/workflows/volume_mounts.md b/docs/examples/workflows/volume_mounts.md index 8e6e951e0..0617ea1cd 100644 --- a/docs/examples/workflows/volume_mounts.md +++ b/docs/examples/workflows/volume_mounts.md @@ -84,19 +84,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/docs/examples/workflows/volume_mounts_nfs.md b/docs/examples/workflows/volume_mounts_nfs.md index bbf756b01..103f0e32d 100644 --- a/docs/examples/workflows/volume_mounts_nfs.md +++ b/docs/examples/workflows/volume_mounts_nfs.md @@ -73,19 +73,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/nfs name: '{{inputs.parameters.vol}}' diff --git a/docs/examples/workflows/volume_mounts_wt.md b/docs/examples/workflows/volume_mounts_wt.md index bab391421..532c59d3d 100644 --- a/docs/examples/workflows/volume_mounts_wt.md +++ b/docs/examples/workflows/volume_mounts_wt.md @@ -84,19 +84,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/docs/examples/workflows/workflow_on_exit.md b/docs/examples/workflows/workflow_on_exit.md index 565d79708..bea6619ff 100644 --- a/docs/examples/workflows/workflow_on_exit.md +++ b/docs/examples/workflows/workflow_on_exit.md @@ -85,20 +85,15 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(s)' + print(s) - dag: tasks: - arguments: diff --git a/docs/examples/workflows/workflow_with_global_params.md b/docs/examples/workflows/workflow_with_global_params.md index 07f12341d..0fc8f9921 100644 --- a/docs/examples/workflows/workflow_with_global_params.md +++ b/docs/examples/workflows/workflow_with_global_params.md @@ -44,19 +44,14 @@ command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: v = json.loads(r'''{{inputs.parameters.v}}''') + except: v = r'''{{inputs.parameters.v}}''' - try: v = json.loads(r''''''{{inputs.parameters.v}}'''''') - - except: v = r''''''{{inputs.parameters.v}}'''''' - - - print(v)' + print(v) ``` diff --git a/examples/workflows/artifacts/artifact-with-fanout.yaml b/examples/workflows/artifacts/artifact-with-fanout.yaml index 0d9d4381e..88a5f6bf8 100644 --- a/examples/workflows/artifacts/artifact-with-fanout.yaml +++ b/examples/workflows/artifacts/artifact-with-fanout.yaml @@ -36,9 +36,14 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - with open('/tmp/file', 'w+') as f:\n for i in range(10):\n f.write(json.dumps(i)\ - \ + '\\n')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + with open('/tmp/file', 'w+') as f: + for i in range(10): + f.write(json.dumps(i) + '\n') - inputs: artifacts: - name: in-art @@ -48,10 +53,17 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import sys\nindices = []\nwith open('/tmp/file', 'r') as f:\n for line\ - \ in f.readlines():\n indices.append(line.strip())\njson.dump(indices,\ - \ sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import sys + indices = [] + with open('/tmp/file', 'r') as f: + for line in f.readlines(): + indices.append(line.strip()) + json.dump(indices, sys.stdout) - inputs: parameters: - name: i @@ -60,17 +72,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: i = json.loads(r'''{{inputs.parameters.i}}''') + except: i = r'''{{inputs.parameters.i}}''' - try: i = json.loads(r''''''{{inputs.parameters.i}}'''''') - - except: i = r''''''{{inputs.parameters.i}}'''''' - - - print(i)' + print(i) diff --git a/examples/workflows/artifacts/artifact.yaml b/examples/workflows/artifacts/artifact.yaml index 200c8f828..c91fb7479 100644 --- a/examples/workflows/artifacts/artifact.yaml +++ b/examples/workflows/artifacts/artifact.yaml @@ -28,8 +28,12 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/file',\ - \ 'w+') as f:\n f.write('Hello, world!')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/file', 'w+') as f: + f.write('Hello, world!') - inputs: artifacts: - name: in-art @@ -39,5 +43,9 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/file',\ - \ 'r') as f:\n print(f.readlines())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/file', 'r') as f: + print(f.readlines()) diff --git a/examples/workflows/daemon.yaml b/examples/workflows/daemon.yaml index 1cfe270c4..fee8afbae 100644 --- a/examples/workflows/daemon.yaml +++ b/examples/workflows/daemon.yaml @@ -23,12 +23,21 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nfrom http.server\ - \ import BaseHTTPRequestHandler, HTTPServer\n\nclass MyServer(BaseHTTPRequestHandler):\n\ - \n def do_GET(self):\n self.send_response(200)\n self.send_header('Content-type',\ - \ 'application/json')\n self.end_headers()\n self.wfile.write(bytes(\"\ - {'name':'John'}\", 'utf-8'))\nwebServer = HTTPServer(('0.0.0.0', 8080), MyServer)\n\ - webServer.serve_forever()" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + from http.server import BaseHTTPRequestHandler, HTTPServer + + class MyServer(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + self.wfile.write(bytes("{'name':'John'}", 'utf-8')) + webServer = HTTPServer(('0.0.0.0', 8080), MyServer) + webServer.serve_forever() - inputs: parameters: - name: ip @@ -37,31 +46,19 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: ip = json.loads(r''''''{{inputs.parameters.ip}}'''''') - - except: ip = r''''''{{inputs.parameters.ip}}'''''' - + try: ip = json.loads(r'''{{inputs.parameters.ip}}''') + except: ip = r'''{{inputs.parameters.ip}}''' import http.client - import os - print(os.environ) - - server_ip = ip.replace(''"'', '''') - - connection = http.client.HTTPConnection(''{server_ip}:8080''.format(server_ip=server_ip)) - - connection.request(''GET'', ''/'') - + server_ip = ip.replace('"', '') + connection = http.client.HTTPConnection('{server_ip}:8080'.format(server_ip=server_ip)) + connection.request('GET', '/') response = connection.getresponse() - - print(response.read())' + print(response.read()) diff --git a/examples/workflows/dags/any-success-all-fail.yaml b/examples/workflows/dags/any-success-all-fail.yaml index b7e66c773..07c1d4795 100644 --- a/examples/workflows/dags/any-success-all-fail.yaml +++ b/examples/workflows/dags/any-success-all-fail.yaml @@ -39,10 +39,18 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: a = json.loads(r'''{{inputs.parameters.a}}''')\nexcept: a = r'''{{inputs.parameters.a}}'''\n\ - \nimport random\nrandom.seed(a)\nif random.random() < 0.5:\n raise Exception('Oh,\ - \ no!')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + + import random + random.seed(a) + if random.random() < 0.5: + raise Exception('Oh, no!') - inputs: parameters: - name: a @@ -51,20 +59,15 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - raise Exception(a)' + raise Exception(a) - inputs: parameters: - name: a @@ -73,17 +76,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) diff --git a/examples/workflows/dags/callable-dag.yaml b/examples/workflows/dags/callable-dag.yaml index 76611c675..90cae3fac 100644 --- a/examples/workflows/dags/callable-dag.yaml +++ b/examples/workflows/dags/callable-dag.yaml @@ -31,20 +31,15 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: name = json.loads(r'''{{inputs.parameters.name}}''') + except: name = r'''{{inputs.parameters.name}}''' - try: name = json.loads(r''''''{{inputs.parameters.name}}'''''') - - except: name = r''''''{{inputs.parameters.name}}'''''' - - - print(''Hello, {name}!''.format(name=name))' + print('Hello, {name}!'.format(name=name)) - dag: tasks: - arguments: diff --git a/examples/workflows/dags/complex-deps.yaml b/examples/workflows/dags/complex-deps.yaml index 5fd5211f7..f9a6624ad 100644 --- a/examples/workflows/dags/complex-deps.yaml +++ b/examples/workflows/dags/complex-deps.yaml @@ -43,6 +43,14 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: p = json.loads(r'''{{inputs.parameters.p}}''')\nexcept: p = r'''{{inputs.parameters.p}}'''\n\ - \nif p < 0.5:\n raise Exception(p)\nprint(42)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: p = json.loads(r'''{{inputs.parameters.p}}''') + except: p = r'''{{inputs.parameters.p}}''' + + if p < 0.5: + raise Exception(p) + print(42) diff --git a/examples/workflows/dags/conditional.yaml b/examples/workflows/dags/conditional.yaml index 09dae8bf1..6ed27c013 100644 --- a/examples/workflows/dags/conditional.yaml +++ b/examples/workflows/dags/conditional.yaml @@ -21,29 +21,32 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport random\n\ - p = random.random()\nif p <= 0.5:\n raise Exception('failure')\nprint('success')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import random + p = random.random() + if p <= 0.5: + raise Exception('failure') + print('success') - name: success script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''success'')' + print('success') - name: failure script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''failure'')' + print('failure') diff --git a/examples/workflows/dags/dag-conditional-on-task-status.yaml b/examples/workflows/dags/dag-conditional-on-task-status.yaml index bc04b483d..871e39318 100644 --- a/examples/workflows/dags/dag-conditional-on-task-status.yaml +++ b/examples/workflows/dags/dag-conditional-on-task-status.yaml @@ -21,29 +21,30 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport random\n\ - if random.randint(0, 1) == 0:\n raise ValueError" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import random + if random.randint(0, 1) == 0: + raise ValueError - name: when-failed script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''It was a failure'')' + print('It was a failure') - name: when-succeeded script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''It was a success'')' + print('It was a success') diff --git a/examples/workflows/dags/dag-conditional-parameters.yaml b/examples/workflows/dags/dag-conditional-parameters.yaml index 5762a92e3..317104cdd 100644 --- a/examples/workflows/dags/dag-conditional-parameters.yaml +++ b/examples/workflows/dags/dag-conditional-parameters.yaml @@ -29,9 +29,9 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - name: heads script: command: diff --git a/examples/workflows/dags/dag-diamond-with-callable-decorators.yaml b/examples/workflows/dags/dag-diamond-with-callable-decorators.yaml index ec3b9bb75..7d4afb3bb 100644 --- a/examples/workflows/dags/dag-diamond-with-callable-decorators.yaml +++ b/examples/workflows/dags/dag-diamond-with-callable-decorators.yaml @@ -43,11 +43,9 @@ spec: command: - python image: python:alpine3.6 - source: 'import json + source: |- + import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) diff --git a/examples/workflows/dags/dag-diamond-with-callable-script.yaml b/examples/workflows/dags/dag-diamond-with-callable-script.yaml index 7274cf13d..952fbe8cb 100644 --- a/examples/workflows/dags/dag-diamond-with-callable-script.yaml +++ b/examples/workflows/dags/dag-diamond-with-callable-script.yaml @@ -13,14 +13,12 @@ spec: command: - python image: python:alpine3.6 - source: 'import json + source: |- + import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) - dag: tasks: - arguments: diff --git a/examples/workflows/dags/dag-with-script-output-param-passing.yaml b/examples/workflows/dags/dag-with-script-output-param-passing.yaml index 3dd9857e7..708bdb36f 100644 --- a/examples/workflows/dags/dag-with-script-output-param-passing.yaml +++ b/examples/workflows/dags/dag-with-script-output-param-passing.yaml @@ -27,8 +27,12 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/test',\ - \ 'w') as f_out:\n f_out.write('test')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/test', 'w') as f_out: + f_out.write('test') - inputs: parameters: - name: a @@ -37,17 +41,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) diff --git a/examples/workflows/dags/dag-with-script-param-passing.yaml b/examples/workflows/dags/dag-with-script-param-passing.yaml index 28bd0ca2a..a86a85c53 100644 --- a/examples/workflows/dags/dag-with-script-param-passing.yaml +++ b/examples/workflows/dags/dag-with-script-param-passing.yaml @@ -22,13 +22,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(42)' + print(42) - inputs: parameters: - name: a @@ -37,17 +35,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - - print(a)' + print(a) diff --git a/examples/workflows/dags/on-exit.yaml b/examples/workflows/dags/on-exit.yaml index ff3032b56..690e25269 100644 --- a/examples/workflows/dags/on-exit.yaml +++ b/examples/workflows/dags/on-exit.yaml @@ -10,13 +10,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Bye Hera'')' + print('Bye Hera') - dag: tasks: - arguments: @@ -42,17 +40,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(''Hello Hera, {s}''.format(s=s))' + print('Hello Hera, {s}'.format(s=s)) diff --git a/examples/workflows/dynamic-resources.yaml b/examples/workflows/dynamic-resources.yaml index 6f63a9a68..42bd8ef53 100644 --- a/examples/workflows/dynamic-resources.yaml +++ b/examples/workflows/dynamic-resources.yaml @@ -35,11 +35,17 @@ spec: command: - python image: python:3.10 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\n\"\"\"Computes\ - \ the resources necessary by the following job, which could be anything.\"\ - \"\"\nimport json\nimport sys\nresources = []\nfor i in range(1, 4):\n \ - \ resources.append({'cpu': i, 'mem': '{v}Mi'.format(v=i * 100)})\njson.dump(resources,\ - \ sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + """Computes the resources necessary by the following job, which could be anything.""" + import json + import sys + resources = [] + for i in range(1, 4): + resources.append({'cpu': i, 'mem': '{v}Mi'.format(v=i * 100)}) + json.dump(resources, sys.stdout) - inputs: parameters: - name: cpu @@ -52,26 +58,18 @@ spec: command: - python image: python:3.10 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: cpu = json.loads(r''''''{{inputs.parameters.cpu}}'''''') - - except: cpu = r''''''{{inputs.parameters.cpu}}'''''' - - try: mem = json.loads(r''''''{{inputs.parameters.mem}}'''''') - - except: mem = r''''''{{inputs.parameters.mem}}'''''' - + try: cpu = json.loads(r'''{{inputs.parameters.cpu}}''') + except: cpu = r'''{{inputs.parameters.cpu}}''' + try: mem = json.loads(r'''{{inputs.parameters.mem}}''') + except: mem = r'''{{inputs.parameters.mem}}''' """Perform some computation.""" - - print(''received cpu {cpu} and mem {mem}''.format(cpu=cpu, mem=mem))' + print('received cpu {cpu} and mem {mem}'.format(cpu=cpu, mem=mem)) - inputs: parameters: - default: '1' @@ -86,23 +84,15 @@ spec: command: - python image: python:3.10 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: cpu = json.loads(r''''''{{inputs.parameters.cpu}}'''''') - - except: cpu = r''''''{{inputs.parameters.cpu}}'''''' - - try: mem = json.loads(r''''''{{inputs.parameters.mem}}'''''') - - except: mem = r''''''{{inputs.parameters.mem}}'''''' - + try: cpu = json.loads(r'''{{inputs.parameters.cpu}}''') + except: cpu = r'''{{inputs.parameters.cpu}}''' + try: mem = json.loads(r'''{{inputs.parameters.mem}}''') + except: mem = r'''{{inputs.parameters.mem}}''' """Perform some computation.""" - - print(''received cpu {cpu} and mem {mem}''.format(cpu=cpu, mem=mem))' + print('received cpu {cpu} and mem {mem}'.format(cpu=cpu, mem=mem)) diff --git a/examples/workflows/dynamic-volumes.yaml b/examples/workflows/dynamic-volumes.yaml index c2d997f16..e0cad23ae 100644 --- a/examples/workflows/dynamic-volumes.yaml +++ b/examples/workflows/dynamic-volumes.yaml @@ -15,15 +15,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import subprocess - - print(subprocess.run(''cd && /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(subprocess.run('cd && /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: v diff --git a/examples/workflows/global-config.yaml b/examples/workflows/global-config.yaml index e01ea8401..c9b428f7d 100644 --- a/examples/workflows/global-config.yaml +++ b/examples/workflows/global-config.yaml @@ -18,10 +18,8 @@ spec: command: - python3 image: image-say - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''hello'')' + print('hello') diff --git a/examples/workflows/hello-world.yaml b/examples/workflows/hello-world.yaml index 6bb6aff26..1e535b515 100644 --- a/examples/workflows/hello-world.yaml +++ b/examples/workflows/hello-world.yaml @@ -17,17 +17,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(''Hello, {s}!''.format(s=s))' + print('Hello, {s}!'.format(s=s)) diff --git a/examples/workflows/loops/dynamic-fanout-extra-kwargs.yaml b/examples/workflows/loops/dynamic-fanout-extra-kwargs.yaml index d5b7275f3..e2a25abbb 100644 --- a/examples/workflows/loops/dynamic-fanout-extra-kwargs.yaml +++ b/examples/workflows/loops/dynamic-fanout-extra-kwargs.yaml @@ -37,17 +37,13 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([i for i in range(10)], sys.stdout)' + json.dump([i for i in range(10)], sys.stdout) - inputs: parameters: - name: value @@ -59,26 +55,16 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: extra_param1 = json.loads(r''''''{{inputs.parameters.extra_param1}}'''''') - - except: extra_param1 = r''''''{{inputs.parameters.extra_param1}}'''''' - - try: extra_param2 = json.loads(r''''''{{inputs.parameters.extra_param2}}'''''') - - except: extra_param2 = r''''''{{inputs.parameters.extra_param2}}'''''' - - try: value = json.loads(r''''''{{inputs.parameters.value}}'''''') - - except: value = r''''''{{inputs.parameters.value}}'''''' - - - print(''Received value={value}, extra_param1={extra_param1}, extra_param2={extra_param2}!''.format(value=value, - extra_param1=extra_param1, extra_param2=extra_param2))' + try: extra_param1 = json.loads(r'''{{inputs.parameters.extra_param1}}''') + except: extra_param1 = r'''{{inputs.parameters.extra_param1}}''' + try: extra_param2 = json.loads(r'''{{inputs.parameters.extra_param2}}''') + except: extra_param2 = r'''{{inputs.parameters.extra_param2}}''' + try: value = json.loads(r'''{{inputs.parameters.value}}''') + except: value = r'''{{inputs.parameters.value}}''' + + print('Received value={value}, extra_param1={extra_param1}, extra_param2={extra_param2}!'.format(value=value, extra_param1=extra_param1, extra_param2=extra_param2)) diff --git a/examples/workflows/loops/dynamic-fanout-fanin.yaml b/examples/workflows/loops/dynamic-fanout-fanin.yaml index f8a342350..7e9db2a11 100644 --- a/examples/workflows/loops/dynamic-fanout-fanin.yaml +++ b/examples/workflows/loops/dynamic-fanout-fanin.yaml @@ -30,17 +30,13 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([{''value'': i} for i in range(10)], sys.stdout)' + json.dump([{'value': i} for i in range(10)], sys.stdout) - inputs: parameters: - name: object @@ -54,10 +50,18 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: object = json.loads(r'''{{inputs.parameters.object}}''')\nexcept: object\ - \ = r'''{{inputs.parameters.object}}'''\n\nprint('Received object: {object}!'.format(object=object))\n\ - value = object['value']\nwith open('/tmp/value', 'w') as f:\n f.write(str(value))" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: object = json.loads(r'''{{inputs.parameters.object}}''') + except: object = r'''{{inputs.parameters.object}}''' + + print('Received object: {object}!'.format(object=object)) + value = object['value'] + with open('/tmp/value', 'w') as f: + f.write(str(value)) - inputs: parameters: - name: values @@ -66,17 +70,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: values = json.loads(r'''{{inputs.parameters.values}}''') + except: values = r'''{{inputs.parameters.values}}''' - try: values = json.loads(r''''''{{inputs.parameters.values}}'''''') - - except: values = r''''''{{inputs.parameters.values}}'''''' - - - print(''Received values: {values}!''.format(values=values))' + print('Received values: {values}!'.format(values=values)) diff --git a/examples/workflows/loops/dynamic-fanout-json-payload.yaml b/examples/workflows/loops/dynamic-fanout-json-payload.yaml index f4d730882..b299ef443 100644 --- a/examples/workflows/loops/dynamic-fanout-json-payload.yaml +++ b/examples/workflows/loops/dynamic-fanout-json-payload.yaml @@ -27,18 +27,13 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([{''p1'': i + 1, ''p2'': i + 2, ''p3'': i + 3} for i in range(10)], - sys.stdout)' + json.dump([{'p1': i + 1, 'p2': i + 2, 'p3': i + 3} for i in range(10)], sys.stdout) - inputs: parameters: - name: p1 @@ -49,25 +44,16 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: p1 = json.loads(r''''''{{inputs.parameters.p1}}'''''') - - except: p1 = r''''''{{inputs.parameters.p1}}'''''' - - try: p2 = json.loads(r''''''{{inputs.parameters.p2}}'''''') - - except: p2 = r''''''{{inputs.parameters.p2}}'''''' - - try: p3 = json.loads(r''''''{{inputs.parameters.p3}}'''''') - - except: p3 = r''''''{{inputs.parameters.p3}}'''''' - - - print(''Received p1={p1}, p2={p2}, p3={p3}''.format(p1=p1, p2=p2, p3=p3))' + try: p1 = json.loads(r'''{{inputs.parameters.p1}}''') + except: p1 = r'''{{inputs.parameters.p1}}''' + try: p2 = json.loads(r'''{{inputs.parameters.p2}}''') + except: p2 = r'''{{inputs.parameters.p2}}''' + try: p3 = json.loads(r'''{{inputs.parameters.p3}}''') + except: p3 = r'''{{inputs.parameters.p3}}''' + + print('Received p1={p1}, p2={p2}, p3={p3}'.format(p1=p1, p2=p2, p3=p3)) diff --git a/examples/workflows/loops/dynamic-fanout.yaml b/examples/workflows/loops/dynamic-fanout.yaml index 0e4e69c8e..6a9458aa5 100644 --- a/examples/workflows/loops/dynamic-fanout.yaml +++ b/examples/workflows/loops/dynamic-fanout.yaml @@ -23,17 +23,13 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - import sys - - json.dump([i for i in range(10)], sys.stdout)' + json.dump([i for i in range(10)], sys.stdout) - inputs: parameters: - name: value @@ -42,17 +38,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: value = json.loads(r'''{{inputs.parameters.value}}''') + except: value = r'''{{inputs.parameters.value}}''' - try: value = json.loads(r''''''{{inputs.parameters.value}}'''''') - - except: value = r''''''{{inputs.parameters.value}}'''''' - - - print(''Received value: {value}!''.format(value=value))' + print('Received value: {value}!'.format(value=value)) diff --git a/examples/workflows/loops/script-loops-maps.yaml b/examples/workflows/loops/script-loops-maps.yaml index f1ab75d3f..97f1f4772 100644 --- a/examples/workflows/loops/script-loops-maps.yaml +++ b/examples/workflows/loops/script-loops-maps.yaml @@ -33,21 +33,14 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: key_1 = json.loads(r'''{{inputs.parameters.key_1}}''') + except: key_1 = r'''{{inputs.parameters.key_1}}''' + try: key_2 = json.loads(r'''{{inputs.parameters.key_2}}''') + except: key_2 = r'''{{inputs.parameters.key_2}}''' - try: key_1 = json.loads(r''''''{{inputs.parameters.key_1}}'''''') - - except: key_1 = r''''''{{inputs.parameters.key_1}}'''''' - - try: key_2 = json.loads(r''''''{{inputs.parameters.key_2}}'''''') - - except: key_2 = r''''''{{inputs.parameters.key_2}}'''''' - - - print(''{key_1}, {key_2}''.format(key_1=key_1, key_2=key_2))' + print('{key_1}, {key_2}'.format(key_1=key_1, key_2=key_2)) diff --git a/examples/workflows/loops/with-sequence.yaml b/examples/workflows/loops/with-sequence.yaml index 907f40cc3..c25a884ce 100644 --- a/examples/workflows/loops/with-sequence.yaml +++ b/examples/workflows/loops/with-sequence.yaml @@ -36,13 +36,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(3)' + print(3) - inputs: parameters: - name: message @@ -51,17 +49,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) diff --git a/examples/workflows/multi-env.yaml b/examples/workflows/multi-env.yaml index f01df69d3..cd2aaa5bd 100644 --- a/examples/workflows/multi-env.yaml +++ b/examples/workflows/multi-env.yaml @@ -22,16 +22,11 @@ spec: - name: c value: '3' image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - - assert os.environ[''a''] == ''1'', os.environ[''a''] - - assert os.environ[''b''] == ''2'', os.environ[''b''] - - assert os.environ[''c''] == ''3'', os.environ[''c'']' + assert os.environ['a'] == '1', os.environ['a'] + assert os.environ['b'] == '2', os.environ['b'] + assert os.environ['c'] == '3', os.environ['c'] diff --git a/examples/workflows/resource-flags.yaml b/examples/workflows/resource-flags.yaml index 4f3773fc6..2297a3546 100644 --- a/examples/workflows/resource-flags.yaml +++ b/examples/workflows/resource-flags.yaml @@ -8,15 +8,29 @@ spec: - name: create-route resource: action: create - manifest: "apiVersion: route.openshift.io/v1\nkind: Route\nmetadata:\n name:\ - \ host-route\nspec:\n to:\n kind: Service\n name: service-name\n" + manifest: | + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: host-route + spec: + to: + kind: Service + name: service-name - name: create-route-without-validation resource: action: create flags: - --validate=false - manifest: "apiVersion: route.openshift.io/v1\nkind: Route\nmetadata:\n name:\ - \ host-route\nspec:\n to:\n kind: Service\n name: service-name\n" + manifest: | + apiVersion: route.openshift.io/v1 + kind: Route + metadata: + name: host-route + spec: + to: + kind: Service + name: service-name - name: resource-validate-example steps: - - continueOn: diff --git a/examples/workflows/scripts/coinflip.yaml b/examples/workflows/scripts/coinflip.yaml index 2cadd019b..081f85e0f 100644 --- a/examples/workflows/scripts/coinflip.yaml +++ b/examples/workflows/scripts/coinflip.yaml @@ -23,38 +23,30 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) - name: heads script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''it was heads'')' + print('it was heads') - name: tails script: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''it was tails'')' + print('it was tails') diff --git a/examples/workflows/scripts/default-param-overwrite.yaml b/examples/workflows/scripts/default-param-overwrite.yaml index bdd60b631..669964279 100644 --- a/examples/workflows/scripts/default-param-overwrite.yaml +++ b/examples/workflows/scripts/default-param-overwrite.yaml @@ -25,13 +25,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Another message for the world!'')' + print('Another message for the world!') - inputs: parameters: - default: Hello, world! @@ -41,17 +39,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: message = json.loads(r'''{{inputs.parameters.message}}''') + except: message = r'''{{inputs.parameters.message}}''' - try: message = json.loads(r''''''{{inputs.parameters.message}}'''''') - - except: message = r''''''{{inputs.parameters.message}}'''''' - - - print(message)' + print(message) diff --git a/examples/workflows/scripts/script-artifact-passing.yaml b/examples/workflows/scripts/script-artifact-passing.yaml index 57271d7bd..ccc5177a2 100644 --- a/examples/workflows/scripts/script-artifact-passing.yaml +++ b/examples/workflows/scripts/script-artifact-passing.yaml @@ -24,8 +24,12 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/hello_world.txt',\ - \ 'w') as f:\n f.write('hello world')" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/hello_world.txt', 'w') as f: + f.write('hello world') - inputs: artifacts: - name: message @@ -35,5 +39,10 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nwith open('/tmp/message',\ - \ 'r') as f:\n message = f.readline()\nprint(message)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + with open('/tmp/message', 'r') as f: + message = f.readline() + print(message) diff --git a/examples/workflows/scripts/script-auto-infer.yaml b/examples/workflows/scripts/script-auto-infer.yaml index e71432589..457986092 100644 --- a/examples/workflows/scripts/script-auto-infer.yaml +++ b/examples/workflows/scripts/script-auto-infer.yaml @@ -26,9 +26,14 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - result = 'foo testing'\nwith open('/tmp/result', 'wb') as f:\n pickle.dump(result,\ - \ f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import pickle + result = 'foo testing' + with open('/tmp/result', 'wb') as f: + pickle.dump(result, f) - inputs: artifacts: - name: i @@ -38,5 +43,11 @@ spec: command: - python image: python:3.8 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport pickle\n\ - with open('/tmp/i', 'rb') as f:\n i = pickle.load(f)\nprint(i)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import pickle + with open('/tmp/i', 'rb') as f: + i = pickle.load(f) + print(i) diff --git a/examples/workflows/scripts/script-variations.yaml b/examples/workflows/scripts/script-variations.yaml index 6d261ca00..83b5e3be5 100644 --- a/examples/workflows/scripts/script-variations.yaml +++ b/examples/workflows/scripts/script-variations.yaml @@ -23,13 +23,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''Hello World!'')' + print('Hello World!') - inputs: parameters: - name: test @@ -39,23 +37,15 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: another_test = json.loads(r''''''{{inputs.parameters.another_test}}'''''') - - except: another_test = r''''''{{inputs.parameters.another_test}}'''''' - - try: test = json.loads(r''''''{{inputs.parameters.test}}'''''') - - except: test = r''''''{{inputs.parameters.test}}'''''' - + try: another_test = json.loads(r'''{{inputs.parameters.another_test}}''') + except: another_test = r'''{{inputs.parameters.another_test}}''' + try: test = json.loads(r'''{{inputs.parameters.test}}''') + except: test = r'''{{inputs.parameters.test}}''' print(test) - - print(another_test)' + print(another_test) diff --git a/examples/workflows/scripts/script-with-default-params.yaml b/examples/workflows/scripts/script-with-default-params.yaml index 995531b6c..ae4a0ea7c 100644 --- a/examples/workflows/scripts/script-with-default-params.yaml +++ b/examples/workflows/scripts/script-with-default-params.yaml @@ -52,25 +52,16 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + try: b = json.loads(r'''{{inputs.parameters.b}}''') + except: b = r'''{{inputs.parameters.b}}''' + try: c = json.loads(r'''{{inputs.parameters.c}}''') + except: c = r'''{{inputs.parameters.c}}''' - try: a = json.loads(r''''''{{inputs.parameters.a}}'''''') - - except: a = r''''''{{inputs.parameters.a}}'''''' - - try: b = json.loads(r''''''{{inputs.parameters.b}}'''''') - - except: b = r''''''{{inputs.parameters.b}}'''''' - - try: c = json.loads(r''''''{{inputs.parameters.c}}'''''') - - except: c = r''''''{{inputs.parameters.c}}'''''' - - - print(a, b, c)' + print(a, b, c) diff --git a/examples/workflows/scripts/script-with-image-pull-policy.yaml b/examples/workflows/scripts/script-with-image-pull-policy.yaml index f6f44329e..ada31e7bf 100644 --- a/examples/workflows/scripts/script-with-image-pull-policy.yaml +++ b/examples/workflows/scripts/script-with-image-pull-policy.yaml @@ -11,10 +11,8 @@ spec: - python image: python:3.8 imagePullPolicy: Always - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''ok'')' + print('ok') diff --git a/examples/workflows/scripts/script-with-resources.yaml b/examples/workflows/scripts/script-with-resources.yaml index be0724a1f..15d3c52db 100644 --- a/examples/workflows/scripts/script-with-resources.yaml +++ b/examples/workflows/scripts/script-with-resources.yaml @@ -13,10 +13,8 @@ spec: resources: requests: memory: 5Gi - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''ok'')' + print('ok') diff --git a/examples/workflows/steps/callable-steps.yaml b/examples/workflows/steps/callable-steps.yaml index 127c0475e..e91caf12d 100644 --- a/examples/workflows/steps/callable-steps.yaml +++ b/examples/workflows/steps/callable-steps.yaml @@ -30,20 +30,15 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: name = json.loads(r'''{{inputs.parameters.name}}''') + except: name = r'''{{inputs.parameters.name}}''' - try: name = json.loads(r''''''{{inputs.parameters.name}}'''''') - - except: name = r''''''{{inputs.parameters.name}}'''''' - - - print(''Hello, {name}!''.format(name=name))' + print('Hello, {name}!'.format(name=name)) - name: calling-steps steps: - - arguments: diff --git a/examples/workflows/upstream/artifact-gc-workflow.yaml b/examples/workflows/upstream/artifact-gc-workflow.yaml index 86769a9b1..1c54944f3 100644 --- a/examples/workflows/upstream/artifact-gc-workflow.yaml +++ b/examples/workflows/upstream/artifact-gc-workflow.yaml @@ -9,11 +9,9 @@ spec: templates: - container: args: - - 'echo "hello world" > /tmp/on-completion.txt - + - | + echo "hello world" > /tmp/on-completion.txt echo "hello world" > /tmp/on-deletion.txt - - ' command: - sh - -c diff --git a/examples/workflows/upstream/artifact-path-placeholders.yaml b/examples/workflows/upstream/artifact-path-placeholders.yaml index 0b4278682..609c8a026 100644 --- a/examples/workflows/upstream/artifact-path-placeholders.yaml +++ b/examples/workflows/upstream/artifact-path-placeholders.yaml @@ -7,17 +7,12 @@ spec: artifacts: - name: text raw: - data: '1 - + data: | + 1 2 - 3 - 4 - 5 - - ' parameters: - name: lines-count value: '3' diff --git a/examples/workflows/upstream/ci-workflowtemplate.yaml b/examples/workflows/upstream/ci-workflowtemplate.yaml index 795f11947..a31292ff6 100644 --- a/examples/workflows/upstream/ci-workflowtemplate.yaml +++ b/examples/workflows/upstream/ci-workflowtemplate.yaml @@ -12,15 +12,11 @@ spec: templates: - container: args: - - 'mkdir -p $(go env GOMODCACHE) - + - | + mkdir -p $(go env GOMODCACHE) [ -e /mnt/GOMODCACHE ] && cp -Rf /mnt/GOMODCACHE $(go env GOMODCACHE) - mkdir -p $(go env GOCACHE) - [ -e /mnt/GOCACHE ] && cp -Rf /mnt/GOCACHE $(go env GOCACHE) - - ' command: - sh - -euxc @@ -48,10 +44,8 @@ spec: name: cache-restore - container: args: - - 'git clone -v -b "{{workflow.parameters.branch}}" --single-branch --depth - 1 https://github.com/golang/example.git . - - ' + - | + git clone -v -b "{{workflow.parameters.branch}}" --single-branch --depth 1 https://github.com/golang/example.git . command: - sh - -euxc @@ -70,9 +64,8 @@ spec: name: clone - container: args: - - 'go mod download -x - - ' + - | + go mod download -x command: - sh - -xuce @@ -91,9 +84,8 @@ spec: name: deps - container: args: - - 'go build ./... - - ' + - | + go build ./... command: - sh - -xuce @@ -112,17 +104,13 @@ spec: name: build - container: args: - - 'go install github.com/jstemmer/go-junit-report@latest - + - | + go install github.com/jstemmer/go-junit-report@latest go install github.com/alexec/junit2html@v0.0.2 - - trap ''cat test.out | go-junit-report | junit2html > test-report.html'' EXIT - + trap 'cat test.out | go-junit-report | junit2html > test-report.html' EXIT go test -v ./... 2>&1 > test.out - - ' command: - sh - -euxc diff --git a/examples/workflows/upstream/coinflip-recursive.yaml b/examples/workflows/upstream/coinflip-recursive.yaml index e1bcb5704..c6ef18e56 100644 --- a/examples/workflows/upstream/coinflip-recursive.yaml +++ b/examples/workflows/upstream/coinflip-recursive.yaml @@ -28,8 +28,7 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + source: |- + import random + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) diff --git a/examples/workflows/upstream/coinflip.yaml b/examples/workflows/upstream/coinflip.yaml index 34989a83c..0f76e7f05 100644 --- a/examples/workflows/upstream/coinflip.yaml +++ b/examples/workflows/upstream/coinflip.yaml @@ -39,8 +39,7 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - - result = ''heads'' if random.randint(0, 1) == 0 else ''tails'' - - print(result)' + source: |- + import random + result = 'heads' if random.randint(0, 1) == 0 else 'tails' + print(result) diff --git a/examples/workflows/upstream/colored-logs.yaml b/examples/workflows/upstream/colored-logs.yaml index 538c547da..6a8ec9f60 100644 --- a/examples/workflows/upstream/colored-logs.yaml +++ b/examples/workflows/upstream/colored-logs.yaml @@ -13,9 +13,10 @@ spec: - name: PYTHONUNBUFFERED value: '1' image: python:3.7 - source: "import time\nimport random\nmessages = ['No Color', '\\x1b[30m%s\\\ - x1b[0m' % 'FG Black', '\\x1b[32m%s\\x1b[0m' % 'FG Green', '\\x1b[34m%s\\x1b[0m'\ - \ % 'FG Blue', '\\x1b[36m%s\\x1b[0m' % 'FG Cyan', '\\x1b[41m%s\\x1b[0m' %\ - \ 'BG Red', '\\x1b[43m%s\\x1b[0m' % 'BG Yellow', '\\x1b[45m%s\\x1b[0m' % 'BG\ - \ Magenta']\nfor i in range(1, 100):\n print(random.choice(messages))\n\ - \ time.sleep(1)" + source: |- + import time + import random + messages = ['No Color', '\x1b[30m%s\x1b[0m' % 'FG Black', '\x1b[32m%s\x1b[0m' % 'FG Green', '\x1b[34m%s\x1b[0m' % 'FG Blue', '\x1b[36m%s\x1b[0m' % 'FG Cyan', '\x1b[41m%s\x1b[0m' % 'BG Red', '\x1b[43m%s\x1b[0m' % 'BG Yellow', '\x1b[45m%s\x1b[0m' % 'BG Magenta'] + for i in range(1, 100): + print(random.choice(messages)) + time.sleep(1) diff --git a/examples/workflows/upstream/conditional-artifacts.yaml b/examples/workflows/upstream/conditional-artifacts.yaml index 982eec9ab..6f51e429b 100644 --- a/examples/workflows/upstream/conditional-artifacts.yaml +++ b/examples/workflows/upstream/conditional-artifacts.yaml @@ -18,9 +18,9 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - name: heads outputs: artifacts: @@ -30,7 +30,9 @@ spec: command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('it was heads')" + source: |- + with open('result.txt', 'w') as f: + f.write('it was heads') - name: tails outputs: artifacts: @@ -40,7 +42,9 @@ spec: command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('it was tails')" + source: |- + with open('result.txt', 'w') as f: + f.write('it was tails') - name: main outputs: artifacts: diff --git a/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml b/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml index 393e98f65..f2d4b5a32 100644 --- a/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml +++ b/examples/workflows/upstream/container-set-template--outputs-result-workflow.yaml @@ -8,9 +8,8 @@ spec: - containerSet: containers: - args: - - 'print("hi") - - ' + - | + print("hi") command: - python - -c diff --git a/examples/workflows/upstream/dag-conditional-parameters.yaml b/examples/workflows/upstream/dag-conditional-parameters.yaml index 88584c6f7..fa816c178 100644 --- a/examples/workflows/upstream/dag-conditional-parameters.yaml +++ b/examples/workflows/upstream/dag-conditional-parameters.yaml @@ -22,9 +22,9 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - - print(''heads'' if random.randint(0, 1) == 0 else ''tails'')' + source: |- + import random + print('heads' if random.randint(0, 1) == 0 else 'tails') - dag: tasks: - name: flip-coin diff --git a/examples/workflows/upstream/exit-handler-with-artifacts.yaml b/examples/workflows/upstream/exit-handler-with-artifacts.yaml index 03d68b876..aa23ff05d 100644 --- a/examples/workflows/upstream/exit-handler-with-artifacts.yaml +++ b/examples/workflows/upstream/exit-handler-with-artifacts.yaml @@ -20,7 +20,9 @@ spec: command: - python image: python:alpine3.6 - source: "with open('result.txt', 'w') as f:\n f.write('Welcome')" + source: |- + with open('result.txt', 'w') as f: + f.write('Welcome') - container: args: - cat /tmp/message diff --git a/examples/workflows/upstream/input-artifact-raw.yaml b/examples/workflows/upstream/input-artifact-raw.yaml index afe3bfd64..15f45a7c8 100644 --- a/examples/workflows/upstream/input-artifact-raw.yaml +++ b/examples/workflows/upstream/input-artifact-raw.yaml @@ -17,11 +17,8 @@ spec: - name: myfile path: /tmp/file raw: - data: 'this is - + data: | + this is the raw file - contents - - ' name: raw-contents diff --git a/examples/workflows/upstream/k8s-json-patch-workflow.yaml b/examples/workflows/upstream/k8s-json-patch-workflow.yaml index eb96e3a7a..0c8c46b91 100644 --- a/examples/workflows/upstream/k8s-json-patch-workflow.yaml +++ b/examples/workflows/upstream/k8s-json-patch-workflow.yaml @@ -11,5 +11,8 @@ spec: flags: - workflow - '{{workflow.name}}' - manifest: "- op: add\n path: /metadata/labels/foo\n value: bar\n" + manifest: | + - op: add + path: /metadata/labels/foo + value: bar mergeStrategy: json diff --git a/examples/workflows/upstream/k8s-resource-log-selector.yaml b/examples/workflows/upstream/k8s-resource-log-selector.yaml index 2d4855a28..3e9786d97 100644 --- a/examples/workflows/upstream/k8s-resource-log-selector.yaml +++ b/examples/workflows/upstream/k8s-resource-log-selector.yaml @@ -9,22 +9,31 @@ spec: resource: action: create failureCondition: status.replicaStatuses.Worker.failed > 0 - manifest: "apiVersion: kubeflow.org/v1\nkind: TFJob\nmetadata:\n name: tfjob-examples\n\ - spec:\n tfReplicaSpecs:\n Worker:\n replicas: 2\n restartPolicy:\ - \ Never\n template:\n metadata:\n # We add this label\ - \ to the pods created by TFJob custom resource to inform Argo Workflows\n\ - \ # that we want to include the logs from the created pods. Once\ - \ the pods are created with this\n # label, you can then use `argo\ - \ logs -c tensorflow` to the logs from this particular container.\n \ - \ # Note that `workflow.name` is a supported global variable provided\ - \ by Argo Workflows.\n #\n # The Kubeflow training controller\ - \ will take this CRD and automatically created worker pods with\n \ - \ # labels, such as `job-role` and `replica-index`. If you'd like to query\ - \ logs for pods with\n # specific labels, you can specify the label\ - \ selector explicitly via `argo logs -l `.\n \ - \ # For example, you can use `argo logs -c tensorflow -l replica-index=0`\ - \ to see the first worker pod's logs.\n labels:\n workflows.argoproj.io/workflow:\ - \ {{workflow.name}}\n spec:\n containers:\n \ - \ - name: tensorflow\n image: \"Placeholder for TensorFlow distributed\ - \ training image\"\n" + manifest: | + apiVersion: kubeflow.org/v1 + kind: TFJob + metadata: + name: tfjob-examples + spec: + tfReplicaSpecs: + Worker: + replicas: 2 + restartPolicy: Never + template: + metadata: + # We add this label to the pods created by TFJob custom resource to inform Argo Workflows + # that we want to include the logs from the created pods. Once the pods are created with this + # label, you can then use `argo logs -c tensorflow` to the logs from this particular container. + # Note that `workflow.name` is a supported global variable provided by Argo Workflows. + # + # The Kubeflow training controller will take this CRD and automatically created worker pods with + # labels, such as `job-role` and `replica-index`. If you'd like to query logs for pods with + # specific labels, you can specify the label selector explicitly via `argo logs -l `. + # For example, you can use `argo logs -c tensorflow -l replica-index=0` to see the first worker pod's logs. + labels: + workflows.argoproj.io/workflow: {{workflow.name}} + spec: + containers: + - name: tensorflow + image: "Placeholder for TensorFlow distributed training image" successCondition: status.replicaStatuses.Worker.succeeded = 2 diff --git a/examples/workflows/upstream/k8s-set-owner-reference.yaml b/examples/workflows/upstream/k8s-set-owner-reference.yaml index 1dd28573c..f2d309d9b 100644 --- a/examples/workflows/upstream/k8s-set-owner-reference.yaml +++ b/examples/workflows/upstream/k8s-set-owner-reference.yaml @@ -8,6 +8,11 @@ spec: - name: k8s-set-owner-reference resource: action: create - manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n generateName: owned-eg-\n\ - data:\n some: value\n" + manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + generateName: owned-eg- + data: + some: value setOwnerReference: true diff --git a/examples/workflows/upstream/loops-arbitrary-sequential-steps.yaml b/examples/workflows/upstream/loops-arbitrary-sequential-steps.yaml index e008f240f..af58795b4 100644 --- a/examples/workflows/upstream/loops-arbitrary-sequential-steps.yaml +++ b/examples/workflows/upstream/loops-arbitrary-sequential-steps.yaml @@ -6,11 +6,15 @@ spec: arguments: parameters: - name: step_params - value: "[\n { \"exit_code\": 0, \"message\": \"succeeds 1\" },\n { \"exit_code\"\ - : 0, \"message\": \"succeeds 2\" },\n { \"exit_code\": 0, \"message\": \"\ - succeeds 3\" },\n { \"exit_code\": 1, \"message\": \"will fail and stop here\"\ - \ },\n { \"exit_code\": 0, \"message\": \"will not run\" },\n { \"exit_code\"\ - : 0, \"message\": \"will not run\" }\n]\n" + value: | + [ + { "exit_code": 0, "message": "succeeds 1" }, + { "exit_code": 0, "message": "succeeds 2" }, + { "exit_code": 0, "message": "succeeds 3" }, + { "exit_code": 1, "message": "will fail and stop here" }, + { "exit_code": 0, "message": "will not run" }, + { "exit_code": 0, "message": "will not run" } + ] entrypoint: loop-arbitrary-sequential-steps-example templates: - container: diff --git a/examples/workflows/upstream/loops-param-result.yaml b/examples/workflows/upstream/loops-param-result.yaml index 7bf46d748..b97049889 100644 --- a/examples/workflows/upstream/loops-param-result.yaml +++ b/examples/workflows/upstream/loops-param-result.yaml @@ -33,8 +33,7 @@ spec: command: - python image: python:alpine3.6 - source: 'import json - + source: |- + import json import sys - - json.dump([i for i in range(20, 31)], sys.stdout)' + json.dump([i for i in range(20, 31)], sys.stdout) diff --git a/examples/workflows/upstream/parallelism-nested.yaml b/examples/workflows/upstream/parallelism-nested.yaml index bb699d6ca..b46414ba9 100644 --- a/examples/workflows/upstream/parallelism-nested.yaml +++ b/examples/workflows/upstream/parallelism-nested.yaml @@ -6,13 +6,11 @@ spec: arguments: parameters: - name: seq-list - value: '["a","b","c","d"] - - ' + value: | + ["a","b","c","d"] - name: parallel-list - value: '[1,2,3,4] - - ' + value: | + [1,2,3,4] entrypoint: parallel-worker templates: - container: diff --git a/examples/workflows/upstream/pod-spec-patch-wf-tmpl.yaml b/examples/workflows/upstream/pod-spec-patch-wf-tmpl.yaml index 246b898a4..cf466cba5 100644 --- a/examples/workflows/upstream/pod-spec-patch-wf-tmpl.yaml +++ b/examples/workflows/upstream/pod-spec-patch-wf-tmpl.yaml @@ -10,8 +10,12 @@ spec: - name: mem-limit value: 100Mi entrypoint: whalesay - podSpecPatch: "containers:\n - name: main\n resources:\n limits:\n \ - \ memory: \"{{workflow.parameters.mem-limit}}\"\n" + podSpecPatch: | + containers: + - name: main + resources: + limits: + memory: "{{workflow.parameters.mem-limit}}" templates: - container: args: diff --git a/examples/workflows/upstream/resource-delete-with-flags.yaml b/examples/workflows/upstream/resource-delete-with-flags.yaml index 107d94e52..3cd0ddaf8 100644 --- a/examples/workflows/upstream/resource-delete-with-flags.yaml +++ b/examples/workflows/upstream/resource-delete-with-flags.yaml @@ -8,8 +8,15 @@ spec: - name: create-configmap resource: action: create - manifest: "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: resource-delete-with-flags\n\ - \ labels:\n cleanup: \"true\"\ndata:\n key: value\n" + manifest: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: resource-delete-with-flags + labels: + cleanup: "true" + data: + key: value - inputs: parameters: - name: selector diff --git a/examples/workflows/upstream/retry-script.yaml b/examples/workflows/upstream/retry-script.yaml index 48a316984..5288db33b 100644 --- a/examples/workflows/upstream/retry-script.yaml +++ b/examples/workflows/upstream/retry-script.yaml @@ -12,10 +12,8 @@ spec: command: - python image: python:alpine3.6 - source: 'import random - + source: |- + import random import sys - exit_code = random.choice([0, 1, 1]) - - sys.exit(exit_code)' + sys.exit(exit_code) diff --git a/examples/workflows/upstream/workflow-of-workflows.yaml b/examples/workflows/upstream/workflow-of-workflows.yaml index 9a436a3e9..706e04988 100644 --- a/examples/workflows/upstream/workflow-of-workflows.yaml +++ b/examples/workflows/upstream/workflow-of-workflows.yaml @@ -12,8 +12,14 @@ spec: resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ workflow-of-workflows-1-\nspec:\n workflowTemplateRef:\n name: {{inputs.parameters.workflowtemplate}}\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: workflow-of-workflows-1- + spec: + workflowTemplateRef: + name: {{inputs.parameters.workflowtemplate}} successCondition: status.phase == Succeeded - inputs: parameters: @@ -23,10 +29,18 @@ spec: resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ workflow-of-workflows-2-\nspec:\n arguments:\n parameters:\n - name:\ - \ message\n value: {{inputs.parameters.message}}\n workflowTemplateRef:\n\ - \ name: {{inputs.parameters.workflowtemplate}}\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: workflow-of-workflows-2- + spec: + arguments: + parameters: + - name: message + value: {{inputs.parameters.message}} + workflowTemplateRef: + name: {{inputs.parameters.workflowtemplate}} successCondition: status.phase == Succeeded - name: main steps: diff --git a/examples/workflows/use_cases/dask.yaml b/examples/workflows/use_cases/dask.yaml index 6d979b145..edf170f48 100644 --- a/examples/workflows/use_cases/dask.yaml +++ b/examples/workflows/use_cases/dask.yaml @@ -20,42 +20,23 @@ spec: command: - python image: ghcr.io/dask/dask:latest - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: n_workers = json.loads(r''''''{{inputs.parameters.n_workers}}'''''') - - except: n_workers = r''''''{{inputs.parameters.n_workers}}'''''' - - try: namespace = json.loads(r''''''{{inputs.parameters.namespace}}'''''') - - except: namespace = r''''''{{inputs.parameters.namespace}}'''''' - + try: n_workers = json.loads(r'''{{inputs.parameters.n_workers}}''') + except: n_workers = r'''{{inputs.parameters.n_workers}}''' + try: namespace = json.loads(r'''{{inputs.parameters.namespace}}''') + except: namespace = r'''{{inputs.parameters.namespace}}''' import subprocess - - subprocess.run([''pip'', ''install'', ''dask-kubernetes'', ''dask[distributed]''], - stdout=subprocess.PIPE, universal_newlines=True) - + subprocess.run(['pip', 'install', 'dask-kubernetes', 'dask[distributed]'], stdout=subprocess.PIPE, universal_newlines=True) import dask.array as da - from dask.distributed import Client - from dask_kubernetes.classic import KubeCluster, make_pod_spec - - cluster = KubeCluster(pod_template=make_pod_spec(image=''ghcr.io/dask/dask:latest'', - memory_limit=''4G'', memory_request=''2G'', cpu_limit=1, cpu_request=1), namespace=namespace, - n_workers=n_workers) - + cluster = KubeCluster(pod_template=make_pod_spec(image='ghcr.io/dask/dask:latest', memory_limit='4G', memory_request='2G', cpu_limit=1, cpu_request=1), namespace=namespace, n_workers=n_workers) client = Client(cluster) - array = da.ones((1000, 1000, 1000)) - - print(''Array mean = {array_mean}, expected = 1.0''.format(array_mean=array.mean().compute())) - - client.close()' + print('Array mean = {array_mean}, expected = 1.0'.format(array_mean=array.mean().compute())) + client.close() diff --git a/examples/workflows/use_cases/map-reduce.yaml b/examples/workflows/use_cases/map-reduce.yaml index 233122d64..6e62af8a0 100644 --- a/examples/workflows/use_cases/map-reduce.yaml +++ b/examples/workflows/use_cases/map-reduce.yaml @@ -46,13 +46,23 @@ spec: command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - try: num_parts = json.loads(r'''{{inputs.parameters.num_parts}}''')\nexcept:\ - \ num_parts = r'''{{inputs.parameters.num_parts}}'''\n\nimport json\nimport\ - \ os\nimport sys\nos.mkdir('/mnt/out')\npart_ids = list(map_(lambda x: str(x),\ - \ range(num_parts)))\nfor (i, part_id) in enumerate(part_ids, start=1):\n\ - \ with open('/mnt/out/' + part_id + '.json', 'w') as f:\n json.dump({'foo':\ - \ i}, f)\njson.dump(part_ids, sys.stdout)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: num_parts = json.loads(r'''{{inputs.parameters.num_parts}}''') + except: num_parts = r'''{{inputs.parameters.num_parts}}''' + + import json + import os + import sys + os.mkdir('/mnt/out') + part_ids = list(map_(lambda x: str(x), range(num_parts))) + for (i, part_id) in enumerate(part_ids, start=1): + with open('/mnt/out/' + part_id + '.json', 'w') as f: + json.dump({'foo': i}, f) + json.dump(part_ids, sys.stdout) - inputs: artifacts: - name: part @@ -70,10 +80,17 @@ spec: command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import os\nos.mkdir('/mnt/out')\nwith open('/mnt/in/part.json') as f:\n \ - \ part = json.load(f)\nwith open('/mnt/out/part.json', 'w') as f:\n json.dump({'bar':\ - \ part['foo'] * 2}, f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import os + os.mkdir('/mnt/out') + with open('/mnt/in/part.json') as f: + part = json.load(f) + with open('/mnt/out/part.json', 'w') as f: + json.dump({'bar': part['foo'] * 2}, f) - inputs: artifacts: - name: results @@ -93,8 +110,16 @@ spec: command: - python image: python:alpine3.6 - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import os\nos.mkdir('/mnt/out')\ntotal = 0\nfor f in list(map_(lambda x: open('/mnt/in/'\ - \ + x), os.listdir('/mnt/in'))):\n result = json.load(f)\n total = total\ - \ + result['bar']\nwith open('/mnt/out/total.json', 'w') as f:\n json.dump({'total':\ - \ total}, f)" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import os + os.mkdir('/mnt/out') + total = 0 + for f in list(map_(lambda x: open('/mnt/in/' + x), os.listdir('/mnt/in'))): + result = json.load(f) + total = total + result['bar'] + with open('/mnt/out/total.json', 'w') as f: + json.dump({'total': total}, f) diff --git a/examples/workflows/use_cases/spacy-inference-pipeline.yaml b/examples/workflows/use_cases/spacy-inference-pipeline.yaml index da428fa8a..fc8a18e25 100644 --- a/examples/workflows/use_cases/spacy-inference-pipeline.yaml +++ b/examples/workflows/use_cases/spacy-inference-pipeline.yaml @@ -22,15 +22,21 @@ spec: requests: cpu: '0.5' memory: 1Gi - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport json\n\ - import subprocess\nfrom spacy.lang.en.examples import sentences\nprint(subprocess.run('cd\ - \ /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode())\n\ - ' the used image does not have `spacy` installed, so we need to install it\ - \ first! '\nsubprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE,\ - \ universal_newlines=True)\n' dumping spacy example sentences data into a\ - \ file\\n replace this with real dataset '\nwith open('/mnt/data/input_data.json',\ - \ 'w') as json_file:\n json.dump(sentences, json_file)\nprint('Data preparation\ - \ completed')\nprint(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + import subprocess + from spacy.lang.en.examples import sentences + print(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode()) + ' the used image does not have `spacy` installed, so we need to install it first! ' + subprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE, universal_newlines=True) + ' dumping spacy example sentences data into a file\n replace this with real dataset ' + with open('/mnt/data/input_data.json', 'w') as json_file: + json.dump(sentences, json_file) + print('Data preparation completed') + print(subprocess.run('cd /mnt/data && ls -l', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/data name: data-dir @@ -43,31 +49,55 @@ spec: requests: cpu: '0.5' memory: 1Gi - source: "import os\nimport sys\nsys.path.append(os.getcwd())\nimport subprocess\n\ - ' the used image does not have `spacy` installed, so we need to install it\ - \ first! '\nsubprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE,\ - \ universal_newlines=True)\nprint(subprocess.run('cd /mnt/data && ls -l ',\ - \ shell=True, capture_output=True).stdout.decode())\nimport json\nfrom typing\ - \ import List\nimport pydantic\nimport spacy\nfrom pydantic import BaseModel\n\ - from spacy.cli import download\n' download and load spacy model https://spacy.io/models/en#en_core_web_lg\ - \ '\nspacy_model_name = 'en_core_web_lg'\ndownload(spacy_model_name)\nnlp\ - \ = spacy.load(spacy_model_name)\n' build pydantic model '\nprint(pydantic.version.version_info())\n\ - \nclass NEROutput(BaseModel):\n input_text: str\n ner_entities: List[str]\ - \ = []\nner_output_list: List[NEROutput] = []\n' read data prepared from previous\ - \ step data_prep '\nwith open('/mnt/data/input_data.json', 'r') as json_file:\n\ - \ input_data = json.load(json_file)\n print(input_data)\n ' iterate\ - \ each sentence in the data and perform NER '\n for sentence in input_data:\n\ - \ print('input text: ' + sentence)\n doc = nlp(sentence)\n \ - \ print('output NER:')\n ner_entities: List[str] = []\n \ - \ for entity in doc.ents:\n ' Print the entity text and its NER\ - \ label '\n ner_entity = entity.text + ' is ' + entity.label_\n\ - \ print(ner_entity)\n ner_entities.append(ner_entity)\n\ - \ print('ner_entities = + ' + ner_entities)\n ner_output = NEROutput(input_text=sentence,\ - \ ner_entities=ner_entities)\n ner_output_list.append(dict(ner_output))\n\ - \ print('ner_output_list = ' + ner_output_list)\nprint('Inference completed')\n\ - ' save output in a file '\nwith open('/mnt/data/output_data.json', 'w') as\ - \ json_file:\n json.dump(ner_output_list, json_file)\nprint(subprocess.run('cd\ - \ /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode())" + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + ' the used image does not have `spacy` installed, so we need to install it first! ' + subprocess.run(['pip', 'install', 'spacy'], stdout=subprocess.PIPE, universal_newlines=True) + print(subprocess.run('cd /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode()) + import json + from typing import List + import pydantic + import spacy + from pydantic import BaseModel + from spacy.cli import download + ' download and load spacy model https://spacy.io/models/en#en_core_web_lg ' + spacy_model_name = 'en_core_web_lg' + download(spacy_model_name) + nlp = spacy.load(spacy_model_name) + ' build pydantic model ' + print(pydantic.version.version_info()) + + class NEROutput(BaseModel): + input_text: str + ner_entities: List[str] = [] + ner_output_list: List[NEROutput] = [] + ' read data prepared from previous step data_prep ' + with open('/mnt/data/input_data.json', 'r') as json_file: + input_data = json.load(json_file) + print(input_data) + ' iterate each sentence in the data and perform NER ' + for sentence in input_data: + print('input text: ' + sentence) + doc = nlp(sentence) + print('output NER:') + ner_entities: List[str] = [] + for entity in doc.ents: + ' Print the entity text and its NER label ' + ner_entity = entity.text + ' is ' + entity.label_ + print(ner_entity) + ner_entities.append(ner_entity) + print('ner_entities = + ' + ner_entities) + ner_output = NEROutput(input_text=sentence, ner_entities=ner_entities) + ner_output_list.append(dict(ner_output)) + print('ner_output_list = ' + ner_output_list) + print('Inference completed') + ' save output in a file ' + with open('/mnt/data/output_data.json', 'w') as json_file: + json.dump(ner_output_list, json_file) + print(subprocess.run('cd /mnt/data && ls -l ', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/data name: data-dir diff --git a/examples/workflows/use_cases/spark.yaml b/examples/workflows/use_cases/spark.yaml index f17318e15..9001cbab0 100644 --- a/examples/workflows/use_cases/spark.yaml +++ b/examples/workflows/use_cases/spark.yaml @@ -56,60 +56,33 @@ spec: requests: cpu: '4' memory: 8Gi - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json - - try: n = json.loads(r''''''{{inputs.parameters.n}}'''''') - - except: n = r''''''{{inputs.parameters.n}}'''''' - + try: n = json.loads(r'''{{inputs.parameters.n}}''') + except: n = r'''{{inputs.parameters.n}}''' import random - import subprocess - import time - - subprocess.run([''pip'', ''install'', ''pyspark'', ''pandas''], stdout=subprocess.PIPE, - universal_newlines=True) - + subprocess.run(['pip', 'install', 'pyspark', 'pandas'], stdout=subprocess.PIPE, universal_newlines=True) import pandas as pd - from pyspark.sql import SparkSession - - spark = SparkSession.builder.master(''local[1]'').appName(''my-spark-example-running-in-hera.com'').getOrCreate() - - (data, columns) = ([random.randint(0, n) for _ in range(n)], [''value'']) - + spark = SparkSession.builder.master('local[1]').appName('my-spark-example-running-in-hera.com').getOrCreate() + (data, columns) = ([random.randint(0, n) for _ in range(n)], ['value']) pandas_df = pd.DataFrame(data=data, columns=columns) - start = time.time() - pandas_result = pandas_df.describe() - pandas_elapsed = time.time() - start - - print(''Pandas dataframe: '') - + print('Pandas dataframe: ') print(pandas_result) - - print(''Pandas dataframe took {pandas_elapsed} seconds to compute''.format(pandas_elapsed=pandas_elapsed)) - + print('Pandas dataframe took {pandas_elapsed} seconds to compute'.format(pandas_elapsed=pandas_elapsed)) spark_df = spark.createDataFrame(data=pandas_df, schema=columns) - start = time.time() - spark_result = spark_df.describe() - spark_elapsed = time.time() - start - - print(''Spark dataframe: '') - + print('Spark dataframe: ') print(spark_result) - - print(''Spark dataframe took {spark_elapsed} seconds to compute''.format(spark_elapsed=spark_elapsed))' + print('Spark dataframe took {spark_elapsed} seconds to compute'.format(spark_elapsed=spark_elapsed)) diff --git a/examples/workflows/use_cases/workflow-of-workflows.yaml b/examples/workflows/use_cases/workflow-of-workflows.yaml index d79d9b94c..d909d9d45 100644 --- a/examples/workflows/use_cases/workflow-of-workflows.yaml +++ b/examples/workflows/use_cases/workflow-of-workflows.yaml @@ -9,19 +9,41 @@ spec: resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ sub-workflow-1-\nspec:\n entrypoint: echo\n templates:\n - container:\n\ - \ args:\n - I'm workflow 1\n command:\n - cowsay\n \ - \ image: docker/whalesay:latest\n name: echo\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: sub-workflow-1- + spec: + entrypoint: echo + templates: + - container: + args: + - I'm workflow 1 + command: + - cowsay + image: docker/whalesay:latest + name: echo successCondition: status.phase == Succeeded - name: w2-resource resource: action: create failureCondition: status.phase in (Failed, Error) - manifest: "apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName:\ - \ sub-workflow-2-\nspec:\n entrypoint: echo\n templates:\n - container:\n\ - \ args:\n - I'm workflow 2\n command:\n - cowsay\n \ - \ image: docker/whalesay:latest\n name: echo\n" + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: sub-workflow-2- + spec: + entrypoint: echo + templates: + - container: + args: + - I'm workflow 2 + command: + - cowsay + image: docker/whalesay:latest + name: echo successCondition: status.phase == Succeeded - name: main steps: diff --git a/examples/workflows/user-container.yaml b/examples/workflows/user-container.yaml index 9d17eaac3..8a6be0b25 100644 --- a/examples/workflows/user-container.yaml +++ b/examples/workflows/user-container.yaml @@ -15,13 +15,11 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - - print(''hi'')' + print('hi') sidecars: - name: sidecar-name volumeMounts: diff --git a/examples/workflows/volume-mounts-nfs.yaml b/examples/workflows/volume-mounts-nfs.yaml index 051cfe1c4..10068514b 100644 --- a/examples/workflows/volume-mounts-nfs.yaml +++ b/examples/workflows/volume-mounts-nfs.yaml @@ -22,19 +22,14 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/nfs name: '{{inputs.parameters.vol}}' diff --git a/examples/workflows/volume-mounts-wt.yaml b/examples/workflows/volume-mounts-wt.yaml index 6ebc2b522..c447e1b7f 100644 --- a/examples/workflows/volume-mounts-wt.yaml +++ b/examples/workflows/volume-mounts-wt.yaml @@ -34,19 +34,14 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/examples/workflows/volume-mounts.yaml b/examples/workflows/volume-mounts.yaml index b4a869d85..56d6efa21 100644 --- a/examples/workflows/volume-mounts.yaml +++ b/examples/workflows/volume-mounts.yaml @@ -34,19 +34,14 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import os - import subprocess - - print(os.listdir(''/mnt'')) - - print(subprocess.run(''cd /mnt && df -h'', shell=True, capture_output=True).stdout.decode())' + print(os.listdir('/mnt')) + print(subprocess.run('cd /mnt && df -h', shell=True, capture_output=True).stdout.decode()) volumeMounts: - mountPath: /mnt/vol name: '{{inputs.parameters.vol}}' diff --git a/examples/workflows/workflow-on-exit.yaml b/examples/workflows/workflow-on-exit.yaml index 454205dd6..e37584709 100644 --- a/examples/workflows/workflow-on-exit.yaml +++ b/examples/workflows/workflow-on-exit.yaml @@ -47,20 +47,15 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: s = json.loads(r'''{{inputs.parameters.s}}''') + except: s = r'''{{inputs.parameters.s}}''' - try: s = json.loads(r''''''{{inputs.parameters.s}}'''''') - - except: s = r''''''{{inputs.parameters.s}}'''''' - - - print(s)' + print(s) - dag: tasks: - arguments: diff --git a/examples/workflows/workflow-with-global-params.yaml b/examples/workflows/workflow-with-global-params.yaml index 973c7bf88..8e9059ef0 100644 --- a/examples/workflows/workflow-with-global-params.yaml +++ b/examples/workflows/workflow-with-global-params.yaml @@ -18,17 +18,12 @@ spec: command: - python image: python:3.8 - source: 'import os - + source: |- + import os import sys - sys.path.append(os.getcwd()) - import json + try: v = json.loads(r'''{{inputs.parameters.v}}''') + except: v = r'''{{inputs.parameters.v}}''' - try: v = json.loads(r''''''{{inputs.parameters.v}}'''''') - - except: v = r''''''{{inputs.parameters.v}}'''''' - - - print(v)' + print(v) From a64faa01eb191a5c544bc1eac5a73757fedb7d83 Mon Sep 17 00:00:00 2001 From: DanCardin Date: Fri, 19 Jan 2024 15:12:08 -0500 Subject: [PATCH 3/3] Update src/hera/_yaml.py Co-authored-by: Elliot Gunton Signed-off-by: DanCardin --- src/hera/_yaml.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hera/_yaml.py b/src/hera/_yaml.py index 1583319c4..57d0b8a2f 100644 --- a/src/hera/_yaml.py +++ b/src/hera/_yaml.py @@ -23,9 +23,12 @@ def str_presenter(dumper, data): _yaml.representer.SafeRepresenter.add_representer(str, str_presenter) -def dump(*args, default_flow_style: bool = False, sort_keys: bool = False, **kwargs) -> str: +def dump(*args, **kwargs) -> str: """Builds the Workflow as an Argo schema Workflow object and returns it as yaml string.""" if not _yaml: raise ImportError("`PyYAML` is not installed. Install `hera[yaml]` to bring in the extra dependency") - return _yaml.dump(*args, default_flow_style=default_flow_style, sort_keys=sort_keys, **kwargs) + # Set some default options if not provided by the user + kwargs.setdefault("default_flow_style", False) + kwargs.setdefault("sort_keys", False) + return _yaml.dump(*args, **kwargs)