diff --git a/docs/examples/workflows/artifact.md b/docs/examples/workflows/artifact.md index 60ac490ed..1880c1abb 100644 --- a/docs/examples/workflows/artifact.md +++ b/docs/examples/workflows/artifact.md @@ -50,7 +50,6 @@ task, consumer, takes this artifact, places it at its own `/file` path, and prin artifacts: - from: '{{tasks.writer.outputs.artifacts.out-art}}' name: in-art - path: /tmp/file depends: writer name: consumer template: consumer diff --git a/docs/examples/workflows/artifact_with_fanout.md b/docs/examples/workflows/artifact_with_fanout.md index 43c7272cb..7deacc5ad 100644 --- a/docs/examples/workflows/artifact_with_fanout.md +++ b/docs/examples/workflows/artifact_with_fanout.md @@ -63,7 +63,6 @@ artifacts: - from: '{{tasks.writer.outputs.artifacts.out-art}}' name: in-art - path: /tmp/file depends: writer name: fanout template: fanout diff --git a/docs/examples/workflows/http_.md b/docs/examples/workflows/http.md similarity index 99% rename from docs/examples/workflows/http_.md rename to docs/examples/workflows/http.md index f9b7f1096..239f0b659 100644 --- a/docs/examples/workflows/http_.md +++ b/docs/examples/workflows/http.md @@ -1,4 +1,4 @@ -# Http +# Http diff --git a/docs/examples/workflows/script_annotations_artifact_custom_volume.md b/docs/examples/workflows/script_annotations_artifact_custom_volume.md new file mode 100644 index 000000000..16be23a83 --- /dev/null +++ b/docs/examples/workflows/script_annotations_artifact_custom_volume.md @@ -0,0 +1,145 @@ +# Script Annotations Artifact Custom Volume + + + +This example will reuse the outputs volume across script steps. + + +=== "Hera" + + ```python linenums="1" + from hera.workflows.artifact import ArtifactLoader + from hera.workflows.volume import Volume + + try: + from typing import Annotated # type: ignore + except ImportError: + from typing_extensions import Annotated # type: ignore + + + from hera.shared import global_config + from hera.workflows import ( + Artifact, + Parameter, + RunnerScriptConstructor, + Steps, + Workflow, + models as m, + script, + ) + + global_config.experimental_features["script_annotations"] = True + global_config.experimental_features["script_runner"] = True + + vol = Volume(name="my-vol", size="1Gi") + + + @script( + constructor=RunnerScriptConstructor(use_volume_for_outputs=vol.name), + volume_mounts=[m.VolumeMount(name="my-vol", mount_path="/mnt/here")], + ) + def output_artifact( + a_number: Annotated[int, Parameter(name="a_number")], + ) -> Annotated[int, Artifact(name="successor_out")]: + return a_number + 1 + + + @script( + constructor=RunnerScriptConstructor(use_volume_for_outputs=vol.name), + volume_mounts=[m.VolumeMount(name="my-vol", mount_path="/mnt/here")], + ) + def use_artifact(successor_in: Annotated[int, Artifact(name="successor_in", loader=ArtifactLoader.json)]): + print(successor_in) + + + with Workflow( + generate_name="test-output-annotations-", + entrypoint="my-steps", + volumes=[vol], + ) as w: + with Steps(name="my-steps") as s: + out = output_artifact(arguments={"a_number": 3}) + use_artifact(arguments=[out.get_artifact("successor_out").as_name("successor_in")]) + ``` + +=== "YAML" + + ```yaml linenums="1" + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: test-output-annotations- + spec: + entrypoint: my-steps + templates: + - name: my-steps + steps: + - - arguments: + parameters: + - name: a_number + value: '3' + name: output-artifact + template: output-artifact + - - arguments: + artifacts: + - from: '{{steps.output-artifact.outputs.artifacts.successor_out}}' + name: successor_in + name: use-artifact + template: use-artifact + - inputs: + parameters: + - name: a_number + name: output-artifact + outputs: + artifacts: + - name: successor_out + path: /hera/outputs/artifacts/successor_out + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_custom_volume:output_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + - name: hera__outputs_directory + value: /hera/outputs + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /mnt/here + name: my-vol + - inputs: + artifacts: + - name: successor_in + name: use-artifact + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_custom_volume:use_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /mnt/here + name: my-vol + volumeClaimTemplates: + - metadata: + name: my-vol + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ``` + diff --git a/docs/examples/workflows/script_annotations_artifact_passing.md b/docs/examples/workflows/script_annotations_artifact_passing.md new file mode 100644 index 000000000..ac96b8d16 --- /dev/null +++ b/docs/examples/workflows/script_annotations_artifact_passing.md @@ -0,0 +1,125 @@ +# Script Annotations Artifact Passing + + + +This example will reuse the outputs volume across script steps. + + +=== "Hera" + + ```python linenums="1" + from hera.workflows.volume import Volume + + try: + from typing import Annotated # type: ignore + except ImportError: + from typing_extensions import Annotated # type: ignore + + + from hera.shared import global_config + from hera.workflows import ( + Artifact, + Parameter, + Steps, + Workflow, + models as m, + script, + ) + + global_config.experimental_features["script_annotations"] = True + global_config.experimental_features["script_runner"] = True + + + @script(constructor="runner") + def output_artifact( + a_number: Annotated[int, Parameter(name="a_number")], + ) -> Annotated[int, Artifact(name="successor_out")]: + return a_number + 1 + + + @script(constructor="runner") + def use_artifact(successor_in: Annotated[int, Artifact(name="successor_in")]): + print(successor_in) + + + with Workflow( + generate_name="annotations-artifact-passing", + entrypoint="my-steps", + ) as w: + with Steps(name="my-steps") as s: + out = output_artifact(arguments={"a_number": 3}) + use_artifact(arguments=[out.get_artifact("successor_out").as_name("successor_in")]) + ``` + +=== "YAML" + + ```yaml linenums="1" + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: annotations-artifact-passing + spec: + entrypoint: my-steps + templates: + - name: my-steps + steps: + - - arguments: + parameters: + - name: a_number + value: '3' + name: output-artifact + template: output-artifact + - - arguments: + artifacts: + - from: '{{steps.output-artifact.outputs.artifacts.successor_out}}' + name: successor_in + name: use-artifact + template: use-artifact + - inputs: + parameters: + - name: a_number + name: output-artifact + outputs: + artifacts: + - name: successor_out + path: /hera/outputs/artifacts/successor_out + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_passing:output_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + - name: hera__outputs_directory + value: /hera/outputs + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /hera/outputs + name: hera-outputs-directory + volumes: + - emptyDir: {} + name: hera-outputs-directory + - inputs: + artifacts: + - name: successor_in + name: use-artifact + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_passing:use_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + image: python:3.8 + source: '{{inputs.parameters}}' + ``` + diff --git a/docs/examples/workflows/script_annotations_outputs.md b/docs/examples/workflows/script_annotations_outputs.md index e825c6609..21fdc083e 100644 --- a/docs/examples/workflows/script_annotations_outputs.md +++ b/docs/examples/workflows/script_annotations_outputs.md @@ -25,7 +25,7 @@ global_config.experimental_features["script_annotations"] = True global_config.experimental_features["script_runner"] = True - global_config.set_class_defaults(RunnerScriptConstructor, outputs_directory="user/chosen/outputs") + global_config.set_class_defaults(RunnerScriptConstructor, outputs_directory="/user/chosen/outputs") @script(constructor="runner") @@ -69,16 +69,16 @@ outputs: artifacts: - name: successor2 - path: user/chosen/outputs/artifacts/successor2 + path: /user/chosen/outputs/artifacts/successor2 - name: successor4 - path: user/chosen/outputs/artifacts/successor4 + path: /user/chosen/outputs/artifacts/successor4 parameters: - name: successor valueFrom: - path: user/chosen/outputs/parameters/successor + path: /user/chosen/outputs/parameters/successor - name: successor3 valueFrom: - path: user/chosen/outputs/parameters/successor3 + path: /user/chosen/outputs/parameters/successor3 script: args: - -m @@ -91,11 +91,11 @@ - name: hera__script_annotations value: '' - name: hera__outputs_directory - value: user/chosen/outputs + value: /user/chosen/outputs image: python:3.8 source: '{{inputs.parameters}}' volumeMounts: - - mountPath: user/chosen/outputs + - mountPath: /user/chosen/outputs name: hera-outputs-directory volumes: - emptyDir: {} diff --git a/docs/examples/workflows/script_artifact_passing.md b/docs/examples/workflows/script_artifact_passing.md index bc58a5741..1442c390f 100644 --- a/docs/examples/workflows/script_artifact_passing.md +++ b/docs/examples/workflows/script_artifact_passing.md @@ -26,10 +26,10 @@ with Workflow(generate_name="artifact-passing-", entrypoint="artifact-example") as w: with Steps(name="artifact-example") as s: - whalesay(name="generate-artifact") + whale_step = whalesay(name="generate-artifact") print_message( name="consume-artifact", - arguments=[Artifact(name="message", from_="{{steps.generate-artifact.outputs.artifacts.hello-art}}")], + arguments=whale_step.get_artifact("hello-art").as_name("message"), ) ``` diff --git a/docs/examples/workflows/upstream/artifact_passing.md b/docs/examples/workflows/upstream/artifact_passing.md index 0ddad0692..b9a042219 100644 --- a/docs/examples/workflows/upstream/artifact_passing.md +++ b/docs/examples/workflows/upstream/artifact_passing.md @@ -30,11 +30,11 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl ) with Steps(name="artifact-example") as s: - Step(name="generate-artifact", template=whalesay) + gen_step = Step(name="generate-artifact", template=whalesay) Step( name="consume-artifact", template=print_message, - arguments=[Artifact(name="message", from_="{{steps.generate-artifact.outputs.artifacts.hello-art}}")], + arguments=gen_step.get_artifact("hello-art").as_name("message"), ) ``` diff --git a/docs/examples/workflows/upstream/volumes_emptydir.md b/docs/examples/workflows/upstream/volumes_emptydir.md index 455627da7..88b17101e 100644 --- a/docs/examples/workflows/upstream/volumes_emptydir.md +++ b/docs/examples/workflows/upstream/volumes_emptydir.md @@ -14,13 +14,14 @@ The upstream example can be [found here](https://github.com/argoproj/argo-workfl from hera.workflows import ( Container, Workflow, + EmptyDirVolume, models as m, ) with Workflow( generate_name="volumes-emptydir-", entrypoint="volumes-emptydir-example", - volumes=[m.Volume(name="workdir", empty_dir=m.EmptyDirVolumeSource())], + volumes=[EmptyDirVolume(name="workdir")], ) as w: empty_dir = Container( name="volumes-emptydir-example", diff --git a/docs/examples/workflows/volume_mounts.md b/docs/examples/workflows/volume_mounts.md index 525ca70be..8e6e951e0 100644 --- a/docs/examples/workflows/volume_mounts.md +++ b/docs/examples/workflows/volume_mounts.md @@ -34,9 +34,9 @@ generate_name="volumes-", entrypoint="d", volumes=[ - Volume(name="v1", mount_path="/mnt/v1", size="1Gi"), - Volume(name="v2", mount_path="/mnt/v2", size="3Gi"), - Volume(name="v3", mount_path="/mnt/v3", size="5Gi"), + Volume(name="v1", size="1Gi"), + Volume(name="v2", size="3Gi"), + Volume(name="v3", size="5Gi"), ], ) as w: with DAG(name="d"): diff --git a/docs/examples/workflows/volume_mounts_wt.md b/docs/examples/workflows/volume_mounts_wt.md index 8c08474ae..bab391421 100644 --- a/docs/examples/workflows/volume_mounts_wt.md +++ b/docs/examples/workflows/volume_mounts_wt.md @@ -34,9 +34,9 @@ generate_name="volumes-", entrypoint="d", volumes=[ - Volume(name="v1", mount_path="/mnt/v1", size="1Gi"), - Volume(name="v2", mount_path="/mnt/v2", size="3Gi"), - Volume(name="v3", mount_path="/mnt/v3", size="5Gi"), + Volume(name="v1", size="1Gi"), + Volume(name="v2", size="3Gi"), + Volume(name="v3", size="5Gi"), ], ) as w: with DAG(name="d"): diff --git a/examples/workflows/artifact-with-fanout.yaml b/examples/workflows/artifact-with-fanout.yaml index d84cb2303..0d9d4381e 100644 --- a/examples/workflows/artifact-with-fanout.yaml +++ b/examples/workflows/artifact-with-fanout.yaml @@ -13,7 +13,6 @@ spec: artifacts: - from: '{{tasks.writer.outputs.artifacts.out-art}}' name: in-art - path: /tmp/file depends: writer name: fanout template: fanout diff --git a/examples/workflows/artifact.yaml b/examples/workflows/artifact.yaml index 7b2dc1d5b..200c8f828 100644 --- a/examples/workflows/artifact.yaml +++ b/examples/workflows/artifact.yaml @@ -13,7 +13,6 @@ spec: artifacts: - from: '{{tasks.writer.outputs.artifacts.out-art}}' name: in-art - path: /tmp/file depends: writer name: consumer template: consumer diff --git a/examples/workflows/http_.py b/examples/workflows/http.py similarity index 100% rename from examples/workflows/http_.py rename to examples/workflows/http.py diff --git a/examples/workflows/http-.yaml b/examples/workflows/http.yaml similarity index 100% rename from examples/workflows/http-.yaml rename to examples/workflows/http.yaml diff --git a/examples/workflows/script-annotations-artifact-custom-volume.yaml b/examples/workflows/script-annotations-artifact-custom-volume.yaml new file mode 100644 index 000000000..9c83fe0ec --- /dev/null +++ b/examples/workflows/script-annotations-artifact-custom-volume.yaml @@ -0,0 +1,76 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: test-output-annotations- +spec: + entrypoint: my-steps + templates: + - name: my-steps + steps: + - - arguments: + parameters: + - name: a_number + value: '3' + name: output-artifact + template: output-artifact + - - arguments: + artifacts: + - from: '{{steps.output-artifact.outputs.artifacts.successor_out}}' + name: successor_in + name: use-artifact + template: use-artifact + - inputs: + parameters: + - name: a_number + name: output-artifact + outputs: + artifacts: + - name: successor_out + path: /hera/outputs/artifacts/successor_out + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_custom_volume:output_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + - name: hera__outputs_directory + value: /hera/outputs + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /mnt/here + name: my-vol + - inputs: + artifacts: + - name: successor_in + name: use-artifact + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_custom_volume:use_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /mnt/here + name: my-vol + volumeClaimTemplates: + - metadata: + name: my-vol + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/examples/workflows/script-annotations-artifact-passing.yaml b/examples/workflows/script-annotations-artifact-passing.yaml new file mode 100644 index 000000000..9df56fc27 --- /dev/null +++ b/examples/workflows/script-annotations-artifact-passing.yaml @@ -0,0 +1,67 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: annotations-artifact-passing +spec: + entrypoint: my-steps + templates: + - name: my-steps + steps: + - - arguments: + parameters: + - name: a_number + value: '3' + name: output-artifact + template: output-artifact + - - arguments: + artifacts: + - from: '{{steps.output-artifact.outputs.artifacts.successor_out}}' + name: successor_in + name: use-artifact + template: use-artifact + - inputs: + parameters: + - name: a_number + name: output-artifact + outputs: + artifacts: + - name: successor_out + path: /hera/outputs/artifacts/successor_out + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_passing:output_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + - name: hera__outputs_directory + value: /hera/outputs + image: python:3.8 + source: '{{inputs.parameters}}' + volumeMounts: + - mountPath: /hera/outputs + name: hera-outputs-directory + volumes: + - emptyDir: {} + name: hera-outputs-directory + - inputs: + artifacts: + - name: successor_in + name: use-artifact + script: + args: + - -m + - hera.workflows.runner + - -e + - examples.workflows.script_annotations_artifact_passing:use_artifact + command: + - python + env: + - name: hera__script_annotations + value: '' + image: python:3.8 + source: '{{inputs.parameters}}' diff --git a/examples/workflows/script-annotations-outputs.yaml b/examples/workflows/script-annotations-outputs.yaml index 3e8c7606c..ec3a6518b 100644 --- a/examples/workflows/script-annotations-outputs.yaml +++ b/examples/workflows/script-annotations-outputs.yaml @@ -20,16 +20,16 @@ spec: outputs: artifacts: - name: successor2 - path: user/chosen/outputs/artifacts/successor2 + path: /user/chosen/outputs/artifacts/successor2 - name: successor4 - path: user/chosen/outputs/artifacts/successor4 + path: /user/chosen/outputs/artifacts/successor4 parameters: - name: successor valueFrom: - path: user/chosen/outputs/parameters/successor + path: /user/chosen/outputs/parameters/successor - name: successor3 valueFrom: - path: user/chosen/outputs/parameters/successor3 + path: /user/chosen/outputs/parameters/successor3 script: args: - -m @@ -42,11 +42,11 @@ spec: - name: hera__script_annotations value: '' - name: hera__outputs_directory - value: user/chosen/outputs + value: /user/chosen/outputs image: python:3.8 source: '{{inputs.parameters}}' volumeMounts: - - mountPath: user/chosen/outputs + - mountPath: /user/chosen/outputs name: hera-outputs-directory volumes: - emptyDir: {} diff --git a/examples/workflows/script_annotations_artifact_custom_volume.py b/examples/workflows/script_annotations_artifact_custom_volume.py new file mode 100644 index 000000000..3858bae4d --- /dev/null +++ b/examples/workflows/script_annotations_artifact_custom_volume.py @@ -0,0 +1,55 @@ +"""This example will reuse the outputs volume across script steps.""" + + +from hera.workflows.artifact import ArtifactLoader +from hera.workflows.volume import Volume + +try: + from typing import Annotated # type: ignore +except ImportError: + from typing_extensions import Annotated # type: ignore + + +from hera.shared import global_config +from hera.workflows import ( + Artifact, + Parameter, + RunnerScriptConstructor, + Steps, + Workflow, + models as m, + script, +) + +global_config.experimental_features["script_annotations"] = True +global_config.experimental_features["script_runner"] = True + +vol = Volume(name="my-vol", size="1Gi") + + +@script( + constructor=RunnerScriptConstructor(use_volume_for_outputs=vol.name), + volume_mounts=[m.VolumeMount(name="my-vol", mount_path="/mnt/here")], +) +def output_artifact( + a_number: Annotated[int, Parameter(name="a_number")], +) -> Annotated[int, Artifact(name="successor_out")]: + return a_number + 1 + + +@script( + constructor=RunnerScriptConstructor(use_volume_for_outputs=vol.name), + volume_mounts=[m.VolumeMount(name="my-vol", mount_path="/mnt/here")], +) +def use_artifact(successor_in: Annotated[int, Artifact(name="successor_in", loader=ArtifactLoader.json)]): + print(successor_in) + + +with Workflow( + generate_name="test-output-annotations-", + entrypoint="my-steps", + volumes=[vol], +) as w: + with Steps(name="my-steps") as s: + out = output_artifact(arguments={"a_number": 3}) + use_artifact(arguments=[out.get_artifact("successor_out").as_name("successor_in")]) diff --git a/examples/workflows/script_annotations_artifact_passing.py b/examples/workflows/script_annotations_artifact_passing.py new file mode 100644 index 000000000..ebf55ac76 --- /dev/null +++ b/examples/workflows/script_annotations_artifact_passing.py @@ -0,0 +1,44 @@ +"""This example will reuse the outputs volume across script steps.""" + + +from hera.workflows.volume import Volume + +try: + from typing import Annotated # type: ignore +except ImportError: + from typing_extensions import Annotated # type: ignore + + +from hera.shared import global_config +from hera.workflows import ( + Artifact, + Parameter, + Steps, + Workflow, + models as m, + script, +) + +global_config.experimental_features["script_annotations"] = True +global_config.experimental_features["script_runner"] = True + + +@script(constructor="runner") +def output_artifact( + a_number: Annotated[int, Parameter(name="a_number")], +) -> Annotated[int, Artifact(name="successor_out")]: + return a_number + 1 + + +@script(constructor="runner") +def use_artifact(successor_in: Annotated[int, Artifact(name="successor_in")]): + print(successor_in) + + +with Workflow( + generate_name="annotations-artifact-passing", + entrypoint="my-steps", +) as w: + with Steps(name="my-steps") as s: + out = output_artifact(arguments={"a_number": 3}) + use_artifact(arguments=[out.get_artifact("successor_out").as_name("successor_in")]) diff --git a/examples/workflows/script_annotations_outputs.py b/examples/workflows/script_annotations_outputs.py index 2278667f1..77c9eaa8a 100644 --- a/examples/workflows/script_annotations_outputs.py +++ b/examples/workflows/script_annotations_outputs.py @@ -15,7 +15,7 @@ global_config.experimental_features["script_annotations"] = True global_config.experimental_features["script_runner"] = True -global_config.set_class_defaults(RunnerScriptConstructor, outputs_directory="user/chosen/outputs") +global_config.set_class_defaults(RunnerScriptConstructor, outputs_directory="/user/chosen/outputs") @script(constructor="runner") diff --git a/examples/workflows/script_artifact_passing.py b/examples/workflows/script_artifact_passing.py index d4675d5f5..8b79234ed 100644 --- a/examples/workflows/script_artifact_passing.py +++ b/examples/workflows/script_artifact_passing.py @@ -16,8 +16,8 @@ def print_message(): with Workflow(generate_name="artifact-passing-", entrypoint="artifact-example") as w: with Steps(name="artifact-example") as s: - whalesay(name="generate-artifact") + whale_step = whalesay(name="generate-artifact") print_message( name="consume-artifact", - arguments=[Artifact(name="message", from_="{{steps.generate-artifact.outputs.artifacts.hello-art}}")], + arguments=whale_step.get_artifact("hello-art").as_name("message"), ) diff --git a/examples/workflows/upstream/artifact_passing.py b/examples/workflows/upstream/artifact_passing.py index e15c6aa84..59af0ad70 100644 --- a/examples/workflows/upstream/artifact_passing.py +++ b/examples/workflows/upstream/artifact_passing.py @@ -17,9 +17,9 @@ ) with Steps(name="artifact-example") as s: - Step(name="generate-artifact", template=whalesay) + gen_step = Step(name="generate-artifact", template=whalesay) Step( name="consume-artifact", template=print_message, - arguments=[Artifact(name="message", from_="{{steps.generate-artifact.outputs.artifacts.hello-art}}")], + arguments=gen_step.get_artifact("hello-art").as_name("message"), ) diff --git a/examples/workflows/upstream/volumes_emptydir.py b/examples/workflows/upstream/volumes_emptydir.py index a10f89736..98ae4ce70 100644 --- a/examples/workflows/upstream/volumes_emptydir.py +++ b/examples/workflows/upstream/volumes_emptydir.py @@ -1,13 +1,14 @@ from hera.workflows import ( Container, Workflow, + EmptyDirVolume, models as m, ) with Workflow( generate_name="volumes-emptydir-", entrypoint="volumes-emptydir-example", - volumes=[m.Volume(name="workdir", empty_dir=m.EmptyDirVolumeSource())], + volumes=[EmptyDirVolume(name="workdir")], ) as w: empty_dir = Container( name="volumes-emptydir-example", diff --git a/examples/workflows/volume_mounts.py b/examples/workflows/volume_mounts.py index 31627ebe8..f983e460d 100644 --- a/examples/workflows/volume_mounts.py +++ b/examples/workflows/volume_mounts.py @@ -24,9 +24,9 @@ def foo(): generate_name="volumes-", entrypoint="d", volumes=[ - Volume(name="v1", mount_path="/mnt/v1", size="1Gi"), - Volume(name="v2", mount_path="/mnt/v2", size="3Gi"), - Volume(name="v3", mount_path="/mnt/v3", size="5Gi"), + Volume(name="v1", size="1Gi"), + Volume(name="v2", size="3Gi"), + Volume(name="v3", size="5Gi"), ], ) as w: with DAG(name="d"): diff --git a/examples/workflows/volume_mounts_wt.py b/examples/workflows/volume_mounts_wt.py index 455fef9d8..2c034c693 100644 --- a/examples/workflows/volume_mounts_wt.py +++ b/examples/workflows/volume_mounts_wt.py @@ -24,9 +24,9 @@ def foo(): generate_name="volumes-", entrypoint="d", volumes=[ - Volume(name="v1", mount_path="/mnt/v1", size="1Gi"), - Volume(name="v2", mount_path="/mnt/v2", size="3Gi"), - Volume(name="v3", mount_path="/mnt/v3", size="5Gi"), + Volume(name="v1", size="1Gi"), + Volume(name="v2", size="3Gi"), + Volume(name="v3", size="5Gi"), ], ) as w: with DAG(name="d"): diff --git a/src/hera/workflows/_mixins.py b/src/hera/workflows/_mixins.py index 0acab4cac..9a1957625 100644 --- a/src/hera/workflows/_mixins.py +++ b/src/hera/workflows/_mixins.py @@ -1061,7 +1061,7 @@ def _get_artifact(self, name: str, subtype: str) -> Artifact: obj = next((output for output in artifacts if output.name == name), None) if obj is not None: - return Artifact(name=name, path=obj.path, from_=f"{{{{{subtype}.{self.name}.outputs.artifacts.{name}}}}}") + return Artifact(name=name, from_=f"{{{{{subtype}.{self.name}.outputs.artifacts.{name}}}}}") raise KeyError(f"No output artifact named `{name}` found") def get_parameters_as(self, name: str) -> Parameter: diff --git a/src/hera/workflows/runner.py b/src/hera/workflows/runner.py index a1ef1874d..720337b93 100644 --- a/src/hera/workflows/runner.py +++ b/src/hera/workflows/runner.py @@ -184,7 +184,7 @@ def _save_annotated_return_outputs( def _get_outputs_path(destination: Union[Parameter, Artifact]) -> Path: """Get the path from the destination annotation using the defined outputs directory.""" - path = Path(os.environ.get("hera__outputs_directory", "/tmp/hera/outputs")) + path = Path(os.environ.get("hera__outputs_directory", "/hera/outputs")) if isinstance(destination, Parameter) and destination.name: path = path / f"parameters/{destination.name}" elif isinstance(destination, Artifact): diff --git a/src/hera/workflows/script.py b/src/hera/workflows/script.py index 5ffc7035f..07bd1ba64 100644 --- a/src/hera/workflows/script.py +++ b/src/hera/workflows/script.py @@ -198,7 +198,8 @@ def _build_script(self) -> _ModelScriptTemplate: ): if not self.constructor.outputs_directory: self.constructor.outputs_directory = self.constructor.DEFAULT_HERA_OUTPUTS_DIRECTORY - self._create_hera_outputs_volume() + if self.constructor.use_volume_for_outputs is None: + self._create_hera_outputs_volume() return self.constructor.transform_script_template_post_build( self, @@ -307,7 +308,7 @@ def _aggregate_callable_io( return current_io def _create_hera_outputs_volume(self) -> None: - """Create the new volume as an EmptyDirVolume if needed for the automatic saving of the hera outputs.""" + """Create a new EmptyDirVolume at the template level, if needed for the automatic saving of the hera outputs.""" assert isinstance(self.constructor, RunnerScriptConstructor) new_volume = EmptyDirVolume(name="hera-outputs-directory", mount_path=self.constructor.outputs_directory) @@ -704,7 +705,10 @@ class RunnerScriptConstructor(ScriptConstructor, ExperimentalMixin): outputs_directory: Optional[str] = None """Used for saving outputs when defined using annotations.""" - DEFAULT_HERA_OUTPUTS_DIRECTORY: str = "/tmp/hera/outputs" + use_volume_for_outputs: Optional[str] = None + """Volume to use if saving outputs when defined using annotations.""" + + DEFAULT_HERA_OUTPUTS_DIRECTORY: str = "/hera/outputs" """Used as the default value for when the outputs_directory is not set""" def transform_values(self, cls: Type[Script], values: Any) -> Any: diff --git a/tests/test_script_annotations.py b/tests/test_script_annotations.py index f3c32201a..1f5961431 100644 --- a/tests/test_script_annotations.py +++ b/tests/test_script_annotations.py @@ -88,19 +88,19 @@ def test_double_default_throws_a_value_error(global_config_fixture): ( "script_annotations_output_param_in_func", {"parameters": [{"name": "a_number"}]}, - {"parameters": [{"name": "successor", "valueFrom": {"path": "/tmp/hera/outputs/parameters/successor"}}]}, + {"parameters": [{"name": "successor", "valueFrom": {"path": "/hera/outputs/parameters/successor"}}]}, ), ( "script_annotations_output_artifact_in_func", {"parameters": [{"name": "a_number"}]}, - {"artifacts": [{"name": "successor", "path": "/tmp/hera/outputs/artifacts/successor"}]}, + {"artifacts": [{"name": "successor", "path": "/hera/outputs/artifacts/successor"}]}, ), ( "script_annotations_output_param_and_artifact_in_func", {"parameters": [{"name": "a_number"}]}, { - "parameters": [{"name": "successor", "valueFrom": {"path": "/tmp/hera/outputs/parameters/successor"}}], - "artifacts": [{"name": "successor2", "path": "/tmp/hera/outputs/artifacts/successor2"}], + "parameters": [{"name": "successor", "valueFrom": {"path": "/hera/outputs/parameters/successor"}}], + "artifacts": [{"name": "successor2", "path": "/hera/outputs/artifacts/successor2"}], }, ), ( @@ -108,12 +108,12 @@ def test_double_default_throws_a_value_error(global_config_fixture): {"parameters": [{"name": "a_number"}]}, { "parameters": [ - {"name": "successor", "valueFrom": {"path": "/tmp/hera/outputs/parameters/successor"}}, - {"name": "successor3", "valueFrom": {"path": "/tmp/hera/outputs/parameters/successor3"}}, + {"name": "successor", "valueFrom": {"path": "/hera/outputs/parameters/successor"}}, + {"name": "successor3", "valueFrom": {"path": "/hera/outputs/parameters/successor3"}}, ], "artifacts": [ - {"name": "successor2", "path": "/tmp/hera/outputs/artifacts/successor2"}, - {"name": "successor4", "path": "/tmp/hera/outputs/artifacts/successor4"}, + {"name": "successor2", "path": "/hera/outputs/artifacts/successor2"}, + {"name": "successor4", "path": "/hera/outputs/artifacts/successor4"}, ], }, ), @@ -121,8 +121,8 @@ def test_double_default_throws_a_value_error(global_config_fixture): "script_annotations_output_in_func_no_name", {"parameters": [{"name": "a_number"}]}, { - "parameters": [{"name": "successor", "valueFrom": {"path": "/tmp/hera/outputs/parameters/successor"}}], - "artifacts": [{"name": "successor2", "path": "/tmp/hera/outputs/artifacts/successor2"}], + "parameters": [{"name": "successor", "valueFrom": {"path": "/hera/outputs/parameters/successor"}}], + "artifacts": [{"name": "successor2", "path": "/hera/outputs/artifacts/successor2"}], }, ), (