Added RawOutputDataConfig in ExecutionSpec (flyteorg#889)

* Added RawOutputDataConfig in ExecutionSpec Signed-off-by: Kevin Su <pingsutw@apache.org> * Updated idl Signed-off-by: Kevin Su <pingsutw@apache.org> * Updated idl Signed-off-by: Kevin Su <pingsutw@apache.org> * Updated idl Signed-off-by: Kevin Su <pingsutw@apache.org> * Updated args Signed-off-by: Kevin Su <pingsutw@apache.org> Signed-off-by: Mike Zhong <mzhong@embarkvet.com>
ProjectAussie · Apr 11, 2022 · 467994a · 467994a
1 parent c2c8866
commit 467994a
Show file tree

Hide file tree

Showing 9 changed files with 90 additions and 98 deletions.
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -32,7 +32,6 @@ certifi==2021.10.8
     #   requests
 cffi==1.15.0
     # via
-    #   -c requirements.txt
     #   bcrypt
     #   cryptography
     #   pynacl
@@ -72,10 +71,7 @@ croniter==1.3.4
     #   -c requirements.txt
     #   flytekit
 cryptography==36.0.1
-    # via
-    #   -c requirements.txt
-    #   paramiko
-    #   secretstorage
+    # via paramiko
 dataclasses-json==0.5.6
     # via
     #   -c requirements.txt
@@ -116,11 +112,11 @@ docstring-parser==0.13
     #   flytekit
 filelock==3.6.0
     # via virtualenv
-flyteidl==0.23.0
+flyteidl==0.23.1
     # via
     #   -c requirements.txt
     #   flytekit
-google-api-core[grpc]==2.5.0
+google-api-core[grpc]==2.7.1
     # via
     #   google-cloud-bigquery
     #   google-cloud-bigquery-storage
@@ -129,15 +125,15 @@ google-auth==2.6.0
     # via
     #   google-api-core
     #   google-cloud-core
-google-cloud-bigquery==2.34.0
+google-cloud-bigquery==2.34.2
     # via -r dev-requirements.in
-google-cloud-bigquery-storage==2.12.0
+google-cloud-bigquery-storage==2.13.0
     # via -r dev-requirements.in
-google-cloud-core==2.2.2
+google-cloud-core==2.2.3
     # via google-cloud-bigquery
 google-crc32c==1.3.0
     # via google-resumable-media
-google-resumable-media==2.3.0
+google-resumable-media==2.3.2
     # via google-cloud-bigquery
 googleapis-common-protos==1.55.0
     # via
@@ -160,17 +156,12 @@ idna==3.3
     # via
     #   -c requirements.txt
     #   requests
-importlib-metadata==4.11.2
+importlib-metadata==4.11.3
     # via
     #   -c requirements.txt
     #   keyring
 iniconfig==1.1.1
     # via pytest
-jeepney==0.7.1
-    # via
-    #   -c requirements.txt
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   -c requirements.txt
@@ -191,11 +182,11 @@ keyring==23.5.0
     # via
     #   -c requirements.txt
     #   flytekit
-markupsafe==2.1.0
+markupsafe==2.1.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.14.1
+marshmallow==3.15.0
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -211,7 +202,7 @@ marshmallow-jsonschema==0.13.0
     #   flytekit
 mock==4.0.3
     # via -r dev-requirements.in
-mypy==0.931
+mypy==0.941
     # via -r dev-requirements.in
 mypy-extensions==0.4.3
     # via
@@ -231,13 +222,15 @@ numpy==1.21.5
     #   pyarrow
 packaging==21.3
     # via
+    #   -c requirements.txt
     #   google-cloud-bigquery
+    #   marshmallow
     #   pytest
 pandas==1.3.5
     # via
     #   -c requirements.txt
     #   flytekit
-paramiko==2.9.2
+paramiko==2.10.2
     # via docker
 platformdirs==2.5.1
     # via virtualenv
@@ -284,13 +277,13 @@ pyasn1==0.4.8
 pyasn1-modules==0.2.8
     # via google-auth
 pycparser==2.21
-    # via
-    #   -c requirements.txt
-    #   cffi
+    # via cffi
 pynacl==1.5.0
     # via paramiko
 pyparsing==3.0.7
-    # via packaging
+    # via
+    #   -c requirements.txt
+    #   packaging
 pyrsistent==0.18.1
     # via
     #   -c requirements.txt
@@ -336,7 +329,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2022.1.18
+regex==2022.3.2
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -350,7 +343,7 @@ requests==2.27.1
     #   google-api-core
     #   google-cloud-bigquery
     #   responses
-responses==0.18.0
+responses==0.19.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -360,10 +353,6 @@ retry==0.9.2
     #   flytekit
 rsa==4.8
     # via google-auth
-secretstorage==3.3.1
-    # via
-    #   -c requirements.txt
-    #   keyring
 six==1.16.0
     # via
     #   -c requirements.txt
@@ -373,6 +362,7 @@ six==1.16.0
     #   google-auth
     #   grpcio
     #   jsonschema
+    #   paramiko
     #   python-dateutil
     #   virtualenv
     #   websocket-client
@@ -414,7 +404,7 @@ urllib3==1.26.8
     #   flytekit
     #   requests
     #   responses
-virtualenv==20.13.2
+virtualenv==20.13.3
     # via pre-commit
 websocket-client==0.59.0
     # via
@@ -424,7 +414,7 @@ wheel==0.37.1
     # via
     #   -c requirements.txt
     #   flytekit
-wrapt==1.13.3
+wrapt==1.14.0
     # via
     #   -c requirements.txt
     #   deprecated

diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -10,7 +10,7 @@ alabaster==0.7.12
     # via sphinx
 arrow==1.2.2
     # via jinja2-time
-astroid==2.10.0
+astroid==2.11.0
     # via sphinx-autoapi
 babel==2.9.1
     # via sphinx
@@ -42,9 +42,7 @@ cookiecutter==1.7.3
 croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
-    # via
-    #   -r doc-requirements.in
-    #   secretstorage
+    # via -r doc-requirements.in
 css-html-js-minify==2.5.5
     # via sphinx-material
 dataclasses-json==0.5.6
@@ -63,7 +61,7 @@ docutils==0.17.1
     # via
     #   sphinx
     #   sphinx-panels
-flyteidl==0.23.0
+flyteidl==0.23.1
     # via flytekit
 furo @ git+https://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
@@ -77,14 +75,10 @@ idna==3.3
     # via requests
 imagesize==1.3.0
     # via sphinx
-importlib-metadata==4.11.2
+importlib-metadata==4.11.3
     # via
     #   keyring
     #   sphinx
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -99,9 +93,9 @@ lazy-object-proxy==1.7.1
     # via astroid
 lxml==4.8.0
     # via sphinx-material
-markupsafe==2.1.0
+markupsafe==2.1.1
     # via jinja2
-marshmallow==3.14.1
+marshmallow==3.15.0
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -114,12 +108,14 @@ mypy-extensions==0.4.3
     # via typing-inspect
 natsort==8.1.0
     # via flytekit
-numpy==1.22.2
+numpy==1.22.3
     # via
     #   pandas
     #   pyarrow
 packaging==21.3
-    # via sphinx
+    # via
+    #   marshmallow
+    #   sphinx
 pandas==1.4.1
     # via flytekit
 poyo==0.5.0
@@ -165,20 +161,18 @@ pytz==2021.3
     #   pandas
 pyyaml==6.0
     # via sphinx-autoapi
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
     #   cookiecutter
     #   flytekit
     #   responses
     #   sphinx
-responses==0.18.0
+responses==0.19.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -245,7 +239,7 @@ typing-extensions==4.1.1
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
-unidecode==1.3.3
+unidecode==1.3.4
     # via
     #   python-slugify
     #   sphinx-autoapi
@@ -256,7 +250,7 @@ urllib3==1.26.8
     #   responses
 wheel==0.37.1
     # via flytekit
-wrapt==1.13.3
+wrapt==1.14.0
     # via
     #   astroid
     #   deprecated

diff --git a/flytekit/models/execution.py b/flytekit/models/execution.py
@@ -82,6 +82,7 @@ def __init__(
         labels=None,
         annotations=None,
         auth_role=None,
+        raw_output_data_config=None,
         max_parallelism=None,
     ):
         """
@@ -92,6 +93,7 @@ def __init__(
         :param flytekit.models.common.Labels labels: Labels to apply to the execution.
         :param flytekit.models.common.Annotations annotations: Annotations to apply to the execution
         :param flytekit.models.common.AuthRole auth_role: The authorization method with which to execute the workflow.
+        :param raw_output_data_config: Optional location of offloaded data for things like S3, etc.
         :param max_parallelism int: Controls the maximum number of tasknodes that can be run in parallel for the entire
             workflow. This is useful to achieve fairness. Note: MapTasks are regarded as one unit, and
             parallelism/concurrency of MapTasks is independent from this.
@@ -104,6 +106,7 @@ def __init__(
         self._labels = labels or _common_models.Labels({})
         self._annotations = annotations or _common_models.Annotations({})
         self._auth_role = auth_role or _common_models.AuthRole()
+        self._raw_output_data_config = raw_output_data_config
         self._max_parallelism = max_parallelism
 
     @property
@@ -156,6 +159,13 @@ def auth_role(self):
         """
         return self._auth_role
 
+    @property
+    def raw_output_data_config(self):
+        """
+        :rtype: flytekit.models.common.RawOutputDataConfig
+        """
+        return self._raw_output_data_config
+
     @property
     def max_parallelism(self) -> int:
         return self._max_parallelism
@@ -172,6 +182,9 @@ def to_flyte_idl(self):
             labels=self.labels.to_flyte_idl(),
             annotations=self.annotations.to_flyte_idl(),
             auth_role=self._auth_role.to_flyte_idl() if self.auth_role else None,
+            raw_output_data_config=self._raw_output_data_config.to_flyte_idl()
+            if self._raw_output_data_config
+            else None,
             max_parallelism=self.max_parallelism,
         )
 
@@ -189,6 +202,9 @@ def from_flyte_idl(cls, p):
             labels=_common_models.Labels.from_flyte_idl(p.labels),
             annotations=_common_models.Annotations.from_flyte_idl(p.annotations),
             auth_role=_common_models.AuthRole.from_flyte_idl(p.auth_role),
+            raw_output_data_config=_common_models.RawOutputDataConfig.from_flyte_idl(p.raw_output_data_config)
+            if p.HasField("raw_output_data_config")
+            else None,
             max_parallelism=p.max_parallelism,
         )
 

diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py
@@ -77,18 +77,20 @@ class Options(object):
            the data for offloaded types is stored.
         auth_role: Specifies the Kubernetes Service account,
            IAM role etc to be used. If not specified defaults will be used.
-        labels:
-        annotations:
-        security_context:
-        max_parallelism:
-        notifications:
-        disable_notifications:
+        labels: Custom labels to be applied to the execution resource
+        annotations: Custom annotations to be applied to the execution resource
+        security_context: Indicates security context for permissions triggered with this launch plan
+        raw_output_data_config: Optional location of offloaded data for things like S3, etc.
+        max_parallelism: Controls the maximum number of tasknodes that can be run in parallel for the entire workflow.
+        notifications: List of notifications for this execution
+        disable_notifications: This should be set to true if all notifications are intended to be disabled for this execution.
     """
 
     raw_data_prefix: typing.Optional[str] = None
     auth_role: typing.Optional[common_models.AuthRole] = None
     labels: typing.Optional[common_models.Labels] = None
     annotations: typing.Optional[common_models.Annotations] = None
+    raw_output_data_config: typing.Optional[common_models.RawOutputDataConfig] = None
     security_context: typing.Optional[security.SecurityContext] = None
     max_parallelism: typing.Optional[int] = None
     notifications: typing.Optional[typing.List[common_models.Notification]] = None
@@ -596,6 +598,7 @@ def _execute(
                     disable_all=options.disable_notifications,
                     labels=options.labels,
                     annotations=options.annotations,
+                    raw_output_data_config=options.raw_output_data_config,
                     auth_role=options.auth_role,
                     max_parallelism=options.max_parallelism,
                 ),