From 749ecd12caeee92697691fa35e85b4b64010ce69 Mon Sep 17 00:00:00 2001 From: Han Wang Date: Mon, 24 Oct 2022 15:49:48 +0800 Subject: [PATCH 1/3] support properties --- .../ml/_schema/component/command_component.py | 1 + .../entities/_component/command_component.py | 5 + .../component/e2etests/test_component.py | 41 +- .../unittests/test_component_schema.py | 8 + ...nd_component_with_properties_e2e_flow.json | 387 ++++++++++++++++++ .../helloworld_component_with_properties.yml | 37 ++ 6 files changed, 478 insertions(+), 1 deletion(-) create mode 100644 sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_properties_e2e_flow.json create mode 100644 sdk/ml/azure-ai-ml/tests/test_configs/components/helloworld_component_with_properties.yml diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py index b06451d98c7b..e501f35741e0 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py @@ -48,6 +48,7 @@ class Meta: ] ), ) + properties = fields.Dict(keys=fields.Str(), values=fields.Raw()) class RestCommandComponentSchema(CommandComponentSchema): diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py index 5ce7514bb049..d1f9c9d175cb 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/command_component.py @@ -54,6 +54,9 @@ class CommandComponent(Component, ParameterizedCommand): :type instance_count: int :param is_deterministic: Whether the command component is deterministic. :type is_deterministic: bool + :param properties: Properties of the component. Contents inside will pass through to backend as a dictionary. + :type properties: dict + :raises ~azure.ai.ml.exceptions.ValidationException: Raised if CommandComponent cannot be successfully validated. Details will be provided in the error message. """ @@ -75,6 +78,7 @@ def __init__( outputs: Dict = None, instance_count: int = None, # promoted property from resources.instance_count is_deterministic: bool = True, + properties: Dict = None, **kwargs, ): # validate init params are valid type @@ -98,6 +102,7 @@ def __init__( inputs=inputs, outputs=outputs, is_deterministic=is_deterministic, + properties=properties, **kwargs, ) diff --git a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py index 06fbb6b683a6..3215fa2350e7 100644 --- a/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py +++ b/sdk/ml/azure-ai-ml/tests/component/e2etests/test_component.py @@ -7,7 +7,6 @@ import pydash import pytest -from test_utilities.utils import _PYTEST_TIMEOUT_METHOD from azure.ai.ml import MLClient, MpiDistribution, load_component, load_environment from azure.ai.ml._restclient.v2022_05_01.models import ComponentContainerData, ListViewType @@ -876,3 +875,43 @@ def test_component_with_default_label( node = default_component() assert node._to_rest_object()["componentId"] == default_component.id + + def test_command_component_with_properties_e2e_flow(self, client: MLClient, randstr: Callable[[str], str]) -> None: + command_component = load_component( + source="./tests/test_configs/components/helloworld_component_with_properties.yml", + ) + expected_dict = { + '$schema': 'https://azuremlschemas.azureedge.net/development/commandComponent.schema.json', + '_source': 'YAML.COMPONENT', + 'command': 'echo Hello World & echo $[[${{inputs.component_in_number}}]] & ' + 'echo ${{inputs.component_in_path}} & echo ' + '${{outputs.component_out_path}} > ' + '${{outputs.component_out_path}}/component_in_number', + 'description': 'This is the basic command component', + 'display_name': 'CommandComponentBasic', + 'inputs': {'component_in_number': {'default': '10.99', + 'description': 'A number', + 'optional': True, + 'type': 'number'}, + 'component_in_path': {'description': 'A path', + 'type': 'uri_folder'}}, + 'is_deterministic': True, + 'outputs': {'component_out_path': {'type': 'uri_folder'}}, + 'properties': {'azureml.pipelines.dynamic': 'true'}, + 'tags': {'owner': 'sdkteam', 'tag': 'tagvalue'}, + 'type': 'command', + } + omit_fields = ["name", "creation_context", "id", "code", "environment", "version"] + rest_component = pydash.omit( + command_component._to_rest_object().as_dict()["properties"]["component_spec"], + omit_fields, + ) + + assert rest_component == expected_dict + + from_rest_component = client.components.create_or_update(command_component, is_anonymous=True) + + previous_dict = pydash.omit(command_component._to_dict(), omit_fields) + current_dict = pydash.omit(from_rest_component._to_dict(), omit_fields) + # TODO(2037030): verify when backend ready + # assert previous_dict == current_dict diff --git a/sdk/ml/azure-ai-ml/tests/component/unittests/test_component_schema.py b/sdk/ml/azure-ai-ml/tests/component/unittests/test_component_schema.py index 80bd741d275a..c5b51a038d29 100644 --- a/sdk/ml/azure-ai-ml/tests/component/unittests/test_component_schema.py +++ b/sdk/ml/azure-ai-ml/tests/component/unittests/test_component_schema.py @@ -294,6 +294,14 @@ def test_anonymous_component_same_name(self, mock_machinelearning_client: MLClie component_hash2 = component_entity2._get_anonymous_hash() assert component_hash1 != component_hash2 + def test_command_component_with_properties(self): + test_path = "./tests/test_configs/components/helloworld_component_with_properties.yml" + component_entity = load_component(source=test_path) + assert component_entity.properties == {"azureml.pipelines.dynamic": "true"} + + validation_result = component_entity._validate() + assert validation_result.passed is True + @pytest.mark.timeout(_COMPONENT_TIMEOUT_SECOND) @pytest.mark.unittest diff --git a/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_properties_e2e_flow.json b/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_properties_e2e_flow.json new file mode 100644 index 000000000000..da78f5d1c219 --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/recordings/component/e2etests/test_component.pyTestComponenttest_command_component_with_properties_e2e_flow.json @@ -0,0 +1,387 @@ +{ + "Entries": [ + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore?api-version=2022-05-01", + "RequestMethod": "GET", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": "azure-ai-ml/1.1.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.8.13 (Windows-10-10.0.19044-SP0)" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Mon, 24 Oct 2022 07:43:13 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:512cc15a-13b5-415b-bfd0-dce7accb6bb1", + "Server-Timing": "traceparent;desc=\u002200-95a055f8ed3ad6c8b10a3a0bad5b455d-d4838fde33f8ea6c-00\u0022", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": [ + "Accept-Encoding", + "Accept-Encoding" + ], + "x-aml-cluster": "vienna-test-westus2-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "6462ea6c-30c9-4bc8-b5f7-d8ccc02d94a4", + "x-ms-ratelimit-remaining-subscription-reads": "11999", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20221024T074314Z:6462ea6c-30c9-4bc8-b5f7-d8ccc02d94a4", + "x-request-time": "0.184" + }, + "ResponseBody": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore", + "name": "workspaceblobstore", + "type": "Microsoft.MachineLearningServices/workspaces/datastores", + "properties": { + "description": null, + "tags": null, + "properties": null, + "isDefault": true, + "credentials": { + "credentialsType": "AccountKey" + }, + "datastoreType": "AzureBlob", + "accountName": "sagvgsoim6nmhbq", + "containerName": "azureml-blobstore-e61cd5e2-512f-475e-9842-5e2a973993b8", + "endpoint": "core.windows.net", + "protocol": "https", + "serviceDataAccessAuthIdentity": "WorkspaceSystemAssignedIdentity" + }, + "systemData": { + "createdAt": "2022-09-22T09:02:03.2629568\u002B00:00", + "createdBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "createdByType": "Application", + "lastModifiedAt": "2022-09-22T09:02:04.166989\u002B00:00", + "lastModifiedBy": "779301c0-18b2-4cdc-801b-a0a3368fee0a", + "lastModifiedByType": "Application" + } + } + }, + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/datastores/workspaceblobstore/listSecrets?api-version=2022-05-01", + "RequestMethod": "POST", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Content-Length": "0", + "User-Agent": "azure-ai-ml/1.1.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.8.13 (Windows-10-10.0.19044-SP0)" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Mon, 24 Oct 2022 07:43:14 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:512cc15a-13b5-415b-bfd0-dce7accb6bb1", + "Server-Timing": "traceparent;desc=\u002200-4cfa436ffe1af8af2beb30a81d984dbb-0ca4d8defd94751c-00\u0022", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": "Accept-Encoding", + "x-aml-cluster": "vienna-test-westus2-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "288fc522-da60-42f5-b570-970d1e1d5268", + "x-ms-ratelimit-remaining-subscription-writes": "1199", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20221024T074315Z:288fc522-da60-42f5-b570-970d1e1d5268", + "x-request-time": "0.173" + }, + "ResponseBody": { + "secretsType": "AccountKey", + "key": "dGhpcyBpcyBmYWtlIGtleQ==" + } + }, + { + "RequestUri": "https://sagvgsoim6nmhbq.blob.core.windows.net/azureml-blobstore-e61cd5e2-512f-475e-9842-5e2a973993b8/LocalUpload/00000000000000000000000000000000/COMPONENT_PLACEHOLDER", + "RequestMethod": "HEAD", + "RequestHeaders": { + "Accept": "application/xml", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": "azsdk-python-storage-blob/12.13.1 Python/3.8.13 (Windows-10-10.0.19044-SP0)", + "x-ms-date": "Mon, 24 Oct 2022 07:43:14 GMT", + "x-ms-version": "2021-08-06" + }, + "RequestBody": null, + "StatusCode": 200, + "ResponseHeaders": { + "Accept-Ranges": "bytes", + "Content-Length": "35", + "Content-MD5": "L/DnSpFIn\u002BjaQWc\u002BsUQdcw==", + "Content-Type": "application/octet-stream", + "Date": "Mon, 24 Oct 2022 07:43:15 GMT", + "ETag": "\u00220x8DA9D48E17467D7\u0022", + "Last-Modified": "Fri, 23 Sep 2022 09:49:17 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "Vary": "Origin", + "x-ms-access-tier": "Hot", + "x-ms-access-tier-inferred": "true", + "x-ms-blob-type": "BlockBlob", + "x-ms-creation-time": "Fri, 23 Sep 2022 09:49:16 GMT", + "x-ms-lease-state": "available", + "x-ms-lease-status": "unlocked", + "x-ms-meta-name": "9c9cfba9-82bd-45db-ad06-07009d1d9672", + "x-ms-meta-upload_status": "completed", + "x-ms-meta-version": "1", + "x-ms-server-encrypted": "true", + "x-ms-version": "2021-08-06" + }, + "ResponseBody": null + }, + { + "RequestUri": "https://sagvgsoim6nmhbq.blob.core.windows.net/azureml-blobstore-e61cd5e2-512f-475e-9842-5e2a973993b8/az-ml-artifacts/00000000000000000000000000000000/COMPONENT_PLACEHOLDER", + "RequestMethod": "HEAD", + "RequestHeaders": { + "Accept": "application/xml", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "User-Agent": "azsdk-python-storage-blob/12.13.1 Python/3.8.13 (Windows-10-10.0.19044-SP0)", + "x-ms-date": "Mon, 24 Oct 2022 07:43:15 GMT", + "x-ms-version": "2021-08-06" + }, + "RequestBody": null, + "StatusCode": 404, + "ResponseHeaders": { + "Date": "Mon, 24 Oct 2022 07:43:15 GMT", + "Server": [ + "Windows-Azure-Blob/1.0", + "Microsoft-HTTPAPI/2.0" + ], + "Transfer-Encoding": "chunked", + "Vary": "Origin", + "x-ms-error-code": "BlobNotFound", + "x-ms-version": "2021-08-06" + }, + "ResponseBody": null + }, + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/9c9cfba9-82bd-45db-ad06-07009d1d9672/versions/1?api-version=2022-05-01", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Content-Length": "288", + "Content-Type": "application/json", + "User-Agent": "azure-ai-ml/1.1.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.8.13 (Windows-10-10.0.19044-SP0)" + }, + "RequestBody": { + "properties": { + "properties": { + "hash_sha256": "0000000000000", + "hash_version": "0000000000000" + }, + "isAnonymous": true, + "isArchived": false, + "codeUri": "https://sagvgsoim6nmhbq.blob.core.windows.net/azureml-blobstore-e61cd5e2-512f-475e-9842-5e2a973993b8/LocalUpload/00000000000000000000000000000000" + } + }, + "StatusCode": 200, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Encoding": "gzip", + "Content-Type": "application/json; charset=utf-8", + "Date": "Mon, 24 Oct 2022 07:43:17 GMT", + "Expires": "-1", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:512cc15a-13b5-415b-bfd0-dce7accb6bb1", + "Server-Timing": "traceparent;desc=\u002200-2c0bc31e770744aefef94805c81e34fc-1e6968de7eea83d2-00\u0022", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Transfer-Encoding": "chunked", + "Vary": [ + "Accept-Encoding", + "Accept-Encoding" + ], + "x-aml-cluster": "vienna-test-westus2-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "b92c4a7d-7478-4f8c-870b-136f93aea9af", + "x-ms-ratelimit-remaining-subscription-writes": "1199", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20221024T074318Z:b92c4a7d-7478-4f8c-870b-136f93aea9af", + "x-request-time": "1.211" + }, + "ResponseBody": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/9c9cfba9-82bd-45db-ad06-07009d1d9672/versions/1", + "name": "1", + "type": "Microsoft.MachineLearningServices/workspaces/codes/versions", + "properties": { + "description": null, + "tags": {}, + "properties": { + "hash_sha256": "0000000000000", + "hash_version": "0000000000000" + }, + "isArchived": false, + "isAnonymous": false, + "codeUri": "https://sagvgsoim6nmhbq.blob.core.windows.net/azureml-blobstore-e61cd5e2-512f-475e-9842-5e2a973993b8/LocalUpload/00000000000000000000000000000000" + }, + "systemData": { + "createdAt": "2022-09-23T09:49:20.984936\u002B00:00", + "createdBy": "Ying Chen", + "createdByType": "User", + "lastModifiedAt": "2022-10-24T07:43:18.2524632\u002B00:00", + "lastModifiedBy": "Han Wang", + "lastModifiedByType": "User" + } + } + }, + { + "RequestUri": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/azureml_anonymous/versions/000000000000000000000?api-version=2022-05-01", + "RequestMethod": "PUT", + "RequestHeaders": { + "Accept": "application/json", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "Content-Length": "1389", + "Content-Type": "application/json", + "User-Agent": "azure-ai-ml/1.1.0 azsdk-python-mgmt-machinelearningservices/0.1.0 Python/3.8.13 (Windows-10-10.0.19044-SP0)" + }, + "RequestBody": { + "properties": { + "description": "This is the basic command component", + "properties": { + "azureml.pipelines.dynamic": "true" + }, + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "isAnonymous": true, + "isArchived": false, + "componentSpec": { + "command": "echo Hello World \u0026 echo $[[${{inputs.component_in_number}}]] \u0026 echo ${{inputs.component_in_path}} \u0026 echo ${{outputs.component_out_path}} \u003E ${{outputs.component_out_path}}/component_in_number", + "code": "azureml:/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/9c9cfba9-82bd-45db-ad06-07009d1d9672/versions/1", + "environment": "azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1", + "name": "azureml_anonymous", + "description": "This is the basic command component", + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "version": "000000000000000000000", + "$schema": "https://azuremlschemas.azureedge.net/development/commandComponent.schema.json", + "display_name": "CommandComponentBasic", + "is_deterministic": true, + "inputs": { + "component_in_number": { + "type": "number", + "optional": true, + "default": "10.99", + "description": "A number" + }, + "component_in_path": { + "type": "uri_folder", + "description": "A path" + } + }, + "outputs": { + "component_out_path": { + "type": "uri_folder" + } + }, + "type": "command", + "properties": { + "azureml.pipelines.dynamic": "true" + }, + "_source": "YAML.COMPONENT" + } + } + }, + "StatusCode": 201, + "ResponseHeaders": { + "Cache-Control": "no-cache", + "Content-Length": "2347", + "Content-Type": "application/json; charset=utf-8", + "Date": "Mon, 24 Oct 2022 07:43:22 GMT", + "Expires": "-1", + "Location": "https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/azureml_anonymous/versions/000000000000000000000?api-version=2022-05-01", + "Pragma": "no-cache", + "Request-Context": "appId=cid-v1:512cc15a-13b5-415b-bfd0-dce7accb6bb1", + "Server-Timing": "traceparent;desc=\u002200-671381b1703e4a9bfb9e40ff08e4b692-2c0fc6ec400630de-00\u0022", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "x-aml-cluster": "vienna-test-westus2-01", + "X-Content-Type-Options": "nosniff", + "x-ms-correlation-request-id": "666d3450-7b5e-41c8-9a54-49a2e57e6af0", + "x-ms-ratelimit-remaining-subscription-writes": "1198", + "x-ms-response-type": "standard", + "x-ms-routing-request-id": "JAPANEAST:20221024T074323Z:666d3450-7b5e-41c8-9a54-49a2e57e6af0", + "x-request-time": "3.861" + }, + "ResponseBody": { + "id": "/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/components/azureml_anonymous/versions/69948cd7-b64e-460c-ab3e-b57ba96f91f4", + "name": "69948cd7-b64e-460c-ab3e-b57ba96f91f4", + "type": "Microsoft.MachineLearningServices/workspaces/components/versions", + "properties": { + "description": null, + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "properties": { + "azureml.pipelines.dynamic": "true" + }, + "isArchived": false, + "isAnonymous": true, + "componentSpec": { + "name": "azureml_anonymous", + "version": "69948cd7-b64e-460c-ab3e-b57ba96f91f4", + "display_name": "CommandComponentBasic", + "is_deterministic": "True", + "type": "command", + "description": "This is the basic command component", + "tags": { + "tag": "tagvalue", + "owner": "sdkteam" + }, + "inputs": { + "component_in_path": { + "type": "uri_folder", + "optional": "False", + "description": "A path" + }, + "component_in_number": { + "type": "number", + "optional": "True", + "default": "10.99", + "description": "A number" + } + }, + "outputs": { + "component_out_path": { + "type": "uri_folder" + } + }, + "code": "azureml:/subscriptions/00000000-0000-0000-0000-000000000/resourceGroups/00000/providers/Microsoft.MachineLearningServices/workspaces/00000/codes/9c9cfba9-82bd-45db-ad06-07009d1d9672/versions/1", + "environment": "azureml://registries/azureml-dev/environments/AzureML-sklearn-0.24-ubuntu18.04-py37-cpu/versions/1", + "resources": { + "instance_count": "1" + }, + "command": "echo Hello World \u0026 echo $[[${{inputs.component_in_number}}]] \u0026 echo ${{inputs.component_in_path}} \u0026 echo ${{outputs.component_out_path}} \u003E ${{outputs.component_out_path}}/component_in_number", + "$schema": "https://azuremlschemas.azureedge.net/development/commandComponent.schema.json" + } + }, + "systemData": { + "createdAt": "2022-10-24T07:37:23.5759217\u002B00:00", + "createdBy": "Han Wang", + "createdByType": "User", + "lastModifiedAt": "2022-10-24T07:37:24.0403904\u002B00:00", + "lastModifiedBy": "Han Wang", + "lastModifiedByType": "User" + } + } + } + ], + "Variables": {} +} diff --git a/sdk/ml/azure-ai-ml/tests/test_configs/components/helloworld_component_with_properties.yml b/sdk/ml/azure-ai-ml/tests/test_configs/components/helloworld_component_with_properties.yml new file mode 100644 index 000000000000..c4b4b0e8498f --- /dev/null +++ b/sdk/ml/azure-ai-ml/tests/test_configs/components/helloworld_component_with_properties.yml @@ -0,0 +1,37 @@ +$schema: https://azuremlschemas.azureedge.net/development/commandComponent.schema.json +type: command + +name: microsoftsamples_command_component_basic +display_name: CommandComponentBasic +description: This is the basic command component +tags: + tag: tagvalue + owner: sdkteam + +version: 0.0.1 + +inputs: + component_in_number: + description: A number + type: number + default: 10.99 + optional: True + component_in_path: + description: A path + type: uri_folder + +outputs: + component_out_path: + type: uri_folder + +# Write some output to work around a bug when pipeline node failed to run with empty dataset as input +command: >- + echo Hello World & + echo $[[${{inputs.component_in_number}}]] & + echo ${{inputs.component_in_path}} & + echo ${{outputs.component_out_path}} > ${{outputs.component_out_path}}/component_in_number + +environment: azureml:AzureML-sklearn-0.24-ubuntu18.04-py37-cpu:1 + +properties: + azureml.pipelines.dynamic: "true" From 643b294ca0b68e11f15036250080e9ffbfe7d16a Mon Sep 17 00:00:00 2001 From: Han Wang Date: Mon, 24 Oct 2022 18:52:13 +0800 Subject: [PATCH 2/3] remove properties if not specified --- .../azure-ai-ml/azure/ai/ml/entities/_component/component.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py index bae695cb8d55..def5d076bcfb 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py @@ -434,6 +434,9 @@ def _to_dict(self) -> Dict: component_schema_dict = self._dump_for_validation() component_schema_dict.pop("base_path", None) + # remove empty properties to keep the component spec unchanged + if not component_schema_dict.get("properties"): + component_schema_dict.pop("properties", None) # TODO: handle other_parameters and remove override from subclass return component_schema_dict From 9bbd05a33bf4d0e2f04256fcb236db62ed22329f Mon Sep 17 00:00:00 2001 From: Han Wang Date: Tue, 25 Oct 2022 12:55:14 +0800 Subject: [PATCH 3/3] remove properties in post dump --- .../azure/ai/ml/_schema/component/command_component.py | 9 ++++++++- .../azure/ai/ml/entities/_component/component.py | 3 --- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py index e501f35741e0..645fcb2ef6dd 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/_schema/component/command_component.py @@ -7,7 +7,7 @@ from copy import deepcopy import yaml -from marshmallow import INCLUDE, fields, post_load +from marshmallow import INCLUDE, fields, post_load, post_dump from azure.ai.ml._schema.assets.asset import AnonymousAssetSchema from azure.ai.ml._schema.component.component import ComponentSchema @@ -50,6 +50,13 @@ class Meta: ) properties = fields.Dict(keys=fields.Str(), values=fields.Raw()) + @post_dump + def remove_unnecessary_fields(self, component_schema_dict, **kwargs): + # remove empty properties to keep the component spec unchanged + if not component_schema_dict.get("properties"): + component_schema_dict.pop("properties", None) + return component_schema_dict + class RestCommandComponentSchema(CommandComponentSchema): """When component load from rest, won't validate on name since there might diff --git a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py index def5d076bcfb..bae695cb8d55 100644 --- a/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py +++ b/sdk/ml/azure-ai-ml/azure/ai/ml/entities/_component/component.py @@ -434,9 +434,6 @@ def _to_dict(self) -> Dict: component_schema_dict = self._dump_for_validation() component_schema_dict.pop("base_path", None) - # remove empty properties to keep the component spec unchanged - if not component_schema_dict.get("properties"): - component_schema_dict.pop("properties", None) # TODO: handle other_parameters and remove override from subclass return component_schema_dict