From 123ee956b6f9f0ba9648fa2abe334e172c6066e3 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Thu, 3 Feb 2022 11:54:49 +0000 Subject: [PATCH] feat: add checksums in Secret Manager (#244) Users can now use checksums for data integrity assurance when adding and accessing SecretVersions. ------ - [ ] Regenerate this pull request now. chore: use gapic-generator-python 0.62.1 fix: resolve DuplicateCredentialArgs error when using credentials_file committer: parthea PiperOrigin-RevId: 425964861 Source-Link: https://github.com/googleapis/googleapis/commit/84b1a5a4f6fb2d04905be58e586b8a7a4310a8cf Source-Link: https://github.com/googleapis/googleapis-gen/commit/4fb761bbd8506ac156f49bac5f18306aa8eb3aa8 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNGZiNzYxYmJkODUwNmFjMTU2ZjQ5YmFjNWYxODMwNmFhOGViM2FhOCJ9 --- .../secret_manager_service/async_client.py | 24 ++--- .../services/secret_manager_service/client.py | 24 ++--- .../secret_manager_service/transports/grpc.py | 5 +- .../transports/grpc_asyncio.py | 5 +- .../cloud/secretmanager_v1/types/resources.py | 34 ++++++- .../test_secret_manager_service.py | 93 ++++++++++++++++++- 6 files changed, 155 insertions(+), 30 deletions(-) diff --git a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/async_client.py b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/async_client.py index da3b66e21c5d..c84f5892748a 100644 --- a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/async_client.py +++ b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/async_client.py @@ -262,7 +262,7 @@ async def list_secrets( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent]) if request is not None and has_flattened_params: @@ -368,7 +368,7 @@ async def create_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent, secret_id, secret]) if request is not None and has_flattened_params: @@ -457,7 +457,7 @@ async def add_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent, payload]) if request is not None and has_flattened_params: @@ -537,7 +537,7 @@ async def get_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -622,7 +622,7 @@ async def update_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([secret, update_mask]) if request is not None and has_flattened_params: @@ -692,7 +692,7 @@ async def delete_secret( sent along with the request as metadata. """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -770,7 +770,7 @@ async def list_secret_versions( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent]) if request is not None and has_flattened_params: @@ -857,7 +857,7 @@ async def get_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -939,7 +939,7 @@ async def access_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1029,7 +1029,7 @@ async def disable_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1109,7 +1109,7 @@ async def enable_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1190,7 +1190,7 @@ async def destroy_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: diff --git a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/client.py b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/client.py index d8dd584bfb33..bd3485515a29 100644 --- a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/client.py +++ b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/client.py @@ -478,7 +478,7 @@ def list_secrets( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent]) if request is not None and has_flattened_params: @@ -584,7 +584,7 @@ def create_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent, secret_id, secret]) if request is not None and has_flattened_params: @@ -673,7 +673,7 @@ def add_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent, payload]) if request is not None and has_flattened_params: @@ -753,7 +753,7 @@ def get_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -838,7 +838,7 @@ def update_secret( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([secret, update_mask]) if request is not None and has_flattened_params: @@ -908,7 +908,7 @@ def delete_secret( sent along with the request as metadata. """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -986,7 +986,7 @@ def list_secret_versions( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([parent]) if request is not None and has_flattened_params: @@ -1073,7 +1073,7 @@ def get_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1155,7 +1155,7 @@ def access_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1235,7 +1235,7 @@ def disable_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1315,7 +1315,7 @@ def enable_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: @@ -1396,7 +1396,7 @@ def destroy_secret_version( """ # Create or coerce a protobuf request object. - # Sanity check: If we got a request object, we should *not* have + # Quick check: If we got a request object, we should *not* have # gotten any keyword arguments that map to the request. has_flattened_params = any([name]) if request is not None and has_flattened_params: diff --git a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc.py b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc.py index 70beab4b9950..4bfe1648d3cc 100644 --- a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc.py +++ b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc.py @@ -168,8 +168,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc_asyncio.py b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc_asyncio.py index 9415311e45c5..356b145cef53 100644 --- a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc_asyncio.py +++ b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/services/secret_manager_service/transports/grpc_asyncio.py @@ -213,8 +213,11 @@ def __init__( if not self._grpc_channel: self._grpc_channel = type(self).create_channel( self._host, + # use the credentials which are saved credentials=self._credentials, - credentials_file=credentials_file, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, scopes=self._scopes, ssl_credentials=self._ssl_channel_credentials, quota_project_id=quota_project_id, diff --git a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/types/resources.py b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/types/resources.py index 1060ad0c1e75..2c9f9e38d50c 100644 --- a/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/types/resources.py +++ b/packages/google-cloud-secret-manager/google/cloud/secretmanager_v1/types/resources.py @@ -154,6 +154,13 @@ class SecretVersion(proto.Message): etag (str): Output only. Etag of the currently stored [SecretVersion][google.cloud.secretmanager.v1.SecretVersion]. + client_specified_payload_checksum (bool): + Output only. True if payload checksum specified in + [SecretPayload][google.cloud.secretmanager.v1.SecretPayload] + object has been received by + [SecretManagerService][google.cloud.secretmanager.v1.SecretManagerService] + on + [SecretManagerService.AddSecretVersion][google.cloud.secretmanager.v1.SecretManagerService.AddSecretVersion]. """ class State(proto.Enum): @@ -176,6 +183,7 @@ class State(proto.Enum): proto.MESSAGE, number=5, message="ReplicationStatus", ) etag = proto.Field(proto.STRING, number=6,) + client_specified_payload_checksum = proto.Field(proto.BOOL, number=7,) class Replication(proto.Message): @@ -281,8 +289,8 @@ class Replica(proto.Message): class CustomerManagedEncryption(proto.Message): - r"""Configuration for encrypting secret payloads using customer- - anaged encryption keys (CMEK). + r"""Configuration for encrypting secret payloads using + customer-managed encryption keys (CMEK). Attributes: kms_key_name (str): @@ -490,9 +498,31 @@ class SecretPayload(proto.Message): data (bytes): The secret data. Must be no larger than 64KiB. + data_crc32c (int): + Optional. If specified, + [SecretManagerService][google.cloud.secretmanager.v1.SecretManagerService] + will verify the integrity of the received + [data][google.cloud.secretmanager.v1.SecretPayload.data] on + [SecretManagerService.AddSecretVersion][google.cloud.secretmanager.v1.SecretManagerService.AddSecretVersion] + calls using the crc32c checksum and store it to include in + future + [SecretManagerService.AccessSecretVersion][google.cloud.secretmanager.v1.SecretManagerService.AccessSecretVersion] + responses. If a checksum is not provided in the + [SecretManagerService.AddSecretVersion][google.cloud.secretmanager.v1.SecretManagerService.AddSecretVersion] + request, the + [SecretManagerService][google.cloud.secretmanager.v1.SecretManagerService] + will generate and store one for you. + + The CRC32C value is encoded as a Int64 for compatibility, + and can be safely downconverted to uint32 in languages that + support this type. + https://cloud.google.com/apis/design/design_patterns#integer_types + + This field is a member of `oneof`_ ``_data_crc32c``. """ data = proto.Field(proto.BYTES, number=1,) + data_crc32c = proto.Field(proto.INT64, number=2, optional=True,) __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/packages/google-cloud-secret-manager/tests/unit/gapic/secretmanager_v1/test_secret_manager_service.py b/packages/google-cloud-secret-manager/tests/unit/gapic/secretmanager_v1/test_secret_manager_service.py index 8abcea7fad93..fd57b41e2422 100644 --- a/packages/google-cloud-secret-manager/tests/unit/gapic/secretmanager_v1/test_secret_manager_service.py +++ b/packages/google-cloud-secret-manager/tests/unit/gapic/secretmanager_v1/test_secret_manager_service.py @@ -539,25 +539,28 @@ def test_secret_manager_service_client_client_options_scopes( @pytest.mark.parametrize( - "client_class,transport_class,transport_name", + "client_class,transport_class,transport_name,grpc_helpers", [ ( SecretManagerServiceClient, transports.SecretManagerServiceGrpcTransport, "grpc", + grpc_helpers, ), ( SecretManagerServiceAsyncClient, transports.SecretManagerServiceGrpcAsyncIOTransport, "grpc_asyncio", + grpc_helpers_async, ), ], ) def test_secret_manager_service_client_client_options_credentials_file( - client_class, transport_class, transport_name + client_class, transport_class, transport_name, grpc_helpers ): # Check the case credentials file is provided. options = client_options.ClientOptions(credentials_file="credentials.json") + with mock.patch.object(transport_class, "__init__") as patched: patched.return_value = None client = client_class(client_options=options, transport=transport_name) @@ -593,6 +596,72 @@ def test_secret_manager_service_client_client_options_from_dict(): ) +@pytest.mark.parametrize( + "client_class,transport_class,transport_name,grpc_helpers", + [ + ( + SecretManagerServiceClient, + transports.SecretManagerServiceGrpcTransport, + "grpc", + grpc_helpers, + ), + ( + SecretManagerServiceAsyncClient, + transports.SecretManagerServiceGrpcAsyncIOTransport, + "grpc_asyncio", + grpc_helpers_async, + ), + ], +) +def test_secret_manager_service_client_create_channel_credentials_file( + client_class, transport_class, transport_name, grpc_helpers +): + # Check the case credentials file is provided. + options = client_options.ClientOptions(credentials_file="credentials.json") + + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client.DEFAULT_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "secretmanager.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=("https://www.googleapis.com/auth/cloud-platform",), + scopes=None, + default_host="secretmanager.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + @pytest.mark.parametrize("request_type", [service.ListSecretsRequest, dict,]) def test_list_secrets(request_type, transport: str = "grpc"): client = SecretManagerServiceClient( @@ -1190,6 +1259,7 @@ def test_add_secret_version(request_type, transport: str = "grpc"): name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) response = client.add_secret_version(request) @@ -1203,6 +1273,7 @@ def test_add_secret_version(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True def test_add_secret_version_empty_call(): @@ -1244,6 +1315,7 @@ async def test_add_secret_version_async( name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) ) response = await client.add_secret_version(request) @@ -1258,6 +1330,7 @@ async def test_add_secret_version_async( assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True @pytest.mark.asyncio @@ -2451,6 +2524,7 @@ def test_get_secret_version(request_type, transport: str = "grpc"): name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) response = client.get_secret_version(request) @@ -2464,6 +2538,7 @@ def test_get_secret_version(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True def test_get_secret_version_empty_call(): @@ -2505,6 +2580,7 @@ async def test_get_secret_version_async( name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) ) response = await client.get_secret_version(request) @@ -2519,6 +2595,7 @@ async def test_get_secret_version_async( assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True @pytest.mark.asyncio @@ -2905,6 +2982,7 @@ def test_disable_secret_version(request_type, transport: str = "grpc"): name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) response = client.disable_secret_version(request) @@ -2918,6 +2996,7 @@ def test_disable_secret_version(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True def test_disable_secret_version_empty_call(): @@ -2959,6 +3038,7 @@ async def test_disable_secret_version_async( name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) ) response = await client.disable_secret_version(request) @@ -2973,6 +3053,7 @@ async def test_disable_secret_version_async( assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True @pytest.mark.asyncio @@ -3138,6 +3219,7 @@ def test_enable_secret_version(request_type, transport: str = "grpc"): name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) response = client.enable_secret_version(request) @@ -3151,6 +3233,7 @@ def test_enable_secret_version(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True def test_enable_secret_version_empty_call(): @@ -3192,6 +3275,7 @@ async def test_enable_secret_version_async( name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) ) response = await client.enable_secret_version(request) @@ -3206,6 +3290,7 @@ async def test_enable_secret_version_async( assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True @pytest.mark.asyncio @@ -3371,6 +3456,7 @@ def test_destroy_secret_version(request_type, transport: str = "grpc"): name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) response = client.destroy_secret_version(request) @@ -3384,6 +3470,7 @@ def test_destroy_secret_version(request_type, transport: str = "grpc"): assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True def test_destroy_secret_version_empty_call(): @@ -3425,6 +3512,7 @@ async def test_destroy_secret_version_async( name="name_value", state=resources.SecretVersion.State.ENABLED, etag="etag_value", + client_specified_payload_checksum=True, ) ) response = await client.destroy_secret_version(request) @@ -3439,6 +3527,7 @@ async def test_destroy_secret_version_async( assert response.name == "name_value" assert response.state == resources.SecretVersion.State.ENABLED assert response.etag == "etag_value" + assert response.client_specified_payload_checksum is True @pytest.mark.asyncio