diff --git a/bigtable/google/cloud/bigtable/cluster.py b/bigtable/google/cloud/bigtable/cluster.py index e22f383bed95..48b335c5196e 100644 --- a/bigtable/google/cloud/bigtable/cluster.py +++ b/bigtable/google/cloud/bigtable/cluster.py @@ -23,7 +23,7 @@ bigtable_instance_admin_pb2 as messages_v2_pb2) from google.cloud.operation import Operation from google.cloud.operation import _compute_type_url -from google.cloud.operation import _register_type_url +from google.cloud.operation import register_type_url _CLUSTER_NAME_RE = re.compile(r'^projects/(?P[^/]+)/' @@ -36,7 +36,7 @@ _UPDATE_CLUSTER_METADATA_URL = _compute_type_url( messages_v2_pb2.UpdateClusterMetadata) -_register_type_url( +register_type_url( _UPDATE_CLUSTER_METADATA_URL, messages_v2_pb2.UpdateClusterMetadata) @@ -218,7 +218,7 @@ def create(self): operation = Operation.from_pb(operation_pb, client) operation.target = self - operation.metadata['request_type'] = 'CreateCluster' + operation.caller_metadata['request_type'] = 'CreateCluster' return operation def update(self): @@ -249,7 +249,7 @@ def update(self): operation = Operation.from_pb(operation_pb, client) operation.target = self - operation.metadata['request_type'] = 'UpdateCluster' + operation.caller_metadata['request_type'] = 'UpdateCluster' return operation def delete(self): diff --git a/bigtable/google/cloud/bigtable/instance.py b/bigtable/google/cloud/bigtable/instance.py index 8730836622d0..afa4066a75b0 100644 --- a/bigtable/google/cloud/bigtable/instance.py +++ b/bigtable/google/cloud/bigtable/instance.py @@ -28,7 +28,7 @@ from google.cloud.bigtable.table import Table from google.cloud.operation import Operation from google.cloud.operation import _compute_type_url -from google.cloud.operation import _register_type_url +from google.cloud.operation import register_type_url _EXISTING_INSTANCE_LOCATION_ID = 'see-existing-cluster' @@ -38,8 +38,10 @@ _CREATE_INSTANCE_METADATA_URL = _compute_type_url( messages_v2_pb2.CreateInstanceMetadata) -_register_type_url( +register_type_url( _CREATE_INSTANCE_METADATA_URL, messages_v2_pb2.CreateInstanceMetadata) +_INSTANCE_METADATA_URL = _compute_type_url(data_v2_pb2.Instance) +register_type_url(_INSTANCE_METADATA_URL, data_v2_pb2.Instance) def _prepare_create_request(instance): @@ -237,7 +239,7 @@ def create(self): operation = Operation.from_pb(operation_pb, self._client) operation.target = self - operation.metadata['request_type'] = 'CreateInstance' + operation.caller_metadata['request_type'] = 'CreateInstance' return operation def update(self): diff --git a/bigtable/unit_tests/test_cluster.py b/bigtable/unit_tests/test_cluster.py index e497a025fd2c..82185cef030f 100644 --- a/bigtable/unit_tests/test_cluster.py +++ b/bigtable/unit_tests/test_cluster.py @@ -257,8 +257,9 @@ def test_create(self): self.assertEqual(result.name, OP_NAME) self.assertIs(result.target, cluster) self.assertIs(result.client, client) - self.assertIsNone(result.pb_metadata) - self.assertEqual(result.metadata, {'request_type': 'CreateCluster'}) + self.assertIsNone(result.metadata) + self.assertEqual(result.caller_metadata, + {'request_type': 'CreateCluster'}) self.assertEqual(len(stub.method_calls), 1) api_name, args, kwargs = stub.method_calls[0] @@ -323,10 +324,11 @@ def test_update(self): self.assertEqual(result.name, OP_NAME) self.assertIs(result.target, cluster) self.assertIs(result.client, client) - self.assertIsInstance(result.pb_metadata, + self.assertIsInstance(result.metadata, messages_v2_pb2.UpdateClusterMetadata) - self.assertEqual(result.pb_metadata.request_time, NOW_PB) - self.assertEqual(result.metadata, {'request_type': 'UpdateCluster'}) + self.assertEqual(result.metadata.request_time, NOW_PB) + self.assertEqual(result.caller_metadata, + {'request_type': 'UpdateCluster'}) self.assertEqual(len(stub.method_calls), 1) api_name, args, kwargs = stub.method_calls[0] diff --git a/bigtable/unit_tests/test_instance.py b/bigtable/unit_tests/test_instance.py index bf47ab4f62a3..223cfd2033ff 100644 --- a/bigtable/unit_tests/test_instance.py +++ b/bigtable/unit_tests/test_instance.py @@ -265,10 +265,11 @@ def test_create(self): self.assertEqual(result.name, self.OP_NAME) self.assertIs(result.target, instance) self.assertIs(result.client, client) - self.assertIsInstance(result.pb_metadata, + self.assertIsInstance(result.metadata, messages_v2_pb2.CreateInstanceMetadata) - self.assertEqual(result.pb_metadata.request_time, NOW_PB) - self.assertEqual(result.metadata, {'request_type': 'CreateInstance'}) + self.assertEqual(result.metadata.request_time, NOW_PB) + self.assertEqual(result.caller_metadata, + {'request_type': 'CreateInstance'}) self.assertEqual(len(stub.method_calls), 1) api_name, args, kwargs = stub.method_calls[0] diff --git a/core/google/cloud/operation.py b/core/google/cloud/operation.py index f359496ba3cd..839e5e3eaf30 100644 --- a/core/google/cloud/operation.py +++ b/core/google/cloud/operation.py @@ -39,7 +39,7 @@ def _compute_type_url(klass, prefix=_GOOGLE_APIS_PREFIX): return '%s/%s' % (prefix, name) -def _register_type_url(type_url, klass): +def register_type_url(type_url, klass): """Register a klass as the factory for a given type URL. :type type_url: str @@ -57,55 +57,102 @@ def _register_type_url(type_url, klass): _TYPE_URL_MAP[type_url] = klass +def _from_any(any_pb): + """Convert an ``Any`` protobuf into the actual class. + + Uses the type URL to do the conversion. + + .. note:: + + This assumes that the type URL is already registered. + + :type any_pb: :class:`google.protobuf.any_pb2.Any` + :param any_pb: An any object to be converted. + + :rtype: object + :returns: The instance (of the correct type) stored in the any + instance. + """ + klass = _TYPE_URL_MAP[any_pb.type_url] + return klass.FromString(any_pb.value) + + class Operation(object): """Representation of a Google API Long-Running Operation. + .. _protobuf: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L80 + .. _service: https://github.com/googleapis/googleapis/blob/\ + 050400df0fdb16f63b63e9dee53819044bffc857/\ + google/longrunning/operations.proto#L38 + .. _JSON: https://cloud.google.com/speech/reference/rest/\ + v1beta1/operations#Operation + + This wraps an operation `protobuf`_ object and attempts to + interact with the long-running operations `service`_ (specific + to a given API). (Some services also offer a `JSON`_ + API that maps the same underlying data type.) + :type name: str :param name: The fully-qualified path naming the operation. :type client: object: must provide ``_operations_stub`` accessor. :param client: The client used to poll for the status of the operation. - :type pb_metadata: object - :param pb_metadata: Instance of protobuf metadata class - - :type kw: dict - :param kw: caller-assigned metadata about the operation + :type caller_metadata: dict + :param caller_metadata: caller-assigned metadata about the operation """ target = None """Instance assocated with the operations: callers may set.""" - def __init__(self, name, client, pb_metadata=None, **kw): + response = None + """Response returned from completed operation. + + Only one of this and :attr:`error` can be populated. + """ + + error = None + """Error that resulted from a failed (complete) operation. + + Only one of this and :attr:`response` can be populated. + """ + + metadata = None + """Metadata about the current operation (as a protobuf). + + Code that uses operations must register the metadata types (via + :func:`register_type_url`) to ensure that the metadata fields can be + converted into the correct types. + """ + + def __init__(self, name, client, **caller_metadata): self.name = name self.client = client - self.pb_metadata = pb_metadata - self.metadata = kw.copy() + self.caller_metadata = caller_metadata.copy() self._complete = False @classmethod - def from_pb(cls, op_pb, client, **kw): + def from_pb(cls, operation_pb, client, **caller_metadata): """Factory: construct an instance from a protobuf. - :type op_pb: :class:`google.longrunning.operations_pb2.Operation` - :param op_pb: Protobuf to be parsed. + :type operation_pb: + :class:`~google.longrunning.operations_pb2.Operation` + :param operation_pb: Protobuf to be parsed. :type client: object: must provide ``_operations_stub`` accessor. :param client: The client used to poll for the status of the operation. - :type kw: dict - :param kw: caller-assigned metadata about the operation + :type caller_metadata: dict + :param caller_metadata: caller-assigned metadata about the operation :rtype: :class:`Operation` :returns: new instance, with attributes based on the protobuf. """ - pb_metadata = None - if op_pb.metadata.type_url: - type_url = op_pb.metadata.type_url - md_klass = _TYPE_URL_MAP.get(type_url) - if md_klass: - pb_metadata = md_klass.FromString(op_pb.metadata.value) - return cls(op_pb.name, client, pb_metadata, **kw) + result = cls(operation_pb.name, client, **caller_metadata) + result._update_state(operation_pb) + return result @property def complete(self): @@ -116,22 +163,46 @@ def complete(self): """ return self._complete + def _get_operation_rpc(self): + """Polls the status of the current operation. + + :rtype: :class:`~google.longrunning.operations_pb2.Operation` + :returns: The latest status of the current operation. + """ + request_pb = operations_pb2.GetOperationRequest(name=self.name) + return self.client._operations_stub.GetOperation(request_pb) + + def _update_state(self, operation_pb): + """Update the state of the current object based on operation. + + :type operation_pb: + :class:`~google.longrunning.operations_pb2.Operation` + :param operation_pb: Protobuf to be parsed. + """ + if operation_pb.done: + self._complete = True + + if operation_pb.HasField('metadata'): + self.metadata = _from_any(operation_pb.metadata) + + result_type = operation_pb.WhichOneof('result') + if result_type == 'error': + self.error = operation_pb.error + elif result_type == 'response': + self.response = _from_any(operation_pb.response) + def poll(self): """Check if the operation has finished. :rtype: bool :returns: A boolean indicating if the current operation has completed. - :raises: :class:`ValueError ` if the operation + :raises: :class:`~exceptions.ValueError` if the operation has already completed. """ if self.complete: raise ValueError('The operation has completed.') - request_pb = operations_pb2.GetOperationRequest(name=self.name) - # We expect a `google.longrunning.operations_pb2.Operation`. - operation_pb = self.client._operations_stub.GetOperation(request_pb) - - if operation_pb.done: - self._complete = True + operation_pb = self._get_operation_rpc() + self._update_state(operation_pb) return self.complete diff --git a/core/unit_tests/test_operation.py b/core/unit_tests/test_operation.py index 234b5d93c749..e67bf21f6f23 100644 --- a/core/unit_tests/test_operation.py +++ b/core/unit_tests/test_operation.py @@ -44,11 +44,11 @@ def test_w_prefix(self): '%s/%s' % (PREFIX, Struct.DESCRIPTOR.full_name)) -class Test__register_type_url(unittest.TestCase): +class Test_register_type_url(unittest.TestCase): def _callFUT(self, type_url, klass): - from google.cloud.operation import _register_type_url - _register_type_url(type_url, klass) + from google.cloud.operation import register_type_url + register_type_url(type_url, klass) def test_simple(self): from google.cloud import operation as MUT @@ -106,19 +106,23 @@ def test_ctor_defaults(self): self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) self.assertIsNone(operation.target) - self.assertIsNone(operation.pb_metadata) - self.assertEqual(operation.metadata, {}) + self.assertIsNone(operation.response) + self.assertIsNone(operation.error) + self.assertIsNone(operation.metadata) + self.assertEqual(operation.caller_metadata, {}) def test_ctor_explicit(self): client = _Client() - pb_metadata = object() operation = self._makeOne( - self.OPERATION_NAME, client, pb_metadata, foo='bar') + self.OPERATION_NAME, client, foo='bar') + self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) self.assertIsNone(operation.target) - self.assertIs(operation.pb_metadata, pb_metadata) - self.assertEqual(operation.metadata, {'foo': 'bar'}) + self.assertIsNone(operation.response) + self.assertIsNone(operation.error) + self.assertIsNone(operation.metadata) + self.assertEqual(operation.caller_metadata, {'foo': 'bar'}) def test_from_pb_wo_metadata_or_kw(self): from google.longrunning import operations_pb2 @@ -130,41 +134,47 @@ def test_from_pb_wo_metadata_or_kw(self): self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) - self.assertIsNone(operation.pb_metadata) - self.assertEqual(operation.metadata, {}) + self.assertIsNone(operation.metadata) + self.assertEqual(operation.caller_metadata, {}) def test_from_pb_w_unknown_metadata(self): from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any - from google.protobuf.struct_pb2 import Struct, Value - TYPE_URI = 'type.googleapis.com/%s' % (Struct.DESCRIPTOR.full_name,) + from google.protobuf.json_format import ParseDict + from google.protobuf.struct_pb2 import Struct + from google.cloud._testing import _Monkey + from google.cloud import operation as MUT + type_url = 'type.googleapis.com/%s' % (Struct.DESCRIPTOR.full_name,) client = _Client() - meta = Struct(fields={'foo': Value(string_value=u'Bar')}) - metadata_pb = Any(type_url=TYPE_URI, value=meta.SerializeToString()) + meta = ParseDict({'foo': 'Bar'}, Struct()) + metadata_pb = Any(type_url=type_url, value=meta.SerializeToString()) operation_pb = operations_pb2.Operation( name=self.OPERATION_NAME, metadata=metadata_pb) klass = self._getTargetClass() - operation = klass.from_pb(operation_pb, client) + with _Monkey(MUT, _TYPE_URL_MAP={type_url: Struct}): + operation = klass.from_pb(operation_pb, client) self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) - self.assertIsNone(operation.pb_metadata) - self.assertEqual(operation.metadata, {}) + self.assertEqual(operation.metadata, meta) + self.assertEqual(operation.caller_metadata, {}) def test_from_pb_w_metadata_and_kwargs(self): from google.longrunning import operations_pb2 from google.protobuf.any_pb2 import Any - from google.protobuf.struct_pb2 import Struct, Value + from google.protobuf.struct_pb2 import Struct + from google.protobuf.struct_pb2 import Value from google.cloud import operation as MUT from google.cloud._testing import _Monkey - TYPE_URI = 'type.googleapis.com/%s' % (Struct.DESCRIPTOR.full_name,) - type_url_map = {TYPE_URI: Struct} + + type_url = 'type.googleapis.com/%s' % (Struct.DESCRIPTOR.full_name,) + type_url_map = {type_url: Struct} client = _Client() meta = Struct(fields={'foo': Value(string_value=u'Bar')}) - metadata_pb = Any(type_url=TYPE_URI, value=meta.SerializeToString()) + metadata_pb = Any(type_url=type_url, value=meta.SerializeToString()) operation_pb = operations_pb2.Operation( name=self.OPERATION_NAME, metadata=metadata_pb) klass = self._getTargetClass() @@ -174,11 +184,8 @@ def test_from_pb_w_metadata_and_kwargs(self): self.assertEqual(operation.name, self.OPERATION_NAME) self.assertIs(operation.client, client) - pb_metadata = operation.pb_metadata - self.assertIsInstance(pb_metadata, Struct) - self.assertEqual(list(pb_metadata.fields), ['foo']) - self.assertEqual(pb_metadata.fields['foo'].string_value, 'Bar') - self.assertEqual(operation.metadata, {'baz': 'qux'}) + self.assertEqual(operation.metadata, meta) + self.assertEqual(operation.caller_metadata, {'baz': 'qux'}) def test_complete_property(self): client = _Client() @@ -198,8 +205,9 @@ def test_poll_already_complete(self): operation.poll() def test_poll_false(self): - from google.longrunning.operations_pb2 import GetOperationRequest - response_pb = _GetOperationResponse(False) + from google.longrunning import operations_pb2 + + response_pb = operations_pb2.Operation(done=False) client = _Client() stub = client._operations_stub stub._get_operation_response = response_pb @@ -208,12 +216,13 @@ def test_poll_false(self): self.assertFalse(operation.poll()) request_pb = stub._get_operation_requested - self.assertIsInstance(request_pb, GetOperationRequest) + self.assertIsInstance(request_pb, operations_pb2.GetOperationRequest) self.assertEqual(request_pb.name, self.OPERATION_NAME) def test_poll_true(self): - from google.longrunning.operations_pb2 import GetOperationRequest - response_pb = _GetOperationResponse(True) + from google.longrunning import operations_pb2 + + response_pb = operations_pb2.Operation(done=True) client = _Client() stub = client._operations_stub stub._get_operation_response = response_pb @@ -222,13 +231,90 @@ def test_poll_true(self): self.assertTrue(operation.poll()) request_pb = stub._get_operation_requested - self.assertIsInstance(request_pb, GetOperationRequest) + self.assertIsInstance(request_pb, operations_pb2.GetOperationRequest) self.assertEqual(request_pb.name, self.OPERATION_NAME) + def test__update_state_done(self): + from google.longrunning import operations_pb2 + + operation = self._makeOne(None, None) + self.assertFalse(operation.complete) + operation_pb = operations_pb2.Operation(done=True) + operation._update_state(operation_pb) + self.assertTrue(operation.complete) + + def test__update_state_metadata(self): + from google.longrunning import operations_pb2 + from google.protobuf.any_pb2 import Any + from google.protobuf.struct_pb2 import Value + from google.cloud._testing import _Monkey + from google.cloud import operation as MUT + + operation = self._makeOne(None, None) + self.assertIsNone(operation.metadata) + + val_pb = Value(number_value=1337) + type_url = 'type.googleapis.com/%s' % (Value.DESCRIPTOR.full_name,) + val_any = Any(type_url=type_url, value=val_pb.SerializeToString()) + operation_pb = operations_pb2.Operation(metadata=val_any) + + with _Monkey(MUT, _TYPE_URL_MAP={type_url: Value}): + operation._update_state(operation_pb) + + self.assertEqual(operation.metadata, val_pb) + + def test__update_state_error(self): + from google.longrunning import operations_pb2 + from google.rpc.status_pb2 import Status + from google.cloud._testing import _Monkey + + operation = self._makeOne(None, None) + self.assertIsNone(operation.error) + self.assertIsNone(operation.response) + + error_pb = Status(code=1) + operation_pb = operations_pb2.Operation(error=error_pb) + operation._update_state(operation_pb) + + self.assertEqual(operation.error, error_pb) + self.assertIsNone(operation.response) + + def test__update_state_response(self): + from google.longrunning import operations_pb2 + from google.protobuf.any_pb2 import Any + from google.protobuf.struct_pb2 import Value + from google.cloud._testing import _Monkey + from google.cloud import operation as MUT + + operation = self._makeOne(None, None) + self.assertIsNone(operation.error) + self.assertIsNone(operation.response) + + response_pb = Value(string_value='totes a response') + type_url = 'type.googleapis.com/%s' % (Value.DESCRIPTOR.full_name,) + response_any = Any(type_url=type_url, + value=response_pb.SerializeToString()) + operation_pb = operations_pb2.Operation(response=response_any) + + with _Monkey(MUT, _TYPE_URL_MAP={type_url: Value}): + operation._update_state(operation_pb) + + self.assertIsNone(operation.error) + self.assertEqual(operation.response, response_pb) + + def test__update_state_no_result(self): + from google.longrunning import operations_pb2 + + operation = self._makeOne(None, None) + self.assertIsNone(operation.error) + self.assertIsNone(operation.response) + + operation_pb = operations_pb2.Operation() + operation._update_state(operation_pb) -class _GetOperationResponse(object): - def __init__(self, done): - self.done = done + # Make sure nothing changed. + self.assertIsNone(operation.error) + self.assertIsNone(operation.response) class _OperationsStub(object):