diff --git a/automl/beta/batch_predict.py b/automl/beta/batch_predict.py new file mode 100644 index 000000000000..7d634d2e0a68 --- /dev/null +++ b/automl/beta/batch_predict.py @@ -0,0 +1,52 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def batch_predict(project_id, model_id, input_uri, output_uri): + """Batch predict""" + # [START automl_batch_predict_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # model_id = "YOUR_MODEL_ID" + # input_uri = "gs://YOUR_BUCKET_ID/path/to/your/input/csv_or_jsonl" + # output_uri = "gs://YOUR_BUCKET_ID/path/to/save/results/" + + prediction_client = automl.PredictionServiceClient() + + # Get the full path of the model. + model_full_id = prediction_client.model_path( + project_id, "us-central1", model_id + ) + + gcs_source = automl.types.GcsSource(input_uris=[input_uri]) + + input_config = automl.types.BatchPredictInputConfig(gcs_source=gcs_source) + gcs_destination = automl.types.GcsDestination(output_uri_prefix=output_uri) + output_config = automl.types.BatchPredictOutputConfig( + gcs_destination=gcs_destination + ) + + response = prediction_client.batch_predict( + model_full_id, input_config, output_config + ) + + print("Waiting for operation to complete...") + print( + "Batch Prediction results saved to Cloud Storage bucket. {}".format( + response.result() + ) + ) + # [END automl_batch_predict_beta] diff --git a/automl/beta/batch_predict_test.py b/automl/beta/batch_predict_test.py new file mode 100644 index 000000000000..2869873a9198 --- /dev/null +++ b/automl/beta/batch_predict_test.py @@ -0,0 +1,47 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific ladnguage governing permissions and +# limitations under the License. + +import datetime +import os + +import batch_predict + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +BUCKET_ID = "{}-lcm".format(PROJECT_ID) +MODEL_ID = "TEN0000000000000000000" +PREFIX = "TEST_EXPORT_OUTPUT_" + datetime.datetime.now().strftime( + "%Y%m%d%H%M%S" +) + + +def test_batch_predict(capsys): + # As batch prediction can take a long time. Try to batch predict on a model + # and confirm that the model was not found, but other elements of the + # request were valid. + try: + input_uri = "gs://{}/entity-extraction/input.jsonl".format(BUCKET_ID) + output_uri = "gs://{}/{}/".format(BUCKET_ID, PREFIX) + batch_predict.batch_predict( + PROJECT_ID, MODEL_ID, input_uri, output_uri + ) + out, _ = capsys.readouterr() + assert ( + "does not exist" + in out + ) + except Exception as e: + assert ( + "does not exist" + in e.message + ) diff --git a/automl/beta/video_classification_create_dataset.py b/automl/beta/video_classification_create_dataset.py new file mode 100644 index 000000000000..19bb271b9d6c --- /dev/null +++ b/automl/beta/video_classification_create_dataset.py @@ -0,0 +1,45 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_dataset(project_id, display_name): + """Create a dataset.""" + # [START automl_video_classification_create_dataset_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # display_name = "your_datasets_display_name" + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, "us-central1") + metadata = automl.types.VideoClassificationDatasetMetadata() + dataset = automl.types.Dataset( + display_name=display_name, + video_classification_dataset_metadata=metadata, + ) + + # Create a dataset with the dataset metadata in the region. + created_dataset = client.create_dataset(project_location, dataset) + + # Display the dataset information + print("Dataset name: {}".format(created_dataset.name)) + # To get the dataset id, you have to parse it out of the `name` field. + # As dataset Ids are required for other methods. + # Name Form: + # `projects/{project_id}/locations/{location_id}/datasets/{dataset_id}` + print("Dataset id: {}".format(created_dataset.name.split("/")[-1])) + # [END automl_video_classification_create_dataset_beta] diff --git a/automl/beta/video_classification_create_dataset_test.py b/automl/beta/video_classification_create_dataset_test.py new file mode 100644 index 000000000000..2851e42aa214 --- /dev/null +++ b/automl/beta/video_classification_create_dataset_test.py @@ -0,0 +1,51 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import video_classification_create_dataset + + +PROJECT_ID = os.environ["AUTOML_PROJECT_ID"] +pytest.DATASET_ID = None + + +@pytest.fixture(scope="function", autouse=True) +def teardown(): + yield + + # Delete the created dataset + client = automl.AutoMlClient() + dataset_full_id = client.dataset_path( + PROJECT_ID, "us-central1", pytest.DATASET_ID + ) + response = client.delete_dataset(dataset_full_id) + response.result() + + +def test_video_classification_create_dataset(capsys): + # create dataset + dataset_name = "test_" + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + video_classification_create_dataset.create_dataset( + PROJECT_ID, dataset_name + ) + out, _ = capsys.readouterr() + assert "Dataset id: " in out + + # Get the the created dataset id for deletion + pytest.DATASET_ID = out.splitlines()[1].split()[2] diff --git a/automl/beta/video_classification_create_model.py b/automl/beta/video_classification_create_model.py new file mode 100644 index 000000000000..7fbdfe738e72 --- /dev/null +++ b/automl/beta/video_classification_create_model.py @@ -0,0 +1,42 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def create_model(project_id, dataset_id, display_name): + """Create a model.""" + # [START automl_video_classification_create_model_beta] + from google.cloud import automl_v1beta1 as automl + + # TODO(developer): Uncomment and set the following variables + # project_id = "YOUR_PROJECT_ID" + # dataset_id = "YOUR_DATASET_ID" + # display_name = "your_models_display_name" + + client = automl.AutoMlClient() + + # A resource that represents Google Cloud Platform location. + project_location = client.location_path(project_id, "us-central1") + metadata = automl.types.VideoClassificationModelMetadata() + model = automl.types.Model( + display_name=display_name, + dataset_id=dataset_id, + video_classification_model_metadata=metadata, + ) + + # Create a model with the model metadata in the region. + response = client.create_model(project_location, model) + + print("Training operation name: {}".format(response.operation.name)) + print("Training started...") + # [END automl_video_classification_create_model_beta] diff --git a/automl/beta/video_classification_create_model_test.py b/automl/beta/video_classification_create_model_test.py new file mode 100644 index 000000000000..a91363007639 --- /dev/null +++ b/automl/beta/video_classification_create_model_test.py @@ -0,0 +1,46 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from google.cloud import automl_v1beta1 as automl +import pytest + +import video_classification_create_model + +PROJECT_ID = os.environ["GCLOUD_PROJECT"] +DATASET_ID = "VCN510437278078730240" +pytest.OPERATION_ID = None + + +@pytest.fixture(scope="function", autouse=True) +def teardown(): + yield + + # Cancel the operation + client = automl.AutoMlClient() + client.transport._operations_client.cancel_operation(pytest.OPERATION_ID) + + +def test_video_classification_create_model(capsys): + video_classification_create_model.create_model( + PROJECT_ID, DATASET_ID, "classification_test_create_model" + ) + out, _ = capsys.readouterr() + assert "Training started" in out + + # Get the the operation id for cancellation + pytest.OPERATION_ID = out.split("Training operation name: ")[1].split( + "\n" + )[0]