Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(datastore): Add Datastore Admin API samples #4121

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions datastore/cloud-client/README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ This directory contains samples for Google Cloud Datastore. `Google Cloud Datast

.. _Google Cloud Datastore: https://cloud.google.com/datastore/docs




Set environment variables:
`GOOGLE_CLOUD_PROJECT` - Google Cloud project id
`CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name

Roles to be set in your Service Account and App Engine default service account:
`Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner`
`Storage Admin`, or `Owner`


Setup
-------------------------------------------------------------------------------

Expand Down
11 changes: 10 additions & 1 deletion datastore/cloud-client/README.rst.in
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,19 @@ product:
name: Google Cloud Datastore
short_name: Cloud Datastore
url: https://cloud.google.com/datastore/docs
description: >
description: >
`Google Cloud Datastore`_ is a NoSQL document database built for automatic
scaling, high performance, and ease of application development.

other_required_steps: >
Set environment variables:
IlyaFaer marked this conversation as resolved.
Show resolved Hide resolved
`GOOGLE_CLOUD_PROJECT` - Google Cloud project id
`CLOUD_STORAGE_BUCKET` - Google Cloud Storage bucket name

Roles to be set in your Service Account and App Engine default service account:
`Datastore Import Export Admin`, or `Cloud Datastore Owner`, or `Owner`
`Storage Admin`, or `Owner`

setup:
- auth
- install_deps
Expand Down
86 changes: 86 additions & 0 deletions datastore/cloud-client/admin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# Copyright 2016, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# [START datastore_admin_client_create]
from google.cloud.datastore_admin_v1.gapic import datastore_admin_client


def client_create():
"""Creates a new Datastore admin client."""
client = datastore_admin_client.DatastoreAdminClient()

print("Admin client created\n")
return client
# [END datastore_admin_client_create]


# [START datastore_admin_entities_export]
def export_entities(project_id, output_url_prefix):
"""
Exports a copy of all or a subset of entities from
Datastore to another storage system, such as Cloud Storage.
"""
# project_id = "project-id"
# output_url_prefix = "gs://bucket-name"
client = datastore_admin_client.DatastoreAdminClient()

op = client.export_entities(project_id, output_url_prefix)
response = op.result(timeout=200)

print("Entities were exported\n")
return response
# [END datastore_admin_entities_export]


# [START datastore_admin_entities_import]
def import_entities(project_id, input_url):
"""Imports entities into Datastore."""
# project_id := "project-id"
# input_url := "gs://bucket-name/overall-export-metadata-file"
client = datastore_admin_client.DatastoreAdminClient()

op = client.import_entities(project_id, input_url)
response = op.result(timeout=200)

print("Entities were imported\n")
return response
# [END datastore_admin_entities_import]


# [START datastore_admin_index_get]
def get_index(project_id, index_id):
"""Gets an index."""
# project_id := "my-project-id"
# index_id := "my-index"
client = datastore_admin_client.DatastoreAdminClient()
index = client.get_index(project_id, index_id)

print("Got index: %v\n", index.index_id)
return index
# [END datastore_admin_index_get]


# [START datastore_admin_index_list]
def list_indexes(project_id):
"""Lists the indexes."""
# project_id := "my-project-id"
client = datastore_admin_client.DatastoreAdminClient()

indexes = []
for index in client.list_indexes(project_id):
indexes.append(index)
print("Got index: %v\n", index.index_id)

print("Got list of indexes\n")
return indexes
# [END datastore_admin_index_list]
48 changes: 48 additions & 0 deletions datastore/cloud-client/admin_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Copyright 2016, Google, Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

import backoff
import pytest

import admin


PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
BUCKET = os.environ["CLOUD_STORAGE_BUCKET"]


class TestDatastoreAdminSnippets:
def test_client_create(self):
assert admin.client_create()

def test_get_index(self):
indexes = admin.list_indexes(PROJECT)
if not indexes:
pytest.skip(
"Skipping datastore test. At least "
"one index should present in database."
)

assert admin.get_index(PROJECT, indexes[0].index_id)

def test_list_index(self):
assert admin.list_indexes(PROJECT)

@backoff.on_exception(backoff.expo, AssertionError, max_tries=3, max_time=540000)
def test_export_import_entities(self):
response = admin.export_entities(PROJECT, "gs://" + BUCKET)
assert response

assert admin.import_entities(PROJECT, response.output_url)