From fdb021811698dc1ed3fee6cb76d570aa3ad98f04 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 10 Aug 2016 14:49:53 -0700 Subject: [PATCH] Add veneer samples for BigQuery. [(#444)](https://github.com/GoogleCloudPlatform/python-docs-samples/issues/444) Put them in their own directory, since they duplicate the content from the existing samples that use the generated libraries. --- samples/snippets/README.md | 5 ++ samples/snippets/async_query.py | 83 ++++++++++++++++++++++++++++ samples/snippets/async_query_test.py | 27 +++++++++ samples/snippets/requirements.txt | 1 + samples/snippets/sync_query.py | 66 ++++++++++++++++++++++ samples/snippets/sync_query_test.py | 27 +++++++++ 6 files changed, 209 insertions(+) create mode 100644 samples/snippets/README.md create mode 100755 samples/snippets/async_query.py create mode 100644 samples/snippets/async_query_test.py create mode 100644 samples/snippets/requirements.txt create mode 100755 samples/snippets/sync_query.py create mode 100644 samples/snippets/sync_query_test.py diff --git a/samples/snippets/README.md b/samples/snippets/README.md new file mode 100644 index 000000000..9c1f9b9fb --- /dev/null +++ b/samples/snippets/README.md @@ -0,0 +1,5 @@ +# BigQuery Google Cloud Client Library Samples + + + + diff --git a/samples/snippets/async_query.py b/samples/snippets/async_query.py new file mode 100755 index 000000000..0ca324240 --- /dev/null +++ b/samples/snippets/async_query.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to perform asynchronous queries in BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python async_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' +""" + +import argparse +import time +import uuid + +from gcloud import bigquery + + +def async_query(query): + client = bigquery.Client() + query_job = client.run_async_query(str(uuid.uuid4()), query) + query_job.use_legacy_sql = False + query_job.begin() + + wait_for_job(query_job) + + # Manually construct the QueryResults. + # TODO: The client library will provide a helper method that does this. + # https://github.com/GoogleCloudPlatform/gcloud-python/issues/2083 + query_results = bigquery.query.QueryResults('', client) + query_results._properties['jobReference'] = { + 'jobId': query_job.name, + 'projectId': query_job.project + } + + # Drain the query results by requesting a page at a time. + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break + + +def wait_for_job(job): + while True: + job.reload() # Refreshes the state via a GET request. + if job.state == 'DONE': + if job.error_result: + raise RuntimeError(job.error_result) + return + time.sleep(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('query', help='BigQuery SQL Query.') + + args = parser.parse_args() + + async_query(args.query) diff --git a/samples/snippets/async_query_test.py b/samples/snippets/async_query_test.py new file mode 100644 index 000000000..810c538a6 --- /dev/null +++ b/samples/snippets/async_query_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from async_query import async_query + + +def test_async_query(cloud_config, capsys): + query = ( + 'SELECT corpus FROM `publicdata.samples.shakespeare` ' + 'GROUP BY corpus;') + + async_query(query) + + out, _ = capsys.readouterr() + + assert 'romeoandjuliet' in out diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt new file mode 100644 index 000000000..2beeafe63 --- /dev/null +++ b/samples/snippets/requirements.txt @@ -0,0 +1 @@ +gcloud==0.18.1 diff --git a/samples/snippets/sync_query.py b/samples/snippets/sync_query.py new file mode 100755 index 000000000..59007b537 --- /dev/null +++ b/samples/snippets/sync_query.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Command-line application to perform synchronous queries in BigQuery. + +For more information, see the README.md under /bigquery. + +Example invocation: + $ python sync_query.py \ + 'SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus' +""" + +import argparse + +# [START sync_query] +from gcloud import bigquery + + +def sync_query(query): + client = bigquery.Client() + query_results = client.run_sync_query(query) + + # Use standard SQL syntax for queries. + # See: https://cloud.google.com/bigquery/sql-reference/ + query_results.use_legacy_sql = False + + query_results.run() + + # Drain the query results by requesting a page at a time. + page_token = None + + while True: + rows, total_rows, page_token = query_results.fetch_data( + max_results=10, + page_token=page_token) + + for row in rows: + print(row) + + if not page_token: + break +# [END sync_query] + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('query', help='BigQuery SQL Query.') + + args = parser.parse_args() + + sync_query(args.query) diff --git a/samples/snippets/sync_query_test.py b/samples/snippets/sync_query_test.py new file mode 100644 index 000000000..6f6b4f5fa --- /dev/null +++ b/samples/snippets/sync_query_test.py @@ -0,0 +1,27 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sync_query import sync_query + + +def test_sync_query(cloud_config, capsys): + query = ( + 'SELECT corpus FROM `publicdata.samples.shakespeare` ' + 'GROUP BY corpus;') + + sync_query(query) + + out, _ = capsys.readouterr() + + assert 'romeoandjuliet' in out