diff --git a/bigquery/README.md b/bigquery/README.md index 5e82a68d630..5c3c32316ac 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -1,6 +1,11 @@ # Getting Started with BigQuery and the Google Java API Client library -Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API Client Libraries. For more information, read the [Getting Started with BigQuery and the Google Java API Client library][1] codelab. +Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries +on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API +Client Libraries. + +For more information, read the [Getting Started with BigQuery and the Google Java API Client +library][1] codelab. ## Quickstart @@ -8,7 +13,7 @@ Install [Maven](http://maven.apache.org/). Build your project with: - mvn package -DskipTests + mvn clean package -DskipTests You can then run a given `ClassName` via: diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java index e44aa537470..e0384da0867 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java @@ -24,12 +24,14 @@ import java.io.IOException; import java.util.Iterator; -import java.util.Scanner; /** * Example of authorizing with BigQuery and reading from a public dataset. */ public class AsyncQuerySample { + private static final String DEFAULT_QUERY = + "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;"; + // [START main] /** * Prompts for all the parameters required to make a query. @@ -39,17 +41,42 @@ public class AsyncQuerySample { * @throws InterruptedException InterruptedException */ public static void main(final String[] args) throws IOException, InterruptedException { - Scanner scanner = new Scanner(System.in); - System.out.println("Enter your project id: "); - String projectId = scanner.nextLine(); - System.out.println("Enter your query string: "); - String queryString = scanner.nextLine(); - System.out.println("Run query in batch mode? [true|false] "); - boolean batch = Boolean.valueOf(scanner.nextLine()); - System.out.println("Enter how often to check if your job is complete " + "(milliseconds): "); - long waitTime = scanner.nextLong(); - scanner.close(); - Iterator pages = run(projectId, queryString, batch, waitTime); + String projectId = System.getProperty("projectId"); + if (projectId == null || projectId.isEmpty()) { + System.err.println("The projectId property must be set."); + System.exit(1); + } + System.out.printf("projectId: %s\n", projectId); + + String queryString = System.getProperty("query"); + if (queryString == null || queryString.isEmpty()) { + System.out.println("The query property was not set, using default."); + queryString = DEFAULT_QUERY; + } + System.out.printf("query: %s\n", queryString); + + String useBatchString = System.getProperty("useBatchMode"); + if (useBatchString == null || useBatchString.isEmpty()) { + useBatchString = "false"; + } + boolean useBatchMode = Boolean.parseBoolean(useBatchString); + System.out.printf("useBatchMode: %b\n", useBatchMode); + + String waitTimeString = System.getProperty("waitTime"); + if (waitTimeString == null || waitTimeString.isEmpty()) { + waitTimeString = "1000"; + } + long waitTime = Long.parseLong(waitTimeString); + System.out.printf("waitTime: %d (milliseconds)\n", waitTime); + + String useLegacySqlString = System.getProperty("useLegacySql"); + if (useLegacySqlString == null || useLegacySqlString.isEmpty()) { + useLegacySqlString = "false"; + } + boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString); + + Iterator pages = + run(projectId, queryString, useBatchMode, waitTime, useLegacySql); while (pages.hasNext()) { BigQueryUtils.printRows(pages.next().getRows(), System.out); } @@ -62,19 +89,24 @@ public static void main(final String[] args) throws IOException, InterruptedExce * * @param projectId Get this from Google Developers console * @param queryString Query we want to run against BigQuery - * @param batch True if you want to batch the queries + * @param useBatchMode True if you want to batch the queries * @param waitTime How long to wait before retries + * @param useLegacySql Boolean that is false if using standard SQL syntax. * @return An iterator to the result of your pages * @throws IOException Thrown if there's an IOException * @throws InterruptedException Thrown if there's an Interrupted Exception */ public static Iterator run( - final String projectId, final String queryString, final boolean batch, final long waitTime) + final String projectId, + final String queryString, + final boolean useBatchMode, + final long waitTime, + final boolean useLegacySql) throws IOException, InterruptedException { Bigquery bigquery = BigQueryServiceFactory.getService(); - Job query = asyncQuery(bigquery, projectId, queryString, batch); + Job query = asyncQuery(bigquery, projectId, queryString, useBatchMode, useLegacySql); Bigquery.Jobs.Get getRequest = bigquery.jobs().get(projectId, query.getJobReference().getJobId()); @@ -96,17 +128,27 @@ public static Iterator run( * @param bigquery an authorized BigQuery client * @param projectId a String containing the project ID * @param querySql the actual query string - * @param batch True if you want to run the query as BATCH + * @param useBatchMode True if you want to run the query as BATCH + * @param useLegacySql Boolean that is false if using standard SQL syntax. * @return a reference to the inserted query job * @throws IOException Thrown if there's a network exception */ public static Job asyncQuery( - final Bigquery bigquery, final String projectId, final String querySql, final boolean batch) + final Bigquery bigquery, + final String projectId, + final String querySql, + final boolean useBatchMode, + final boolean useLegacySql) throws IOException { - JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(querySql); + JobConfigurationQuery queryConfig = + new JobConfigurationQuery() + .setQuery(querySql) + // Set the useLegacySql parameter to false to use standard SQL syntax. See: + // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql + .setUseLegacySql(useLegacySql); - if (batch) { + if (useBatchMode) { queryConfig.setPriority("BATCH"); } diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java index 3bfb30d9f64..bf3b0eb8bed 100644 --- a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java +++ b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java @@ -22,11 +22,14 @@ import java.io.IOException; import java.util.Iterator; -import java.util.Scanner; + /** * Runs a synchronous query against Bigtable. */ public class SyncQuerySample { + private static final String DEFAULT_QUERY = + "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;"; + private static final long TEN_SECONDS_MILLIS = 10000; /** * Protected because this is a collection of static methods. @@ -41,18 +44,39 @@ protected SyncQuerySample() {} * @throws IOException ioexceptino */ public static void main(final String[] args) throws IOException { - Scanner scanner = new Scanner(System.in); - System.out.println("Enter your project id: "); - String projectId = scanner.nextLine(); - System.out.println("Enter your query string: "); - String queryString = scanner.nextLine(); - System.out.println( - "Enter how long to wait for the query to complete" - + " (in milliseconds):\n " - + "(if longer than 10 seconds, use an asynchronous query)"); - long waitTime = scanner.nextLong(); - scanner.close(); - Iterator pages = run(projectId, queryString, waitTime); + String projectId = System.getProperty("projectId"); + if (projectId == null || projectId.isEmpty()) { + System.err.println("The projectId property must be set."); + System.exit(1); + } + System.out.printf("projectId: %s\n", projectId); + + String queryString = System.getProperty("query"); + if (queryString == null || queryString.isEmpty()) { + System.out.println("The query property was not set, using default."); + queryString = DEFAULT_QUERY; + } + System.out.printf("query: %s\n", queryString); + + String waitTimeString = System.getProperty("waitTime"); + if (waitTimeString == null || waitTimeString.isEmpty()) { + waitTimeString = "1000"; + } + long waitTime = Long.parseLong(waitTimeString); + System.out.printf("waitTime: %d (milliseconds)\n", waitTime); + if (waitTime > TEN_SECONDS_MILLIS) { + System.out.println( + "WARNING: If the query is going to take longer than 10 seconds to complete, use an" + + " asynchronous query."); + } + + String useLegacySqlString = System.getProperty("useLegacySql"); + if (useLegacySqlString == null || useLegacySqlString.isEmpty()) { + useLegacySqlString = "false"; + } + boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString); + + Iterator pages = run(projectId, queryString, waitTime, useLegacySql); while (pages.hasNext()) { BigQueryUtils.printRows(pages.next().getRows(), System.out); } @@ -65,23 +89,33 @@ public static void main(final String[] args) throws IOException { * @param projectId project id from developer console * @param queryString query to run * @param waitTime Timeout in milliseconds before we abort + * @param useLegacySql Boolean that is false if using standard SQL syntax. * @return Iterator that pages through the results of the query * @throws IOException ioexception */ // [START run] public static Iterator run( - final String projectId, final String queryString, final long waitTime) throws IOException { + final String projectId, + final String queryString, + final long waitTime, + final boolean useLegacySql) throws IOException { Bigquery bigquery = BigQueryServiceFactory.getService(); - //Wait until query is done with 10 second timeout, at most 5 retries on error + + // Wait until query is done with `waitTime` millisecond timeout, at most 5 retries on error. QueryResponse query = bigquery .jobs() - .query(projectId, new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString)) + .query( + projectId, + new QueryRequest() + .setTimeoutMs(waitTime) + .setQuery(queryString) + // Set the useLegacySql parameter to false to use standard SQL syntax. See: + // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql + .setUseLegacySql(useLegacySql)) .execute(); - //Make a request to get the results of the query - //(timeout is zero since job should be complete) - + // Make a request to get the results of the query. GetQueryResults getRequest = bigquery .jobs() diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java index a19c98b9f5c..a71e472acdb 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java @@ -38,17 +38,41 @@ public class AsyncQuerySampleTest { @Test public void testInteractive() throws IOException, InterruptedException { Iterator pages = - AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, false, 5000); + AsyncQuerySample.run( + Constants.PROJECT_ID, + "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;", + false /* useBatchMode */, + 5000, + false /* useLegacySql */); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); } } @Test - @Ignore // Batches can take up to 3 hours to run, probably shouldn't use this + public void testInteractiveLegacySql() throws IOException, InterruptedException { + Iterator pages = + AsyncQuerySample.run( + Constants.PROJECT_ID, + Constants.QUERY, + false /* useBatchMode */, + 5000, + true /* useLegacySql */); + while (pages.hasNext()) { + assertThat(pages.next().getRows()).isNotEmpty(); + } + } + + @Test + @Ignore // Batches can take up to 3 hours to run, don't run during the system tests. public void testBatch() throws IOException, InterruptedException { Iterator pages = - AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, true, 5000); + AsyncQuerySample.run( + Constants.PROJECT_ID, + Constants.QUERY, + true /* useBatchMode */, + 5000, + true /* useLegacySql */); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); } diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java index f8a9441bc05..e40d02f30f5 100644 --- a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java +++ b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java @@ -37,7 +37,20 @@ public class SyncQuerySampleTest { @Test public void testSyncQuery() throws IOException { Iterator pages = - SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000); + SyncQuerySample.run( + Constants.PROJECT_ID, + "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;", + 10000, + false /* useLegacySql */); + while (pages.hasNext()) { + assertThat(pages.next().getRows()).isNotEmpty(); + } + } + + @Test + public void testSyncQueryLegacySql() throws IOException { + Iterator pages = + SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000, true /* useLegacySql */); while (pages.hasNext()) { assertThat(pages.next().getRows()).isNotEmpty(); }