Merge pull request #305 from GoogleCloudPlatform/tswast-bq-standard-sql

BigQuery query samples: command line parameters.
GoogleCloudPlatform · Aug 18, 2016 · 44824dd · 44824dd
2 parents 74c2d8f + 2d7fb47
commit 44824dd
Show file tree

Hide file tree

Showing 5 changed files with 162 additions and 44 deletions.
diff --git a/bigquery/README.md b/bigquery/README.md
@@ -1,14 +1,19 @@
 # Getting Started with BigQuery and the Google Java API Client library
 
-Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API Client Libraries. For more information, read the [Getting Started with BigQuery and the Google Java API Client library][1] codelab.
+Google's BigQuery Service features a REST-based API that allows developers to create applications to run ad-hoc queries
+on massive datasets. These sample Java applications demonstrate how to access the BigQuery API using the Google Java API
+Client Libraries.
+
+For more information, read the [Getting Started with BigQuery and the Google Java API Client
+library][1] codelab.
 
 ## Quickstart
 
 Install [Maven](http://maven.apache.org/).
 
 Build your project with:
 
-	mvn package -DskipTests
+	mvn clean package -DskipTests
 
 You can then run a given `ClassName` via:
 

diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/AsyncQuerySample.java
@@ -24,12 +24,14 @@
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.Scanner;
 
 /**
  * Example of authorizing with BigQuery and reading from a public dataset.
  */
 public class AsyncQuerySample {
+  private static final String DEFAULT_QUERY =
+      "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;";
+
   // [START main]
   /**
    * Prompts for all the parameters required to make a query.
@@ -39,17 +41,42 @@ public class AsyncQuerySample {
    * @throws InterruptedException InterruptedException
    */
   public static void main(final String[] args) throws IOException, InterruptedException {
-    Scanner scanner = new Scanner(System.in);
-    System.out.println("Enter your project id: ");
-    String projectId = scanner.nextLine();
-    System.out.println("Enter your query string: ");
-    String queryString = scanner.nextLine();
-    System.out.println("Run query in batch mode? [true|false] ");
-    boolean batch = Boolean.valueOf(scanner.nextLine());
-    System.out.println("Enter how often to check if your job is complete " + "(milliseconds): ");
-    long waitTime = scanner.nextLong();
-    scanner.close();
-    Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, batch, waitTime);
+    String projectId = System.getProperty("projectId");
+    if (projectId == null || projectId.isEmpty()) {
+      System.err.println("The projectId property must be set.");
+      System.exit(1);
+    }
+    System.out.printf("projectId: %s\n", projectId);
+
+    String queryString = System.getProperty("query");
+    if (queryString == null || queryString.isEmpty()) {
+      System.out.println("The query property was not set, using default.");
+      queryString = DEFAULT_QUERY;
+    }
+    System.out.printf("query: %s\n", queryString);
+
+    String useBatchString = System.getProperty("useBatchMode");
+    if (useBatchString == null || useBatchString.isEmpty()) {
+      useBatchString = "false";
+    }
+    boolean useBatchMode = Boolean.parseBoolean(useBatchString);
+    System.out.printf("useBatchMode: %b\n", useBatchMode);
+
+    String waitTimeString = System.getProperty("waitTime");
+    if (waitTimeString == null || waitTimeString.isEmpty()) {
+      waitTimeString = "1000";
+    }
+    long waitTime = Long.parseLong(waitTimeString);
+    System.out.printf("waitTime: %d (milliseconds)\n", waitTime);
+
+    String useLegacySqlString = System.getProperty("useLegacySql");
+    if (useLegacySqlString == null || useLegacySqlString.isEmpty()) {
+      useLegacySqlString = "false";
+    }
+    boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString);
+
+    Iterator<GetQueryResultsResponse> pages =
+        run(projectId, queryString, useBatchMode, waitTime, useLegacySql);
     while (pages.hasNext()) {
       BigQueryUtils.printRows(pages.next().getRows(), System.out);
     }
@@ -62,19 +89,24 @@ public static void main(final String[] args) throws IOException, InterruptedExce
    *
    * @param projectId Get this from Google Developers console
    * @param queryString Query we want to run against BigQuery
-   * @param batch True if you want to batch the queries
+   * @param useBatchMode True if you want to batch the queries
    * @param waitTime How long to wait before retries
+   * @param useLegacySql Boolean that is false if using standard SQL syntax.
    * @return An iterator to the result of your pages
    * @throws IOException Thrown if there's an IOException
    * @throws InterruptedException Thrown if there's an Interrupted Exception
    */
   public static Iterator<GetQueryResultsResponse> run(
-      final String projectId, final String queryString, final boolean batch, final long waitTime)
+      final String projectId,
+      final String queryString,
+      final boolean useBatchMode,
+      final long waitTime,
+      final boolean useLegacySql)
       throws IOException, InterruptedException {
 
     Bigquery bigquery = BigQueryServiceFactory.getService();
 
-    Job query = asyncQuery(bigquery, projectId, queryString, batch);
+    Job query = asyncQuery(bigquery, projectId, queryString, useBatchMode, useLegacySql);
     Bigquery.Jobs.Get getRequest =
         bigquery.jobs().get(projectId, query.getJobReference().getJobId());
 
@@ -96,17 +128,27 @@ public static Iterator<GetQueryResultsResponse> run(
    * @param bigquery an authorized BigQuery client
    * @param projectId a String containing the project ID
    * @param querySql  the actual query string
-   * @param batch True if you want to run the query as BATCH
+   * @param useBatchMode True if you want to run the query as BATCH
+   * @param useLegacySql Boolean that is false if using standard SQL syntax.
    * @return a reference to the inserted query job
    * @throws IOException Thrown if there's a network exception
    */
   public static Job asyncQuery(
-      final Bigquery bigquery, final String projectId, final String querySql, final boolean batch)
+      final Bigquery bigquery,
+      final String projectId,
+      final String querySql,
+      final boolean useBatchMode,
+      final boolean useLegacySql)
       throws IOException {
 
-    JobConfigurationQuery queryConfig = new JobConfigurationQuery().setQuery(querySql);
+    JobConfigurationQuery queryConfig =
+        new JobConfigurationQuery()
+            .setQuery(querySql)
+            // Set the useLegacySql parameter to false to use standard SQL syntax. See:
+            // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql
+            .setUseLegacySql(useLegacySql);
 
-    if (batch) {
+    if (useBatchMode) {
       queryConfig.setPriority("BATCH");
     }
 

diff --git a/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java b/bigquery/src/main/java/com/google/cloud/bigquery/samples/SyncQuerySample.java
@@ -22,11 +22,14 @@
 
 import java.io.IOException;
 import java.util.Iterator;
-import java.util.Scanner;
+
 /**
  * Runs a synchronous query against Bigtable.
  */
 public class SyncQuerySample {
+  private static final String DEFAULT_QUERY =
+      "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;";
+  private static final long TEN_SECONDS_MILLIS = 10000;
 
   /**
    * Protected because this is a collection of static methods.
@@ -41,18 +44,39 @@ protected SyncQuerySample() {}
    * @throws IOException ioexceptino
    */
   public static void main(final String[] args) throws IOException {
-    Scanner scanner = new Scanner(System.in);
-    System.out.println("Enter your project id: ");
-    String projectId = scanner.nextLine();
-    System.out.println("Enter your query string: ");
-    String queryString = scanner.nextLine();
-    System.out.println(
-        "Enter how long to wait for the query to complete"
-            + " (in milliseconds):\n "
-            + "(if longer than 10 seconds, use an asynchronous query)");
-    long waitTime = scanner.nextLong();
-    scanner.close();
-    Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, waitTime);
+    String projectId = System.getProperty("projectId");
+    if (projectId == null || projectId.isEmpty()) {
+      System.err.println("The projectId property must be set.");
+      System.exit(1);
+    }
+    System.out.printf("projectId: %s\n", projectId);
+
+    String queryString = System.getProperty("query");
+    if (queryString == null || queryString.isEmpty()) {
+      System.out.println("The query property was not set, using default.");
+      queryString = DEFAULT_QUERY;
+    }
+    System.out.printf("query: %s\n", queryString);
+
+    String waitTimeString = System.getProperty("waitTime");
+    if (waitTimeString == null || waitTimeString.isEmpty()) {
+      waitTimeString = "1000";
+    }
+    long waitTime = Long.parseLong(waitTimeString);
+    System.out.printf("waitTime: %d (milliseconds)\n", waitTime);
+    if (waitTime > TEN_SECONDS_MILLIS) {
+      System.out.println(
+          "WARNING: If the query is going to take longer than 10 seconds to complete, use an"
+          + " asynchronous query.");
+    }
+
+    String useLegacySqlString = System.getProperty("useLegacySql");
+    if (useLegacySqlString == null || useLegacySqlString.isEmpty()) {
+      useLegacySqlString = "false";
+    }
+    boolean useLegacySql = Boolean.parseBoolean(useLegacySqlString);
+
+    Iterator<GetQueryResultsResponse> pages = run(projectId, queryString, waitTime, useLegacySql);
     while (pages.hasNext()) {
       BigQueryUtils.printRows(pages.next().getRows(), System.out);
     }
@@ -65,23 +89,33 @@ public static void main(final String[] args) throws IOException {
    * @param projectId project id from developer console
    * @param queryString query to run
    * @param waitTime Timeout in milliseconds before we abort
+   * @param useLegacySql Boolean that is false if using standard SQL syntax.
    * @return Iterator that pages through the results of the query
    * @throws IOException ioexception
    */
   // [START run]
   public static Iterator<GetQueryResultsResponse> run(
-      final String projectId, final String queryString, final long waitTime) throws IOException {
+      final String projectId,
+      final String queryString,
+      final long waitTime,
+      final boolean useLegacySql) throws IOException {
     Bigquery bigquery = BigQueryServiceFactory.getService();
-    //Wait until query is done with 10 second timeout, at most 5 retries on error
+
+    // Wait until query is done with `waitTime` millisecond timeout, at most 5 retries on error.
     QueryResponse query =
         bigquery
             .jobs()
-            .query(projectId, new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString))
+            .query(
+                projectId,
+                new QueryRequest()
+                    .setTimeoutMs(waitTime)
+                    .setQuery(queryString)
+                    // Set the useLegacySql parameter to false to use standard SQL syntax. See:
+                    // https://cloud.google.com/bigquery/sql-reference/enabling-standard-sql
+                    .setUseLegacySql(useLegacySql))
             .execute();
 
-    //Make a request to get the results of the query
-    //(timeout is zero since job should be complete)
-
+    // Make a request to get the results of the query.
     GetQueryResults getRequest =
         bigquery
             .jobs()

diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/AsyncQuerySampleTest.java
@@ -38,17 +38,41 @@ public class AsyncQuerySampleTest {
   @Test
   public void testInteractive() throws IOException, InterruptedException {
     Iterator<GetQueryResultsResponse> pages =
-        AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, false, 5000);
+        AsyncQuerySample.run(
+            Constants.PROJECT_ID,
+            "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;",
+            false /* useBatchMode */,
+            5000,
+            false /* useLegacySql */);
     while (pages.hasNext()) {
       assertThat(pages.next().getRows()).isNotEmpty();
     }
   }
 
   @Test
-  @Ignore // Batches can take up to 3 hours to run, probably shouldn't use this
+  public void testInteractiveLegacySql() throws IOException, InterruptedException {
+    Iterator<GetQueryResultsResponse> pages =
+        AsyncQuerySample.run(
+            Constants.PROJECT_ID,
+            Constants.QUERY,
+            false /* useBatchMode */,
+            5000,
+            true /* useLegacySql */);
+    while (pages.hasNext()) {
+      assertThat(pages.next().getRows()).isNotEmpty();
+    }
+  }
+
+  @Test
+  @Ignore // Batches can take up to 3 hours to run, don't run during the system tests.
   public void testBatch() throws IOException, InterruptedException {
     Iterator<GetQueryResultsResponse> pages =
-        AsyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, true, 5000);
+        AsyncQuerySample.run(
+            Constants.PROJECT_ID,
+            Constants.QUERY,
+            true /* useBatchMode */,
+            5000,
+            true /* useLegacySql */);
     while (pages.hasNext()) {
       assertThat(pages.next().getRows()).isNotEmpty();
     }

diff --git a/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java b/bigquery/src/test/java/com/google/cloud/bigquery/samples/test/SyncQuerySampleTest.java
@@ -37,7 +37,20 @@ public class SyncQuerySampleTest {
   @Test
   public void testSyncQuery() throws IOException {
     Iterator<GetQueryResultsResponse> pages =
-        SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000);
+        SyncQuerySample.run(
+            Constants.PROJECT_ID,
+            "SELECT corpus FROM `publicdata.samples.shakespeare` GROUP BY corpus;",
+            10000,
+            false /* useLegacySql */);
+    while (pages.hasNext()) {
+      assertThat(pages.next().getRows()).isNotEmpty();
+    }
+  }
+
+  @Test
+  public void testSyncQueryLegacySql() throws IOException {
+    Iterator<GetQueryResultsResponse> pages =
+        SyncQuerySample.run(Constants.PROJECT_ID, Constants.QUERY, 10000, true /* useLegacySql */);
     while (pages.hasNext()) {
       assertThat(pages.next().getRows()).isNotEmpty();
     }