tweak BigQuery default project detection (#195)

* use DefaultProjectFactory to detect project * detect default project in BigQueryClient Conflicts: scio-bigquery/src/main/scala/com/spotify/scio/bigquery/BigQueryClient.scala scio-repl/src/main/scala/com/spotify/scio/repl/ScioILoop.scala
spotify · Jul 19, 2016 · c6b7456 · c6b7456
1 parent 9fb497f
commit c6b7456
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 113 deletions.
diff --git a/build.sbt b/build.sbt
@@ -36,7 +36,6 @@ val guavaVersion = "19.0"
 val hadoopVersion = "2.7.2"
 val hamcrestVersion = "1.3"
 val hbaseVersion = "1.0.2"
-val ini4jVersion = "0.5.4"
 val javaLshVersion = "0.9"
 val jodaConvertVersion = "1.8.1"
 val junitVersion = "4.12"
@@ -195,8 +194,7 @@ lazy val scioCore: Project = Project(
       "com.twitter" %% "chill" % chillVersion,
       "com.twitter" % "chill-protobuf" % chillVersion,
       "commons-io" % "commons-io" % commonsIoVersion,
-      "org.apache.commons" % "commons-math3" % commonsMath3Version,
-      "org.ini4j" % "ini4j" % ini4jVersion
+      "org.apache.commons" % "commons-math3" % commonsMath3Version
     )
   )
 ).dependsOn(

diff --git a/scio-bigquery/src/main/scala/com/spotify/scio/bigquery/BigQueryClient.scala b/scio-bigquery/src/main/scala/com/spotify/scio/bigquery/BigQueryClient.scala
@@ -37,6 +37,7 @@ import org.apache.beam.sdk.io.BigQueryIO
 import org.apache.beam.sdk.io.BigQueryIO.Write.CreateDisposition._
 import org.apache.beam.sdk.io.BigQueryIO.Write.WriteDisposition._
 import org.apache.beam.sdk.io.BigQueryIO.Write.{CreateDisposition, WriteDisposition}
+import org.apache.beam.sdk.options.GcpOptions.DefaultProjectFactory
 import org.apache.beam.sdk.util.{BigQueryTableInserter, BigQueryTableRowIterator}
 import org.apache.commons.io.FileUtils
 import org.joda.time.format.{DateTimeFormat, PeriodFormatterBuilder}
@@ -498,12 +499,17 @@ object BigQueryClient {
    */
   def defaultInstance(): BigQueryClient = {
     if (instance == null) {
-      val project = sys.props(PROJECT_KEY)
-      if (project == null) {
-        throw new RuntimeException(
-          s"Property $PROJECT_KEY not set. Use -D$PROJECT_KEY=<BILLING_PROJECT>")
+      instance = if (sys.props(PROJECT_KEY) != null) {
+        BigQueryClient(sys.props(PROJECT_KEY))
+      } else {
+        val project = new DefaultProjectFactory().create(null)
+        if (project != null) {
+          BigQueryClient(project)
+        } else {
+          throw new RuntimeException(
+            s"Property $PROJECT_KEY not set. Use -D$PROJECT_KEY=<BILLING_PROJECT>")
+        }
       }
-      instance = BigQueryClient(project)
     }
     instance
   }

diff --git a/scio-core/src/main/scala/com/spotify/scio/util/GCloudConfigUtils.scala b/scio-core/src/main/scala/com/spotify/scio/util/GCloudConfigUtils.scala
diff --git a/scio-repl/src/main/scala/com/spotify/scio/repl/ScioILoop.scala b/scio-repl/src/main/scala/com/spotify/scio/repl/ScioILoop.scala
@@ -20,10 +20,10 @@ package com.spotify.scio.repl
 import java.io.BufferedReader
 
 import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions
+import org.apache.beam.sdk.options.GcpOptions.DefaultProjectFactory
 import org.apache.beam.sdk.options.PipelineOptionsFactory
 import com.spotify.scio.bigquery.BigQueryClient
 import com.spotify.scio.scioVersion
-import com.spotify.scio.util.GCloudConfigUtils
 
 import scala.tools.nsc.GenericRunnerSettings
 import scala.tools.nsc.interpreter.{IR, JPrintWriter}
@@ -213,20 +213,18 @@ class ScioILoop(scioClassLoader: ScioReplClassLoader,
 
     val key = BigQueryClient.PROJECT_KEY
 
-    if (sys.props(key) == null) {
-      val project = GCloudConfigUtils.getGCloudProjectId
-      project match {
-        case Some(project_id) =>
-          echo(s"Using '$project_id' as your BigQuery project.")
-          sys.props(key) = project_id
-          create(project_id)
-        case None =>
-          echo(s"System property '$key' not set. BigQueryClient is not available.")
-          echo(s"Set it with '-D$key=<PROJECT-NAME>' command line argument.")
-          IR.Success
-      }
-    } else {
+    if (sys.props(key) != null) {
       create(sys.props(key))
+    } else {
+      val defaultProject = new DefaultProjectFactory().create(null)
+      if (defaultProject != null) {
+        echo(s"Using '$defaultProject' as your BigQuery project.")
+        create(defaultProject)
+      } else {
+        echo(s"System property '$key' not set. BigQueryClient is not available.")
+        echo(s"Set it with '-D$key=<PROJECT-NAME>' command line argument.")
+        IR.Success
+      }
     }
   }