Merge pull request #10723 from ibzib/spark-test-script

[BEAM-9211] upload missing Spark portable jar test script
apache · Feb 14, 2020 · d9099fb · d9099fb
2 parents 561b221 + 9990cdb
commit d9099fb
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 8 deletions.
diff --git a/runners/flink/job-server/flink_job_server.gradle b/runners/flink/job-server/flink_job_server.gradle
@@ -212,13 +212,14 @@ def addTestJavaJarCreator(String pyVersion) {
       exec {
         executable "sh"
         def options = [
-                "--flink_job_server_jar ${shadowJar.archivePath}",
+                "--runner FlinkRunner",
+                "--job_server_jar ${shadowJar.archivePath}",
                 "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
                 "--python_root_dir ${project.rootDir}/sdks/python",
                 "--python_version ${pyVersion}",
                 "--python_container_image apachebeam/python${pyVersion}_sdk:${project.sdk_version}",
         ]
-        args "-c", "../../job-server/test_pipeline_jar.sh ${options.join(' ')}"
+        args "-c", "../../../portability/test_pipeline_jar.sh ${options.join(' ')}"
       }
     }
   }

diff --git a/...ers/flink/job-server/test_pipeline_jar.sh → runners/portability/test_pipeline_jar.sh b/...ers/flink/job-server/test_pipeline_jar.sh → runners/portability/test_pipeline_jar.sh
@@ -23,8 +23,13 @@ while [[ $# -gt 0 ]]
 do
 key="$1"
 case $key in
-    --flink_job_server_jar)
-        FLINK_JOB_SERVER_JAR="$2"
+    --job_server_jar)
+        JOB_SERVER_JAR="$2"
+        shift # past argument
+        shift # past value
+        ;;
+    --runner)
+        RUNNER="$2"
         shift # past argument
         shift # past value
         ;;
@@ -96,11 +101,17 @@ result = pipeline.run()
 result.wait_until_finish()
 "
 
+if [[ "$RUNNER" = "FlinkRunner" ]]; then
+  INPUT_JAR_ARG="flink_job_server_jar"
+else
+  INPUT_JAR_ARG="spark_job_server_jar"
+fi
+
 # Create the jar
 OUTPUT_JAR=flink-test-$(date +%Y%m%d-%H%M%S).jar
 (python -c "$PIPELINE_PY" \
-  --runner FlinkRunner \
-  --flink_job_server_jar $FLINK_JOB_SERVER_JAR \
+  --runner "$RUNNER" \
+  --"$INPUT_JAR_ARG" "$JOB_SERVER_JAR" \
   --output_executable_path $OUTPUT_JAR \
   --parallelism 1 \
   --sdk_worker_parallelism 1 \

diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle
@@ -159,13 +159,14 @@ def addTestJavaJarCreator(String pyVersion) {
       exec {
         executable "sh"
         def options = [
-                "--spark_job_server_jar ${shadowJar.archivePath}",
+                "--runner SparkRunner",
+                "--job_server_jar ${shadowJar.archivePath}",
                 "--env_dir ${project.rootProject.buildDir}/gradleenv/${project.path.hashCode()}",
                 "--python_root_dir ${project.rootDir}/sdks/python",
                 "--python_version ${pyVersion}",
                 "--python_container_image apachebeam/python${pyVersion}_sdk:${project.sdk_version}",
         ]
-        args "-c", "./test_spark_pipeline_jar.sh ${options.join(' ')}"
+        args "-c", "../../portability/test_pipeline_jar.sh ${options.join(' ')}"
       }
     }
   }