From a16a7caa476953d4eeb99eba0a2428758b3c0fea Mon Sep 17 00:00:00 2001 From: Christopher Lambert Date: Wed, 18 Sep 2024 15:52:57 +0200 Subject: [PATCH] support customizing the crossengine namespace (#632) --- .../integtests/crossengine/ITCrossEngine.java | 24 ++++++++++++++----- .../crossengine/ITCrossEngineDremio.java | 2 +- .../ITCrossEngineDremioPermissions.java | 2 +- .../flink/IcebergFlinkExtension.java | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngine.java b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngine.java index 13d36434..5fd155b2 100644 --- a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngine.java +++ b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngine.java @@ -50,12 +50,18 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ITCrossEngine { + private static final String NAMESPACE = System.getProperty("nesqueit.namespace", "db"); + + private static String sparkTableIdentifier(String tableName) { + return String.format("nessie.%s.%s", NAMESPACE, tableName); + } + @Order(20) @Test public void createNamespace(@NessieAPI NessieApiV1 nessie, @NessieDefaultBranch String branch) throws Exception { try { - nessie.createNamespace().namespace("db").refName(branch).create(); + nessie.createNamespace().namespace(NAMESPACE).refName(branch).create(); } catch (NessieNamespaceAlreadyExistsException ignore) { // ignore } @@ -64,7 +70,7 @@ public void createNamespace(@NessieAPI NessieApiV1 nessie, @NessieDefaultBranch @Order(100) @Test public void createTables(@Spark SparkSession spark) { - spark.sql("CREATE TABLE nessie.db.from_spark (id int, val string)"); + spark.sql(format("CREATE TABLE %s (id int, val string)", sparkTableIdentifier("from_spark"))); } @Order(101) @@ -87,7 +93,7 @@ private static void tableAddRow(String table, Object... values) { @ParameterizedTest @MethodSource("tables") public void insertIntoSpark(String table, @Spark SparkSession spark) { - spark.sql(format("INSERT INTO nessie.db.%s select 100, \"from-spark\"", table)); + spark.sql(format("INSERT INTO %s select 100, \"from-spark\"", sparkTableIdentifier(table))); tableAddRow(table, 100, "from-spark"); } @@ -105,7 +111,10 @@ public void insertIntoFlink(String table, @Flink FlinkHelper flink) { @MethodSource("tables") public void selectFromSpark(String table, @Spark SparkSession spark) { assertThat( - spark.sql(format("SELECT id, val FROM nessie.db.%s", table)).collectAsList().stream() + spark + .sql(format("SELECT id, val FROM %s", sparkTableIdentifier(table))) + .collectAsList() + .stream() .map(r -> asList(r.get(0), r.get(1)))) .containsExactlyInAnyOrderElementsOf(tableRows.get(table)); } @@ -123,7 +132,7 @@ public void selectFromFlink(String table, @Flink FlinkHelper flink) { @ParameterizedTest @MethodSource("tables") public void insertIntoSpark2(String table, @Spark SparkSession spark) { - spark.sql(format("INSERT INTO nessie.db.%s select 101, \"from-spark\"", table)); + spark.sql(format("INSERT INTO %s select 101, \"from-spark\"", sparkTableIdentifier(table))); tableAddRow(table, 101, "from-spark"); } @@ -141,7 +150,10 @@ public void insertIntoFlink2(String table, @Flink FlinkHelper flink) { @MethodSource("tables") public void selectFromSpark2(String table, @Spark SparkSession spark) { assertThat( - spark.sql(format("SELECT id, val FROM nessie.db.%s", table)).collectAsList().stream() + spark + .sql(format("SELECT id, val FROM %s", sparkTableIdentifier(table))) + .collectAsList() + .stream() .map(r -> asList(r.get(0), r.get(1)))) .containsExactlyInAnyOrderElementsOf(tableRows.get(table)); } diff --git a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremio.java b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremio.java index 27f91b64..1c783c16 100644 --- a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremio.java +++ b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremio.java @@ -51,7 +51,7 @@ }) public class ITCrossEngineDremio { - private static final String NAMESPACE = "db"; + private static final String NAMESPACE = System.getProperty("nesqueit.namespace", "db"); private static final String DREMIO_TABLE = "from_dremio"; private static final String SPARK_TABLE = "from_spark"; private static final String FLINK_TABLE = "from_flink"; diff --git a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremioPermissions.java b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremioPermissions.java index 6a7fa2b2..2863b4dd 100644 --- a/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremioPermissions.java +++ b/nqeit-cross-engine/src/test/java/org/projectnessie/integtests/crossengine/ITCrossEngineDremioPermissions.java @@ -46,7 +46,7 @@ }) public class ITCrossEngineDremioPermissions { - private static final String NAMESPACE = "db"; + private static final String NAMESPACE = System.getProperty("nesqueit.namespace", "db"); private static final String READ_ONLY_TABLE = "dremio_readonly"; // pre-created on the outside private static final String EXPECTED_PERMISSION_ERROR = String.format( diff --git a/nqeit-iceberg-flink-extension/src/main/java/org/projectnessie/integtests/flink/IcebergFlinkExtension.java b/nqeit-iceberg-flink-extension/src/main/java/org/projectnessie/integtests/flink/IcebergFlinkExtension.java index 12fa6089..2972968e 100644 --- a/nqeit-iceberg-flink-extension/src/main/java/org/projectnessie/integtests/flink/IcebergFlinkExtension.java +++ b/nqeit-iceberg-flink-extension/src/main/java/org/projectnessie/integtests/flink/IcebergFlinkExtension.java @@ -102,7 +102,7 @@ public Object resolveParameter( private static class FlinkPerRun implements CloseableResource, FlinkHelper { private final FlinkHolder flinkHolder; private final String catalogName; - private final String databaseName = "db"; + private final String databaseName = System.getProperty("nesqueit.namespace", "db"); static FlinkPerRun get(ExtensionContext extensionContext, Flink flinkConfig) { return extensionContext