From 3b30c969206489bce336859f3f3135c49515f6db Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 12:33:36 +0200 Subject: [PATCH 1/8] #755 - spark 3.5.0 version --- .github/workflows/ci.yml | 2 +- README.md | 33 ++++----- build.sbt | 71 +++++++++++++++++-- .../scala/frameless/functions/package.scala | 2 +- .../apache/spark/sql/reflection/package.scala | 9 ++- 5 files changed, 92 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d97e41a6..cbc8fad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,7 +162,7 @@ jobs: - name: Submit Dependencies uses: scalacenter/sbt-dependency-submission@v2 with: - modules-ignore: root-spark33_2.13 root-spark33_2.12 docs_2.13 docs_2.12 root-spark34_2.13 root-spark34_2.12 root-spark32_2.13 root-spark32_2.12 + modules-ignore: root-spark33_2.13 root-spark33_2.12 docs_2.13 docs_2.12 root-spark34_2.13 root-spark34_2.12 root-spark35_2.13 root-spark35_2.12 root-spark32_2.13 root-spark32_2.12 configs-ignore: test scala-tool scala-doc-tool test-internal site: diff --git a/README.md b/README.md index cd5ba25b..a28c1612 100644 --- a/README.md +++ b/README.md @@ -25,22 +25,23 @@ associated channels (e.g. GitHub, Discord) to be a safe and friendly environment The compatible versions of [Spark](http://spark.apache.org/) and [cats](https://github.com/typelevel/cats) are as follows: -| Frameless | Spark | Cats | Cats-Effect | Scala | -|-----------|-----------------------|----------|-------------|-------------| -| 0.14.1 | 3.4.0 / 3.3.0 / 3.2.2 | 2.x | 3.x | 2.12 / 2.13 | -| 0.14.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 | -| 0.13.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 | -| 0.12.0 | 3.2.1 / 3.1.3 / 3.0.3 | 2.x | 3.x | 2.12 / 2.13 | -| 0.11.1 | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 | -| 0.11.0* | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 | -| 0.10.1 | 3.1.0 | 2.x | 2.x | 2.12 | -| 0.9.0 | 3.0.0 | 1.x | 1.x | 2.12 | -| 0.8.0 | 2.4.0 | 1.x | 1.x | 2.11 / 2.12 | -| 0.7.0 | 2.3.1 | 1.x | 1.x | 2.11 | -| 0.6.1 | 2.3.0 | 1.x | 0.8 | 2.11 | -| 0.5.2 | 2.2.1 | 1.x | 0.8 | 2.11 | -| 0.4.1 | 2.2.0 | 1.x | 0.8 | 2.11 | -| 0.4.0 | 2.2.0 | 1.0.0-IF | 0.4 | 2.11 | +| Frameless | Spark | Cats | Cats-Effect | Scala | +|-----------|-------------------------------|----------|-------------|-------------| +| 0.14.2 | 3.5.0 / 3.4.0 / 3.3.0 / 3.2.2 | 2.x | 3.x | 2.12 / 2.13 | +| 0.14.1 | 3.4.0 / 3.3.0 / 3.2.2 | 2.x | 3.x | 2.12 / 2.13 | +| 0.14.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 | +| 0.13.0 | 3.3.0 / 3.2.2 / 3.1.3 | 2.x | 3.x | 2.12 / 2.13 | +| 0.12.0 | 3.2.1 / 3.1.3 / 3.0.3 | 2.x | 3.x | 2.12 / 2.13 | +| 0.11.1 | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 | +| 0.11.0* | 3.2.0 / 3.1.2 / 3.0.1 | 2.x | 2.x | 2.12 / 2.13 | +| 0.10.1 | 3.1.0 | 2.x | 2.x | 2.12 | +| 0.9.0 | 3.0.0 | 1.x | 1.x | 2.12 | +| 0.8.0 | 2.4.0 | 1.x | 1.x | 2.11 / 2.12 | +| 0.7.0 | 2.3.1 | 1.x | 1.x | 2.11 | +| 0.6.1 | 2.3.0 | 1.x | 0.8 | 2.11 | +| 0.5.2 | 2.2.1 | 1.x | 0.8 | 2.11 | +| 0.4.1 | 2.2.0 | 1.x | 0.8 | 2.11 | +| 0.4.0 | 2.2.0 | 1.0.0-IF | 0.4 | 2.11 | _\* 0.11.0 has broken Spark 3.1.2 and 3.0.1 artifacts published._ diff --git a/build.sbt b/build.sbt index 24c66880..d36743ac 100644 --- a/build.sbt +++ b/build.sbt @@ -1,4 +1,5 @@ -val sparkVersion = "3.4.1" +val sparkVersion = "3.5.0" +val spark34Version = "3.4.1" val spark33Version = "3.3.3" val spark32Version = "3.2.4" val catsCoreVersion = "2.10.0" @@ -24,12 +25,23 @@ lazy val root = project .in(file(".")) .enablePlugins(NoPublishPlugin) .settings(crossScalaVersions := Nil) - .aggregate(`root-spark34`, `root-spark33`, `root-spark32`, docs) + .aggregate(`root-spark35`, `root-spark34`, `root-spark33`, `root-spark32`, docs) + +lazy val `root-spark35` = project + .in(file(".spark35")) + .enablePlugins(NoPublishPlugin) + .aggregate(core, cats, dataset, refined, ml) lazy val `root-spark34` = project .in(file(".spark34")) .enablePlugins(NoPublishPlugin) - .aggregate(core, cats, dataset, refined, ml) + .aggregate( + core, + `cats-spark34`, + `dataset-spark34`, + `refined-spark34`, + `ml-spark34` + ) lazy val `root-spark33` = project .in(file(".spark33")) @@ -61,6 +73,15 @@ lazy val cats = project .settings(catsSettings) .dependsOn(dataset % "test->test;compile->compile;provided->provided") +lazy val `cats-spark34` = project + .settings(name := "frameless-cats-spark34") + .settings(sourceDirectory := (cats / sourceDirectory).value) + .settings(catsSettings) + .settings(spark34Settings) + .dependsOn( + `dataset-spark34` % "test->test;compile->compile;provided->provided" + ) + lazy val `cats-spark33` = project .settings(name := "frameless-cats-spark33") .settings(sourceDirectory := (cats / sourceDirectory).value) @@ -91,6 +112,19 @@ lazy val dataset = project .settings(sparkDependencies(sparkVersion)) .dependsOn(core % "test->test;compile->compile") +lazy val `dataset-spark34` = project + .settings(name := "frameless-dataset-spark34") + .settings( + Compile / unmanagedSourceDirectories += baseDirectory.value / "src" / "main" / "spark-3.4+" + ) + .settings( + Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "spark-3.3+" + ) + .settings(datasetSettings) + .settings(sparkDependencies(spark34Version)) + .settings(spark34Settings) + .dependsOn(core % "test->test;compile->compile") + lazy val `dataset-spark33` = project .settings(name := "frameless-dataset-spark33") .settings(sourceDirectory := (dataset / sourceDirectory).value) @@ -124,6 +158,15 @@ lazy val refined = project .settings(refinedSettings) .dependsOn(dataset % "test->test;compile->compile;provided->provided") +lazy val `refined-spark34` = project + .settings(name := "frameless-refined-spark34") + .settings(sourceDirectory := (refined / sourceDirectory).value) + .settings(refinedSettings) + .settings(spark34Settings) + .dependsOn( + `dataset-spark34` % "test->test;compile->compile;provided->provided" + ) + lazy val `refined-spark33` = project .settings(name := "frameless-refined-spark33") .settings(sourceDirectory := (refined / sourceDirectory).value) @@ -151,6 +194,17 @@ lazy val ml = project dataset % "test->test;compile->compile;provided->provided" ) +lazy val `ml-spark34` = project + .settings(name := "frameless-ml-spark34") + .settings(sourceDirectory := (ml / sourceDirectory).value) + .settings(mlSettings) + .settings(sparkMlDependencies(spark34Version)) + .settings(spark34Settings) + .dependsOn( + core % "test->test;compile->compile", + `dataset-spark33` % "test->test;compile->compile;provided->provided" + ) + lazy val `ml-spark33` = project .settings(name := "frameless-ml-spark33") .settings(sourceDirectory := (ml / sourceDirectory).value) @@ -326,9 +380,18 @@ lazy val spark32Settings = Seq( tlVersionIntroduced := Map("2.12" -> "0.13.0", "2.13" -> "0.13.0") ) +lazy val spark34Settings = Seq[Setting[_]]( + tlVersionIntroduced := Map("2.12" -> "0.14.1", "2.13" -> "0.14.1"), + mimaPreviousArtifacts := Set( + organization.value %% moduleName.value + .split("-") + .dropRight(1) + .mkString("-") % "0.14.1" + ) +) + lazy val spark33Settings = Seq[Setting[_]]( tlVersionIntroduced := Map("2.12" -> "0.13.0", "2.13" -> "0.13.0"), - // frameless-dataset-spark33 was originally frameless-dataset mimaPreviousArtifacts := Set( organization.value %% moduleName.value .split("-") diff --git a/dataset/src/main/scala/frameless/functions/package.scala b/dataset/src/main/scala/frameless/functions/package.scala index 21b5bc99..c76cda1a 100644 --- a/dataset/src/main/scala/frameless/functions/package.scala +++ b/dataset/src/main/scala/frameless/functions/package.scala @@ -7,7 +7,7 @@ import shapeless.labelled.FieldType import shapeless.ops.hlist.IsHCons import shapeless.ops.record.{Keys, Values} -import org.apache.spark.sql.catalyst.ScalaReflection +import org.apache.spark.sql.{reflection => ScalaReflection} import org.apache.spark.sql.catalyst.expressions.Literal package object functions extends Udf with UnaryFunctions { diff --git a/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala b/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala index d03c49d6..6d9ca574 100644 --- a/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala +++ b/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala @@ -3,7 +3,6 @@ package org.apache.spark.sql import org.apache.spark.sql.catalyst.ScalaReflection.{cleanUpReflectionObjects, getClassFromType, localTypeOf} import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, CalendarIntervalType, DataType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, NullType, ObjectType, ShortType} import org.apache.spark.unsafe.types.CalendarInterval -import org.apache.spark.sql.catalyst.ScalaReflection.{isNativeType => oisNativeType} /** * Copy of spark's pre 3.4 reflection based encoding @@ -11,9 +10,13 @@ import org.apache.spark.sql.catalyst.ScalaReflection.{isNativeType => oisNativeT package object reflection { /** - * forwards to keep the rest of the code looking the same + * copy of pre 3.5.0 isNativeType, https://issues.apache.org/jira/browse/SPARK-44343 removed it */ - def isNativeType(dt: DataType): Boolean = oisNativeType(dt) + def isNativeType(dt: DataType): Boolean = dt match { + case NullType | BooleanType | ByteType | ShortType | IntegerType | LongType | + FloatType | DoubleType | BinaryType | CalendarIntervalType => true + case _ => false + } private object ScalaSubtypeLock From 3faa8ab13666ebcb7b54f04f71d4c09fa050a7e4 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 12:41:34 +0200 Subject: [PATCH 2/8] #755 - spark 3.5.0 version - re-formatting only --- .../scala/frameless/functions/package.scala | 110 ++++++++++++------ .../apache/spark/sql/reflection/package.scala | 54 ++++++--- 2 files changed, 110 insertions(+), 54 deletions(-) diff --git a/dataset/src/main/scala/frameless/functions/package.scala b/dataset/src/main/scala/frameless/functions/package.scala index c76cda1a..1a57101e 100644 --- a/dataset/src/main/scala/frameless/functions/package.scala +++ b/dataset/src/main/scala/frameless/functions/package.scala @@ -5,9 +5,9 @@ import scala.reflect.ClassTag import shapeless._ import shapeless.labelled.FieldType import shapeless.ops.hlist.IsHCons -import shapeless.ops.record.{Keys, Values} +import shapeless.ops.record.{ Keys, Values } -import org.apache.spark.sql.{reflection => ScalaReflection} +import org.apache.spark.sql.{ reflection => ScalaReflection } import org.apache.spark.sql.catalyst.expressions.Literal package object functions extends Udf with UnaryFunctions { @@ -15,26 +15,40 @@ package object functions extends Udf with UnaryFunctions { object aggregate extends AggregateFunctions object nonAggregate extends NonAggregateFunctions - /** Creates a [[frameless.TypedAggregate]] of literal value. If A is to be encoded using an Injection make - * sure the injection instance is in scope. - * - * apache/spark - */ - def litAggr[A, T](value: A)(implicit i0: TypedEncoder[A], i1: Refute[IsValueClass[A]]): TypedAggregate[T, A] = + /** + * Creates a [[frameless.TypedAggregate]] of literal value. If A is to be encoded using an Injection make + * sure the injection instance is in scope. + * + * apache/spark + */ + def litAggr[A, T]( + value: A + )(implicit + i0: TypedEncoder[A], + i1: Refute[IsValueClass[A]] + ): TypedAggregate[T, A] = new TypedAggregate[T, A](lit(value).expr) - /** Creates a [[frameless.TypedColumn]] of literal value. If A is to be encoded using an Injection make - * sure the injection instance is in scope. - * - * apache/spark - * - * @tparam A the literal value type - * @tparam T the row type - */ - def lit[A, T](value: A)( - implicit encoder: TypedEncoder[A]): TypedColumn[T, A] = { - - if (ScalaReflection.isNativeType(encoder.jvmRepr) && encoder.catalystRepr == encoder.jvmRepr) { + /** + * Creates a [[frameless.TypedColumn]] of literal value. If A is to be encoded using an Injection make + * sure the injection instance is in scope. + * + * apache/spark + * + * @tparam A the literal value type + * @tparam T the row type + */ + def lit[A, T]( + value: A + )(implicit + encoder: TypedEncoder[A] + ): TypedColumn[T, A] = { + + if ( + ScalaReflection.isNativeType( + encoder.jvmRepr + ) && encoder.catalystRepr == encoder.jvmRepr + ) { val expr = Literal(value, encoder.catalystRepr) new TypedColumn(expr) @@ -52,14 +66,24 @@ package object functions extends Udf with UnaryFunctions { } } - /** Creates a [[frameless.TypedColumn]] of literal value - * for a Value class `A`. - * - * @tparam A the value class - * @tparam T the row type - */ - def litValue[A : IsValueClass, T, G <: ::[_, HNil], H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], K <: Symbol, V, KS <: ::[_ <: Symbol, HNil], VS <: HList](value: A)( - implicit + /** + * Creates a [[frameless.TypedColumn]] of literal value + * for a Value class `A`. + * + * @tparam A the value class + * @tparam T the row type + */ + def litValue[ + A: IsValueClass, + T, + G <: ::[_, HNil], + H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], + K <: Symbol, + V, + KS <: ::[_ <: Symbol, HNil], + VS <: HList + ](value: A + )(implicit i0: LabelledGeneric.Aux[A, G], i1: DropUnitValues.Aux[G, H], i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil], @@ -69,7 +93,7 @@ package object functions extends Udf with UnaryFunctions { i6: IsHCons.Aux[VS, V, HNil], i7: TypedEncoder[V], i8: ClassTag[A] - ): TypedColumn[T, A] = { + ): TypedColumn[T, A] = { val expr = { val field: H = i1(i0.to(value)) val v: V = i6.head(i4(field)) @@ -90,14 +114,24 @@ package object functions extends Udf with UnaryFunctions { ) } - /** Creates a [[frameless.TypedColumn]] of literal value - * for an optional Value class `A`. - * - * @tparam A the value class - * @tparam T the row type - */ - def litValue[A : IsValueClass, T, G <: ::[_, HNil], H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], K <: Symbol, V, KS <: ::[_ <: Symbol, HNil], VS <: HList](value: Option[A])( - implicit + /** + * Creates a [[frameless.TypedColumn]] of literal value + * for an optional Value class `A`. + * + * @tparam A the value class + * @tparam T the row type + */ + def litValue[ + A: IsValueClass, + T, + G <: ::[_, HNil], + H <: ::[_ <: FieldType[_ <: Symbol, _], HNil], + K <: Symbol, + V, + KS <: ::[_ <: Symbol, HNil], + VS <: HList + ](value: Option[A] + )(implicit i0: LabelledGeneric.Aux[A, G], i1: DropUnitValues.Aux[G, H], i2: IsHCons.Aux[H, _ <: FieldType[K, V], HNil], @@ -107,7 +141,7 @@ package object functions extends Udf with UnaryFunctions { i6: IsHCons.Aux[VS, V, HNil], i7: TypedEncoder[V], i8: ClassTag[A] - ): TypedColumn[T, Option[A]] = { + ): TypedColumn[T, Option[A]] = { val expr = value match { case Some(some) => { val field: H = i1(i0.to(some)) diff --git a/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala b/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala index 6d9ca574..07090a8d 100644 --- a/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala +++ b/dataset/src/main/scala/org/apache/spark/sql/reflection/package.scala @@ -1,7 +1,26 @@ package org.apache.spark.sql -import org.apache.spark.sql.catalyst.ScalaReflection.{cleanUpReflectionObjects, getClassFromType, localTypeOf} -import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, CalendarIntervalType, DataType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, LongType, NullType, ObjectType, ShortType} +import org.apache.spark.sql.catalyst.ScalaReflection.{ + cleanUpReflectionObjects, + getClassFromType, + localTypeOf +} +import org.apache.spark.sql.types.{ + BinaryType, + BooleanType, + ByteType, + CalendarIntervalType, + DataType, + Decimal, + DecimalType, + DoubleType, + FloatType, + IntegerType, + LongType, + NullType, + ObjectType, + ShortType +} import org.apache.spark.unsafe.types.CalendarInterval /** @@ -13,14 +32,16 @@ package object reflection { * copy of pre 3.5.0 isNativeType, https://issues.apache.org/jira/browse/SPARK-44343 removed it */ def isNativeType(dt: DataType): Boolean = dt match { - case NullType | BooleanType | ByteType | ShortType | IntegerType | LongType | - FloatType | DoubleType | BinaryType | CalendarIntervalType => true + case NullType | BooleanType | ByteType | ShortType | IntegerType | + LongType | FloatType | DoubleType | BinaryType | CalendarIntervalType => + true case _ => false } private object ScalaSubtypeLock - val universe: scala.reflect.runtime.universe.type = scala.reflect.runtime.universe + val universe: scala.reflect.runtime.universe.type = + scala.reflect.runtime.universe import universe._ @@ -32,7 +53,7 @@ package object reflection { * Unlike `schemaFor`, this function doesn't do any massaging of types into the Spark SQL type * system. As a result, ObjectType will be returned for things like boxed Integers */ - def dataTypeFor[T : TypeTag]: DataType = dataTypeFor(localTypeOf[T]) + def dataTypeFor[T: TypeTag]: DataType = dataTypeFor(localTypeOf[T]) /** * Synchronize to prevent concurrent usage of `<:<` operator. @@ -49,18 +70,19 @@ package object reflection { private def dataTypeFor(tpe: `Type`): DataType = cleanUpReflectionObjects { tpe.dealias match { - case t if isSubtype(t, definitions.NullTpe) => NullType - case t if isSubtype(t, definitions.IntTpe) => IntegerType - case t if isSubtype(t, definitions.LongTpe) => LongType - case t if isSubtype(t, definitions.DoubleTpe) => DoubleType - case t if isSubtype(t, definitions.FloatTpe) => FloatType - case t if isSubtype(t, definitions.ShortTpe) => ShortType - case t if isSubtype(t, definitions.ByteTpe) => ByteType - case t if isSubtype(t, definitions.BooleanTpe) => BooleanType + case t if isSubtype(t, definitions.NullTpe) => NullType + case t if isSubtype(t, definitions.IntTpe) => IntegerType + case t if isSubtype(t, definitions.LongTpe) => LongType + case t if isSubtype(t, definitions.DoubleTpe) => DoubleType + case t if isSubtype(t, definitions.FloatTpe) => FloatType + case t if isSubtype(t, definitions.ShortTpe) => ShortType + case t if isSubtype(t, definitions.ByteTpe) => ByteType + case t if isSubtype(t, definitions.BooleanTpe) => BooleanType case t if isSubtype(t, localTypeOf[Array[Byte]]) => BinaryType - case t if isSubtype(t, localTypeOf[CalendarInterval]) => CalendarIntervalType + case t if isSubtype(t, localTypeOf[CalendarInterval]) => + CalendarIntervalType case t if isSubtype(t, localTypeOf[Decimal]) => DecimalType.SYSTEM_DEFAULT - case _ => + case _ => /* original Spark code checked for scala.Array vs ObjectType, this (and associated code) isn't needed due to TypedEncoders arrayEncoder */ val clazz = getClassFromType(tpe) From dc84310c48b914f57aceeee2c4b72403e7a80869 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 12:45:49 +0200 Subject: [PATCH 3/8] #755 - spark 3.5.0 version - re-formatting only --- build.sbt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d36743ac..7114748d 100644 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,13 @@ lazy val root = project .in(file(".")) .enablePlugins(NoPublishPlugin) .settings(crossScalaVersions := Nil) - .aggregate(`root-spark35`, `root-spark34`, `root-spark33`, `root-spark32`, docs) + .aggregate( + `root-spark35`, + `root-spark34`, + `root-spark33`, + `root-spark32`, + docs + ) lazy val `root-spark35` = project .in(file(".spark35")) From 89ec302d1c7d193918fbf5ada35601719253e18c Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 13:08:30 +0200 Subject: [PATCH 4/8] #755 - spark 3.5.0 version - build root 34 fix --- build.sbt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 7114748d..a8bc9a58 100644 --- a/build.sbt +++ b/build.sbt @@ -120,11 +120,12 @@ lazy val dataset = project lazy val `dataset-spark34` = project .settings(name := "frameless-dataset-spark34") + .settings(sourceDirectory := (dataset / sourceDirectory).value) .settings( - Compile / unmanagedSourceDirectories += baseDirectory.value / "src" / "main" / "spark-3.4+" + Compile / unmanagedSourceDirectories += (dataset / baseDirectory).value / "src" / "main" / "spark-3.4+" ) .settings( - Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "spark-3.3+" + Test / unmanagedSourceDirectories += (dataset / baseDirectory).value / "src" / "test" / "spark-3.3+" ) .settings(datasetSettings) .settings(sparkDependencies(spark34Version)) From 1881df1a7ecd27f8d241db666bdfb014e26eaad6 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 13:27:14 +0200 Subject: [PATCH 5/8] #755 - spark 3.5.0 version - build root 34 fix - gc --- .jmvopts | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .jmvopts diff --git a/.jmvopts b/.jmvopts new file mode 100644 index 00000000..6374ca28 --- /dev/null +++ b/.jmvopts @@ -0,0 +1,2 @@ +-Xms1g +-Xmx4g \ No newline at end of file From 85ccf93d8d30a4df0640547a0dd86bde34a1bd70 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Mon, 25 Sep 2023 13:39:13 +0200 Subject: [PATCH 6/8] #755 - spark 3.5.0 version - build root 34 fix - gc oops --- .jmvopts => .jvmopts | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .jmvopts => .jvmopts (100%) diff --git a/.jmvopts b/.jvmopts similarity index 100% rename from .jmvopts rename to .jvmopts From dfd3ebc00bf21e4c0c6ce13ce2ad6488553eec60 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Thu, 28 Sep 2023 07:34:19 +0200 Subject: [PATCH 7/8] #755 - remove jvmopts, add ci param, per cchantep --- .github/workflows/ci.yml | 1 + .jvmopts | 2 -- build.sbt | 2 ++ 3 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 .jvmopts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbc8fad0..6054d067 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SBT_OPTS: '-Xms1g -Xmx4g' SPARK_LOCAL_IP: localhost diff --git a/.jvmopts b/.jvmopts deleted file mode 100644 index 6374ca28..00000000 --- a/.jvmopts +++ /dev/null @@ -1,2 +0,0 @@ --Xms1g --Xmx4g \ No newline at end of file diff --git a/build.sbt b/build.sbt index a8bc9a58..f157a7fc 100644 --- a/build.sbt +++ b/build.sbt @@ -469,3 +469,5 @@ ThisBuild / githubWorkflowBuildMatrixAdditions += "project" -> roots ThisBuild / githubWorkflowBuildMatrixExclusions ++= roots.init.map { project => MatrixExclude(Map("scala" -> "2.13", "project" -> project)) } + +ThisBuild / githubWorkflowEnv += "SBT_OPTS" -> "-Xms1g -Xmx4g" From dc728a9a2f02cc4fc410e21dab33eab0769a7834 Mon Sep 17 00:00:00 2001 From: Chris Twiner Date: Thu, 28 Sep 2023 08:07:11 +0200 Subject: [PATCH 8/8] #755 - tlbaseVersion tag was bumped by cchantep --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index f157a7fc..b0e6bd06 100644 --- a/build.sbt +++ b/build.sbt @@ -16,7 +16,7 @@ val nakedFSVersion = "0.1.0" val Scala212 = "2.12.18" val Scala213 = "2.13.12" -ThisBuild / tlBaseVersion := "0.14" +ThisBuild / tlBaseVersion := "0.15" ThisBuild / crossScalaVersions := Seq(Scala213, Scala212) ThisBuild / scalaVersion := Scala212