diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ab356fd2cde..967a3cacfdf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -72,6 +72,12 @@ jobs: spark-archive: '-Pscala-2.13' exclude-tags: '' comment: 'normal' + - java: 25 + python: '3.11' + spark: '4.2' + spark-archive: '-Pscala-2.13' + exclude-tags: '' + comment: 'normal' - java: 8 python: '3.9' spark: '3.5' @@ -96,6 +102,12 @@ jobs: spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.1.1 -Dspark.archive.name=spark-4.1.1-bin-hadoop3.tgz' exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-4.1-binary' + - java: 17 + python: '3.11' + spark: '3.5' + spark-archive: '-Pscala-2.13 -Dspark.archive.mirror=https://www.apache.org/dyn/closer.lua/spark/spark-4.2.0-preview3 -Dspark.archive.name=spark-4.2.0-preview3-bin-hadoop3.tgz' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' + comment: 'verify-on-spark-4.2-binary' env: SPARK_LOCAL_IP: localhost steps: @@ -214,6 +226,16 @@ jobs: spark-compile: "3.5" spark-runtime: "4.0" comment: "normal" + - java: 21 + scala: "2.13" + spark-compile: "3.5" + spark-runtime: "4.1" + comment: "normal" + - java: 25 + scala: "2.13" + spark-compile: "3.5" + spark-runtime: "4.2" + comment: "normal" env: SPARK_LOCAL_IP: localhost TEST_MODULES: "extensions/spark/kyuubi-spark-connector-hive,\ diff --git a/bin/load-kyuubi-env.sh b/bin/load-kyuubi-env.sh index aa7cee023eb..79f1144698f 100755 --- a/bin/load-kyuubi-env.sh +++ b/bin/load-kyuubi-env.sh @@ -68,7 +68,6 @@ if [[ -z ${JAVA_HOME} ]]; then fi KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions" -KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS -Dio.netty.tryReflectionSetAccessible=true" KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/java.lang=ALL-UNNAMED" KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" @@ -85,10 +84,12 @@ KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/sun.security.action=AL KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/sun.security.tools.keytool=ALL-UNNAMED" KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/sun.security.x509=ALL-UNNAMED" KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --add-opens=java.base/sun.util.calendar=ALL-UNNAMED" +KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS -Djdk.reflect.useDirectMethodHandle=false" +KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --enable-native-access=ALL-UNNAMED" +KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS --sun-misc-unsafe-memory-access=allow" export KYUUBI_JAVA_OPTS="$KYUUBI_JAVA_OPTS" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions" -KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS -Dio.netty.tryReflectionSetAccessible=true" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/java.lang=ALL-UNNAMED" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/java.lang.reflect=ALL-UNNAMED" @@ -105,6 +106,9 @@ KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/sun.security.a KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/sun.security.tools.keytool=ALL-UNNAMED" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/sun.security.x509=ALL-UNNAMED" KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --add-opens=java.base/sun.util.calendar=ALL-UNNAMED" +KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS -Djdk.reflect.useDirectMethodHandle=false" +KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --sun-misc-unsafe-memory-access=allow" +KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS --enable-native-access=ALL-UNNAMED" export KYUUBI_CTL_JAVA_OPTS="$KYUUBI_CTL_JAVA_OPTS" export KYUUBI_SCALA_VERSION="${KYUUBI_SCALA_VERSION:-"2.12"}" diff --git a/dev/dependencyList b/dev/dependencyList index f77abbbe84b..414074152f0 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -47,8 +47,8 @@ grpc-stub/1.76.2//grpc-stub-1.76.2.jar grpc-util/1.76.2//grpc-util-1.76.2.jar gson/2.11.0//gson-2.11.0.jar guava/33.4.8-jre//guava-33.4.8-jre.jar -hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar -hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar +hadoop-client-api/3.4.3//hadoop-client-api-3.4.3.jar +hadoop-client-runtime/3.4.3//hadoop-client-runtime-3.4.3.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala index 64faa2bc15e..b8cbf09e21e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala @@ -17,7 +17,10 @@ package org.apache.kyuubi.plugin.spark.authz +import java.net.URI + import scala.reflect.io.File +import scala.util.Try import org.apache.spark.sql.{DataFrame, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.TableIdentifier @@ -35,6 +38,7 @@ import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ import org.apache.kyuubi.util.AssertionUtils._ +import org.apache.kyuubi.util.reflect.DynConstructors abstract class PrivilegesBuilderSuite extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll with BeforeAndAfterEach { @@ -1451,6 +1455,33 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(accessType1 == AccessType.WRITE) } + private val emptyCatalogStorageFormat = Try { + DynConstructors.builder() + .impl( + classOf[CatalogStorageFormat], + classOf[Option[URI]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Boolean], + classOf[Map[String, String]]) + .build[CatalogStorageFormat]() + .newInstance(None, None, None, None, None, Boolean.box(false), Map.empty) + }.recover { case _: Exception => + DynConstructors.builder() + .impl( + classOf[CatalogStorageFormat], + classOf[Option[URI]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Boolean], + classOf[Map[String, String]]) + .build[CatalogStorageFormat]() + .newInstance(None, None, None, None, Boolean.box(false), Map.empty) + }.get + test("InsertIntoDataSourceCommand") { val tableName = "InsertIntoDataSourceTable" withTable(tableName) { _ => @@ -1461,13 +1492,7 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val newTable = CatalogTable( identifier = TableIdentifier(tableName, None), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat( - locationUri = None, - inputFormat = None, - outputFormat = None, - serde = None, - compressed = false, - properties = Map.empty), + storage = emptyCatalogStorageFormat, schema = schema, provider = Some(classOf[SimpleInsertSource].getName)) diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala index 86be8bb0d8f..acae656a87f 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala @@ -17,7 +17,10 @@ package org.apache.kyuubi.plugin.lineage.helper +import java.net.URI + import scala.reflect.io.File +import scala.util.Try import org.apache.spark.SparkConf import org.apache.spark.kyuubi.lineage.{LineageConf, SparkContextHelper} @@ -29,6 +32,7 @@ import org.apache.spark.sql.types.{IntegerType, StringType, StructType} import org.apache.kyuubi.KyuubiFunSuite import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.util.reflect.DynConstructors abstract class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite with SparkListenerExtensionTest { @@ -323,6 +327,33 @@ abstract class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite } } + private val emptyCatalogStorageFormat = Try { + DynConstructors.builder() + .impl( + classOf[CatalogStorageFormat], + classOf[Option[URI]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Boolean], + classOf[Map[String, String]]) + .build[CatalogStorageFormat]() + .newInstance(None, None, None, None, None, Boolean.box(false), Map.empty) + }.recover { case _: Exception => + DynConstructors.builder() + .impl( + classOf[CatalogStorageFormat], + classOf[Option[URI]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Option[String]], + classOf[Boolean], + classOf[Map[String, String]]) + .build[CatalogStorageFormat]() + .newInstance(None, None, None, None, Boolean.box(false), Map.empty) + }.get + test("columns lineage extract - InsertIntoDataSourceCommand") { val tableName = "insertintodatasourcecommand" withTable(tableName) { _ => @@ -332,13 +363,7 @@ abstract class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite val newTable = CatalogTable( identifier = TableIdentifier(tableName, None), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat( - locationUri = None, - inputFormat = None, - outputFormat = None, - serde = None, - compressed = false, - properties = Map.empty), + storage = emptyCatalogStorageFormat, schema = schema, provider = Some(classOf[SimpleInsertSource].getName)) spark.sessionState.catalog.createTable(newTable, ignoreIfExists = false) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala index ad056a0643d..203fa9950f0 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/ui/EngineTabSuite.scala @@ -20,10 +20,14 @@ package org.apache.spark.ui import org.apache.http.client.methods.HttpGet import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils +import org.apache.spark.SPARK_VERSION +import org.scalactic.source.Position +import org.scalatest.Tag import org.apache.kyuubi.engine.spark.WithSparkSQLEngine import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.session.SessionHandle +import org.apache.kyuubi.util.SemanticVersion class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { override def withKyuubiConf: Map[String, String] = Map( @@ -31,6 +35,16 @@ class EngineTabSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { "spark.ui.port" -> "0", "spark.sql.redaction.string.regex" -> "(?i)url|access|secret|password") + override protected def test(testName: String, testTags: Tag*)(testBody: => Any)(implicit + pos: Position): Unit = { + // SPARK-47086 (4.2.0) Upgrade to Jetty 12 and Servlet 6.0 + if (SemanticVersion(SPARK_VERSION) >= "4.2") { + ignore(s"$testName (excluded)")(testBody) + } else { + super.test(testName, testTags: _*)(testBody) + } + } + override protected def beforeEach(): Unit = { super.beforeEach() startSparkEngine() diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerGroupSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerGroupSuite.scala index 173b4d2932a..936c3a7d4ca 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerGroupSuite.scala +++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerGroupSuite.scala @@ -68,6 +68,7 @@ class KyuubiOperationPerGroupSuite extends WithKyuubiServer with SparkQueryTests } test("kyuubi defined function - system_user/session_user") { + assume(SPARK_ENGINE_RUNTIME_VERSION < "4.2") withSessionConf(Map("hive.server2.proxy.user" -> "user1"))(Map.empty)(Map.empty) { withJdbcStatement() { statement => val res = statement.executeQuery("select system_user() as c1, session_user() as c2") diff --git a/pom.xml b/pom.xml index 412d1060b99..b54f4a49656 100644 --- a/pom.xml +++ b/pom.xml @@ -150,7 +150,7 @@ 1.76.2 33.4.8-jre 1.0.3 - 3.3.6 + 3.4.3 4.0.3 1.1.1 3.1.3 @@ -301,7 +301,8 @@ --add-opens=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false - -Dio.netty.tryReflectionSetAccessible=true + --enable-native-access=ALL-UNNAMED + --sun-misc-unsafe-memory-access=allow -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 @@ -1968,6 +1969,21 @@ + + java-25 + + 25 + + + + 2.43.0 + true + 1.22.0 + + + scala-2.12 @@ -2083,6 +2099,32 @@ + + spark-4.2 + + + extensions/spark/kyuubi-spark-connector-hive + + + 17 + 17 + 4.2.0-preview3 + 4.0 + 4.13.1 + 4.0.0 + delta-spark_${scala.binary.version} + + hudi-spark3.5-bundle_${scala.binary.version} + + paimon-common + org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.HudiTest + spark-${spark.version}-bin-hadoop3.tgz + + + spark-master