[SPARK-36796][BUILD][CORE][SQL] Pass all sql/core and dependent modules UTs with JDK 17 except one case in postgreSQL/text.sql

core/src/main/scala/org/apache/spark/SparkContext.scala

@@ -399,6 +399,8 @@ class SparkContext(config: SparkConf) extends Logging {
     // This should be set as early as possible.
     SparkContext.fillMissingMagicCommitterConfsIfNeeded(_conf)
 
+    JavaModuleUtils.supplementJava17ModuleOptsIfNeeded(_conf)
+
     _driverLogger = DriverLogger(_conf)
 
     val resourcesFileOpt = conf.get(DRIVER_RESOURCES_FILE)

core/src/main/scala/org/apache/spark/util/JavaModuleUtils.scala

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.util
+
+import scala.collection.mutable
+
+import org.apache.commons.lang3.{JavaVersion, SystemUtils}
+
+import org.apache.spark.SparkConf
+import org.apache.spark.internal.config.{DRIVER_JAVA_OPTIONS, EXECUTOR_JAVA_OPTIONS, OptionalConfigEntry}
+
+object JavaModuleUtils {
+
+  private val javaModuleOptions = Set("--add-opens java.base/java.nio=ALL-UNNAMED",
+    "--add-opens java.base/sun.nio.ch=ALL-UNNAMED",
+    "--add-opens java.base/java.lang.invoke=ALL-UNNAMED",
+    "--add-opens java.base/java.nio=ALL-UNNAMED",
+    "--add-opens java.base/sun.nio.ch=ALL-UNNAMED",
+    "--add-opens java.base/java.lang.invoke=ALL-UNNAMED",
+    "--add-opens java.base/java.util=ALL-UNNAMED",
+    "--add-opens java.base/sun.security.action=ALL-UNNAMED",
+    "--add-opens java.base/sun.util.calendar=ALL-UNNAMED",
+    "--add-opens java.base/java.lang=ALL-UNNAMED",
+    "--add-opens java.base/sun.nio.cs=ALL-UNNAMED",
+    "--add-opens java.base/java.net=ALL-UNNAMED",
+    "--add-opens java.base/java.io=ALL-UNNAMED",
+    "--add-opens java.base/java.util.concurrent=ALL-UNNAMED",
+    "--add-exports java.base/jdk.internal.util.random=ALL-UNNAMED")
+
+  def isJavaVersionAtLeast17: Boolean = SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_17)
+
+  def defaultModuleOptions(): String = javaModuleOptions.mkString(" ", " ", " ")
+
+
+  def supplementJava17ModuleOptsIfNeeded(conf: SparkConf): Unit = {
+
+    def supplementModuleOpts(configEntry: OptionalConfigEntry[String]): Unit = {
+      val v = conf.get(configEntry) match {
+        case Some(opts) => JavaModuleUtils.defaultModuleOptions() + opts
+        case None => JavaModuleUtils.defaultModuleOptions()
+      }
+      conf.set(configEntry.key, v)
+    }
+
+    if (Utils.isTesting && isJavaVersionAtLeast17) {
+      supplementModuleOpts(DRIVER_JAVA_OPTIONS)
+      supplementModuleOpts(EXECUTOR_JAVA_OPTIONS)
+    }
+  }
+
+  def supplementJava17ModuleOptsIfNeeded(args: Seq[String]): Seq[String] = {
+
+    def supplementModuleOpts(buffer: mutable.Buffer[String], key: String): Unit = {
+      val index = buffer.indexWhere(_.startsWith(s"$key}="))
+      if (index != -1) {
+        buffer.update(index, buffer(index) + JavaModuleUtils.defaultModuleOptions())
+      } else {
+        buffer.prependAll(Seq("--conf", s"$key=${JavaModuleUtils.defaultModuleOptions()}"))
+      }
+    }
+
+    if (Utils.isTesting && isJavaVersionAtLeast17) {
+      val buffer = args.toBuffer
+      supplementModuleOpts(buffer, DRIVER_JAVA_OPTIONS.key)
+      supplementModuleOpts(buffer, EXECUTOR_JAVA_OPTIONS.key)
+      buffer.toSeq
+    } else args
+  }
+}

core/src/test/scala/org/apache/spark/deploy/SparkSubmitTestUtils.scala

@@ -30,7 +30,7 @@ import org.scalatest.time.SpanSugar._
 
 import org.apache.spark.ProcessTestUtils.ProcessOutputCapturer
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{JavaModuleUtils, Utils}
 
 trait SparkSubmitTestUtils extends SparkFunSuite with TimeLimits {
 
@@ -53,7 +53,9 @@ trait SparkSubmitTestUtils extends SparkFunSuite with TimeLimits {
     } else {
       new File(new File(sparkHome, "bin"), "spark-submit")
     }
-    val commands = Seq(sparkSubmit.getCanonicalPath) ++ args
+
+    val commands = Seq(sparkSubmit.getCanonicalPath) ++
+      JavaModuleUtils.supplementJava17ModuleOptsIfNeeded(args)
     val commandLine = commands.mkString("'", "' '", "'")
 
     val builder = new ProcessBuilder(commands: _*).directory(new File(sparkHome))

core/src/test/scala/org/apache/spark/launcher/LauncherBackendSuite.scala

@@ -27,7 +27,7 @@ import org.scalatest.matchers.should.Matchers._
 
 import org.apache.spark._
 import org.apache.spark.internal.config.UI.UI_ENABLED
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{JavaModuleUtils, Utils}
 
 class LauncherBackendSuite extends SparkFunSuite with Matchers {
 
@@ -46,15 +46,22 @@ class LauncherBackendSuite extends SparkFunSuite with Matchers {
   private def testWithMaster(master: String): Unit = {
     val env = new java.util.HashMap[String, String]()
     env.put("SPARK_PRINT_LAUNCH_COMMAND", "1")
-    val handle = new SparkLauncher(env)
+    val launcher = new SparkLauncher(env)
       .setSparkHome(sys.props("spark.test.home"))
       .setConf(SparkLauncher.DRIVER_EXTRA_CLASSPATH, System.getProperty("java.class.path"))
       .setConf(UI_ENABLED.key, "false")
       .setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, s"-Dtest.appender=console")
       .setMaster(master)
       .setAppResource(SparkLauncher.NO_RESOURCE)
       .setMainClass(TestApp.getClass.getName().stripSuffix("$"))
-      .startApplication()
+
+    if(JavaModuleUtils.isJavaVersionAtLeast17) {
+      launcher.setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS,
+        s"${JavaModuleUtils.defaultModuleOptions()} -Dtest.appender=console")
+    } else {
+      launcher.setConf(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, s"-Dtest.appender=console")
+    }
+    val handle = launcher.startApplication()
 
     try {
       eventually(timeout(30.seconds), interval(100.milliseconds)) {

pom.xml

@@ -208,6 +208,8 @@
 
     <test.java.home>${java.home}</test.java.home>
 
+    <extraJavaTestArgs></extraJavaTestArgs>
+
     <!-- Some UI tests require Chrome and Chrome driver installed so those tests are disabled by default. -->
     <test.default.exclude.tags>org.apache.spark.tags.ChromeUITest</test.default.exclude.tags>
     <test.exclude.tags></test.exclude.tags>
@@ -2686,7 +2688,7 @@
               <include>**/*Suite.java</include>
             </includes>
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
             <environmentVariables>
               <!--
                 Setting SPARK_DIST_CLASSPATH is a simple way to make sure any child processes
@@ -2737,7 +2739,7 @@
             <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
             <junitxml>.</junitxml>
             <filereports>SparkTestSuite.txt</filereports>
-            <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+            <argLine>-ea -Xmx5g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
             <stderr/>
             <environmentVariables>
               <!--
@@ -3431,6 +3433,26 @@
 
     </profile>
 
+    <profile>
+      <id>jdk-17</id>
+      <properties>
+        <extraJavaTestArgs>
+          --add-opens java.base/java.nio=ALL-UNNAMED
+          --add-opens java.base/sun.nio.ch=ALL-UNNAMED
+          --add-opens java.base/java.lang.invoke=ALL-UNNAMED
+          --add-opens java.base/java.util=ALL-UNNAMED
+          --add-opens java.base/sun.security.action=ALL-UNNAMED
+          --add-opens java.base/sun.util.calendar=ALL-UNNAMED
+          --add-opens java.base/java.lang=ALL-UNNAMED
+          --add-opens java.base/java.io=ALL-UNNAMED
+          --add-opens java.base/sun.nio.cs=ALL-UNNAMED
+          --add-opens java.base/java.net=ALL-UNNAMED
+          --add-opens java.base/java.util.concurrent=ALL-UNNAMED
+          --add-exports java.base/jdk.internal.util.random=ALL-UNNAMED
+        </extraJavaTestArgs>
+      </properties>
+    </profile>
+
     <!--
      This is a profile to enable the use of the ASF snapshot and staging repositories
      during a build. It is useful when testing against nightly or RC releases of dependencies.

sql/catalyst/pom.xml

@@ -158,7 +158,7 @@
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest-maven-plugin</artifactId>
         <configuration>
-          <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+          <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
         </configuration>
       </plugin>
       <plugin>

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala

@@ -280,6 +280,7 @@ private object DateTimeFormatterHelper {
   // 2.4, the SimpleDateFormat uses Monday as the first day of week.
   final val weekBasedLetters = Set('Y', 'W', 'w', 'u', 'e', 'c')
   final val unsupportedLetters = Set('A', 'n', 'N', 'p')
+  final val unknownPatternLetters: Set[Char] = Set('B')
   // The quarter fields will also be parsed strangely, e.g. when the pattern contains `yMd` and can
   // be directly resolved then the `q` do check for whether the month is valid, but if the date
   // fields is incomplete, e.g. `yM`, the checking will be bypassed.
@@ -322,6 +323,9 @@ private object DateTimeFormatterHelper {
             (isParsing && unsupportedLettersForParsing.contains(c))) {
             throw new IllegalArgumentException(s"Illegal pattern character: $c")
           }
+          for (c <- patternPart if unknownPatternLetters.contains(c)) {
+            throw new IllegalArgumentException(s"Unknown pattern letter: $c")
+          }
           for (style <- unsupportedPatternLengths if patternPart.contains(style)) {
             throw new IllegalArgumentException(s"Too many pattern letters: ${style.head}")
           }

sql/core/pom.xml

@@ -215,7 +215,7 @@
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest-maven-plugin</artifactId>
         <configuration>
-          <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+          <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} ${extraJavaTestArgs} -Dio.netty.tryReflectionSetAccessible=true</argLine>
         </configuration>
       </plugin>
       <plugin>