Ability to build for all Scala versions

Allows building for all Spark/Scala version combinations. Introduces new system properties to control the behavior:
* `allScalaVersions=true` lets Spark build for all Scala versions
* the previous property `scalaVersion` is now a version list via the `scalaVersions` property, provides backwards compatibility
* the previous property `defaultScalaVersion` is now a version list via the `defaultScalaVersions` property, provides backwards compatibility

The defaults, via `gradle.properties` do not change.

Also...
* Unify processing of the xyzVersions everywhere
* Fix compilation bug in Flink 1.19 jmh code (was not built before)
* Update release jobs/script
* Remove "hack" for `:iceberg-bom`
* Simplify and fix related usages of the build system properties

.github/workflows/delta-conversion-ci.yml

@@ -88,7 +88,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersions=2.12 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:
@@ -117,7 +117,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersions=2.13 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

.github/workflows/java-ci.yml

@@ -101,7 +101,7 @@ jobs:
       with:
         distribution: zulu
         java-version: ${{ matrix.jvm }}
-    - run: ./gradlew -DallModules build -x test -x javadoc -x integrationTest
+    - run: ./gradlew -DallModules=true -DallScalaVersions=true build -x test -x javadoc -x integrationTest
 
   build-javadoc:
     runs-on: ubuntu-22.04

.github/workflows/publish-snapshot.yml

@@ -40,5 +40,4 @@ jobs:
           java-version: 8
       - run: |
           ./gradlew printVersion
-          ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
-          ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+          ./gradlew -DallModules=true -DallScalaVersions=true publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

.github/workflows/spark-ci.yml

@@ -94,7 +94,7 @@ jobs:
           tool-cache: false
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
       - run: |
-          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= \
+          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersions=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= \
             :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check \
             :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check \
             :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check \

README.md

@@ -56,7 +56,7 @@ Iceberg is built using Gradle with Java 8, 11, or 17.
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`
 * To fix code style for default versions: `./gradlew spotlessApply`
-* To fix code style for all versions of Spark/Hive/Flink:`./gradlew spotlessApply -DallModules`
+* To fix code style for all versions of Spark/Hive/Flink:`./gradlew spotlessApply -DallModules=true`
 
 Iceberg table support is organized in library modules:
 

build.gradle

@@ -45,10 +45,6 @@ buildscript {
   }
 }
 
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
-String sparkVersionsString = System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")
-List<String> sparkVersions = sparkVersionsString != null && !sparkVersionsString.isEmpty() ? sparkVersionsString.split(",") : []
-
 try {
   // apply these plugins in a try-catch block so that we can handle cases without .git directory
   apply plugin: 'com.palantir.git-version'
@@ -578,6 +574,10 @@ project(':iceberg-delta-lake') {
     integrationRuntime.extendsFrom testRuntimeOnly
   }
 
+  List<String> sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",").toList().findAll { !it.empty }
+  List<String> scalaVersions = System.getProperty("scalaVersions").split(",").findAll { !it.empty }
+  String scalaVersion = scalaVersions[0]
+
   dependencies {
     implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow')
     api project(':iceberg-api')
@@ -1037,41 +1037,6 @@ apply from: 'tasks.gradle'
 
 project(':iceberg-bom') {
   apply plugin: 'java-platform'
-
-  dependencies {
-    constraints {
-      // The Iceberg-Build builds for only one Scala version at a time, so the BOM would also
-      // only contain artifacts for that single Scala version. The following code ensures that
-      // the BOM references the artifacts for all Scala versions.
-      def sparkScalaPattern = ~"(.*)-([0-9][.][0-9]+)_([0-9][.][0-9]+)"
-      def sparkScalaVersions = [
-        "3.3": ["2.12", "2.13"],
-        "3.4": ["2.12", "2.13"],
-        "3.5": ["2.12", "2.13"],
-      ]
-      rootProject.allprojects.forEach {
-        // Do not include ':iceberg-spark', the bom itself and the root project.
-        if (it.name != 'iceberg-bom' && it != rootProject && it.childProjects.isEmpty()) {
-          if (it.name.startsWith("iceberg-spark-")) {
-            def sparkScalaMatcher = sparkScalaPattern.matcher(it.name)
-            if (!sparkScalaMatcher.find()) {
-              throw new GradleException("Expected a Spark/Scala version combination in Gradle project name ${it.name}")
-            }
-            def prjName = sparkScalaMatcher.group(1)
-            def sparkVer = sparkScalaMatcher.group(2)
-            for (def scalaVer in sparkScalaVersions[sparkVer]) {
-              add("api", "${it.group}:$prjName-${sparkVer}_$scalaVer:${it.version}")
-            }
-          } else {
-            add("api", project(it.path))
-          }
-        }
-      }
-    }
-  }
-
-  // Needed to get the "faked" Scala artifacts into the bom
-  javaPlatform { allowDependencies() }
 }
 
 def cleanSite = tasks.register("cleanSite", Delete) {

dev/stage-binaries.sh

@@ -18,16 +18,4 @@
 # under the License.
 #
 
-SCALA_VERSION=2.12
-FLINK_VERSIONS=1.17,1.18,1.19
-SPARK_VERSIONS=3.3,3.4,3.5
-HIVE_VERSIONS=2,3
-
-./gradlew -Prelease -DscalaVersion=$SCALA_VERSION -DflinkVersions=$FLINK_VERSIONS -DsparkVersions=$SPARK_VERSIONS -DhiveVersions=$HIVE_VERSIONS publishApachePublicationToMavenRepository
-
-# Also publish Scala 2.13 Artifacts for versions that support it.
-# Flink does not yet support 2.13 (and is largely dropping a user-facing dependency on Scala). Hive doesn't need a Scala specification.
-./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.3 :iceberg-spark:iceberg-spark-3.3_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.3_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.3_2.13:publishApachePublicationToMavenRepository
-./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.4 :iceberg-spark:iceberg-spark-3.4_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.4_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.4_2.13:publishApachePublicationToMavenRepository
-./gradlew -Prelease -DscalaVersion=2.13 -DsparkVersions=3.5 :iceberg-spark:iceberg-spark-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-extensions-3.5_2.13:publishApachePublicationToMavenRepository :iceberg-spark:iceberg-spark-runtime-3.5_2.13:publishApachePublicationToMavenRepository
-
+./gradlew -Prelease -DallModules=true -DallScalaVersions=true publishApachePublicationToMavenRepository

flink/v1.17/build.gradle

@@ -18,7 +18,8 @@
  */
 
 String flinkMajorVersion = '1.17'
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+List<String> scalaVersions = System.getProperty("scalaVersions").split(",").findAll { !it.empty }
+String scalaVersion = scalaVersions[0]
 
 project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
 

flink/v1.18/build.gradle

@@ -18,7 +18,8 @@
  */
 
 String flinkMajorVersion = '1.18'
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+List<String> scalaVersions = System.getProperty("scalaVersions").split(",").findAll { !it.empty }
+String scalaVersion = scalaVersions[0]
 
 project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
 

flink/v1.19/build.gradle

@@ -18,7 +18,8 @@
  */
 
 String flinkMajorVersion = '1.19'
-String scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+List<String> scalaVersions = System.getProperty("scalaVersions").split(",").findAll { !it.empty }
+String scalaVersion = scalaVersions[0]
 
 project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") {
 

flink/v1.19/flink/src/jmh/java/org/apache/iceberg/flink/sink/shuffle/MapRangePartitionerBenchmark.java

@@ -82,10 +82,9 @@ public void setupBenchmark() {
           mapStatistics.put(sortKey, weight);
         });
 
-    MapDataStatistics dataStatistics = new MapDataStatistics(mapStatistics);
     this.partitioner =
         new MapRangePartitioner(
-            SCHEMA, SortOrder.builderFor(SCHEMA).asc("id").build(), dataStatistics, 2);
+            SCHEMA, SortOrder.builderFor(SCHEMA).asc("id").build(), mapStatistics, 2);
 
     List<Integer> keys = Lists.newArrayList(weights.keySet().iterator());
     long[] weightsCDF = new long[keys.size()];

gradle.properties

@@ -16,13 +16,43 @@
 jmhOutputPath=build/reports/jmh/human-readable-output.txt
 jmhJsonOutputPath=build/reports/jmh/results.json
 jmhIncludeRegex=.*
+
+# The following settings define which versions of Flink, Hive, Spark
+# and Scala are known to Iceberg.
+#
+# The 'allModules' system property instructs the Gradle build to build
+# all Flink, Spark and Hive versions.
+systemProp.allModules=false
+#
+# The 'all*Versions' flag defines whether all known versions should
+# be included in the build, overriding the *Versions system properties.
+#
+
+# Flink
 systemProp.defaultFlinkVersions=1.19
 systemProp.knownFlinkVersions=1.17,1.18,1.19
+systemProp.allFlinkVersions=false
+#
+# Hive
 systemProp.defaultHiveVersions=2
 systemProp.knownHiveVersions=2,3
+systemProp.allHiveVersions=false
+#
+# Spark
 systemProp.defaultSparkVersions=3.5
 systemProp.knownSparkVersions=3.3,3.4,3.5
-systemProp.defaultScalaVersion=2.12
+systemProp.allSparkVersions=false
+systemProp.sparkScalaVersions_3.3=2.12,2.13
+systemProp.sparkScalaVersions_3.4=2.12,2.13
+systemProp.sparkScalaVersions_3.5=2.12,2.13
+#
+# Scala
+systemProp.defaultScalaVersions=2.12
 systemProp.knownScalaVersions=2.12,2.13
+# Within IntelliJ, the 'allScalaVersions' system property has no effect,
+# because IJ cannot use the same set of sources against different Scala versions.
+systemProp.allScalaVersions=false
+#
+
 org.gradle.parallel=true
 org.gradle.jvmargs=-Xmx1024m

hive-runtime/build.gradle

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-def hiveVersions = (System.getProperty("hiveVersions") != null ? System.getProperty("hiveVersions") : System.getProperty("defaultHiveVersions")).split(",")
+def hiveVersions = (System.getProperty("hiveVersions") != null ? System.getProperty("hiveVersions") : System.getProperty("defaultHiveVersions")).split(",").findAll { !it.empty }
 
 project(':iceberg-hive-runtime') {
   apply plugin: 'io.github.goooler.shadow'

jmh.gradle

@@ -21,36 +21,19 @@ if (jdkVersion != '8' && jdkVersion != '11' && jdkVersion != '17') {
   throw new GradleException("The JMH benchmarks must be run with JDK 8 or JDK 11 or JDK 17")
 }
 
-def flinkVersions = (System.getProperty("flinkVersions") != null ? System.getProperty("flinkVersions") : System.getProperty("defaultFlinkVersions")).split(",")
-def sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",")
-def scalaVersion = System.getProperty("scalaVersion") != null ? System.getProperty("scalaVersion") : System.getProperty("defaultScalaVersion")
+List<String> flinkVersions = System.getProperty("flinkVersions").split(",").toList().findAll { !it.empty }
+List<String> sparkVersions = System.getProperty("sparkVersions").split(",").toList().findAll { !it.empty }
+List<String> scalaVersions = System.getProperty("scalaVersions").split(",").findAll { !it.empty }
+String scalaVersion = scalaVersions[0]
 def jmhProjects = [project(":iceberg-core"), project(":iceberg-data")]
 
-if (flinkVersions.contains("1.16")) {
-  jmhProjects.add(project(":iceberg-flink:iceberg-flink-1.16"))
+for (ver in flinkVersions) {
+  jmhProjects.add(project(":iceberg-flink:iceberg-flink-${ver}"))
 }
 
-if (flinkVersions.contains("1.17")) {
-  jmhProjects.add(project(":iceberg-flink:iceberg-flink-1.17"))
-}
-
-if (flinkVersions.contains("1.18")) {
-  jmhProjects.add(project(":iceberg-flink:iceberg-flink-1.18"))
-}
-
-if (sparkVersions.contains("3.3")) {
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.3_${scalaVersion}"))
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.3_${scalaVersion}"))
-}
-
-if (sparkVersions.contains("3.4")) {
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.4_${scalaVersion}"))
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.4_${scalaVersion}"))
-}
-
-if (sparkVersions.contains("3.5")) {
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.5_${scalaVersion}"))
-  jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-3.5_${scalaVersion}"))
+for (ver in sparkVersions) {
+  jmhProjects.add(project(":iceberg-spark:iceberg-spark-${ver}_${scalaVersion}"))
+  jmhProjects.add(project(":iceberg-spark:iceberg-spark-extensions-${ver}_${scalaVersion}"))
 }
 
 configure(jmhProjects) {