Merge branch 'main' into keep-glue-column-desc

Incorporate latest changes from main, including updated test code

.asf.yaml

@@ -35,6 +35,9 @@ github:
 
   protected_branches:
     main:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+
       required_linear_history: true
 
   features:

.baseline/checkstyle/checkstyle.xml

@@ -88,6 +88,14 @@
         <property name="format" value="Preconditions\.checkArgument\([^;]+%d[^;]+\);"/>
         <property name="message" value="Preconditions.checkArgument does not support %d. use %s instead"/>
     </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="^\s*import\s+static\s+(?!\Qorg.assertj.core.api.Assertions.\E).*\.assertThatThrownBy;"/>
+        <property name="message" value="assertThatThrownBy() should be statically imported from org.assertj.core.api.Assertions"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="^\s*import\s+\Qorg.assertj.core.api.Assertions;\E" />
+        <property name="message" value="org.assertj.core.api.Assertions should only be used with static imports" />
+    </module>
     <module name="SuppressionFilter"> <!-- baseline-gradle: README.md -->
         <property name="file" value="${config_loc}/checkstyle-suppressions.xml"/>
     </module>

.github/workflows/api-binary-compatibility.yml

@@ -27,6 +27,9 @@ on:
       - 'apache-iceberg-**'
   pull_request:
     paths:
+      - '.github/workflows/api-binary-compatibility.yml'
+      - '*.gradle'
+      - 'gradle*'
       - 'api/**'
       - '.palantir/revapi.yml'
 

.github/workflows/flink-ci.yml

@@ -71,8 +71,12 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11]
+        jvm: [8, 11, 17]
         flink: ['1.17', '1.18', '1.19']
+        exclude:
+          # Flink 1.17 does not support Java 17.
+          - jvm: 17
+            flink: '1.17'
     env:
       SPARK_LOCAL_IP: localhost
     steps:

.github/workflows/hive-ci.yml

@@ -94,14 +94,17 @@ jobs:
 
   hive3-tests:
     runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        jvm: [8, 11, 17]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-java@v4
       with:
         distribution: zulu
-        java-version: 8
+        java-version: ${{ matrix.jvm }}
     - uses: actions/cache@v4
       with:
         path: |

.github/workflows/java-ci.yml

@@ -96,7 +96,7 @@ jobs:
       with:
         distribution: zulu
         java-version: 8
-    - run: ./gradlew -DallVersions build -x test -x javadoc -x integrationTest
+    - run: ./gradlew -DallModules build -x test -x javadoc -x integrationTest
 
   build-javadoc:
     runs-on: ubuntu-22.04

.github/workflows/publish-snapshot.yml

@@ -40,5 +40,5 @@ jobs:
           java-version: 8
       - run: |
           ./gradlew printVersion
-          ./gradlew -DallVersions publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+          ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
           ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

.github/workflows/site-ci.yml

@@ -22,6 +22,7 @@ on:
     branches:
       - main
     paths:
+      - docs/**
       - site/**
   workflow_dispatch:
 jobs:

.github/workflows/spark-ci.yml

@@ -65,12 +65,13 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  spark-3x-scala-2-12-tests:
+  spark-tests:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11]
+        jvm: [8, 11, 17]
         spark: ['3.3', '3.4', '3.5']
+        scala: ['2.12', '2.13']
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -86,77 +87,16 @@ jobs:
             ~/.gradle/wrapper
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
-      - uses: jlumbroso/free-disk-space@v1.3.1
+      - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
         with:
           tool-cache: false
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.12:check -Pquick=true -x javadoc
-      - uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: test logs
-          path: |
-            **/build/testlogs
-
-  spark-3x-scala-2-13-tests:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        jvm: [8, 11]
-        spark: ['3.3','3.4','3.5']
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
-        with:
-          distribution: zulu
-          java-version: ${{ matrix.jvm }}
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: ${{ runner.os }}-gradle-
-      - uses: jlumbroso/free-disk-space@v1.3.1
-        with:
-          tool-cache: false
-      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_2.13:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_2.13:check -Pquick=true -x javadoc
-      - uses: actions/upload-artifact@v4
-        if: failure()
-        with:
-          name: test logs
-          path: |
-            **/build/testlogs
-
-  spark-3x-java-17-tests:
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        spark: ['3.3','3.4', '3.5']
-        scala-version: ['2.12', '2.13']
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-java@v4
-        with:
-          distribution: zulu
-          java-version: 17
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ~/.gradle/caches
-            ~/.gradle/wrapper
-          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
-          restore-keys: ${{ runner.os }}-gradle-
-      - uses: jlumbroso/free-disk-space@v1.3.1
-        with:
-          tool-cache: false
-      - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala-version }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala-version }}:check :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala-version }}:check :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala-version }}:check -Pquick=true -x javadoc
+      - run: |
+          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= \
+            :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check \
+            :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check \
+            :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check \
+            -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

.palantir/revapi.yml

@@ -1018,6 +1018,44 @@ acceptedBreaks:
       old: "method void org.apache.iceberg.PositionDeletesTable.PositionDeletesBatchScan::<init>(org.apache.iceberg.Table,\
         \ org.apache.iceberg.Schema, org.apache.iceberg.TableScanContext)"
       justification: "Removing deprecated code"
+  "1.5.0":
+    org.apache.iceberg:iceberg-api:
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.types.Types.NestedField"
+      new: "class org.apache.iceberg.types.Types.NestedField"
+      justification: "new Constructor added"
+    org.apache.iceberg:iceberg-core:
+    - code: "java.method.visibilityReduced"
+      old: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      new: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      justification: "Deprecations for 1.6.0 release"
+    - code: "java.element.noLongerDeprecated"
+      old: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      new: "method void org.apache.iceberg.encryption.PlaintextEncryptionManager::<init>()"
+      justification: "Constructor became private as part of deprecations cleanup for 1.6.0 release"
+    - code: "java.element.noLongerDeprecated"
+      old: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, java.lang.String, java.lang.String, java.lang.String,\
+        \ java.lang.String)"
+      new: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, org.apache.iceberg.rest.auth.AuthConfig)"
+      justification: "This is actually a removal of a deprecated constructor as part\
+        \ of deprecations for 1.6.0 release, apparently mis-reported by rev-api"
+    - code: "java.method.numberOfParametersChanged"
+      old: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, java.lang.String, java.lang.String, java.lang.String,\
+        \ java.lang.String)"
+      new: "method void org.apache.iceberg.rest.auth.OAuth2Util.AuthSession::<init>(java.util.Map<java.lang.String,\
+        \ java.lang.String>, org.apache.iceberg.rest.auth.AuthConfig)"
+      justification: "This is actually a removal of a deprecated constructor as part\
+        \ of deprecations for 1.6.0 release, apparently mis-reported by rev-api"
+    - code: "java.method.removed"
+      old: "method java.util.List<java.lang.String> org.apache.iceberg.ReachableFileUtil::statisticsFilesLocations(org.apache.iceberg.Table,\
+        \ java.util.function.Predicate<org.apache.iceberg.StatisticsFile>)"
+      justification: "Deprecations for 1.6.0 release"
+    - code: "java.method.removed"
+      old: "method org.apache.iceberg.DataFiles.Builder org.apache.iceberg.DataFiles.Builder::withEqualityFieldIds(java.util.List<java.lang.Integer>)"
+      justification: "Deprecations for 1.6.0 release"
   apache-iceberg-0.14.0:
     org.apache.iceberg:iceberg-api:
     - code: "java.class.defaultSerializationChanged"

LICENSE

@@ -227,7 +227,6 @@ This product includes code from Apache Parquet.
 
 * DynMethods.java
 * DynConstructors.java
-* AssertHelpers.java
 * IOUtil.java readFully and tests
 * ByteBufferInputStream implementations and tests
 

NOTICE

@@ -1,6 +1,6 @@
 
 Apache Iceberg
-Copyright 2017-2022 The Apache Software Foundation
+Copyright 2017-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

README.md

@@ -56,7 +56,7 @@ Iceberg is built using Gradle with Java 8, 11, or 17.
 * To invoke a build and run tests: `./gradlew build`
 * To skip tests: `./gradlew build -x test -x integrationTest`
 * To fix code style for default versions: `./gradlew spotlessApply`
-* To fix code style for all versions of Spark/Hive/Flink:`./gradlew spotlessApply -DallVersions`
+* To fix code style for all versions of Spark/Hive/Flink:`./gradlew spotlessApply -DallModules`
 
 Iceberg table support is organized in library modules:
 

aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSURI.java

@@ -19,6 +19,7 @@
 package org.apache.iceberg.aliyun.oss;
 
 import com.aliyun.oss.internal.OSSUtils;
+import java.util.Locale;
 import java.util.Set;
 import org.apache.iceberg.exceptions.ValidationException;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -64,7 +65,7 @@ public OSSURI(String location) {
 
     String scheme = schemeSplit[0];
     ValidationException.check(
-        VALID_SCHEMES.contains(scheme.toLowerCase()),
+        VALID_SCHEMES.contains(scheme.toLowerCase(Locale.ROOT)),
         "Invalid scheme: %s in OSS location %s",
         scheme,
         location);

aliyun/src/test/java/org/apache/iceberg/aliyun/TestAliyunClientFactories.java

@@ -18,31 +18,32 @@
  */
 package org.apache.iceberg.aliyun;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import com.aliyun.oss.OSS;
 import java.util.Map;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
-import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 public class TestAliyunClientFactories {
 
   @Test
   public void testLoadDefault() {
-    Assertions.assertThat(AliyunClientFactories.defaultFactory())
+    assertThat(AliyunClientFactories.defaultFactory())
         .as("Default client should be singleton")
         .isEqualTo(AliyunClientFactories.defaultFactory());
 
     AliyunClientFactory defaultFactory = AliyunClientFactories.from(Maps.newHashMap());
-    Assertions.assertThat(defaultFactory)
+    assertThat(defaultFactory)
         .as("Should load default when factory impl not configured")
         .isInstanceOf(AliyunClientFactories.DefaultAliyunClientFactory.class);
 
-    Assertions.assertThat(defaultFactory.aliyunProperties().accessKeyId())
+    assertThat(defaultFactory.aliyunProperties().accessKeyId())
         .as("Should have no Aliyun properties set")
         .isNull();
 
-    Assertions.assertThat(defaultFactory.aliyunProperties().securityToken())
+    assertThat(defaultFactory.aliyunProperties().securityToken())
         .as("Should have no security token")
         .isNull();
 
@@ -53,15 +54,15 @@ public void testLoadDefault() {
                 "key",
                 AliyunProperties.CLIENT_SECURITY_TOKEN,
                 "token"));
-    Assertions.assertThat(defaultFactoryWithConfig)
+    assertThat(defaultFactoryWithConfig)
         .as("Should load default when factory impl not configured")
         .isInstanceOf(AliyunClientFactories.DefaultAliyunClientFactory.class);
 
-    Assertions.assertThat(defaultFactoryWithConfig.aliyunProperties().accessKeyId())
+    assertThat(defaultFactoryWithConfig.aliyunProperties().accessKeyId())
         .as("Should have access key set")
         .isEqualTo("key");
 
-    Assertions.assertThat(defaultFactoryWithConfig.aliyunProperties().securityToken())
+    assertThat(defaultFactoryWithConfig.aliyunProperties().securityToken())
         .as("Should have security token set")
         .isEqualTo("token");
   }
@@ -70,7 +71,7 @@ public void testLoadDefault() {
   public void testLoadCustom() {
     Map<String, String> properties = Maps.newHashMap();
     properties.put(AliyunProperties.CLIENT_FACTORY, CustomFactory.class.getName());
-    Assertions.assertThat(AliyunClientFactories.from(properties))
+    assertThat(AliyunClientFactories.from(properties))
         .as("Should load custom class")
         .isInstanceOf(CustomFactory.class);
   }