Bump to Iceberg 1.4.0 (#93)

* Bump to Iceberg 1.4.0 and Spark 3.5 * Update Dockerfile * WIP * Fix the properties * Fix last property * Bump to RC2 * Update Dockerfile * Update spark/Dockerfile Co-authored-by: issac-lim <sungkooc.lim@gmail.com> * Update Dockerfile * Oops * Update Dockerfile --------- Co-authored-by: issac-lim <sungkooc.lim@gmail.com>
tabular-io · Oct 10, 2023 · 5291ffe · 5291ffe
1 parent f571e1f
commit 5291ffe
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 10 deletions.
diff --git a/spark/Dockerfile b/spark/Dockerfile
@@ -50,7 +50,7 @@ WORKDIR ${SPARK_HOME}
 
 ENV SPARK_VERSION=3.4.1
 ENV SPARK_MAJOR_VERSION=3.4
-ENV ICEBERG_VERSION=1.3.1
+ENV ICEBERG_VERSION=1.4.0
 
 # Download spark
 RUN mkdir -p ${SPARK_HOME} \
@@ -61,12 +61,8 @@ RUN mkdir -p ${SPARK_HOME} \
 # Download iceberg spark runtime
 RUN curl https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-spark-runtime-${SPARK_MAJOR_VERSION}_2.12-${ICEBERG_VERSION}.jar
 
-# Download Java AWS SDK
-ENV AWSSDK_VERSION=2.20.18
-RUN curl https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWSSDK_VERSION}/bundle-${AWSSDK_VERSION}.jar -Lo /opt/spark/jars/bundle-${AWSSDK_VERSION}.jar
-
-# Download URL connection client required for S3FileIO
-RUN curl https://repo1.maven.org/maven2/software/amazon/awssdk/url-connection-client/${AWSSDK_VERSION}/url-connection-client-${AWSSDK_VERSION}.jar -Lo /opt/spark/jars/url-connection-client-${AWSSDK_VERSION}.jar
+# Download AWS bundle
+RUN curl -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
 # Install AWS CLI
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \

diff --git a/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API.ipynb b/spark/notebooks/Iceberg - An Introduction to the Iceberg Java API.ipynb
@@ -35,15 +35,15 @@
     "import org.apache.hadoop.conf.Configuration;\n",
     "import org.apache.iceberg.CatalogProperties;\n",
     "import org.apache.iceberg.rest.RESTCatalog;\n",
-    "import org.apache.iceberg.aws.AwsProperties;\n",
+    "import org.apache.iceberg.aws.s3.S3FileIOProperties;\n",
     "\n",
     "Map<String, String> properties = new HashMap<>();\n",
     "\n",
     "properties.put(CatalogProperties.CATALOG_IMPL, \"org.apache.iceberg.rest.RESTCatalog\");\n",
     "properties.put(CatalogProperties.URI, \"http://rest:8181\");\n",
     "properties.put(CatalogProperties.WAREHOUSE_LOCATION, \"s3a://warehouse/wh\");\n",
     "properties.put(CatalogProperties.FILE_IO_IMPL, \"org.apache.iceberg.aws.s3.S3FileIO\");\n",
-    "properties.put(AwsProperties.S3FILEIO_ENDPOINT, \"http://minio:9000\");\n",
+    "properties.put(S3FileIOProperties.ENDPOINT, \"http://minio:9000\");\n",
     "\n",
     "RESTCatalog catalog = new RESTCatalog();\n",
     "Configuration conf = new Configuration();\n",
@@ -202,7 +202,7 @@
     "properties.put(CatalogProperties.URI, \"http://rest:8181\");\n",
     "properties.put(CatalogProperties.WAREHOUSE_LOCATION, \"s3a://warehouse/wh/\");\n",
     "properties.put(CatalogProperties.FILE_IO_IMPL, \"org.apache.iceberg.aws.s3.S3FileIO\");\n",
-    "properties.put(AwsProperties.S3FILEIO_ENDPOINT, \"http://minio:9000\");\n",
+    "properties.put(S3FileIOProperties.ENDPOINT, \"http://minio:9000\");\n",
     "\n",
     "RESTCatalog catalog = new RESTCatalog();\n",
     "Configuration conf = new Configuration();\n",