diff --git a/api_validation/pom.xml b/api_validation/pom.xml index 66bb5e40d2a..892bd83ef24 100644 --- a/api_validation/pom.xml +++ b/api_validation/pom.xml @@ -77,53 +77,11 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - diff --git a/datagen/pom.xml b/datagen/pom.xml index 1a540a303f5..2414152b6f3 100644 --- a/datagen/pom.xml +++ b/datagen/pom.xml @@ -71,54 +71,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -131,54 +89,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark330cdh.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark330cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -196,93 +112,10 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-annotation_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-unsafe_${scala.binary.version} - ${spark.version} - provided - - - org.scala-lang - scala-reflect - ${scala.version} - provided - - - com.esotericsoftware.kryo - kryo-shaded-db - ${spark.version} - provided - - - org.apache.arrow - arrow-format - ${spark.version} - compile - - - org.apache.arrow - arrow-memory - ${spark.version} - compile - - - org.apache.arrow - arrow-vector - ${spark.version} - compile - - - com.fasterxml.jackson.core - jackson-annotations - ${spark.version} - provided - - - org.apache.hive - hive-exec - ${spark.version} - provided - - - org.apache.hive - hive-serde - ${spark.version} - provided - - - org.apache.commons - commons-io - ${spark.version} - provided - - - org.apache.hadoop - hadoop-client - ${hadoop.client.version} + com.nvidia + rapids-4-spark-db-bom + ${project.version} + pom provided diff --git a/dist/pom.xml b/dist/pom.xml index 42f07c6f53d..e40de9e7d33 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -164,144 +164,6 @@ - - release321cdh - - - buildver - 321cdh - - - - ${buildver} - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - - - - - cloudera-repo - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - - release330cdh - - - buildver - 330cdh - - - - ${buildver} - - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark330cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark330cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - - - - - cloudera-repo - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - individual diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml index abe5bf52a4c..2bd20772065 100644 --- a/integration_tests/pom.xml +++ b/integration_tests/pom.xml @@ -86,54 +86,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -146,54 +104,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark330cdh.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark330cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -211,111 +127,10 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-annotation_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-unsafe_${scala.binary.version} - ${spark.version} - provided - - - org.scala-lang - scala-reflect - ${scala.version} - provided - - - com.esotericsoftware.kryo - kryo-shaded-db - ${spark.version} - provided - - - org.apache.arrow - arrow-format - ${spark.version} - compile - - - org.apache.arrow - arrow-memory - ${spark.version} - compile - - - org.apache.arrow - arrow-vector - ${spark.version} - compile - - - com.fasterxml.jackson.core - jackson-annotations - ${spark.version} - provided - - - org.apache.hive - hive-exec - ${spark.version} - provided - - - org.apache.hive - hive-serde - ${spark.version} - provided - - - org.json4s - json4s-ast_${scala.binary.version} - ${spark.version} - provided - - - org.json4s - json4s-core_${scala.binary.version} - ${spark.version} - provided - - - org.json4s - json4s-jackson_${scala.binary.version} - ${spark.version} - provided - - - org.apache.commons - commons-io - ${spark.version} - provided - - - org.apache.hadoop - hadoop-client - ${hadoop.client.version} + com.nvidia + rapids-4-spark-db-bom + ${project.version} + pom provided diff --git a/pom.xml b/pom.xml index f83bd8ac076..02b59f0cd1a 100644 --- a/pom.xml +++ b/pom.xml @@ -194,9 +194,22 @@ cloudera-repo https://repository.cloudera.com/artifactory/cloudera-repos/ + + true + + + true + + + + cloudera-repo + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + shim-deps/cloudera delta-lake/delta-20x @@ -281,6 +294,7 @@ 1.12.0 + shim-deps/databricks delta-lake/delta-spark321db @@ -432,7 +446,14 @@ + + + cloudera-repo + https://repository.cloudera.com/artifactory/cloudera-repos/ + + + shim-deps/cloudera delta-lake/delta-21x delta-lake/delta-22x @@ -465,6 +486,7 @@ ${spark330.iceberg.version} + shim-deps/databricks delta-lake/delta-spark330db @@ -496,6 +518,7 @@ ${spark330.iceberg.version} + shim-deps/databricks delta-lake/delta-spark332db diff --git a/shim-deps/cloudera/pom.xml b/shim-deps/cloudera/pom.xml new file mode 100644 index 00000000000..9b075aa6cdb --- /dev/null +++ b/shim-deps/cloudera/pom.xml @@ -0,0 +1,90 @@ + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent + 23.10.0-SNAPSHOT + ../../pom.xml + + rapids-4-spark-cdh-bom + pom + CDH Shim Dependencies + 23.10.0-SNAPSHOT + + + + org.apache.spark + spark-sql_${scala.binary.version} + compile + + + org.apache.arrow + arrow-vector + + + + + org.apache.spark + spark-hive_${scala.binary.version} + compile + + + + org.apache.spark + spark-core_${scala.binary.version} + + + org.apache.arrow + arrow-vector + + + + + org.apache.arrow + arrow-vector + ${arrow.cdh.version} + compile + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + io.netty + netty-common + + + + + org.apache.spark + spark-avro_${scala.binary.version} + compile + + + \ No newline at end of file diff --git a/shim-deps/databricks/pom.xml b/shim-deps/databricks/pom.xml new file mode 100644 index 00000000000..80cfcf7c3c1 --- /dev/null +++ b/shim-deps/databricks/pom.xml @@ -0,0 +1,266 @@ + + + + 4.0.0 + + + com.nvidia + rapids-4-spark-parent + 23.10.0-SNAPSHOT + ../../pom.xml + + rapids-4-spark-db-bom + pom + Databricks Shim Dependencies + 23.10.0-SNAPSHOT + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-avro_${scala.binary.version} + ${spark.version} + compile + + + org.apache.avro + avro-mapred + ${spark.version} + compile + + + org.apache.avro + avro + ${spark.version} + compile + + + org.apache.hive + hive-exec + ${spark.version} + compile + + + org.apache.hive + hive-metastore-client-patched + ${spark.version} + compile + + + org.apache.hive + hive-serde + ${spark.version} + compile + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + compile + + + com.fasterxml.jackson.core + jackson-core + ${spark.version} + compile + + + com.fasterxml.jackson.core + jackson-annotations + ${spark.version} + compile + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-annotation_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-network-common_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-network-shuffle_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-launcher_${scala.binary.version} + ${spark.version} + compile + + + org.apache.spark + spark-unsafe_${scala.binary.version} + ${spark.version} + compile + + + org.json4s + json4s-ast_${scala.binary.version} + ${spark.version} + compile + + + org.json4s + json4s-core_${scala.binary.version} + ${spark.version} + compile + + + org.json4s + json4s-jackson_${scala.binary.version} + ${spark.version} + compile + + + org.apache.commons + commons-io + ${spark.version} + compile + + + org.scala-lang + scala-reflect + ${scala.version} + compile + + + org.apache.commons + commons-lang3 + ${spark.version} + compile + + + com.esotericsoftware.kryo + kryo-shaded-db + ${spark.version} + compile + + + org.apache.parquet + parquet-hadoop + ${spark.version} + compile + + + org.apache.parquet + parquet-common + ${spark.version} + compile + + + org.apache.parquet + parquet-column + ${spark.version} + compile + + + org.apache.parquet + parquet-format + ${spark.version} + compile + + + org.apache.arrow + arrow-format + ${spark.version} + compile + + + org.apache.arrow + arrow-memory + ${spark.version} + compile + + + org.apache.arrow + arrow-vector + ${spark.version} + compile + + + org.apache.hadoop + hadoop-client + ${hadoop.client.version} + compile + + + org.apache.orc + orc-core + ${spark.version} + compile + + + org.apache.orc + orc-shims + ${spark.version} + compile + + + org.apache.orc + orc-mapreduce + ${spark.version} + compile + + + org.apache.hive + hive-storage-api + ${spark.version} + compile + + + com.google.protobuf + protobuf-java + ${spark.version} + compile + + + org.javaassist + javaassist + ${spark.version} + compile + + + \ No newline at end of file diff --git a/shuffle-plugin/pom.xml b/shuffle-plugin/pom.xml index bcf8c8e3599..8ff9c2e8686 100644 --- a/shuffle-plugin/pom.xml +++ b/shuffle-plugin/pom.xml @@ -82,54 +82,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -147,15 +105,10 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} + com.nvidia + rapids-4-spark-db-bom + ${project.version} + pom provided diff --git a/sql-plugin/pom.xml b/sql-plugin/pom.xml index b7b0c94ea4a..11cbdca82fa 100644 --- a/sql-plugin/pom.xml +++ b/sql-plugin/pom.xml @@ -113,53 +113,11 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - @@ -173,53 +131,11 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark330cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark330cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - @@ -272,209 +188,10 @@ - org.apache.spark - spark-sql_${scala.binary.version} - - - org.apache.spark - spark-avro_${scala.binary.version} - ${spark.version} - provided - - - org.apache.avro - avro-mapred - ${spark.version} - provided - - - org.apache.avro - avro - ${spark.version} - provided - - - org.apache.hive - hive-exec - ${spark.version} - provided - - - org.apache.hive - hive-metastore-client-patched - ${spark.version} - provided - - - org.apache.hive - hive-serde - ${spark.version} - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - - - com.fasterxml.jackson.core - jackson-core - ${spark.version} - provided - - - com.fasterxml.jackson.core - jackson-annotations - ${spark.version} - provided - - - org.apache.spark - spark-core_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-annotation_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-catalyst_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-network-common_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-network-shuffle_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-launcher_${scala.binary.version} - ${spark.version} - provided - - - org.apache.spark - spark-unsafe_${scala.binary.version} - ${spark.version} - provided - - - org.json4s - json4s-ast_${scala.binary.version} - ${spark.version} - provided - - - org.json4s - json4s-core_${scala.binary.version} - ${spark.version} - provided - - - org.apache.commons - commons-io - ${spark.version} - provided - - - org.scala-lang - scala-reflect - ${scala.version} - provided - - - org.apache.commons - commons-lang3 - ${spark.version} - provided - - - com.esotericsoftware.kryo - kryo-shaded-db - ${spark.version} - provided - - - org.apache.parquet - parquet-hadoop - ${spark.version} - provided - - - org.apache.parquet - parquet-common - ${spark.version} - provided - - - org.apache.parquet - parquet-column - ${spark.version} - provided - - - org.apache.parquet - parquet-format - ${spark.version} - provided - - - org.apache.arrow - arrow-memory - ${spark.version} - provided - - - org.apache.arrow - arrow-vector - ${spark.version} - provided - - - org.apache.hadoop - hadoop-client - ${hadoop.client.version} - provided - - - org.apache.orc - orc-core - ${spark.version} - provided - - - org.apache.orc - orc-shims - ${spark.version} - provided - - - org.apache.orc - orc-mapreduce - ${spark.version} - provided - - - org.apache.hive - hive-storage-api - ${spark.version} - provided - - - com.google.protobuf - protobuf-java - ${spark.version} + com.nvidia + rapids-4-spark-db-bom + ${project.version} + pom provided diff --git a/tests/pom.xml b/tests/pom.xml index 335cd130b3e..df74c595db3 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -115,54 +115,12 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark321cdh.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark321cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - @@ -178,61 +136,13 @@ - org.apache.spark - spark-sql_${scala.binary.version} - ${spark330cdh.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.spark - spark-hive_${scala.binary.version} - ${spark330cdh.version} - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - - - - cloudera-repo - https://repository.cloudera.com/artifactory/cloudera-repos/ - - - - org.apache.spark - spark-core_${scala.binary.version} - - - org.apache.arrow - arrow-vector - - - provided - - - org.apache.arrow - arrow-vector - ${arrow.cdh.version} + com.nvidia + rapids-4-spark-cdh-bom + ${project.version} + pom provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - io.netty - netty-common - - - - - cloudera-repo - https://repository.cloudera.com/artifactory/cloudera-repos/ - -