diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 9663e03ee6a74..ee5906feb1c02 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -121,7 +121,7 @@ private[hive] object IsolatedClientLoader extends Logging { val hiveArtifacts = version.extraDeps ++ Seq("hive-metastore", "hive-exec", "hive-common", "hive-serde") .map(a => s"org.apache.hive:$a:${version.fullVersion}") ++ - Seq("com.google.guava:guava:14.0.1") ++ hadoopJarNames + Seq("com.google.guava:guava:14.0.1") val extraExclusions = if (hadoopVersion.startsWith("3")) { // this introduced from lower version of Hive could conflict with jars in Hadoop 3.2+, so @@ -131,7 +131,7 @@ private[hive] object IsolatedClientLoader extends Logging { Seq.empty } - val classpath = quietly { + val hiveJarsClasspath = quietly { SparkSubmitUtils.resolveMavenCoordinates( hiveArtifacts.mkString(","), SparkSubmitUtils.buildIvySettings( @@ -139,7 +139,19 @@ private[hive] object IsolatedClientLoader extends Logging { ivyPath), exclusions = version.exclusions ++ extraExclusions) } - val allFiles = classpath.split(",").map(new File(_)).toSet + + val hadoopJarsClasspath = quietly { + SparkSubmitUtils.resolveMavenCoordinates( + hadoopJarNames.mkString(","), + SparkSubmitUtils.buildIvySettings( + Some(remoteRepos), + ivyPath), + exclusions = version.exclusions ++ extraExclusions) + } + + val allFiles = + (hiveJarsClasspath.split(",") ++ hadoopJarsClasspath.split(",")) + .map(new File(_)).toSet // TODO: Remove copy logic. val tempDir = Utils.createTempDir(namePrefix = s"hive-${version}") diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index cfdd862189e31..27ba3eca81948 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -103,7 +103,6 @@ package object client { // Since HIVE-14496, Hive materialized view need calcite-core. // For spark, only VersionsSuite currently creates a hive materialized view for testing. case object v2_3 extends HiveVersion("2.3.7", - extraDeps = Seq("org.apache.zookeeper:zookeeper:3.4.6"), exclusions = Seq("org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", "org.apache.curator:*",