test

apache · Nov 21, 2020 · b853f5e · b853f5e
1 parent 668db8b
commit b853f5e
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 4 deletions.
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala
@@ -121,7 +121,7 @@ private[hive] object IsolatedClientLoader extends Logging {
     val hiveArtifacts = version.extraDeps ++
       Seq("hive-metastore", "hive-exec", "hive-common", "hive-serde")
         .map(a => s"org.apache.hive:$a:${version.fullVersion}") ++
-      Seq("com.google.guava:guava:14.0.1") ++ hadoopJarNames
+      Seq("com.google.guava:guava:14.0.1")
 
     val extraExclusions = if (hadoopVersion.startsWith("3")) {
       // this introduced from lower version of Hive could conflict with jars in Hadoop 3.2+, so
@@ -131,15 +131,27 @@ private[hive] object IsolatedClientLoader extends Logging {
       Seq.empty
     }
 
-    val classpath = quietly {
+    val hiveJarsClasspath = quietly {
       SparkSubmitUtils.resolveMavenCoordinates(
         hiveArtifacts.mkString(","),
         SparkSubmitUtils.buildIvySettings(
           Some(remoteRepos),
           ivyPath),
         exclusions = version.exclusions ++ extraExclusions)
     }
-    val allFiles = classpath.split(",").map(new File(_)).toSet
+
+    val hadoopJarsClasspath = quietly {
+      SparkSubmitUtils.resolveMavenCoordinates(
+        hadoopJarNames.mkString(","),
+        SparkSubmitUtils.buildIvySettings(
+          Some(remoteRepos),
+          ivyPath),
+        exclusions = version.exclusions ++ extraExclusions)
+    }
+
+    val allFiles =
+      (hiveJarsClasspath.split(",") ++ hadoopJarsClasspath.split(","))
+        .map(new File(_)).toSet
 
     // TODO: Remove copy logic.
     val tempDir = Utils.createTempDir(namePrefix = s"hive-${version}")

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala
@@ -103,7 +103,6 @@ package object client {
     // Since HIVE-14496, Hive materialized view need calcite-core.
     // For spark, only VersionsSuite currently creates a hive materialized view for testing.
     case object v2_3 extends HiveVersion("2.3.7",
-      extraDeps = Seq("org.apache.zookeeper:zookeeper:3.4.6"),
       exclusions = Seq("org.apache.calcite:calcite-druid",
         "org.apache.calcite.avatica:avatica",
         "org.apache.curator:*",