Include dependency of common jar in tools jar (#5270)

* Revert "Fix tools depending on the common jar (#5239)" This reverts commit cec1b7d. * generate reduced pom Signed-off-by: Niranjan Artal <nartal@nvidia.com> * add minimizeJar to include classes Signed-off-by: Niranjan Artal <nartal@nvidia.com> * update spark-nightly-build.sh to to deploy dependency reduced pom Signed-off-by: Niranjan Artal <nartal@nvidia.com>
NVIDIA · Apr 20, 2022 · 55177e4 · 55177e4
1 parent aaae73d
commit 55177e4
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 58 deletions.
diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh
@@ -99,7 +99,7 @@ TOOL_FPATH="deployjars/$TOOL_ART_ID-$TOOL_ART_VER"
 TOOL_DOC_JARS="-Dsources=${TOOL_FPATH}-sources.jar -Djavadoc=${TOOL_FPATH}-javadoc.jar"
 $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
             $TOOL_DOC_JARS \
-            -Dfile=$TOOL_FPATH.jar -DpomFile=${TOOL_PL}/pom.xml
+            -Dfile=$TOOL_FPATH.jar -DpomFile=${TOOL_PL}/dependency-reduced-pom.xml
 
 ###### Deploy Spark 2.x explain meta jar ######
 SPARK2_PL=${SPARK2_PL:-"spark2-sql-plugin"}

diff --git a/jenkins/spark-nightly-build.sh b/jenkins/spark-nightly-build.sh
@@ -22,6 +22,7 @@ set -ex
 ## export 'M2DIR' so that shims can get the correct Spark dependency info
 export M2DIR="$WORKSPACE/.m2"
 
+TOOL_PL=${TOOL_PL:-"tools"}
 DIST_PL="dist"
 function mvnEval {
     mvn help:evaluate -q -pl $DIST_PL $MVN_URM_MIRROR -Prelease311 -Dmaven.repo.local=$M2DIR -Dcuda.version=$CUDA_CLASSIFIER -DforceStdout -Dexpression=$1
@@ -120,7 +121,8 @@ if [[ $SKIP_DEPLOY != 'true' ]]; then
     mvn -B deploy -pl '!dist' \
         -Dbuildver=$SPARK_BASE_SHIM_VERSION \
         $MVN_URM_MIRROR -Dmaven.repo.local=$M2DIR \
-        -Dcuda.version=$CUDA_CLASSIFIER
+        -Dcuda.version=$CUDA_CLASSIFIER \
+        -DpomFile=${TOOL_PL}/dependency-reduced-pom.xml
 fi
 
 # Parse Spark files from local mvn repo

diff --git a/tools/pom.xml b/tools/pom.xml
@@ -40,6 +40,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>com.nvidia</groupId>
+            <artifactId>rapids-4-spark-common_${scala.binary.version}</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
@@ -97,9 +102,11 @@
                             <goal>shade</goal>
                         </goals>
                         <configuration>
+                            <minimizeJar>true</minimizeJar>
                             <artifactSet>
                                 <includes>
                                     <include>org.rogach:scallop_${scala.binary.version}</include>
+                                    <include>com.nvidia:rapids-4-spark-common_${scala.binary.version}</include>
                                 </includes>
                             </artifactSet>
                             <filters>
@@ -118,7 +125,7 @@
                                     <mainClass>com.nvidia.spark.rapids.tool.profiling.ProfileMain</mainClass>
                                 </transformer>
                             </transformers>
-                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <createDependencyReducedPom>true</createDependencyReducedPom>
                         </configuration>
                     </execution>
                 </executions>

diff --git a/tools/src/main/scala/com/nvidia/spark/rapids/tool/ThreadFactoryBuilder.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/ThreadFactoryBuilder.scala
diff --git a/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Profiler.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/profiling/Profiler.scala
@@ -22,8 +22,8 @@ import scala.collection.JavaConverters._
 import scala.collection.mutable.{ArrayBuffer, HashMap}
 import scala.util.control.NonFatal
 
+import com.nvidia.spark.rapids.ThreadFactoryBuilder
 import com.nvidia.spark.rapids.tool.{EventLogInfo, EventLogPathProcessor}
-import com.nvidia.spark.rapids.tool.ThreadFactoryBuilder
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.internal.Logging

diff --git a/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala b/tools/src/main/scala/com/nvidia/spark/rapids/tool/qualification/Qualification.scala
@@ -20,8 +20,8 @@ import java.util.concurrent.{ConcurrentLinkedQueue, Executors, ThreadPoolExecuto
 
 import scala.collection.JavaConverters._
 
+import com.nvidia.spark.rapids.ThreadFactoryBuilder
 import com.nvidia.spark.rapids.tool.EventLogInfo
-import com.nvidia.spark.rapids.tool.ThreadFactoryBuilder
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.spark.internal.Logging

diff --git a/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppFilterImpl.scala b/tools/src/main/scala/org/apache/spark/sql/rapids/tool/AppFilterImpl.scala
@@ -22,8 +22,8 @@ import java.util.regex.PatternSyntaxException
 
 import scala.collection.JavaConverters._
 
+import com.nvidia.spark.rapids.ThreadFactoryBuilder
 import com.nvidia.spark.rapids.tool.EventLogInfo
-import com.nvidia.spark.rapids.tool.ThreadFactoryBuilder
 import com.nvidia.spark.rapids.tool.qualification.QualificationArgs
 import org.apache.hadoop.conf.Configuration