From 5d0375527c55c71349170e0defe17ccd08a26375 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Wed, 1 Dec 2021 23:16:41 -0800 Subject: [PATCH 1/5] Use intransitive dependency:get for artifacts Fixes #4253 Signed-off-by: Gera Shegalov --- jenkins/spark-tests.sh | 2 +- jenkins/version-def.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 9f6379e87c3..8b5fe11112d 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -23,7 +23,7 @@ nvidia-smi ARTF_ROOT="$WORKSPACE/jars" MVN_GET_CMD="mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:get -B \ - -Dmaven.repo.local=$WORKSPACE/.m2 \ + -Dmaven.repo.local=$WORKSPACE/.m2 -Dtransitive=false \ $MVN_URM_MIRROR -Ddest=$ARTF_ROOT" rm -rf $ARTF_ROOT && mkdir -p $ARTF_ROOT diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index 6e8786eaf6b..975507efe21 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -21,7 +21,7 @@ set -e # $OVERWRITE_PARAMS patten 'abc=123;def=456;' PRE_IFS=$IFS IFS=";" -for VAR in $OVERWRITE_PARAMS;do +for VAR in $OVERWRITE_PARAMS; do echo $VAR && export $VAR done IFS=$PRE_IFS From 485f75987a1199cd64e54b6ef5259d1b368b9e11 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 2 Dec 2021 15:08:22 -0800 Subject: [PATCH 2/5] Generate reduced pom Signed-off-by: Gera Shegalov --- dist/maven-antrun/build-parallel-worlds.xml | 24 +++++++++++++++++++++ dist/pom.xml | 24 ++++++++++++++++----- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/dist/maven-antrun/build-parallel-worlds.xml b/dist/maven-antrun/build-parallel-worlds.xml index bff21cf5faf..6c52c18602a 100644 --- a/dist/maven-antrun/build-parallel-worlds.xml +++ b/dist/maven-antrun/build-parallel-worlds.xml @@ -123,5 +123,29 @@ + + + Generating dependency-reduced-pom.xml + + <dependency> + <groupId>com.nvidia</groupId> + <artifactId>rapids-4-spark-aggregator_\S+?</artifactId> + <version>\S+?</version> + <classifier>\S+?</classifier> + <scope>\S+?</scope> + </dependency> + + + + Generated regex to remove aggregator dependencies: + ${aggregatorDependencyRegex} + + + + + + \ No newline at end of file diff --git a/dist/pom.xml b/dist/pom.xml index c505e9cdb03..a61c4377686 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -34,11 +34,7 @@ rapids-4-spark-aggregator_${scala.binary.version} ${project.version} ${spark.version.classifier} - - provided + compile + META-INF/maven/** + @@ -299,6 +299,20 @@ + + verify + + run + + reduce-pom-deps-in-the-jar + + + + + + update_config_docs verify From 18461ae363b03b2e9fcc295162f2da439d7a5ba3 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 2 Dec 2021 19:10:25 -0800 Subject: [PATCH 3/5] Update deploy pom file Signed-off-by: Gera Shegalov --- jenkins/deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh index e218c736964..359e1291f12 100755 --- a/jenkins/deploy.sh +++ b/jenkins/deploy.sh @@ -53,6 +53,7 @@ ART_ID=`mvn help:evaluate -q -pl $DIST_PL -Dexpression=project.artifactId -Dforc ART_VER=`mvn help:evaluate -q -pl $DIST_PL -Dexpression=project.version -DforceStdout` FPATH="$DIST_PL/target/$ART_ID-$ART_VER" +POM_FPATH="$DIST_PL/target/extra-resources/META-INF/maven/com.nvidia/$ART_ID/pom.xml" echo "Plan to deploy ${FPATH}.jar to $SERVER_URL (ID:$SERVER_ID)" @@ -83,7 +84,8 @@ $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ # Distribution jar is a shaded artifact so use the reduced dependency pom. $DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ $SRC_DOC_JARS \ - -Dfile=$FPATH.jar -DgroupId=com.nvidia -DartifactId=$ART_ID -Dversion=$ART_VER -DpomFile=./dist/pom.xml + -Dfile=$FPATH.jar -DgroupId=com.nvidia -DartifactId=$ART_ID -Dversion=$ART_VER \ + -DpomFile="$POM_FPATH" ###### Deploy integration tests jar(s) ###### TESTS_ART_ID=`mvn help:evaluate -q -pl $TESTS_PL -Dexpression=project.artifactId -DforceStdout` From eb47e7a69228b24b2f4393913537c4508b01a942 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 2 Dec 2021 19:17:45 -0800 Subject: [PATCH 4/5] Undo -Dtransitive=false --- dist/pom.xml | 3 +-- jenkins/spark-tests.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dist/pom.xml b/dist/pom.xml index a61c4377686..0a2606482d9 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -308,8 +308,7 @@ + destfile="${project.build.directory}/${project.artifactId}-${project.version}.jar"/> diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh index 8b5fe11112d..9f6379e87c3 100755 --- a/jenkins/spark-tests.sh +++ b/jenkins/spark-tests.sh @@ -23,7 +23,7 @@ nvidia-smi ARTF_ROOT="$WORKSPACE/jars" MVN_GET_CMD="mvn org.apache.maven.plugins:maven-dependency-plugin:2.8:get -B \ - -Dmaven.repo.local=$WORKSPACE/.m2 -Dtransitive=false \ + -Dmaven.repo.local=$WORKSPACE/.m2 \ $MVN_URM_MIRROR -Ddest=$ARTF_ROOT" rm -rf $ARTF_ROOT && mkdir -p $ARTF_ROOT From 08a91056fce6e35317df51cd738eaaac8c82de80 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Thu, 2 Dec 2021 19:24:56 -0800 Subject: [PATCH 5/5] Formatting --- dist/maven-antrun/build-parallel-worlds.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dist/maven-antrun/build-parallel-worlds.xml b/dist/maven-antrun/build-parallel-worlds.xml index 6c52c18602a..66fedb8671f 100644 --- a/dist/maven-antrun/build-parallel-worlds.xml +++ b/dist/maven-antrun/build-parallel-worlds.xml @@ -140,8 +140,9 @@ Generated regex to remove aggregator dependencies: ${aggregatorDependencyRegex} - +