Integration-test tests jar for hive UDF tests (#4890)

1, Add commit version info for the integration-test tests jar. 2, Upload Databricks shims of integration-test tests jar to internal maven repo. 3, Download integration-test tests jar for the hive UDF integration tests. Signed-off-by: Tim Liu <timl@nvidia.com>
NVIDIA · Mar 3, 2022 · 1d0004c · 1d0004c
1 parent bec04a8
commit 1d0004c
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 5 deletions.
diff --git a/integration_tests/pom.xml b/integration_tests/pom.xml
@@ -248,6 +248,23 @@
                             <goal>run</goal>
                         </goals>
                     </execution>
+                    <execution>
+                        <id>generate-test-build-info</id>
+                        <phase>process-test-classes</phase>
+                        <configuration>
+                            <!-- Execute the shell script to generate the plugin build information. -->
+                            <target>
+                                <exec executable="bash" failonerror="true" output="${project.build.testOutputDirectory}/rapids4spark-version-info.properties" >
+                                    <arg value="${spark.rapids.source.basedir}/build/build-info"/>
+                                    <arg value="${project.version}"/>
+                                    <arg value="${cudf.version}"/>
+                                </exec>
+                            </target>
+                        </configuration>
+                        <goals>
+                            <goal>run</goal>
+                        </goals>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>

diff --git a/jenkins/databricks/deploy.sh b/jenkins/databricks/deploy.sh
@@ -38,3 +38,7 @@ mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERV
 DBINTTESTJARFPATH=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}.jar
 mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
     -Dfile=$DBINTTESTJARFPATH -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME
+# Deploy integration-tests tests jar for hive UDF tests
+HIVEUDFTESTSJAR=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}'tests'.jar
+mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
+    -Dfile=$HIVEUDFTESTSJAR -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME'tests'
diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh
@@ -54,6 +54,12 @@ RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/"
 # The version of pytest.tar.gz that is uploaded is the one built against spark301 but its being pushed without classifier for now
 RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz"
 
+# For the HIVE UDF integration tests
+$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \
+    -Dtransitive=false \
+    -DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=$SHUFFLE_SPARK_SHIM'tests'
+RAPIDS_HIVE_UDF_TESTS_JAR=="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM'tests'.jar"
+
 tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'}
 rm -rf "$tmp_info"
 TEE_CMD="tee -a $tmp_info"
@@ -74,20 +80,22 @@ getRevision() {
 set +x
 echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info"
 echo "-------------------- cudf JNI BUILD INFO --------------------" >> "$tmp_info"
-c_ver=$(getRevision $JARS_PATH/$CUDF_JAR cudf-java-version-info.properties)
+c_ver=$(getRevision $CUDF_JAR cudf-java-version-info.properties)
 echo "-------------------- rapids-4-spark BUILD INFO --------------------" >> "$tmp_info"
-p_ver=$(getRevision $JARS_PATH/$RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties)
+p_ver=$(getRevision $RAPIDS_PLUGIN_JAR rapids4spark-version-info.properties)
 echo "-------------------- rapids-4-spark-integration-tests BUILD INFO --------------------" >> "$tmp_info"
-it_ver=$(getRevision $JARS_PATH/$RAPIDS_TEST_JAR rapids4spark-version-info.properties)
+it_ver=$(getRevision $RAPIDS_TEST_JAR rapids4spark-version-info.properties)
 echo "-------------------- rapids-4-spark-integration-tests pytest BUILD INFO --------------------" >> "$tmp_info"
-pt_ver=$(getRevision $JARS_PATH/$RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties)
+pt_ver=$(getRevision $RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties)
+echo "-------------------- rapids-4-spark-integration-tests tests BUILD INFO --------------------" >> "$tmp_info"
+hu_ver=$(getRevision $RAPIDS_HIVE_UDF_TESTS_JAR rapids4spark-version-info.properties)
 echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info"
 set -x
 cat "$tmp_info" || true
 
 SKIP_REVISION_CHECK=${SKIP_REVISION_CHECK:-'false'}
 if [[ "$SKIP_REVISION_CHECK" != "true" && (-z "$c_ver" || -z "$p_ver"|| \
-      "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver") ]]; then
+      "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver"  || "$p_ver" != "$hu_ver") ]]; then
   echo "Artifacts revisions are inconsistent!"
   exit 1
 fi