Relocate Hive UDF Classes (#4904)

* Revert "Integration-test tests jar for hive UDF tests (#4890)"

This reverts commit 1d0004c.

* Move Hive UDFs from test jar to main jar

Signed-off-by: Chong Gao <res_life@163.com>

integration_tests/pom.xml

@@ -248,23 +248,6 @@
                             <goal>run</goal>
                         </goals>
                     </execution>
-                    <execution>
-                        <id>generate-test-build-info</id>
-                        <phase>process-test-classes</phase>
-                        <configuration>
-                            <!-- Execute the shell script to generate the plugin build information. -->
-                            <target>
-                                <exec executable="bash" failonerror="true" output="${project.build.testOutputDirectory}/rapids4spark-version-info.properties" >
-                                    <arg value="${spark.rapids.source.basedir}/build/build-info"/>
-                                    <arg value="${project.version}"/>
-                                    <arg value="${cudf.version}"/>
-                                </exec>
-                            </target>
-                        </configuration>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                    </execution>
                 </executions>
             </plugin>
             <plugin>

integration_tests/run_pyspark_from_build.sh

@@ -41,12 +41,16 @@ else
     if [ -d "$LOCAL_JAR_PATH" ]; then
         CUDF_JARS=$(echo "$LOCAL_JAR_PATH"/cudf-*.jar)
         PLUGIN_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark_*.jar)
-        TEST_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER*.jar)
+        # the integration-test-spark3xx.jar, should not include the integration-test-spark3xxtest.jar
+        TEST_JARS=$(echo "$LOCAL_JAR_PATH"/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER.jar)
     else
         CUDF_JARS=$(echo "$SCRIPTPATH"/target/dependency/cudf-*.jar)
         PLUGIN_JARS=$(echo "$SCRIPTPATH"/../dist/target/rapids-4-spark_*.jar)
-        TEST_JARS=$(echo "$SCRIPTPATH"/target/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER*.jar)
+        # the integration-test-spark3xx.jar, should not include the integration-test-spark3xxtest.jar
+        TEST_JARS=$(echo "$SCRIPTPATH"/target/rapids-4-spark-integration-tests*-$SPARK_SHIM_VER.jar)
     fi
+
+    # Only 3 jars: cudf.jar dist.jar integration-test.jar
     ALL_JARS="$CUDF_JARS $PLUGIN_JARS $TEST_JARS"
     echo "AND PLUGIN JARS: $ALL_JARS"
     if [[ "${TEST}" != "" ]];

jenkins/databricks/deploy.sh

@@ -38,7 +38,3 @@ mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERV
 DBINTTESTJARFPATH=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}.jar
 mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
     -Dfile=$DBINTTESTJARFPATH -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME
-# Deploy integration-tests tests jar for hive UDF tests
-HIVEUDFTESTSJAR=./integration_tests/target/rapids-4-spark-integration-tests_$SCALA_VERSION-$SPARK_PLUGIN_JAR_VERSION-${DB_SHIM_NAME}'tests'.jar
-mvn -B deploy:deploy-file $MVN_URM_MIRROR -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
-    -Dfile=$HIVEUDFTESTSJAR -DpomFile=integration_tests/pom.xml -Dclassifier=$DB_SHIM_NAME'tests'

jenkins/spark-tests.sh

@@ -54,12 +54,6 @@ RAPIDS_INT_TESTS_HOME="$ARTF_ROOT/integration_tests/"
 # The version of pytest.tar.gz that is uploaded is the one built against spark301 but its being pushed without classifier for now
 RAPIDS_INT_TESTS_TGZ="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-pytest.tar.gz"
 
-# For the HIVE UDF integration tests
-$MVN_GET_CMD -DremoteRepositories=$PROJECT_TEST_REPO \
-    -Dtransitive=false \
-    -DgroupId=com.nvidia -DartifactId=rapids-4-spark-integration-tests_$SCALA_BINARY_VER -Dversion=$PROJECT_TEST_VER -Dclassifier=$SHUFFLE_SPARK_SHIM'tests'
-RAPIDS_HIVE_UDF_TESTS_JAR=="$ARTF_ROOT/rapids-4-spark-integration-tests_${SCALA_BINARY_VER}-$PROJECT_TEST_VER-$SHUFFLE_SPARK_SHIM'tests'.jar"
-
 tmp_info=${TMP_INFO_FILE:-'/tmp/artifacts-build.info'}
 rm -rf "$tmp_info"
 TEE_CMD="tee -a $tmp_info"
@@ -87,15 +81,13 @@ echo "-------------------- rapids-4-spark-integration-tests BUILD INFO ---------
 it_ver=$(getRevision $RAPIDS_TEST_JAR rapids4spark-version-info.properties)
 echo "-------------------- rapids-4-spark-integration-tests pytest BUILD INFO --------------------" >> "$tmp_info"
 pt_ver=$(getRevision $RAPIDS_INT_TESTS_TGZ integration_tests/rapids4spark-version-info.properties)
-echo "-------------------- rapids-4-spark-integration-tests tests BUILD INFO --------------------" >> "$tmp_info"
-hu_ver=$(getRevision $RAPIDS_HIVE_UDF_TESTS_JAR rapids4spark-version-info.properties)
 echo -e "\n==================== ARTIFACTS BUILD INFO ====================\n" >> "$tmp_info"
 set -x
 cat "$tmp_info" || true
 
 SKIP_REVISION_CHECK=${SKIP_REVISION_CHECK:-'false'}
 if [[ "$SKIP_REVISION_CHECK" != "true" && (-z "$c_ver" || -z "$p_ver"|| \
-      "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver"  || "$p_ver" != "$hu_ver") ]]; then
+      "$p_ver" != "$it_ver" || "$p_ver" != "$pt_ver") ]]; then
   echo "Artifacts revisions are inconsistent!"
   exit 1
 fi