From 9582bc87859a7d45985eda1c52281fc16fd88d88 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 10 Sep 2020 14:20:26 -0500 Subject: [PATCH] Add some more checks to databricks build scripts (#721) * Add some more checks to databricks build scripts Signed-off-by: Thomas Graves * remove extra newline * use the right -gt for bash --- jenkins/databricks/build.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 9f8a282483e..5b9bb401c40 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -91,7 +91,9 @@ mvn -B '-Pdatabricks,!snapshot-shims' clean package -DskipTests # Copy so we pick up new built jar and latesty CuDF jar. Note that the jar names has to be # exactly what is in the staticly setup Databricks cluster we use. +echo "Copying rapids jars: dist/target/$RAPIDS_BUILT_JAR $DB_RAPIDS_JAR_LOC" sudo cp dist/target/$RAPIDS_BUILT_JAR $DB_RAPIDS_JAR_LOC +echo "Copying cudf jars: $CUDF_JAR $DB_CUDF_JAR_LOC" sudo cp $CUDF_JAR $DB_CUDF_JAR_LOC # tests @@ -106,6 +108,16 @@ sudo ln -s /databricks/jars/ $SPARK_HOME/jars || true sudo chmod 777 /databricks/data/logs/ sudo chmod 777 /databricks/data/logs/* echo { \"port\":\"15002\" } > ~/.databricks-connect +if [ `ls $DB_JAR_LOC/rapids* | wc -l` -gt 1 ]; then + echo "ERROR: Too many rapids jars in $DB_JAR_LOC" + ls $DB_JAR_LOC/rapids* + exit 1 +fi +if [ `ls $DB_JAR_LOC/cudf* | wc -l` -gt 1 ]; then + echo "ERROR: Too many cudf jars in $DB_JAR_LOC" + ls $DB_JAR_LOC/cudf* + exit 1 +fi $SPARK_HOME/bin/spark-submit ./runtests.py --runtime_env="databricks" cd /home/ubuntu tar -zcvf spark-rapids-built.tgz spark-rapids