From 275a2c4c20972012608faeec1bacfd650ce886e8 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 10 Sep 2020 10:52:54 -0500 Subject: [PATCH 1/3] Add some more checks to databricks build scripts Signed-off-by: Thomas Graves --- jenkins/databricks/build.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 9f8a282483e..977df3a375c 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -89,9 +89,12 @@ mvn -B install:install-file \ mvn -B '-Pdatabricks,!snapshot-shims' clean package -DskipTests + # Copy so we pick up new built jar and latesty CuDF jar. Note that the jar names has to be # exactly what is in the staticly setup Databricks cluster we use. +echo "Copying rapids jars: dist/target/$RAPIDS_BUILT_JAR $DB_RAPIDS_JAR_LOC" sudo cp dist/target/$RAPIDS_BUILT_JAR $DB_RAPIDS_JAR_LOC +echo "Copying cudf jars: $CUDF_JAR $DB_CUDF_JAR_LOC" sudo cp $CUDF_JAR $DB_CUDF_JAR_LOC # tests @@ -106,6 +109,16 @@ sudo ln -s /databricks/jars/ $SPARK_HOME/jars || true sudo chmod 777 /databricks/data/logs/ sudo chmod 777 /databricks/data/logs/* echo { \"port\":\"15002\" } > ~/.databricks-connect +if [ `ls $DB_JAR_LOC/rapids* | wc -l` > 1 ]; then + echo "ERROR: Too many rapids jars in $DB_JAR_LOC" + ls $DB_JAR_LOC/rapids* + exit 1 +fi +if [ `ls $DB_JAR_LOC/cudf* | wc -l` > 1 ]; then + echo "ERROR: Too many cudf jars in $DB_JAR_LOC" + ls $DB_JAR_LOC/cudf* + exit 1 +fi $SPARK_HOME/bin/spark-submit ./runtests.py --runtime_env="databricks" cd /home/ubuntu tar -zcvf spark-rapids-built.tgz spark-rapids From ddbd1fad608c53c8f48efa11994fdffc1f1d66d4 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 10 Sep 2020 10:53:59 -0500 Subject: [PATCH 2/3] remove extra newline --- jenkins/databricks/build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 977df3a375c..21944bef493 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -89,7 +89,6 @@ mvn -B install:install-file \ mvn -B '-Pdatabricks,!snapshot-shims' clean package -DskipTests - # Copy so we pick up new built jar and latesty CuDF jar. Note that the jar names has to be # exactly what is in the staticly setup Databricks cluster we use. echo "Copying rapids jars: dist/target/$RAPIDS_BUILT_JAR $DB_RAPIDS_JAR_LOC" From 72fac135d2c7b5544dd9144bf81864120df23183 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 10 Sep 2020 11:09:58 -0500 Subject: [PATCH 3/3] use the right -gt for bash --- jenkins/databricks/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/databricks/build.sh b/jenkins/databricks/build.sh index 21944bef493..5b9bb401c40 100755 --- a/jenkins/databricks/build.sh +++ b/jenkins/databricks/build.sh @@ -108,12 +108,12 @@ sudo ln -s /databricks/jars/ $SPARK_HOME/jars || true sudo chmod 777 /databricks/data/logs/ sudo chmod 777 /databricks/data/logs/* echo { \"port\":\"15002\" } > ~/.databricks-connect -if [ `ls $DB_JAR_LOC/rapids* | wc -l` > 1 ]; then +if [ `ls $DB_JAR_LOC/rapids* | wc -l` -gt 1 ]; then echo "ERROR: Too many rapids jars in $DB_JAR_LOC" ls $DB_JAR_LOC/rapids* exit 1 fi -if [ `ls $DB_JAR_LOC/cudf* | wc -l` > 1 ]; then +if [ `ls $DB_JAR_LOC/cudf* | wc -l` -gt 1 ]; then echo "ERROR: Too many cudf jars in $DB_JAR_LOC" ls $DB_JAR_LOC/cudf* exit 1