From 38ccae38286a1174d8b2bdb87a5b9bccf940f306 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Wed, 8 Jul 2020 23:22:35 +0800 Subject: [PATCH 1/2] Integration tests on spark 3.0.1-SNAPSHOT & 3.10.-SNAPSHOT Add Jenkinsfile for integration tests on spark 3.0.1-SNAPSHOT & 3.10.-SNAPSHOT --- jenkins/Jenkinsfile.301.integration | 99 +++++++++++++++++++++++++++++ jenkins/Jenkinsfile.310.integration | 99 +++++++++++++++++++++++++++++ jenkins/version-def.sh | 2 +- 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 jenkins/Jenkinsfile.301.integration create mode 100644 jenkins/Jenkinsfile.310.integration diff --git a/jenkins/Jenkinsfile.301.integration b/jenkins/Jenkinsfile.301.integration new file mode 100644 index 00000000000..67cb5a5aa18 --- /dev/null +++ b/jenkins/Jenkinsfile.301.integration @@ -0,0 +1,99 @@ +#!/usr/local/env groovy +/* + * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* +* Jenkins file for running spark3.0 integration tests +* +*/ + +@Library('shared-libs') _ + +def urmUrl="https://${ArtifactoryConstants.ARTIFACTORY_NAME}/artifactory/sw-spark-maven" + +pipeline { + agent none + + options { + ansiColor('xterm') + timestamps() + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + parameters { + string(name: 'OVERWRITE_PARAMS', defaultValue: '', + description: 'parameters format XXX_VER=xxx;YYY_VER=yyy;') + string(name: 'REF', defaultValue: 'branch-0.2', description: 'Commit to build') + } + + environment { + JENKINS_ROOT = 'jenkins' + TEST_SCRIPT = '$JENKINS_ROOT/spark-tests.sh' + LIBCUDF_KERNEL_CACHE_PATH='/tmp/.cudf' + ARTIFACTORY_NAME = "${ArtifactoryConstants.ARTIFACTORY_NAME}" + URM_URL = "${urmUrl}" + MVN_URM_MIRROR='-s jenkins/settings.xml -P mirror-apache-to-urm' + } + + stages { + stage('IT on 3.0.1-SNAPSHOT') { + agent { label 'docker-gpu' } + environment {SPARK_VER='3.0.1-SNAPSHOT'} + steps { + script { + def CUDA_NAME=sh(returnStdout: true, + script: '. jenkins/version-def.sh>&2 && echo -n $CUDA_CLASSIFIER | sed "s/-/./g"') + def IMAGE_NAME="$ARTIFACTORY_NAME/sw-spark-docker/plugin:it-centos7-$CUDA_NAME" + def CUDA_VER="$CUDA_NAME" - "cuda" + sh "docker pull $IMAGE_NAME" + docker.image(IMAGE_NAME).inside("--runtime=nvidia -v ${HOME}/.zinc:${HOME}/.zinc:rw") { + sh "bash $TEST_SCRIPT" + } + } + } + } + } // end of stages + post { + always { + script { + def status = "failed" + if (currentBuild.currentResult == "SUCCESS") { + status = "success" + slack("#rapidsai-spark-cicd", "Success", color: "#33CC33") + } + else { + slack("#rapidsai-spark-cicd", "Failed", color: "#FF0000") + } + } + echo 'Pipeline finished!' + } + } +} // end of pipeline + +void slack(Map params = [:], String channel, String message) { + Map defaultParams = [ + color: "#000000", + baseUrl: "https://nvidia.slack.com/services/hooks/jenkins-ci/", + tokenCredentialId: "slack_token" + ] + + params["channel"] = channel + params["message"] = "${BUILD_URL}\n" + message + + slackSend(defaultParams << params) +} diff --git a/jenkins/Jenkinsfile.310.integration b/jenkins/Jenkinsfile.310.integration new file mode 100644 index 00000000000..538cc7a0f1a --- /dev/null +++ b/jenkins/Jenkinsfile.310.integration @@ -0,0 +1,99 @@ +#!/usr/local/env groovy +/* + * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* +* Jenkins file for running spark3.0 integration tests +* +*/ + +@Library('shared-libs') _ + +def urmUrl="https://${ArtifactoryConstants.ARTIFACTORY_NAME}/artifactory/sw-spark-maven" + +pipeline { + agent none + + options { + ansiColor('xterm') + timestamps() + timeout(time: 240, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + parameters { + string(name: 'OVERWRITE_PARAMS', defaultValue: '', + description: 'parameters format XXX_VER=xxx;YYY_VER=yyy;') + string(name: 'REF', defaultValue: 'branch-0.2', description: 'Commit to build') + } + + environment { + JENKINS_ROOT = 'jenkins' + TEST_SCRIPT = '$JENKINS_ROOT/spark-tests.sh' + LIBCUDF_KERNEL_CACHE_PATH='/tmp/.cudf' + ARTIFACTORY_NAME = "${ArtifactoryConstants.ARTIFACTORY_NAME}" + URM_URL = "${urmUrl}" + MVN_URM_MIRROR='-s jenkins/settings.xml -P mirror-apache-to-urm' + } + + stages { + stage('IT on 3.1.0-SNAPSHOT') { + agent { label 'docker-gpu' } + environment {SPARK_VER='3.0.1-SNAPSHOT'} + steps { + script { + def CUDA_NAME=sh(returnStdout: true, + script: '. jenkins/version-def.sh>&2 && echo -n $CUDA_CLASSIFIER | sed "s/-/./g"') + def IMAGE_NAME="$ARTIFACTORY_NAME/sw-spark-docker/plugin:it-centos7-$CUDA_NAME" + def CUDA_VER="$CUDA_NAME" - "cuda" + sh "docker pull $IMAGE_NAME" + docker.image(IMAGE_NAME).inside("--runtime=nvidia -v ${HOME}/.zinc:${HOME}/.zinc:rw") { + sh "bash $TEST_SCRIPT" + } + } + } + } + } // end of stages + post { + always { + script { + def status = "failed" + if (currentBuild.currentResult == "SUCCESS") { + status = "success" + slack("#rapidsai-spark-cicd", "Success", color: "#33CC33") + } + else { + slack("#rapidsai-spark-cicd", "Failed", color: "#FF0000") + } + } + echo 'Pipeline finished!' + } + } +} // end of pipeline + +void slack(Map params = [:], String channel, String message) { + Map defaultParams = [ + color: "#000000", + baseUrl: "https://nvidia.slack.com/services/hooks/jenkins-ci/", + tokenCredentialId: "slack_token" + ] + + params["channel"] = channel + params["message"] = "${BUILD_URL}\n" + message + + slackSend(defaultParams << params) +} diff --git a/jenkins/version-def.sh b/jenkins/version-def.sh index 3947cc38fe0..25b5a8c0408 100755 --- a/jenkins/version-def.sh +++ b/jenkins/version-def.sh @@ -22,7 +22,7 @@ set -e PRE_IFS=$IFS IFS=";" for VAR in $OVERWRITE_PARAMS;do - echo $VAR && declare $VAR + echo $VAR && export $VAR done IFS=$PRE_IFS From 4aa647dfd37ef8fbbbac32f538fcb70e5ed4fa17 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Fri, 10 Jul 2020 10:22:09 +0800 Subject: [PATCH 2/2] Fix copyright and spark version anotations --- jenkins/Jenkinsfile.301.integration | 4 ++-- jenkins/Jenkinsfile.310.integration | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/jenkins/Jenkinsfile.301.integration b/jenkins/Jenkinsfile.301.integration index 67cb5a5aa18..1b477b27b55 100644 --- a/jenkins/Jenkinsfile.301.integration +++ b/jenkins/Jenkinsfile.301.integration @@ -1,6 +1,6 @@ #!/usr/local/env groovy /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2020, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ /** * -* Jenkins file for running spark3.0 integration tests +* Jenkins file for running spark3.0.1 integration tests * */ diff --git a/jenkins/Jenkinsfile.310.integration b/jenkins/Jenkinsfile.310.integration index 538cc7a0f1a..c1dc0f6a52c 100644 --- a/jenkins/Jenkinsfile.310.integration +++ b/jenkins/Jenkinsfile.310.integration @@ -1,6 +1,6 @@ #!/usr/local/env groovy /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2020, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ /** * -* Jenkins file for running spark3.0 integration tests +* Jenkins file for running spark3.1.0 integration tests * */ @@ -53,7 +53,7 @@ pipeline { stages { stage('IT on 3.1.0-SNAPSHOT') { agent { label 'docker-gpu' } - environment {SPARK_VER='3.0.1-SNAPSHOT'} + environment {SPARK_VER='3.1.0-SNAPSHOT'} steps { script { def CUDA_NAME=sh(returnStdout: true,