From 609b33a3957f73fcd7cd3ac5b594913f747dce8e Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Tue, 16 Jun 2020 12:30:27 +0800 Subject: [PATCH 1/2] Support public release for plugin jar Add a Jenkinsfile for public release job. Add a deploy script Add rat exclusions in pom file for temp files generated by deployment job Add profiles to generate javadoc and sources jars Add neccessary sections(scm, licenses ...) in root pom file for public release --- dist/pom.xml | 1 + jenkins/Jenkinsfile.release | 103 ++++++++++++++++++++++++++++++++++++ jenkins/deploy.sh | 58 ++++++++++++++++++++ jenkins/settings.xml | 5 ++ pom.xml | 65 +++++++++++++++++++++++ 5 files changed, 232 insertions(+) create mode 100644 jenkins/Jenkinsfile.release create mode 100755 jenkins/deploy.sh diff --git a/dist/pom.xml b/dist/pom.xml index ededbc20549..da46c7d00a5 100644 --- a/dist/pom.xml +++ b/dist/pom.xml @@ -100,6 +100,7 @@ dependency-reduced-pom.xml + pom.xml.asc diff --git a/jenkins/Jenkinsfile.release b/jenkins/Jenkinsfile.release new file mode 100644 index 00000000000..ba1ad518cf5 --- /dev/null +++ b/jenkins/Jenkinsfile.release @@ -0,0 +1,103 @@ +#!/usr/local/env groovy +/* + * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** +* +* Jenkinsfile for building and deploy rapids-plugin to public repo +* +*/ + +def SERVERS_MAP = [ + Local: 'http://apt-sh04:8081/repository/xgboost-spark-release', + Sonatype: 'https://oss.sonatype.org/service/local/staging/deploy/maven2' +] + +def SEC_IDS = [ + Local: ['local-gpg-passphrase', 'local-gpg-private-key', 'local-username-password'], + Sonatype: ['rapids-gpg-passphrase', 'rapids-gpg-private-key', 'sonatype-username-password'] +] + +pipeline { + agent { label 'vanilla||docker-deploy||docker-gpu' } + + options { + ansiColor('xterm') + timeout(time: 120, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '10')) + } + + parameters { + choice(name: 'DEPLOY_TO', choices: ['Local', 'Sonatype'], + description: 'Where to deploy artifacts to') + string(name: 'REF', defaultValue: 'branch-0.1', description: 'Commit to build') + } + + environment { + JENKINS_ROOT='jenkins' + IMAGE_NAME="urm.nvidia.com/sw-spark-docker/plugin:dev-ubuntu16-cuda10.1" + LIBCUDF_KERNEL_CACHE_PATH='/tmp/.cudf' + MVN_MIRROR='-s jenkins/settings.xml -P mirror-apache-to-urm' + URM_CREDS = credentials("svcngcc_artifactory") + DIST_PL='dist' + SQL_PL='sql-plugin' + } + + stages { + stage('Build') { + steps { + script { + def DOCKER_CMD="docker --config $WORKSPACE/.docker" + sh """ + echo $URM_CREDS_PSW | $DOCKER_CMD login https://urm.nvidia.com -u $URM_CREDS_USR --password-stdin + $DOCKER_CMD pull $IMAGE_NAME + $DOCKER_CMD logout https://urm.nvidia.com + """ + docker.image("$IMAGE_NAME").inside("--runtime=nvidia -v ${HOME}/.m2:${HOME}/.m2:rw \ + -v ${HOME}/.zinc:${HOME}/.zinc:rw \ + -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group") { + sh "mvn -U -B clean install $MVN_MIRROR -P source-javadoc" + } + } + } + } + + stage("Deploy") { + environment { + SERVER_ID='ossrh' + SERVER_URL="${SERVERS_MAP["$DEPLOY_TO"]}" + GPG_PASSPHRASE=credentials("${SEC_IDS["$DEPLOY_TO"][0]}") + GPG_FILE=credentials("${SEC_IDS["$DEPLOY_TO"][1]}") + SONATYPE=credentials("${SEC_IDS["$DEPLOY_TO"][2]}") + GNUPGHOME="${WORKSPACE}/.gnupg" + } + steps { + script { + docker.image("$IMAGE_NAME").inside("-v ${HOME}/.m2:${HOME}/.m2:rw \ + -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group") { + sh 'rm -rf $GNUPGHOME' + sh 'gpg --import $GPG_FILE' + retry (3) { + sh "bash $JENKINS_ROOT/deploy.sh true" + } + } + } + } + } + } // End of stages + +} + diff --git a/jenkins/deploy.sh b/jenkins/deploy.sh new file mode 100755 index 00000000000..eb9cfebd168 --- /dev/null +++ b/jenkins/deploy.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +SIGN_FILE=$1 + +###### Build the path of jar(s) to be deployed ###### + +cd $WORKSPACE +ART_ID=`mvn exec:exec -q -pl $DIST_PL -Dexec.executable=echo -Dexec.args='${project.artifactId}'` +ART_VER=`mvn exec:exec -q -pl $DIST_PL -Dexec.executable=echo -Dexec.args='${project.version}'` + +FPATH="$DIST_PL/target/$ART_ID-$ART_VER" + +echo "Plan to deploy ${FPATH}.jar to $SERVER_URL (ID:$SERVER_ID)" + + +###### Choose the deploy command ###### + +if [ "$SIGN_FILE" == true ]; then + # No javadoc and sources jar is generated for shade artifact only. Use 'sql-plugin' instead + SQL_ART_ID=`mvn exec:exec -q -pl $SQL_PL -Dexec.executable=echo -Dexec.args='${project.artifactId}'` + SQL_ART_VER=`mvn exec:exec -q -pl $SQL_PL -Dexec.executable=echo -Dexec.args='${project.version}'` + JS_FPATH="${SQL_PL}/target/${SQL_ART_ID}-${SQL_ART_VER}" + SRC_DOC_JARS="-Dsources=${JS_FPATH}-sources.jar -Djavadoc=${JS_FPATH}-javadoc.jar" + DEPLOY_CMD="mvn -B gpg:sign-and-deploy-file -s jenkins/settings.xml -Dgpg.passphrase=$GPG_PASSPHRASE" +else + DEPLOY_CMD="mvn -B deploy:deploy-file -s jenkins/settings.xml" +fi + +echo "Deploy CMD: $DEPLOY_CMD" + + +###### Deploy the parent pom file ###### + +$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ + -Dfile=./pom.xml -DpomFile=./pom.xml + +###### Deploy the artifact jar(s) ###### + +$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \ + $SRC_DOC_JARS \ + -Dfile=$FPATH.jar -DpomFile=${DIST_PL}/pom.xml + diff --git a/jenkins/settings.xml b/jenkins/settings.xml index 7f10a90c8eb..4cbdffbed8c 100644 --- a/jenkins/settings.xml +++ b/jenkins/settings.xml @@ -27,6 +27,11 @@ ${env.URM_CREDS_PSW} snapshots + + ossrh + ${env.SONATYPE_USR} + ${env.SONATYPE_PSW} + diff --git a/pom.xml b/pom.xml index cfdcff5b66c..0926eedb46e 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,32 @@ 0.1-SNAPSHOT pom + https://github.com/NVIDIA + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + scm:git:https://github.com/NVIDIA/spark-rapids.git + scm:git:git@github.com:NVIDIA/spark-rapids.git + HEAD + https://github.com/NVIDIA/spark-rapids + + + + revans2 + Robert Evans + bobby@apache.org + + Committer + + -6 + + + dist shuffle-plugin @@ -42,6 +68,42 @@ true + + source-javadoc + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.0 + + + attach-source + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.0.0 + + + attach-javadoc + + jar + + + + + -Xdoclint:none + + + + + @@ -388,6 +450,9 @@ **/*.md NOTICE-binary docs/dev/idea-code-style-settings.xml + **/.m2/** + .gnupg/** + pom.xml.asc From 7af1950feb650a36e831dfd65466759ce83e9dec Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Thu, 18 Jun 2020 10:19:23 +0800 Subject: [PATCH 2/2] Disable the choice to local --- jenkins/Jenkinsfile.release | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/Jenkinsfile.release b/jenkins/Jenkinsfile.release index ba1ad518cf5..b94ce69c17f 100644 --- a/jenkins/Jenkinsfile.release +++ b/jenkins/Jenkinsfile.release @@ -22,7 +22,7 @@ */ def SERVERS_MAP = [ - Local: 'http://apt-sh04:8081/repository/xgboost-spark-release', + Local: ' ', Sonatype: 'https://oss.sonatype.org/service/local/staging/deploy/maven2' ] @@ -41,7 +41,7 @@ pipeline { } parameters { - choice(name: 'DEPLOY_TO', choices: ['Local', 'Sonatype'], + choice(name: 'DEPLOY_TO', choices: ['Sonatype'], description: 'Where to deploy artifacts to') string(name: 'REF', defaultValue: 'branch-0.1', description: 'Commit to build') }