Merge pull request NVIDIA#798 from NVIDIA/bot-auto-merge-branch-22.12

[auto-merge] bot-auto-merge-branch-22.12 to branch-23.02 [skip ci] [bot]

ci/Jenkinsfile.premerge

@@ -0,0 +1,286 @@
+#!/usr/local/env groovy
+/*
+ * Copyright (c) 2022, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ *
+ * Jenkinsfile for building spark-rapids-jni on blossom
+ *
+ */
+import hudson.model.Result
+import hudson.model.Run
+import jenkins.model.CauseOfInterruption.UserInterruption
+
+@Library(['blossom-lib']) _
+@Library('blossom-github-lib@master')
+import ipp.blossom.*
+
+def githubHelper // blossom github helper
+def TEMP_IMAGE_BUILD = true
+def IMAGE_PREMERGE = "${common.ARTIFACTORY_NAME}/sw-spark-docker/plugin-jni:centos7-cuda11.5.0-blossom"
+def cpuImage = pod.getCPUYAML(IMAGE_PREMERGE)
+def PREMERGE_DOCKERFILE = 'ci/Dockerfile'
+def PREMERGE_TAG
+def skipped = false
+def major_ver // major version extracted from project version
+def minor_ver // minor version extracted from project version
+
+pipeline {
+    agent {
+        kubernetes {
+            label "premerge-init-${BUILD_TAG}"
+            cloud 'sc-ipp-blossom-prod'
+            yaml cpuImage
+        }
+    }
+
+    options {
+        ansiColor('xterm')
+        buildDiscarder(logRotator(numToKeepStr: '50'))
+        skipDefaultCheckout true
+        timeout(time: 12, unit: 'HOURS')
+    }
+
+    parameters {
+        string(name: 'PARALLEL_LEVEL', defaultValue: '18',
+            description: 'Parallel build cudf cpp with -DCPP_PARALLEL_LEVEL')
+        string(name: 'REF', defaultValue: '',
+            description: 'Merged commit of specific PR')
+        string(name: 'GITHUB_DATA', defaultValue: '',
+            description: 'Json-formatted github data from upstream blossom-ci')
+    }
+
+    environment {
+        JENKINS_ROOT = 'jenkins'
+        GITHUB_TOKEN = credentials("github-token")
+        ART_CREDS = credentials("urm_creds")
+        ART_URL = "https://${common.ARTIFACTORY_NAME}/artifactory/sw-spark-maven"
+        MVN_MIRROR = '-s ci/settings.xml -P mirror-apache-to-urm'
+        ARTIFACTORY_NAME = "${common.ARTIFACTORY_NAME}"
+        PVC = credentials("pvc")
+        CUSTOM_WORKSPACE = "/home/jenkins/agent/workspace/${BUILD_TAG}"
+        CUDA_CLASSIFIER = 'cuda11'
+    }
+
+    stages {
+        stage("Init githubHelper") {
+            steps {
+                script {
+                    githubHelper = GithubHelper.getInstance("${GITHUB_TOKEN}", params.GITHUB_DATA)
+                    // desc contains the PR ID and can be accessed from different builds
+                    currentBuild.description = githubHelper.getBuildDescription()
+                    try {
+                        // quiet period here in case the first build of two close dup triggers has not set desc
+                        sleep(time: 30, unit: "SECONDS")
+                        // abort duplicate running builds of the same PR (based on build desc)
+                        abortDupBuilds()
+                    } catch (e) { // do not block following build if abort failure
+                        echo "failed to try abort duplicate builds: " + e.toString()
+                    }
+
+                    def title = githubHelper.getIssue().title
+                    if (title ==~ /.*\[skip ci\].*/) {
+                        githubHelper.updateCommitStatus("$BUILD_URL", "Skipped", GitHubCommitState.SUCCESS)
+                        currentBuild.result == "SUCCESS"
+                        skipped = true
+                        return
+                    }
+                }
+            }
+        } // end of Init githubHelper
+
+        stage('Build docker image') {
+            when {
+                beforeAgent true
+                expression {
+                    !skipped
+                }
+            }
+
+            agent {
+                kubernetes {
+                    label "premerge-docker-${BUILD_TAG}"
+                    cloud 'sc-ipp-blossom-prod'
+                    yaml pod.getDockerBuildYAML()
+                    workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
+                    customWorkspace "${CUSTOM_WORKSPACE}"
+                }
+            }
+
+            steps {
+                script {
+                    githubHelper.updateCommitStatus("$BUILD_URL", "Running", GitHubCommitState.PENDING)
+                    checkout(
+                        changelog: false,
+                        poll: true,
+                        scm: [
+                            $class           : 'GitSCM', branches: [[name: githubHelper.getMergedSHA()]],
+                            submoduleCfg     : [],
+                            userRemoteConfigs: [[
+                                                    credentialsId: 'github-token',
+                                                    url          : githubHelper.getCloneUrl(),
+                                                    refspec      : '+refs/pull/*/merge:refs/remotes/origin/pr/*']]
+                        ]
+                    )
+
+                    stash(name: "source_tree", includes: "**")
+
+                    container('docker-build') {
+                        // check if pre-merge dockerfile modified
+                        def dockerfileModified = sh(returnStdout: true,
+                            script: 'BASE=$(git --no-pager log --oneline -1 | awk \'{ print $NF }\'); ' +
+                                'git --no-pager diff --name-only HEAD $(git merge-base HEAD $BASE) ' +
+                                "-- ${PREMERGE_DOCKERFILE} || true")
+                        if (!dockerfileModified?.trim()) {
+                            TEMP_IMAGE_BUILD = false
+                        }
+
+                        if (TEMP_IMAGE_BUILD) {
+                            PREMERGE_TAG = "centos7-cuda11.5.0-blossom-dev-${BUILD_TAG}"
+                            IMAGE_PREMERGE = "${ARTIFACTORY_NAME}/sw-spark-docker-local/plugin-jni:${PREMERGE_TAG}"
+                            docker.build(IMAGE_PREMERGE, "-f ${PREMERGE_DOCKERFILE} -t $IMAGE_PREMERGE .")
+                            uploadDocker(IMAGE_PREMERGE)
+                        }
+                    }
+                }
+            }
+        } // end of Build docker image
+
+        stage("Determine Project Version") {
+            when {
+                expression {
+                    !skipped
+                }
+            }
+
+            steps {
+                script {
+                    unstash "source_tree"
+                    container('cpu') {
+                        // Retrieve PROJECT_VER from pom
+                        PROJECT_VER = sh(returnStdout: true, script: "mvn help:evaluate -q " +
+                            "-Dexpression=project.version -DforceStdout | cut -d'.' -f1,2")
+                        echo PROJECT_VER
+                    }
+
+                    def versions = PROJECT_VER.split('\\.')
+                    major_ver = versions[0].toInteger()
+                    minor_ver = versions[1].toInteger()
+                }
+            }
+        }
+
+        stage('Premerge Test') {
+            when {
+                beforeAgent true
+                beforeOptions true
+                expression {
+                    !skipped
+                }
+            }
+            options {
+                // We have to use params to pass the resource label in options block,
+                // this is a limitation of declarative pipeline. And we need to lock resource before agent start
+                lock(label: "${params.GPU_POOL}", quantity: 1, variable: 'GPU_RESOURCE')
+            }
+            agent {
+                kubernetes {
+                    label "premerge-ci-1-${BUILD_TAG}"
+                    cloud 'sc-ipp-blossom-prod'
+                    yaml pod.getGPUYAML("${IMAGE_PREMERGE}", "${env.GPU_RESOURCE}", '8', '32Gi')
+                    workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
+                    customWorkspace "${CUSTOM_WORKSPACE}"
+                }
+            }
+
+            steps {
+                script {
+                    container('gpu') {
+                        timeout(time: 3, unit: 'HOURS') { // step only timeout for test run
+                            common.resolveIncompatibleDriverIssue(this)
+                            sh 'scl enable devtoolset-9 "ci/premerge-build.sh"'
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            script {
+                if (skipped) {
+                    return
+                }
+
+                if (currentBuild.currentResult == "SUCCESS") {
+                    githubHelper.updateCommitStatus("$BUILD_URL", "Success", GitHubCommitState.SUCCESS)
+                } else {
+                    // upload log only in case of build failure
+                    def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com"]
+                    guardWords.add("nvidia-smi(?s)(.*?)(?=git)") // hide GPU info
+                    githubHelper.uploadLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)
+
+                    githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
+                }
+
+                if (TEMP_IMAGE_BUILD) {
+                    container('cpu') {
+                        deleteDockerTempTag("${PREMERGE_TAG}") // clean premerge temp image
+                    }
+                }
+            }
+        }
+    }
+} // end of pipeline
+
+void uploadDocker(String IMAGE_NAME) {
+    def DOCKER_CMD = "docker --config $WORKSPACE/.docker"
+    retry(3) {
+        sleep(time: 10, unit: "SECONDS")
+        sh """
+            echo $ART_CREDS_PSW | $DOCKER_CMD login $ARTIFACTORY_NAME -u $ART_CREDS_USR --password-stdin
+            $DOCKER_CMD push $IMAGE_NAME
+            $DOCKER_CMD logout $ARTIFACTORY_NAME
+        """
+    }
+}
+
+void deleteDockerTempTag(String tag) {
+    if (!tag?.trim()) { // return if the tag is null or empty
+        return
+    }
+    sh "curl -u $ART_CREDS_USR:$ART_CREDS_PSW -XDELETE " +
+        "https://${ARTIFACTORY_NAME}/artifactory/sw-spark-docker-local/plugin-jni/${tag} || true"
+}
+
+void abortDupBuilds() {
+    Run prevBuild = currentBuild.rawBuild.getPreviousBuildInProgress()
+    while (prevBuild != null) {
+        if (prevBuild.isInProgress()) {
+            def prevDesc = prevBuild.description?.trim()
+            if (prevDesc && prevDesc == currentBuild.description?.trim()) {
+                def prevExecutor = prevBuild.getExecutor()
+                if (prevExecutor != null) {
+                    echo "...Aborting duplicate Build #${prevBuild.number}"
+                    prevExecutor.interrupt(Result.ABORTED,
+                        new UserInterruption("Build #${currentBuild.number}"))
+                }
+            }
+        }
+        prevBuild = prevBuild.getPreviousBuildInProgress()
+    }
+}