[CI] Pipeline refactoring

.ci/Jenkinsfile_coverage

@@ -3,99 +3,91 @@
 library 'kibana-pipeline-library'
 kibanaLibrary.load() // load from the Jenkins instance
 
-stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a little bit
-  timeout(time: 180, unit: 'MINUTES') {
-    timestamps {
-      ansiColor('xterm') {
-        catchError {
+kibanaPipeline(timeoutMinutes: 180) {
+  catchErrors {
+    withEnv([
+      'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
+    ]) {
+      parallel([
+        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
+        'x-pack-intake-agent': {
           withEnv([
-            'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
+            'NODE_ENV=test' // Needed for jest tests only
           ]) {
-            parallel([
-              'kibana-intake-agent': {
-                kibanaPipeline.intakeWorker('kibana-intake', './test/scripts/jenkins_unit.sh')()
-              },
-              'x-pack-intake-agent': {
-                withEnv([
-                  'NODE_ENV=test' // Needed for jest tests only
-                ]) {
-                  kibanaPipeline.intakeWorker('x-pack-intake', './test/scripts/jenkins_xpack.sh')()
-                }
-              },
-              'kibana-oss-agent': kibanaPipeline.withWorkers('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-                'oss-ciGroup1': kibanaPipeline.getOssCiGroupWorker(1),
-                'oss-ciGroup2': kibanaPipeline.getOssCiGroupWorker(2),
-                'oss-ciGroup3': kibanaPipeline.getOssCiGroupWorker(3),
-                'oss-ciGroup4': kibanaPipeline.getOssCiGroupWorker(4),
-                'oss-ciGroup5': kibanaPipeline.getOssCiGroupWorker(5),
-                'oss-ciGroup6': kibanaPipeline.getOssCiGroupWorker(6),
-                'oss-ciGroup7': kibanaPipeline.getOssCiGroupWorker(7),
-                'oss-ciGroup8': kibanaPipeline.getOssCiGroupWorker(8),
-                'oss-ciGroup9': kibanaPipeline.getOssCiGroupWorker(9),
-                'oss-ciGroup10': kibanaPipeline.getOssCiGroupWorker(10),
-                'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
-                'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
-              ]),
-              'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-                'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
-                'xpack-ciGroup2': kibanaPipeline.getXpackCiGroupWorker(2),
-                'xpack-ciGroup3': kibanaPipeline.getXpackCiGroupWorker(3),
-                'xpack-ciGroup4': kibanaPipeline.getXpackCiGroupWorker(4),
-                'xpack-ciGroup5': kibanaPipeline.getXpackCiGroupWorker(5),
-                'xpack-ciGroup6': kibanaPipeline.getXpackCiGroupWorker(6),
-                'xpack-ciGroup7': kibanaPipeline.getXpackCiGroupWorker(7),
-                'xpack-ciGroup8': kibanaPipeline.getXpackCiGroupWorker(8),
-                'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
-                'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
-              ]),
-            ])
-            kibanaPipeline.jobRunner('tests-l', false) {
-              kibanaPipeline.downloadCoverageArtifacts()
-              kibanaPipeline.bash(
-                '''
-                  # bootstrap from x-pack folder
-                  source src/dev/ci_setup/setup_env.sh
-                  cd x-pack
-                  yarn kbn bootstrap --prefer-offline
-                  cd ..
-                  # extract archives
-                  mkdir -p /tmp/extracted_coverage
-                  echo extracting intakes
-                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
-                  tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
-                  echo extracting kibana-oss-tests
-                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
-                  echo extracting kibana-xpack-tests
-                  tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
-                  # replace path in json files to have valid html report
-                  pwd=$(pwd)
-                  du -sh /tmp/extracted_coverage/target/kibana-coverage/
-                  echo replacing path in json files
-                  for i in {1..9}; do
-                    sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json &
-                  done
-                  wait
-                  # merge oss & x-pack reports
-                  echo merging coverage reports
-                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary
-                  yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary
-                  echo copy mocha reports
-                  mkdir -p target/kibana-coverage/mocha-combined
-                  cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined
-                ''',
-                "run `yarn kbn bootstrap && merge coverage`"
-              )
-              sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
-              kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
-              sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
-              kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
-              sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
-              kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
-            }
+            workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh')()
           }
-        }
-        kibanaPipeline.sendMail()
+        },
+        'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
+          'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
+          'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
+          'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
+          'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
+          'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
+          'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
+          'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
+          'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
+          'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
+          'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
+          'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
+          'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
+        ]),
+        'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
+          'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
+          'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
+          'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
+          'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
+          'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
+          'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
+          'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
+          'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
+          'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
+          'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
+        ]),
+      ])
+      workers.base(name: 'coverage-worker', label: 'tests-l', ramDisk: false, bootstrapped: false) {
+        kibanaPipeline.downloadCoverageArtifacts()
+        kibanaPipeline.bash(
+          '''
+            # bootstrap from x-pack folder
+            source src/dev/ci_setup/setup_env.sh
+            cd x-pack
+            yarn kbn bootstrap --prefer-offline
+            cd ..
+            # extract archives
+            mkdir -p /tmp/extracted_coverage
+            echo extracting intakes
+            tar -xzf /tmp/downloaded_coverage/coverage/kibana-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+            tar -xzf /tmp/downloaded_coverage/coverage/x-pack-intake/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+            echo extracting kibana-oss-tests
+            tar -xzf /tmp/downloaded_coverage/coverage/kibana-oss-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+            echo extracting kibana-xpack-tests
+            tar -xzf /tmp/downloaded_coverage/coverage/kibana-xpack-tests/kibana-coverage.tar.gz -C /tmp/extracted_coverage
+            # replace path in json files to have valid html report
+            pwd=$(pwd)
+            du -sh /tmp/extracted_coverage/target/kibana-coverage/
+            echo replacing path in json files
+            for i in {1..9}; do
+              sed -i "s|/dev/shm/workspace/kibana|$pwd|g" /tmp/extracted_coverage/target/kibana-coverage/functional/${i}*.json &
+            done
+            wait
+            # merge oss & x-pack reports
+            echo merging coverage reports
+            yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/jest --report-dir target/kibana-coverage/jest-combined --reporter=html --reporter=json-summary
+            yarn nyc report --temp-dir /tmp/extracted_coverage/target/kibana-coverage/functional --report-dir target/kibana-coverage/functional-combined --reporter=html --reporter=json-summary
+            echo copy mocha reports
+            mkdir -p target/kibana-coverage/mocha-combined
+            cp -r /tmp/extracted_coverage/target/kibana-coverage/mocha target/kibana-coverage/mocha-combined
+          ''',
+          "run `yarn kbn bootstrap && merge coverage`"
+        )
+        sh 'tar -czf kibana-jest-coverage.tar.gz target/kibana-coverage/jest-combined/*'
+        kibanaPipeline.uploadCoverageArtifacts("coverage/jest-combined", 'kibana-jest-coverage.tar.gz')
+        sh 'tar -czf kibana-functional-coverage.tar.gz target/kibana-coverage/functional-combined/*'
+        kibanaPipeline.uploadCoverageArtifacts("coverage/functional-combined", 'kibana-functional-coverage.tar.gz')
+        sh 'tar -czf kibana-mocha-coverage.tar.gz target/kibana-coverage/mocha-combined/*'
+        kibanaPipeline.uploadCoverageArtifacts("coverage/mocha-combined", 'kibana-mocha-coverage.tar.gz')
       }
     }
   }
+  kibanaPipeline.sendMail()
 }

.ci/Jenkinsfile_flaky

@@ -21,53 +21,47 @@ def workerFailures = []
 currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
 currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"
 
-stage("Kibana Pipeline") {
-  timeout(time: 180, unit: 'MINUTES') {
-    timestamps {
-      ansiColor('xterm') {
-        def agents = [:]
-        for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
-          def agentNumberInside = agentNumber
-          def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
-          agents["agent-${agentNumber}"] = {
-            catchError {
-              print "Agent ${agentNumberInside} - ${agentExecutions} executions"
-
-              kibanaPipeline.withWorkers('flaky-test-runner', {
-                if (NEED_BUILD) {
-                  if (!IS_XPACK) {
-                    kibanaPipeline.buildOss()
-                    if (CI_GROUP == '1') {
-                      runbld("./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
-                    }
-                  } else {
-                    kibanaPipeline.buildXpack()
-                  }
-                }
-              }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
+kibanaPipeline(timeoutMinutes: 180) {
+  def agents = [:]
+  for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
+    def agentNumberInside = agentNumber
+    def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
+    agents["agent-${agentNumber}"] = {
+      catchErrors {
+        print "Agent ${agentNumberInside} - ${agentExecutions} executions"
+
+        workers.functional('flaky-test-runner', {
+          if (NEED_BUILD) {
+            if (!IS_XPACK) {
+              kibanaPipeline.buildOss()
+              if (CI_GROUP == '1') {
+                runbld("./test/scripts/jenkins_build_kbn_tp_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
+              }
+            } else {
+              kibanaPipeline.buildXpack()
             }
           }
-        }
+        }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
+      }
+    }
+  }
 
-        parallel(agents)
+  parallel(agents)
 
-        currentBuild.description += ", Failures: ${workerFailures.size()}"
+  currentBuild.description += ", Failures: ${workerFailures.size()}"
 
-        if (workerFailures.size() > 0) {
-          print "There were ${workerFailures.size()} test suite failures."
-          print "The executions that failed were:"
-          print workerFailures.join("\n")
-          print "Please check 'Test Result' and 'Pipeline Steps' pages for more info"
-        }
-      }
-    }
+  if (workerFailures.size() > 0) {
+    print "There were ${workerFailures.size()} test suite failures."
+    print "The executions that failed were:"
+    print workerFailures.join("\n")
+    print "Please check 'Test Result' and 'Pipeline Steps' pages for more info"
   }
 }
 
 def getWorkerFromParams(isXpack, job, ciGroup) {
   if (!isXpack) {
     if (job == 'serverMocha') {
-      return kibanaPipeline.getPostBuildWorker('serverMocha', {
+      return kibanaPipeline.functionalTestProcess('serverMocha', {
         kibanaPipeline.bash(
           """
             source src/dev/ci_setup/setup_env.sh
@@ -77,20 +71,20 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
         )
       })
     } else if (job == 'firefoxSmoke') {
-      return kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld('./test/scripts/jenkins_firefox_smoke.sh', 'Execute kibana-firefoxSmoke') })
+      return kibanaPipeline.functionalTestProcess('firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh')
     } else if(job == 'visualRegression') {
-      return kibanaPipeline.getPostBuildWorker('visualRegression', { runbld('./test/scripts/jenkins_visual_regression.sh', 'Execute kibana-visualRegression') })
+      return kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh')
     } else {
-      return kibanaPipeline.getOssCiGroupWorker(ciGroup)
+      return kibanaPipeline.ossCiGroupProcess(ciGroup)
     }
   }
 
   if (job == 'firefoxSmoke') {
-    return kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld('./test/scripts/jenkins_xpack_firefox_smoke.sh', 'Execute xpack-firefoxSmoke') })
+    return kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh')
   } else if(job == 'visualRegression') {
-    return kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld('./test/scripts/jenkins_xpack_visual_regression.sh', 'Execute xpack-visualRegression') })
+    return kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')
   } else {
-    return kibanaPipeline.getXpackCiGroupWorker(ciGroup)
+    return kibanaPipeline.xpackCiGroupProcess(ciGroup)
   }
 }
 
@@ -105,10 +99,9 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor
       for(def j = 0; j < workerExecutions; j++) {
         print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
         withEnv([
-          "JOB=agent-${agentNumber}-worker-${workerNumber}-${j}",
           "REMOVE_KIBANA_INSTALL_DIR=1",
         ]) {
-          catchError {
+          catchErrors {
             try {
               worker(workerNumber)
             } catch (ex) {

.ci/es-snapshots/Jenkinsfile_build_es

@@ -26,7 +26,7 @@ timeout(time: 120, unit: 'MINUTES') {
   timestamps {
     ansiColor('xterm') {
       node('linux && immutable') {
-        catchError {
+        catchErrors {
           def VERSION
           def SNAPSHOT_ID
           def DESTINATION