Merge branch 'main' into bsearch/abort-again

.buildkite/ftr_configs.yml

@@ -225,7 +225,6 @@ enabled:
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
@@ -461,6 +460,8 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts
 
 
 

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipelines/artifacts.yml

@@ -71,6 +71,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
     soft_fail:

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'

.buildkite/pipelines/security_solution/base.yml

@@ -0,0 +1,36 @@
+steps:
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless
+    label: 'Serverless MKI QA Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+    label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+    label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/build_kibana.sh

@@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross
 is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages")
 is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime")
 is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts")
+is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets")
 
 echo "> node scripts/build" "${BUILD_ARGS[@]}"
 node scripts/build "${BUILD_ARGS[@]}"
@@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-initialize \
   --skip-generic-folders \
   --skip-platform-folders \
+  --skip-cdn-assets \
   --skip-archives \
   --docker-images \
   --docker-tag-qualifier="$GIT_COMMIT" \

.buildkite/scripts/lifecycle/post_command.sh

@@ -14,6 +14,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-security-solution/**/management/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -137,8 +137,8 @@ const uploadPipeline = (pipelineContent: string | object) => {
     }
 
     if (
-      GITHUB_PR_LABELS.includes('ci:project-deploy-es') ||
-      GITHUB_PR_LABELS.includes('ci:project-deploy-oblt') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-security')
     ) {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));

.buildkite/scripts/pipelines/security_solution_quality_gate/mki_security_solution_cypress.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No target script from the package.json file, is supplied"
+    exit 1
+fi
+
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+cd x-pack/test/security_solution_cypress
+set +e
+
+QA_API_KEY=$(retry 5 5 vault read -field=qa_api_key secret/kibana-issues/dev/security-solution-qg-enc-key)
+
+CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,19 +1,5 @@
 #!/bin/bash
-set -euo pipefail
-
-source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
-.buildkite/scripts/bootstrap.sh
-
-export JOB=kibana-security-solution-chrome
 
-buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
-
-echo "--- Serverless Security Second Quality Gate"
-cd x-pack/test/security_solution_cypress
-set +e
-
-VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
-ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+set -euo pipefail
 
-CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status
+ts-node .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+import fs from 'fs';
+
+const getPipeline = (filename: string, removeSteps = true) => {
+  const str = fs.readFileSync(filename).toString();
+  return removeSteps ? str.replace(/^steps:/, '') : str;
+};
+
+const uploadPipeline = (pipelineContent: string | object) => {
+  const str =
+    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
+
+  execSync('buildkite-agent pipeline upload', {
+    input: str,
+    stdio: ['pipe', 'inherit', 'inherit'],
+  });
+};
+
+(async () => {
+  try {
+    const pipeline = [];
+
+    pipeline.push(getPipeline('.buildkite/pipelines/security_solution/base.yml', false));
+    // remove duplicated steps
+    uploadPipeline([...new Set(pipeline)].join('\n'));
+  } catch (ex) {
+    console.error('PR pipeline generation error', ex.message);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -11,25 +11,43 @@ KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
 
 echo "--- Create contexts"
 mkdir -p target
-node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --docker-context-use-local-artifact "${BUILD_ARGS[@]}"
+node scripts/build --skip-initialize --skip-generic-folders --skip-platform-folders --skip-archives --skip-cdn-assets --docker-context-use-local-artifact
 
 echo "--- Setup context"
 DOCKER_BUILD_FOLDER=$(mktemp -d)
+DOCKER_BUILD_ARGS=''
+case $KIBANA_DOCKER_CONTEXT in
+  default)
+    DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
+  cloud)
+    DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
+  ubi8)
+    DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
+  ubi9)
+    DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
+  ironbank)
+    DOCKER_CONTEXT_FILE="kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"
+    DOCKER_BUILD_ARGS='--build-arg BASE_REGISTRY=docker.elastic.co --build-arg BASE_IMAGE=ubi9/ubi --build-arg BASE_TAG=latest'
+
+    # See src/dev/build/tasks/os_packages/docker_generator/templates/ironbank/hardening_manifest.yaml
+    curl --retry 8 -sS -L -o "$DOCKER_BUILD_FOLDER/tini" "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64"
+    echo "8053cc21a3a9bdd6042a495349d1856ae8d3b3e7664c9654198de0087af031f5d41139ec85a2f5d7d2febd22ec3f280767ff23b9d5f63d490584e2b7ad3c218c  $DOCKER_BUILD_FOLDER/tini" | sha512sum -c -
+    curl --retry 8 -sS -L -o "$DOCKER_BUILD_FOLDER/NotoSansCJK-Regular.ttc" https://github.com/googlefonts/noto-cjk/raw/NotoSansV2.001/NotoSansCJK-Regular.ttc
+    echo "0ce56bde1853fed3e53282505bac65707385275a27816c29712ab04c187aa249797c82c58759b2b36c210d4e2683eda92359d739a8045cb8385c2c34d37cc9e1  $DOCKER_BUILD_FOLDER/NotoSansCJK-Regular.ttc" | sha512sum -c -
+
+    echo "Warning: this will build an approximation of the Iron Bank context, using a swapped base image"
+  ;;
+esac
 
-if [[ "$KIBANA_DOCKER_CONTEXT" == "default" ]]; then
-  DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
-elif [[ "$KIBANA_DOCKER_CONTEXT" == "cloud" ]]; then
-  DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
-elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi8" ]]; then
-  DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
-elif [[ "$KIBANA_DOCKER_CONTEXT" == "ubi9" ]]; then
-  DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
-fi
 
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
 download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
-docker build .
+docker build $DOCKER_BUILD_ARGS .

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -37,7 +37,8 @@ node scripts/build \
   --skip-docker-ubuntu \
   --skip-docker-ubi \
   --skip-docker-cloud \
-  --skip-docker-contexts
+  --skip-docker-contexts \
+  --skip-cdn-assets
 
 echo "--- Tag images"
 docker rmi "$KIBANA_IMAGE"

.buildkite/scripts/steps/cloud/purge_deployments.sh

@@ -2,4 +2,8 @@
 
 set -euo pipefail
 
+echo '--- Purging Cloud deployments'
 ts-node .buildkite/scripts/steps/cloud/purge_deployments.ts
+
+echo '--- Purging Project deployments'
+ts-node .buildkite/scripts/steps/cloud/purge_projects.ts