Merge branch 'main' of github.com:elastic/kibana into refactor-execut…

…ion-handler-stage-1

.buildkite/ftr_configs.yml

@@ -11,12 +11,13 @@ disabled:
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.trial.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.edr_workflows.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.basic.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/serverless/config.base.edr_workflows.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.essentials.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/config.base.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.base.ts
-  - x-pack/test/security_solution_endpoint/config.base.ts
-  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
+  - x-pack/test/security_solution_endpoint/configs/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -408,6 +409,10 @@ enabled:
   - x-pack/test/security_functional/insecure_cluster_warning.config.ts
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
+  - x-pack/test/security_solution_endpoint/configs/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/configs/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/configs/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -578,9 +583,17 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/security_and_spaces/configs/ess.trial.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint_api_int/configs/serverless.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.endpoint.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/integrations.config.ts
-  - x-pack/test/security_solution_api_integration/test_suites/security_solution_endpoint/configs/serverless.integrations.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/artifacts/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/authentication/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/metadata/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/package/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/policy_response/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/resolver/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/edr_workflows/response_actions/trial_license_complete_tier/configs/serverless.config.ts

.buildkite/pipeline-utils/agent_images.ts

@@ -52,4 +52,19 @@ function getAgentImageConfig({ returnYaml = false } = {}): string | AgentImageCo
   return config;
 }
 
-export { getAgentImageConfig };
+const expandAgentQueue = (queueName: string = 'n2-4-spot') => {
+  const [kind, cores, addition] = queueName.split('-');
+  const additionalProps =
+    {
+      spot: { preemptible: true },
+      virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+    }[addition] || {};
+
+  return {
+    ...getAgentImageConfig(),
+    machineType: `${kind}-standard-${cores}`,
+    ...additionalProps,
+  };
+};
+
+export { getAgentImageConfig, expandAgentQueue };

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -16,27 +16,10 @@ import { BuildkiteClient, BuildkiteStep } from '../buildkite';
 import { CiStatsClient, TestGroupRunOrderResponse } from './client';
 
 import DISABLED_JEST_CONFIGS from '../../disabled_jest_configs.json';
-import { getAgentImageConfig } from '#pipeline-utils';
+import { expandAgentQueue } from '#pipeline-utils';
 
 type RunGroup = TestGroupRunOrderResponse['types'][0];
 
-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (process.env?.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld') {
-    const [kind, cores, spot] = queueName.split('-');
-    return {
-      ...getAgentImageConfig(),
-      machineType: `${kind}-standard-${cores}`,
-      preemptible: spot === 'spot',
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 const getRequiredEnv = (name: string) => {
   const value = process.env[name];
   if (typeof value !== 'string' || !value) {
@@ -436,7 +419,7 @@ export async function pickTestGroupRunOrder() {
             parallelism: unit.count,
             timeout_in_minutes: 120,
             key: 'jest',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -454,7 +437,7 @@ export async function pickTestGroupRunOrder() {
             parallelism: integration.count,
             timeout_in_minutes: 120,
             key: 'jest-integration',
-            agents: getAgentRule('n2-4-spot'),
+            agents: expandAgentQueue('n2-4-spot'),
             retry: {
               automatic: [
                 { exit_status: '-1', limit: 3 },
@@ -488,7 +471,7 @@ export async function pickTestGroupRunOrder() {
                   label: title,
                   command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
                   timeout_in_minutes: 90,
-                  agents: getAgentRule(queue),
+                  agents: expandAgentQueue(queue),
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,
                     ...FTR_EXTRA_ARGS,

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -7,7 +7,7 @@
  */
 
 import { groups } from './groups.json';
-import { BuildkiteStep } from '#pipeline-utils';
+import { BuildkiteStep, expandAgentQueue } from '#pipeline-utils';
 
 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@@ -32,34 +32,6 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
-// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
-/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
-const getAgentRule = (queueName: string = 'n2-4-spot') => {
-  if (
-    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
-    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
-  ) {
-    const [kind, cores, addition] = queueName.split('-');
-    const additionalProps =
-      {
-        spot: { preemptible: true },
-        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
-      }[addition] || {};
-
-    return {
-      provider: 'gcp',
-      image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-prod',
-      machineType: `${kind}-standard-${cores}`,
-      ...additionalProps,
-    };
-  } else {
-    return {
-      queue: queueName,
-    };
-  }
-};
-
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -150,7 +122,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: getAgentRule('c2-8'),
+  agents: expandAgentQueue('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -173,7 +145,7 @@ for (const testSuite of testSuites) {
       concurrency,
       concurrency_group: process.env.UUID,
       concurrency_method: 'eager',
-      agents: getAgentRule('n2-4-spot'),
+      agents: expandAgentQueue('n2-4-spot'),
       depends_on: 'build',
       timeout_in_minutes: 150,
       cancel_on_build_failing: true,
@@ -197,7 +169,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: getAgentRule(agentQueue),
+        agents: expandAgentQueue(agentQueue),
         key: `cypress-suite-${suiteIndex++}`,
         depends_on: 'build',
         timeout_in_minutes: 150,
@@ -233,7 +205,7 @@ pipeline.steps.push({
 pipeline.steps.push({
   command: 'ts-node .buildkite/pipelines/flaky_tests/post_stats_on_pr.ts',
   label: 'Post results on Github pull request',
-  agents: getAgentRule('n2-4-spot'),
+  agents: expandAgentQueue('n2-4-spot'),
   timeout_in_minutes: 15,
   retry: {
     automatic: [{ exit_status: '-1', limit: 3 }],

.buildkite/pipelines/on_merge.yml

@@ -16,28 +16,6 @@ steps:
           limit: 1
   - wait
 
-  - label: 'Triggering changes-based pipelines'
-    branches: main
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      machineType: n2-standard-2
-    # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15
-    plugins:
-      - chronotc/monorepo-diff#v2.0.4:
-          watch:
-            - path:
-                - 'versions.json'
-              config:
-                command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main'
-                label: 'Trigger pipeline deploy'
-                agents:
-                  image: family/kibana-ubuntu-2004
-                  imageProject: elastic-images-prod
-                  provider: gcp
-                  machineType: n2-standard-2
-
   - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
     label: Build Kibana Distribution and Plugins
     agents:

.buildkite/pipelines/security_solution_quality_gate/mki_periodic/mki_periodic_defend_workflows.yml

@@ -1,18 +1,143 @@
 steps:
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run
-    label: "Cypress MKI - Defend Workflows "
-    key: test_defend_workflows
-    agents:
-      image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-prod
-      provider: gcp
-      enableNestedVirtualization: true
-      localSsds: 1
-      localSsdInterface: nvme
-      machineType: n2-standard-4
-    timeout_in_minutes: 300
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: "*"
-          limit: 1
+  - group: "Cypress MKI - Defend Workflows"
+    key: cypress_test_defend_workflows
+    steps:
+      - label: "Running cypress:dw:qa:serverless:run"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh cypress:dw:qa:serverless:run
+        key: test_defend_workflows
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 300
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: "*"
+              limit: 1
+
+  - group: "API MKI - Defend Workflows"
+    key: api_test_defend_workflows
+    steps:
+#      - label: "Running edr_workflows:artifacts:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:artifacts:qa:serverless
+#        key: edr_workflows:artifacts:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:authentication:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:authentication:qa:serverless
+#        key: edr_workflows:authentication:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:metadata:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:metadata:qa:serverless
+#        key: edr_workflows:metadata:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+#
+#      - label: "Running edr_workflows:package:qa:serverless"
+#        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:package:qa:serverless
+#        key: edr_workflows:package:qa:serverless
+#        agents:
+#          image: family/kibana-ubuntu-2004
+#          imageProject: elastic-images-prod
+#          provider: gcp
+#          enableNestedVirtualization: true
+#          localSsds: 1
+#          localSsdInterface: nvme
+#          machineType: n2-standard-4
+#        timeout_in_minutes: 120
+#        retry:
+#          automatic:
+#            - exit_status: "1"
+#              limit: 1
+
+      - label: "Running edr_workflows:policy_response:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:policy_response:qa:serverless
+        key: edr_workflows:policy_response:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1
+
+      - label: "Running edr_workflows:resolver:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:resolver:qa:serverless
+        key: edr_workflows:resolver:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1
+
+      - label: "Running edr_workflows:response_actions:qa:serverless"
+        command: .buildkite/scripts/pipelines/security_solution_quality_gate/api_integration/api-integration-tests.sh edr_workflows:response_actions:qa:serverless
+        key: edr_workflows:response_actions:qa:serverless
+        agents:
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-prod
+          provider: gcp
+          enableNestedVirtualization: true
+          localSsds: 1
+          localSsdInterface: nvme
+          machineType: n2-standard-4
+        timeout_in_minutes: 120
+        retry:
+          automatic:
+            - exit_status: "1"
+              limit: 1