Merge branch 'main' into renovate/main-openapi-spec

.github/workflows/codeql.yml

@@ -32,14 +32,13 @@ jobs:
         languages: ${{ matrix.language }}
         config-file: ./.github/codeql/codeql-config.yml
 
-    # TODO: Possibly required to follow all call paths, however, when enabled, the step below runs out of memory.
-    # Possible workarounds: Apply for access to the GitHub beta where we can use beefier machines, or run it ourselves on Buildkite
-    # - name: yarn kbn bootstrap
-    #   run: |
-    #     mkdir ~/.npm-global
-    #     npm config set prefix '~/.npm-global'
-    #     export PATH=~/.npm-global/bin:$PATH
-    #     yarn kbn bootstrap --no-validate --no-vscode
+    - name: setup node
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: '.nvmrc'
+    - name: yarn kbn bootstrap
+      run: |
+        yarn kbn bootstrap --no-validate --no-vscode
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@883d8588e56d1753a8a58c1c86e88976f0c23449 # v3.26.3

docs/settings/alert-action-settings.asciidoc

@@ -141,8 +141,12 @@ A boolean value indicating that a footer with a relevant link should be added to
 `xpack.actions.enabledActionTypes` {ess-icon}::
 A list of action types that are enabled. It defaults to `["*"]`, enabling all types. The names for built-in {kib} action types are prefixed with a `.` and include: `.email`, `.index`, `.jira`, `.opsgenie`, `.pagerduty`, `.resilient`, `.server-log`, `.servicenow`, .`servicenow-itom`, `.servicenow-sir`, `.slack`, `.swimlane`, `.teams`, `.thehive`, `.tines`, `.torq`, `.xmatters`,  `.gen-ai`,  `.bedrock`, `.gemini`,  `.d3security`, and `.webhook`. An empty list `[]` will disable all action types.
 +
+--
 Disabled action types will not appear as an option when creating new connectors, but existing connectors and actions of that type will remain in {kib} and will not function.
 
+IMPORTANT: <<pre-configured-connectors,Preconfigured connectors>> are not affected by this setting.
+--
+
 `xpack.actions.microsoftExchangeUrl`::
 The URL for the Microsoft Azure Active Directory endpoint to use for MS Exchange email authentication. Default: `https://login.microsoftonline.com`.
 

examples/controls_example/public/app/react_control_example/serialized_control_group_state.ts

@@ -90,12 +90,12 @@ const initialSerializedControlGroupState = {
   } as object,
   references: [
     {
-      name: `controlGroup_${rangeSliderControlId}:${RANGE_SLIDER_CONTROL}DataView`,
+      name: `controlGroup_${rangeSliderControlId}:rangeSliderDataView`,
       type: 'index-pattern',
       id: WEB_LOGS_DATA_VIEW_ID,
     },
     {
-      name: `controlGroup_${optionsListId}:${OPTIONS_LIST_CONTROL}DataView`,
+      name: `controlGroup_${optionsListId}:optionsListDataView`,
       type: 'index-pattern',
       id: WEB_LOGS_DATA_VIEW_ID,
     },

examples/portable_dashboards_example/public/dashboard_with_controls_example.tsx

@@ -11,8 +11,7 @@ import React, { useEffect, useState } from 'react';
 import { ViewMode } from '@kbn/embeddable-plugin/public';
 import type { DataView } from '@kbn/data-views-plugin/public';
 import { EuiPanel, EuiSpacer, EuiText, EuiTitle } from '@elastic/eui';
-import { controlGroupInputBuilder } from '@kbn/controls-plugin/public';
-import { getDefaultControlGroupInput } from '@kbn/controls-plugin/common';
+import { controlGroupStateBuilder } from '@kbn/controls-plugin/public';
 import {
   AwaitingDashboardAPI,
   DashboardRenderer,
@@ -63,16 +62,15 @@ export const DashboardWithControlsExample = ({ dataView }: { dataView: DataView
       <EuiPanel hasBorder={true}>
         <DashboardRenderer
           getCreationOptions={async (): Promise<DashboardCreationOptions> => {
-            const builder = controlGroupInputBuilder;
-            const controlGroupInput = getDefaultControlGroupInput();
-            await builder.addDataControlFromField(controlGroupInput, {
+            const controlGroupState = {};
+            await controlGroupStateBuilder.addDataControlFromField(controlGroupState, {
               dataViewId: dataView.id ?? '',
               title: 'Destintion country',
               fieldName: 'geo.dest',
               width: 'medium',
               grow: false,
             });
-            await builder.addDataControlFromField(controlGroupInput, {
+            await controlGroupStateBuilder.addDataControlFromField(controlGroupState, {
               dataViewId: dataView.id ?? '',
               fieldName: 'bytes',
               width: 'medium',
@@ -85,7 +83,7 @@ export const DashboardWithControlsExample = ({ dataView }: { dataView: DataView
               getInitialInput: () => ({
                 timeRange: { from: 'now-30d', to: 'now' },
                 viewMode: ViewMode.VIEW,
-                controlGroupInput,
+                controlGroupState,
               }),
             };
           }}

packages/kbn-ebt-tools/BUILD.bazel

@@ -0,0 +1,36 @@
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+
+SRCS = glob(
+  [
+    "**/*.ts",
+    "**/*.tsx",
+  ],
+  exclude = [
+    "**/*.config.js",
+    "**/*.mock.*",
+    "**/*.test.*",
+    "**/*.stories.*",
+    "**/__snapshots__/**",
+    "**/integration_tests/**",
+    "**/mocks/**",
+    "**/scripts/**",
+    "**/storybook/**",
+    "**/test_fixtures/**",
+    "**/test_helpers/**",
+  ],
+)
+
+SHARED_DEPS = [
+  "@npm//@elastic/ebt",
+  "@npm//@elastic/apm-rum-core",
+  "@npm//react",
+  "@npm//react-router-dom",
+]
+
+js_library(
+  name = "kbn-ebt-tools",
+  package_name = "@kbn/ebt-tools",
+  srcs = ["package.json"] + SRCS,
+  deps = SHARED_DEPS,
+  visibility = ["//visibility:public"],
+)

packages/kbn-esql-utils/README.md

@@ -8,4 +8,4 @@ This package contains utilities for ES|QL.
 - *removeDropCommandsFromESQLQuery*: Use this function to remove all the occurences of the `drop` command from the query.
 - *appendToESQLQuery*: Use this function to append more pipes in an existing ES|QL query. It adds the additional commands in a new line. 
 - *appendWhereClauseToESQLQuery*: Use this function to append where clause in an existing query. 
-
+- *retieveMetadataColumns*: Use this function to get if there is a metadata option in the from command, and retrieve the columns if so

packages/kbn-esql-utils/index.ts

@@ -24,6 +24,7 @@ export {
   getTimeFieldFromESQLQuery,
   getStartEndParams,
   hasStartEndParams,
+  retieveMetadataColumns,
   TextBasedLanguages,
 } from './src';
 

packages/kbn-esql-utils/src/index.ts

@@ -16,6 +16,7 @@ export {
   removeDropCommandsFromESQLQuery,
   hasTransformationalCommand,
   getTimeFieldFromESQLQuery,
+  retieveMetadataColumns,
 } from './utils/query_parsing_helpers';
 export { appendToESQLQuery, appendWhereClauseToESQLQuery } from './utils/append_to_query';
 export {

packages/kbn-esql-utils/src/utils/query_parsing_helpers.test.ts

@@ -12,6 +12,7 @@ import {
   removeDropCommandsFromESQLQuery,
   hasTransformationalCommand,
   getTimeFieldFromESQLQuery,
+  retieveMetadataColumns,
 } from './query_parsing_helpers';
 
 describe('esql query helpers', () => {
@@ -175,4 +176,17 @@ describe('esql query helpers', () => {
       ).toBe('event.timefield');
     });
   });
+
+  describe('retieveMetadataColumns', () => {
+    it('should return metadata columns if they exist', () => {
+      expect(retieveMetadataColumns('from a  metadata _id, _ignored | eval b = 1')).toStrictEqual([
+        '_id',
+        '_ignored',
+      ]);
+    });
+
+    it('should return empty columns if metadata doesnt exist', () => {
+      expect(retieveMetadataColumns('from a | eval b = 1')).toStrictEqual([]);
+    });
+  });
 });

packages/kbn-esql-utils/src/utils/query_parsing_helpers.ts

@@ -5,7 +5,13 @@
  * in compliance with, at your election, the Elastic License 2.0 or the Server
  * Side Public License, v 1.
  */
-import type { ESQLSource, ESQLFunction, ESQLColumn, ESQLSingleAstItem } from '@kbn/esql-ast';
+import type {
+  ESQLSource,
+  ESQLFunction,
+  ESQLColumn,
+  ESQLSingleAstItem,
+  ESQLCommandOption,
+} from '@kbn/esql-ast';
 import { getAstAndSyntaxErrors, Walker, walk } from '@kbn/esql-ast';
 
 const DEFAULT_ESQL_LIMIT = 1000;
@@ -105,3 +111,14 @@ export const getTimeFieldFromESQLQuery = (esql: string) => {
 
   return column?.name;
 };
+
+export const retieveMetadataColumns = (esql: string): string[] => {
+  const { ast } = getAstAndSyntaxErrors(esql);
+  const options: ESQLCommandOption[] = [];
+
+  walk(ast, {
+    visitCommandOption: (node) => options.push(node),
+  });
+  const metadataOptions = options.find(({ name }) => name === 'metadata');
+  return metadataOptions?.args.map((column) => (column as ESQLColumn).name) ?? [];
+};

packages/kbn-ui-shared-deps-src/BUILD.bazel

@@ -24,6 +24,7 @@ webpack_cli(
     "//packages/kbn-ui-theme",
     "//packages/kbn-i18n",
     "//packages/kbn-i18n-react",
+    "//packages/kbn-ebt-tools",
     "//packages/kbn-esql-ast",
     "//packages/kbn-monaco",
     "//packages/kbn-datemath",

packages/kbn-ui-shared-deps-src/src/definitions.js

@@ -31,7 +31,7 @@ const jsFilename = 'kbn-ui-shared-deps-src.js';
 const cssDistFilename = 'kbn-ui-shared-deps-src.css';
 
 /**
- * Externals mapping inteded to be used in a webpack config
+ * Externals mapping intended to be used in a webpack config
  */
 const externals = {
   /**
@@ -102,6 +102,7 @@ const externals = {
   '@tanstack/react-query-devtools': '__kbnSharedDeps__.ReactQueryDevtools',
   '@kbn/code-editor': '__kbnSharedDeps__.KbnCodeEditor',
   '@kbn/esql-ast': '__kbnSharedDeps__.KbnEsqlAst',
+  '@kbn/ebt-tools': '__kbnSharedDeps__.KbnEbtTools',
   '@elastic/apm-rum-core': '__kbnSharedDeps__.ElasticApmRumCore',
 };
 

packages/kbn-ui-shared-deps-src/src/entry.js

@@ -75,4 +75,5 @@ export const ReactQuery = require('@tanstack/react-query');
 export const ReactQueryDevtools = require('@tanstack/react-query-devtools');
 export const KbnCodeEditor = require('@kbn/code-editor');
 export const KbnEsqlAst = require('@kbn/esql-ast');
+export const KbnEbtTools = require('@kbn/ebt-tools');
 export const ElasticApmRumCore = require('@elastic/apm-rum-core');

packages/presentation/presentation_containers/index.ts

@@ -25,23 +25,24 @@ export {
   type CanDuplicatePanels,
   type CanExpandPanels,
 } from './interfaces/panel_management';
+export {
+  canTrackContentfulRender,
+  type TrackContentfulRender,
+  type TracksQueryPerformance,
+} from './interfaces/performance_trackers';
 export {
   apiIsPresentationContainer,
+  combineCompatibleChildrenApis,
   getContainerParentFromAPI,
   listenForCompatibleApi,
-  combineCompatibleChildrenApis,
   type PanelPackage,
   type PresentationContainer,
 } from './interfaces/presentation_container';
+export { apiPublishesSettings, type PublishesSettings } from './interfaces/publishes_settings';
 export {
   apiHasSerializableState,
   type HasSerializableState,
   type HasSnapshottableState,
   type SerializedPanelState,
 } from './interfaces/serialized_state';
 export { tracksOverlays, type TracksOverlays } from './interfaces/tracks_overlays';
-export {
-  canTrackContentfulRender,
-  type TrackContentfulRender,
-  type TracksQueryPerformance,
-} from './interfaces/performance_trackers';

packages/presentation/presentation_publishing/tsconfig.json

@@ -10,6 +10,6 @@
     "@kbn/es-query",
     "@kbn/data-views-plugin",
     "@kbn/expressions-plugin",
-    "@kbn/core-execution-context-common",
+    "@kbn/core-execution-context-common"
   ]
 }

src/plugins/controls/common/control_group/control_group_persistence.ts

@@ -9,7 +9,6 @@
 import deepEqual from 'fast-deep-equal';
 import { SerializableRecord } from '@kbn/utility-types';
 
-import { v4 } from 'uuid';
 import { pick, omit, xor } from 'lodash';
 
 import {
@@ -23,7 +22,6 @@ import {
 } from './control_group_panel_diff_system';
 import { ControlGroupInput } from '..';
 import {
-  ControlsPanels,
   PersistableControlGroupInput,
   persistableControlGroupInputKeys,
   RawControlGroupAttributes,
@@ -103,32 +101,6 @@ const getPanelsAreEqual = (
   return true;
 };
 
-export const controlGroupInputToRawControlGroupAttributes = (
-  controlGroupInput: Omit<ControlGroupInput, 'id'>
-): RawControlGroupAttributes => {
-  return {
-    controlStyle: controlGroupInput.controlStyle,
-    chainingSystem: controlGroupInput.chainingSystem,
-    showApplySelections: controlGroupInput.showApplySelections,
-    panelsJSON: JSON.stringify(controlGroupInput.panels),
-    ignoreParentSettingsJSON: JSON.stringify(controlGroupInput.ignoreParentSettings),
-  };
-};
-
-export const generateNewControlIds = (controlGroupInput?: PersistableControlGroupInput) => {
-  if (!controlGroupInput?.panels) return;
-
-  const newPanelsMap: ControlsPanels = {};
-  for (const panel of Object.values(controlGroupInput.panels)) {
-    const newId = v4();
-    newPanelsMap[newId] = {
-      ...panel,
-      explicitInput: { ...panel.explicitInput, id: newId },
-    };
-  }
-  return { ...controlGroupInput, panels: newPanelsMap };
-};
-
 export const rawControlGroupAttributesToControlGroupInput = (
   rawControlGroupAttributes: RawControlGroupAttributes
 ): PersistableControlGroupInput | undefined => {

src/plugins/controls/common/index.ts

@@ -22,14 +22,12 @@ export {
   persistableControlGroupInputKeys,
 } from './control_group/types';
 export {
-  controlGroupInputToRawControlGroupAttributes,
   rawControlGroupAttributesToControlGroupInput,
   rawControlGroupAttributesToSerializable,
   serializableToRawControlGroupAttributes,
   getDefaultControlGroupPersistableInput,
   persistableControlGroupInputIsEqual,
   getDefaultControlGroupInput,
-  generateNewControlIds,
 } from './control_group/control_group_persistence';
 
 export {

src/plugins/controls/public/react_controls/control_group/components/control_group.tsx

@@ -121,6 +121,7 @@ export function ControlGroup({
       paddingSize="none"
       color={draggingId ? 'success' : 'transparent'}
       className="controlsWrapper"
+      data-test-subj="controls-group-wrapper"
     >
       <EuiFlexGroup
         gutterSize="s"

src/plugins/controls/public/react_controls/control_group/control_group_unsaved_changes_api.ts

@@ -24,11 +24,7 @@ import { apiPublishesAsyncFilters } from '../controls/data_controls/publishes_as
 
 export type ControlGroupComparatorState = Pick<
   ControlGroupRuntimeState,
-  | 'autoApplySelections'
-  | 'chainingSystem'
-  | 'ignoreParentSettings'
-  | 'initialChildControlState'
-  | 'labelPosition'
+  'autoApplySelections' | 'chainingSystem' | 'ignoreParentSettings' | 'labelPosition'
 > & {
   controlsInOrder: ControlsInOrder;
 };
@@ -38,6 +34,7 @@ export function initializeControlGroupUnsavedChanges(
   children$: PresentationContainer['children$'],
   comparators: StateComparators<ControlGroupComparatorState>,
   snapshotControlsRuntimeState: () => ControlPanelsState,
+  resetControlsUnsavedChanges: () => void,
   parentApi: unknown,
   lastSavedRuntimeState: ControlGroupRuntimeState
 ) {
@@ -47,7 +44,6 @@ export function initializeControlGroupUnsavedChanges(
       chainingSystem: lastSavedRuntimeState.chainingSystem,
       controlsInOrder: getControlsInOrder(lastSavedRuntimeState.initialChildControlState),
       ignoreParentSettings: lastSavedRuntimeState.ignoreParentSettings,
-      initialChildControlState: lastSavedRuntimeState.initialChildControlState,
       labelPosition: lastSavedRuntimeState.labelPosition,
     },
     parentApi,
@@ -72,6 +68,7 @@ export function initializeControlGroupUnsavedChanges(
       ),
       asyncResetUnsavedChanges: async () => {
         controlGroupUnsavedChanges.api.resetUnsavedChanges();
+        resetControlsUnsavedChanges();
 
         const filtersReadyPromises: Array<Promise<void>> = [];
         Object.values(children$.value).forEach((controlApi) => {