Merge branch '8.11' into backport/8.11/pr-168611

package.json

@@ -903,6 +903,7 @@
     "getopts": "^2.2.5",
     "getos": "^3.1.0",
     "globby": "^11.1.0",
+    "gpt-tokenizer": "^2.1.2",
     "handlebars": "4.7.7",
     "he": "^1.2.0",
     "history": "^4.9.0",

packages/kbn-search-connectors/types/connectors.ts

@@ -68,7 +68,7 @@ export type ConnectorConfiguration = Record<
 > & {
   extract_full_html?: { label: string; value: boolean }; // This only exists for Crawler
   use_document_level_security?: ConnectorConfigProperties;
-  use_text_extraction_service?: ConnectorConfigProperties; // This only exists for SharePoint Online
+  use_text_extraction_service?: ConnectorConfigProperties;
 };
 
 export interface ConnectorScheduling {

packages/kbn-search-connectors/types/native_connectors.ts

@@ -444,6 +444,21 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         ],
         value: 50,
       },
+      use_document_level_security: {
+        default_value: null,
+        depends_on: [],
+        display: DisplayType.TOGGLE,
+        label: ENABLE_DOCUMENT_LEVEL_SECURITY_LABEL,
+        options: [],
+        order: 6,
+        required: true,
+        sensitive: false,
+        tooltip: ENABLE_DOCUMENT_LEVEL_SECURITY_TOOLTIP,
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: false,
+      },
       use_text_extraction_service: {
         default_value: false,
         depends_on: [],
@@ -730,7 +745,6 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         ui_restrictions: ['advanced'],
         validations: [],
         value: 3,
-        display_value: 3,
       },
       use_text_extraction_service: {
         default_value: null,
@@ -1204,6 +1218,21 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         ],
         value: 100,
       },
+      use_document_level_security: {
+        default_value: null,
+        depends_on: [],
+        display: DisplayType.TOGGLE,
+        label: ENABLE_DOCUMENT_LEVEL_SECURITY_LABEL,
+        options: [],
+        order: 6,
+        required: true,
+        sensitive: false,
+        tooltip: ENABLE_DOCUMENT_LEVEL_SECURITY_TOOLTIP,
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: false,
+      },
       use_text_extraction_service: {
         default_value: false,
         depends_on: [],
@@ -1845,6 +1874,21 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         validations: [],
         value: '',
       },
+      use_document_level_security: {
+        default_value: null,
+        depends_on: [],
+        display: DisplayType.TOGGLE,
+        label: ENABLE_DOCUMENT_LEVEL_SECURITY_LABEL,
+        options: [],
+        order: 6,
+        required: true,
+        sensitive: false,
+        tooltip: ENABLE_DOCUMENT_LEVEL_SECURITY_TOOLTIP,
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: false,
+      },
     },
     features: {
       [FeatureName.SYNC_RULES]: {
@@ -2559,6 +2603,56 @@ export const NATIVE_CONNECTOR_DEFINITIONS: Record<string, NativeConnector | unde
         validations: [],
         value: true,
       },
+      enumerate_all_sites: {
+        default_value: true,
+        depends_on: [],
+        display: DisplayType.TOGGLE,
+        label: i18n.translate(
+          'searchConnectors.nativeConnectors.sharepoint_online.configuration.enumerateAllSitesLabel',
+          { defaultMessage: 'Enumerate all sites?' }
+        ),
+        options: [],
+        order: 6,
+        required: false,
+        sensitive: false,
+        tooltip: i18n.translate(
+          'searchConnectors.nativeConnectors.sharepoint_online.configuration.enumerateAllSitesTooltip',
+          {
+            defaultMessage:
+              'If enabled, sites will be fetched in bulk, then filtered down to the configured list of sites. This is efficient when syncing many sites. If disabled, each configured site will be fetched with an individual request. This is efficient when syncing fewer sites.',
+          }
+        ),
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: true,
+      },
+      fetch_subsites: {
+        default_value: false,
+        depends_on: [{ field: 'enumerate_all_sites', value: false }],
+        display: DisplayType.TOGGLE,
+        label: i18n.translate(
+          'searchConnectors.nativeConnectors.sharepoint_online.configuration.fetchSubsitesLabel',
+          {
+            defaultMessage: 'Fetch sub-sites of configured sites?',
+          }
+        ),
+        options: [],
+        order: 7,
+        required: false,
+        sensitive: false,
+        tooltip: i18n.translate(
+          'searchConnectors.nativeConnectors.sharepoint_online.configuration.fetchSubsitesTooltip',
+          {
+            defaultMessage:
+              'Whether subsites of the configured site(s) should be automatically fetched.',
+          }
+        ),
+        type: FieldType.BOOLEAN,
+        ui_restrictions: [],
+        validations: [],
+        value: true,
+      },
     },
     features: {
       [FeatureName.SYNC_RULES]: {

packages/kbn-test/jest-preset.js

@@ -105,7 +105,7 @@ module.exports = {
   transformIgnorePatterns: [
     // ignore all node_modules except monaco-editor and react-monaco-editor which requires babel transforms to handle dynamic import()
     // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
-    '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|langchain|langsmith|@cfworker))[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](byte-size|monaco-editor|monaco-yaml|vscode-languageserver-types|react-monaco-editor|d3-interpolate|d3-color|langchain|langsmith|@cfworker|gpt-tokenizer))[/\\\\].+\\.js$',
     'packages/kbn-pm/dist/index.js',
     '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$',
     '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$',

packages/kbn-test/jest_integration_node/jest-preset.js

@@ -22,7 +22,7 @@ module.exports = {
   // An array of regexp pattern strings that are matched against, matched files will skip transformation:
   transformIgnorePatterns: [
     // since ESM modules are not natively supported in Jest yet (https://github.com/facebook/jest/issues/4842)
-    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))[/\\\\].+\\.js$',
+    '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith|gpt-tokenizer))[/\\\\].+\\.js$',
     '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/[/\\\\].+\\.js$',
     '[/\\\\]node_modules(?![\\/\\\\](langchain|langsmith))/dist/util/[/\\\\].+\\.js$',
   ],

packages/kbn-text-based-editor/src/editor_footer.tsx

@@ -58,14 +58,12 @@ export function ErrorsWarningsPopover({
   isPopoverOpen,
   items,
   type,
-  refreshErrors,
   setIsPopoverOpen,
   onErrorClick,
 }: {
   isPopoverOpen: boolean;
   items: MonacoError[];
   type: 'error' | 'warning';
-  refreshErrors: () => void;
   setIsPopoverOpen: (flag: boolean) => void;
   onErrorClick: (error: MonacoError) => void;
 }) {
@@ -89,7 +87,6 @@ export function ErrorsWarningsPopover({
                   }
                 `}
                 onClick={() => {
-                  refreshErrors();
                   setIsPopoverOpen(!isPopoverOpen);
                 }}
               >
@@ -184,7 +181,6 @@ export const EditorFooter = memo(function EditorFooter({
               isPopoverOpen={isPopoverOpen}
               items={errors}
               type="error"
-              refreshErrors={refreshErrors}
               setIsPopoverOpen={setIsPopoverOpen}
               onErrorClick={onErrorClick}
             />
@@ -194,7 +190,6 @@ export const EditorFooter = memo(function EditorFooter({
               isPopoverOpen={isPopoverOpen}
               items={warning}
               type="warning"
-              refreshErrors={refreshErrors}
               setIsPopoverOpen={setIsPopoverOpen}
               onErrorClick={onErrorClick}
             />

src/plugins/data/server/search/strategies/esql_search/esql_search_strategy.ts

@@ -11,6 +11,8 @@ import type { Logger } from '@kbn/core/server';
 import { getKbnServerError, KbnServerError } from '@kbn/kibana-utils-plugin/server';
 import type { ISearchStrategy } from '../../types';
 
+const ES_TIMEOUT_IN_MS = 120000;
+
 export const esqlSearchStrategyProvider = (
   logger: Logger,
   useInternalUser: boolean = false
@@ -23,6 +25,17 @@ export const esqlSearchStrategyProvider = (
    * @returns `Observable<IEsSearchResponse<any>>`
    */
   search: (request, { abortSignal, ...options }, { esClient, uiSettingsClient }) => {
+    const abortController = new AbortController();
+    // We found out that there are cases where we are not aborting correctly
+    // For this reasons we want to manually cancel he abort signal after 2 mins
+
+    abortSignal?.addEventListener('abort', () => {
+      abortController.abort();
+    });
+
+    // Also abort after two mins
+    setTimeout(() => abortController.abort(), ES_TIMEOUT_IN_MS);
+
     // Only default index pattern type is supported here.
     // See ese for other type support.
     if (request.indexType) {
@@ -41,8 +54,10 @@ export const esqlSearchStrategyProvider = (
             },
           },
           {
-            signal: abortSignal,
+            signal: abortController.signal,
             meta: true,
+            // we don't want the ES client to retry (default value is 3)
+            maxRetries: 0,
           }
         );
         return {

src/plugins/unified_histogram/public/chart/histogram.tsx

@@ -107,6 +107,7 @@ export function Histogram({
     timeRange: getTimeRange(),
     timeInterval,
     isPlainRecord,
+    timeField: dataView.timeFieldName,
   });
   const chartRef = useRef<HTMLDivElement | null>(null);
   const { height: containerHeight, width: containerWidth } = useResizeObserver(chartRef.current);

src/plugins/unified_histogram/public/chart/hooks/use_time_range.test.tsx

@@ -245,6 +245,7 @@ describe('useTimeRange', () => {
         timeRange,
         timeInterval,
         isPlainRecord: true,
+        timeField: '@timestamp',
       })
     );
     expect(result.current.timeRangeDisplay).toMatchInlineSnapshot(`
@@ -267,4 +268,17 @@ describe('useTimeRange', () => {
       </EuiText>
     `);
   });
+
+  it('should not render a text for text based languages when not timeField is provided', () => {
+    const { result } = renderHook(() =>
+      useTimeRange({
+        uiSettings,
+        bucketInterval,
+        timeRange,
+        timeInterval,
+        isPlainRecord: true,
+      })
+    );
+    expect(result.current.timeRangeDisplay).toBeNull();
+  });
 });

src/plugins/unified_histogram/public/chart/hooks/use_time_range.tsx

@@ -21,12 +21,14 @@ export const useTimeRange = ({
   timeRange: { from, to },
   timeInterval,
   isPlainRecord,
+  timeField,
 }: {
   uiSettings: IUiSettingsClient;
   bucketInterval?: UnifiedHistogramBucketInterval;
   timeRange: TimeRange;
   timeInterval?: string;
   isPlainRecord?: boolean;
+  timeField?: string;
 }) => {
   const dateFormat = useMemo(() => uiSettings.get('dateFormat'), [uiSettings]);
 
@@ -44,6 +46,10 @@ export const useTimeRange = ({
   );
 
   const timeRangeText = useMemo(() => {
+    if (!timeField && isPlainRecord) {
+      return '';
+    }
+
     const timeRange = {
       from: dateMath.parse(from),
       to: dateMath.parse(to, { roundUp: true }),
@@ -70,18 +76,18 @@ export const useTimeRange = ({
         });
 
     return `${toMoment(timeRange.from)} - ${toMoment(timeRange.to)} ${intervalText}`;
-  }, [bucketInterval?.description, from, isPlainRecord, timeInterval, to, toMoment]);
+  }, [bucketInterval?.description, from, isPlainRecord, timeField, timeInterval, to, toMoment]);
 
   const { euiTheme } = useEuiTheme();
   const timeRangeCss = css`
     padding: 0 ${euiTheme.size.s} 0 ${euiTheme.size.s};
   `;
 
-  let timeRangeDisplay = (
+  let timeRangeDisplay = timeRangeText ? (
     <EuiText size="xs" textAlign="center" css={timeRangeCss}>
       {timeRangeText}
     </EuiText>
-  );
+  ) : null;
 
   if (bucketInterval?.scaled) {
     const toolTipTitle = i18n.translate('unifiedHistogram.timeIntervalWithValueWarning', {