Merge branch 'main' into fix-yarn-env-vars-for-lib-mirrors-override

.buildkite/pipelines/pull_request/security_solution.yml

@@ -12,3 +12,14 @@ steps:
           limit: 1
     artifact_paths:
       - "target/kibana-security-solution/**/*"
+
+  - command: .buildkite/scripts/steps/functional/security_solution_burn.sh
+    label: 'Security Solution Cypress tests, burning changed specs'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 120
+    parallelism: 1
+    soft_fail: true
+    artifact_paths:
+      - "target/kibana-security-solution/**/*"

.buildkite/scripts/steps/functional/security_solution_burn.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'false'
+
+echo "--- Security Solution Cypress tests, burning changed specs (Chrome)"
+
+yarn --cwd x-pack/plugins/security_solution cypress:changed-specs-only

package.json

@@ -837,7 +837,7 @@
     "deep-freeze-strict": "^1.1.1",
     "deepmerge": "^4.2.2",
     "del": "^6.1.0",
-    "elastic-apm-node": "^3.49.0",
+    "elastic-apm-node": "^3.49.1",
     "email-addresses": "^5.0.0",
     "execa": "^4.0.2",
     "expiry-js": "0.1.7",
@@ -1444,6 +1444,7 @@
     "faker": "^5.1.0",
     "fetch-mock": "^7.3.9",
     "file-loader": "^4.2.0",
+    "find-cypress-specs": "^1.35.1",
     "form-data": "^4.0.0",
     "geckodriver": "^4.0.0",
     "gulp-brotli": "^3.0.0",
@@ -1526,6 +1527,7 @@
     "sinon": "^7.4.2",
     "sort-package-json": "^1.53.1",
     "source-map": "^0.7.4",
+    "spec-change": "^1.7.1",
     "string-replace-loader": "^2.2.0",
     "style-loader": "^1.1.3",
     "stylelint": "^14.9.1",

src/plugins/data_views/common/data_views/data_views.test.ts

@@ -629,4 +629,36 @@ describe('IndexPatterns', () => {
       expect(apiClient.getFieldsForWildcard.mock.calls[0][0].allowNoIndex).toBe(true);
     });
   });
+
+  describe('getExistingIndices', () => {
+    test('getExistingIndices returns the valid matched indices', async () => {
+      apiClient.getFieldsForWildcard = jest
+        .fn()
+        .mockResolvedValueOnce({ fields: ['length'] })
+        .mockResolvedValue({ fields: [] });
+      const patternList = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
+      expect(apiClient.getFieldsForWildcard).toBeCalledTimes(2);
+      expect(patternList.length).toBe(1);
+    });
+
+    test('getExistingIndices checks the positive pattern if provided with a negative pattern', async () => {
+      const mockFn = jest.fn().mockResolvedValue({ fields: ['length'] });
+      apiClient.getFieldsForWildcard = mockFn;
+      const patternList = await indexPatterns.getExistingIndices(['-filebeat-*', 'filebeat-*']);
+      expect(mockFn.mock.calls[0][0].pattern).toEqual('filebeat-*');
+      expect(mockFn.mock.calls[1][0].pattern).toEqual('filebeat-*');
+      expect(patternList).toEqual(['-filebeat-*', 'filebeat-*']);
+    });
+
+    test('getExistingIndices handles an error', async () => {
+      apiClient.getFieldsForWildcard = jest
+        .fn()
+        .mockImplementationOnce(async () => {
+          throw new DataViewMissingIndices('Catch me if you can!');
+        })
+        .mockImplementation(() => Promise.resolve({ fields: ['length'] }));
+      const patternList = await indexPatterns.getExistingIndices(['packetbeat-*', 'filebeat-*']);
+      expect(patternList).toEqual(['filebeat-*']);
+    });
+  });
 });

src/plugins/data_views/common/data_views/data_views.ts

@@ -7,10 +7,12 @@
  */
 
 import { i18n } from '@kbn/i18n';
+import { defer, from } from 'rxjs';
 import type { PublicMethodsOf } from '@kbn/utility-types';
 import { castEsToKbnFieldTypeName } from '@kbn/field-types';
 import { FieldFormatsStartCommon, FORMATS_UI_SETTINGS } from '@kbn/field-formats-plugin/common';
 import { v4 as uuidv4 } from 'uuid';
+import { rateLimitingForkJoin } from './utils';
 import { PersistenceAPI } from '../types';
 
 import { createDataViewCache } from '.';
@@ -236,6 +238,12 @@ export interface DataViewsServicePublicMethods {
    * @param options - options for getting fields
    */
   getFieldsForWildcard: (options: GetFieldsOptions) => Promise<FieldSpec[]>;
+  /**
+   * Get existing index pattern list by providing string array index pattern list.
+   * @param indices - index pattern list
+   * @returns index pattern list of index patterns that match indices
+   */
+  getExistingIndices: (indices: string[]) => Promise<string[]>;
   /**
    * Get list of data view ids.
    * @param refresh - clear cache and fetch from server
@@ -505,6 +513,41 @@ export class DataViewsService {
     return fields;
   };
 
+  /**
+   * Get existing index pattern list by providing string array index pattern list.
+   * @param indices index pattern list
+   * @returns index pattern list
+   */
+  getExistingIndices = async (indices: string[]): Promise<string[]> => {
+    const indicesObs = indices.map((pattern) => {
+      // when checking a negative pattern, check if the positive pattern exists
+      const indexToQuery = pattern.trim().startsWith('-')
+        ? pattern.trim().substring(1)
+        : pattern.trim();
+      return defer(() =>
+        from(
+          this.getFieldsForWildcard({
+            // check one field to keep request fast/small
+            fields: ['_id'],
+            // true so no errors thrown in browser
+            allowNoIndex: true,
+            pattern: indexToQuery,
+          })
+        )
+      );
+    });
+
+    return new Promise<boolean[]>((resolve) => {
+      rateLimitingForkJoin(indicesObs, 3, []).subscribe((value) => {
+        resolve(value.map((v) => v.length > 0));
+      });
+    })
+      .then((allPatterns: boolean[]) =>
+        indices.filter((pattern, i, self) => self.indexOf(pattern) === i && allPatterns[i])
+      )
+      .catch(() => indices);
+  };
+
   /**
    * Get field list by providing an index patttern (or spec).
    * @param options options for getting field list

src/plugins/data_views/common/data_views/utils.ts

@@ -6,6 +6,8 @@
  * Side Public License, v 1.
  */
 
+import { catchError, from, Observable, of } from 'rxjs';
+import { mergeMap, last, map, toArray } from 'rxjs/operators';
 import type { RuntimeField, RuntimeFieldSpec, RuntimePrimitiveTypes } from '../types';
 
 export const removeFieldAttrs = (runtimeField: RuntimeField): RuntimeFieldSpec => {
@@ -23,3 +25,30 @@ export const removeFieldAttrs = (runtimeField: RuntimeField): RuntimeFieldSpec =
     ...fieldsTypeOnly,
   };
 };
+
+const MAX_CONCURRENT_REQUESTS = 3;
+/**
+ * Helper function to run forkJoin
+ * with restrictions on how many input observables can be subscribed to concurrently
+ */
+export function rateLimitingForkJoin<T>(
+  observables: Array<Observable<T>>,
+  maxConcurrentRequests = MAX_CONCURRENT_REQUESTS,
+  failValue: T
+): Observable<T[]> {
+  return from(observables).pipe(
+    mergeMap(
+      (observable, index) =>
+        observable.pipe(
+          last(),
+          map((value) => ({ index, value })),
+          catchError(() => of({ index, value: failValue }))
+        ),
+      maxConcurrentRequests
+    ),
+    toArray(),
+    map((indexedObservables) =>
+      indexedObservables.sort((l, r) => l.index - r.index).map((obs) => obs.value)
+    )
+  );
+}

typings/index.d.ts

@@ -20,3 +20,5 @@ declare module 'react-syntax-highlighter/dist/cjs/prism-light';
 declare module 'monaco-editor/esm/vs/basic-languages/markdown/markdown';
 declare module 'monaco-editor/esm/vs/basic-languages/css/css';
 declare module 'monaco-editor/esm/vs/basic-languages/yaml/yaml';
+
+declare module 'find-cypress-specs';

x-pack/plugins/enterprise_search/jest.config.dev.js

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../',
+  projects: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/**/jest.config.js',
+    '<rootDir>/x-pack/plugins/enterprise_search/common/**/jest.config.js',
+    '<rootDir>/x-pack/plugins/enterprise_search/server/**/jest.config.js',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/analytics/jest.config.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: ['<rootDir>/x-pack/plugins/enterprise_search/public/applications/analytics'],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/analytics',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/app_search/jest.config.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: ['<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search'],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/app_search',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/applications/jest.config.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: ['<rootDir>/x-pack/plugins/enterprise_search/public/applications/applications'],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/applications',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/elasticsearch/jest.config.js

@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: ['<rootDir>/x-pack/plugins/enterprise_search/public/applications/elasticsearch'],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/elasticsearch',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/jest.config.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content',
+  ],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};

x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview/jest.config.js

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+module.exports = {
+  preset: '@kbn/test',
+  rootDir: '../../../../../..',
+  roots: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview',
+  ],
+  collectCoverage: true,
+  coverageReporters: ['text', 'html'],
+  collectCoverageFrom: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/**/*.{ts,tsx}',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/server/*.ts',
+    '!<rootDir>/x-pack/plugins/enterprise_search/public/applications/test_helpers/**/*.{ts,tsx}',
+  ],
+  coverageDirectory:
+    '<rootDir>/target/kibana-coverage/jest/x-pack/plugins/enterprise_search/public/applications/enterprise_search_overview',
+  modulePathIgnorePatterns: [
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/app_search/cypress',
+    '<rootDir>/x-pack/plugins/enterprise_search/public/applications/workplace_search/cypress',
+  ],
+};