From 348472cd59e6730d2ec19db812912ec819311ade Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 1 Mar 2021 10:19:13 -0700 Subject: [PATCH 01/24] [Reporting] Clean up test helpers and mocks (#92550) * [Reporting] Clean up logger instances and mocks * revert logging changes, just keep test changes * remove fluff * clean up too much logger.clone Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/config/create_config.test.ts | 110 +++++++++--------- .../export_types/png/execute_job/index.ts | 5 +- .../printable_pdf/execute_job/index.ts | 7 +- .../server/routes/diagnostic/browser.test.ts | 10 +- .../server/routes/diagnostic/config.test.ts | 8 +- .../routes/diagnostic/screenshot.test.ts | 10 +- .../server/routes/generation.test.ts | 9 +- .../reporting/server/routes/jobs.test.ts | 13 ++- .../create_mock_reportingplugin.ts | 39 +++---- .../server/test_helpers/create_mock_server.ts | 35 ------ .../reporting/server/test_helpers/index.ts | 2 +- x-pack/plugins/reporting/server/types.ts | 4 +- 12 files changed, 113 insertions(+), 139 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/test_helpers/create_mock_server.ts diff --git a/x-pack/plugins/reporting/server/config/create_config.test.ts b/x-pack/plugins/reporting/server/config/create_config.test.ts index a940e62a7225b1..c649fff446a224 100644 --- a/x-pack/plugins/reporting/server/config/create_config.test.ts +++ b/x-pack/plugins/reporting/server/config/create_config.test.ts @@ -5,36 +5,11 @@ * 2.0. */ -import * as Rx from 'rxjs'; import { CoreSetup, PluginInitializerContext } from 'src/core/server'; +import { coreMock } from 'src/core/server/mocks'; import { LevelLogger } from '../lib'; +import { createMockConfigSchema } from '../test_helpers'; import { createConfig$ } from './create_config'; -import { ReportingConfigType } from './schema'; - -interface KibanaServer { - hostname?: string; - port?: number; - protocol?: string; -} - -const makeMockInitContext = (config: { - capture?: Partial; - encryptionKey?: string; - kibanaServer: Partial; -}): PluginInitializerContext => - ({ - config: { - create: () => - Rx.of({ - ...config, - capture: config.capture || { browser: { chromium: { disableSandbox: false } } }, - kibanaServer: config.kibanaServer || {}, - }), - }, - } as PluginInitializerContext); - -const makeMockCoreSetup = (serverInfo: KibanaServer): CoreSetup => - ({ http: { getServerInfo: () => serverInfo } } as any); describe('Reporting server createConfig$', () => { let mockCoreSetup: CoreSetup; @@ -42,10 +17,10 @@ describe('Reporting server createConfig$', () => { let mockLogger: LevelLogger; beforeEach(() => { - mockCoreSetup = makeMockCoreSetup({ hostname: 'kibanaHost', port: 5601, protocol: 'http' }); - mockInitContext = makeMockInitContext({ - kibanaServer: {}, - }); + mockCoreSetup = coreMock.createSetup(); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ kibanaServer: {} }) + ); mockLogger = ({ warn: jest.fn(), debug: jest.fn(), @@ -58,14 +33,18 @@ describe('Reporting server createConfig$', () => { }); it('creates random encryption key and default config using host, protocol, and port from server info', async () => { + mockInitContext = coreMock.createPluginInitializerContext({ + ...createMockConfigSchema({ kibanaServer: {} }), + encryptionKey: undefined, + }); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); expect(result.encryptionKey).toMatch(/\S{32,}/); // random 32 characters expect(result.kibanaServer).toMatchInlineSnapshot(` Object { - "hostname": "kibanaHost", - "port": 5601, + "hostname": "localhost", + "port": 80, "protocol": "http", } `); @@ -76,10 +55,11 @@ describe('Reporting server createConfig$', () => { }); it('uses the user-provided encryption key', async () => { - mockInitContext = makeMockInitContext({ - encryptionKey: 'iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii', - kibanaServer: {}, - }); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ + encryptionKey: 'iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii', + }) + ); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); expect(result.encryptionKey).toMatch('iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii'); @@ -87,14 +67,16 @@ describe('Reporting server createConfig$', () => { }); it('uses the user-provided encryption key, reporting kibanaServer settings to override server info', async () => { - mockInitContext = makeMockInitContext({ - encryptionKey: 'iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii', - kibanaServer: { - hostname: 'reportingHost', - port: 5677, - protocol: 'httpsa', - }, - }); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ + encryptionKey: 'iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii', + kibanaServer: { + hostname: 'reportingHost', + port: 5677, + protocol: 'httpsa', + }, + }) + ); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); @@ -103,26 +85,36 @@ describe('Reporting server createConfig$', () => { "capture": Object { "browser": Object { "chromium": Object { - "disableSandbox": false, + "disableSandbox": true, }, }, }, + "csv": Object {}, "encryptionKey": "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii", + "index": ".reporting", "kibanaServer": Object { "hostname": "reportingHost", "port": 5677, "protocol": "httpsa", }, + "queue": Object { + "indexInterval": "week", + "pollEnabled": true, + "pollInterval": 3000, + "timeout": 120000, + }, } `); expect((mockLogger.warn as any).mock.calls.length).toBe(0); }); it('uses user-provided disableSandbox: false', async () => { - mockInitContext = makeMockInitContext({ - encryptionKey: '888888888888888888888888888888888', - capture: { browser: { chromium: { disableSandbox: false } } }, - } as ReportingConfigType); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ + encryptionKey: '888888888888888888888888888888888', + capture: { browser: { chromium: { disableSandbox: false } } }, + }) + ); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); @@ -131,10 +123,12 @@ describe('Reporting server createConfig$', () => { }); it('uses user-provided disableSandbox: true', async () => { - mockInitContext = makeMockInitContext({ - encryptionKey: '888888888888888888888888888888888', - capture: { browser: { chromium: { disableSandbox: true } } }, - } as ReportingConfigType); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ + encryptionKey: '888888888888888888888888888888888', + capture: { browser: { chromium: { disableSandbox: true } } }, + }) + ); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); @@ -143,9 +137,11 @@ describe('Reporting server createConfig$', () => { }); it('provides a default for disableSandbox', async () => { - mockInitContext = makeMockInitContext({ - encryptionKey: '888888888888888888888888888888888', - } as ReportingConfigType); + mockInitContext = coreMock.createPluginInitializerContext( + createMockConfigSchema({ + encryptionKey: '888888888888888888888888888888888', + }) + ); const mockConfig$: any = mockInitContext.config.create(); const result = await createConfig$(mockCoreSetup, mockConfig$, mockLogger).toPromise(); diff --git a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts index 4f439f494015d8..c65e7bdf7a5cae 100644 --- a/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/png/execute_job/index.ts @@ -25,7 +25,6 @@ export const runTaskFnFactory: RunTaskFnFactory< > = function executeJobFactoryFn(reporting, parentLogger) { const config = reporting.getConfig(); const encryptionKey = config.get('encryptionKey'); - const logger = parentLogger.clone([PNG_JOB_TYPE, 'execute']); return async function runTask(jobId, job, cancellationToken) { const apmTrans = apm.startTransaction('reporting execute_job png', 'reporting'); @@ -33,9 +32,9 @@ export const runTaskFnFactory: RunTaskFnFactory< let apmGeneratePng: { end: () => void } | null | undefined; const generatePngObservable = await generatePngObservableFactory(reporting); - const jobLogger = logger.clone([jobId]); + const jobLogger = parentLogger.clone([PNG_JOB_TYPE, 'execute', jobId]); const process$: Rx.Observable = Rx.of(1).pipe( - mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, logger)), + mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), mergeMap((conditionalHeaders) => { diff --git a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts index 2c4ad288e681b0..8e215f87b52e0b 100644 --- a/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts +++ b/x-pack/plugins/reporting/server/export_types/printable_pdf/execute_job/index.ts @@ -28,20 +28,19 @@ export const runTaskFnFactory: RunTaskFnFactory< const encryptionKey = config.get('encryptionKey'); return async function runTask(jobId, job, cancellationToken) { - const logger = parentLogger.clone([PDF_JOB_TYPE, 'execute-job', jobId]); + const jobLogger = parentLogger.clone([PDF_JOB_TYPE, 'execute-job', jobId]); const apmTrans = apm.startTransaction('reporting execute_job pdf', 'reporting'); const apmGetAssets = apmTrans?.startSpan('get_assets', 'setup'); let apmGeneratePdf: { end: () => void } | null | undefined; const generatePdfObservable = await generatePdfObservableFactory(reporting); - const jobLogger = logger.clone([jobId]); const process$: Rx.Observable = Rx.of(1).pipe( - mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, logger)), + mergeMap(() => decryptJobHeaders(encryptionKey, job.headers, jobLogger)), map((decryptedHeaders) => omitBlockedHeaders(decryptedHeaders)), map((filteredHeaders) => getConditionalHeaders(config, filteredHeaders)), mergeMap((conditionalHeaders) => - getCustomLogo(reporting, conditionalHeaders, job.spaceId, logger) + getCustomLogo(reporting, conditionalHeaders, job.spaceId, jobLogger) ), mergeMap(({ logo, conditionalHeaders }) => { const urls = getFullUrls(config, job); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts index 1267a7a9a69d74..d80be2d7f0f429 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/browser.test.ts @@ -11,7 +11,11 @@ import { createInterface } from 'readline'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '../..'; -import { createMockLevelLogger, createMockReportingCore } from '../../test_helpers'; +import { + createMockLevelLogger, + createMockPluginSetup, + createMockReportingCore, +} from '../../test_helpers'; import { registerDiagnoseBrowser } from './browser'; import type { ReportingRequestHandlerContext } from '../../types'; @@ -55,12 +59,12 @@ describe('POST /diagnose/browser', () => { () => ({}) ); - const mockSetupDeps = ({ + const mockSetupDeps = createMockPluginSetup({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, }, router: httpSetup.createRouter(''), - } as unknown) as any; + }); core = await createMockReportingCore(config, mockSetupDeps); diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts index 3f95dd86d2af5a..f35d8f5910da03 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/config.test.ts @@ -9,7 +9,11 @@ import { UnwrapPromise } from '@kbn/utility-types'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '../..'; -import { createMockReportingCore, createMockLevelLogger } from '../../test_helpers'; +import { + createMockReportingCore, + createMockLevelLogger, + createMockPluginSetup, +} from '../../test_helpers'; import { registerDiagnoseConfig } from './config'; import type { ReportingRequestHandlerContext } from '../../types'; @@ -33,7 +37,7 @@ describe('POST /diagnose/config', () => { () => ({}) ); - mockSetupDeps = ({ + mockSetupDeps = createMockPluginSetup({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, }, diff --git a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts index 754a116af79991..6c723764d9f0ad 100644 --- a/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts +++ b/x-pack/plugins/reporting/server/routes/diagnostic/screenshot.test.ts @@ -9,7 +9,11 @@ import { UnwrapPromise } from '@kbn/utility-types'; import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '../..'; -import { createMockReportingCore, createMockLevelLogger } from '../../test_helpers'; +import { + createMockReportingCore, + createMockLevelLogger, + createMockPluginSetup, +} from '../../test_helpers'; import { registerDiagnoseScreenshot } from './screenshot'; import type { ReportingRequestHandlerContext } from '../../types'; @@ -52,12 +56,12 @@ describe('POST /diagnose/screenshot', () => { () => ({}) ); - const mockSetupDeps = ({ + const mockSetupDeps = createMockPluginSetup({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, }, router: httpSetup.createRouter(''), - } as unknown) as any; + }); core = await createMockReportingCore(config, mockSetupDeps); }); diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index 70a5e2475ab926..490b9b7e45664c 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -12,7 +12,8 @@ import { setupServer } from 'src/core/server/test_utils'; import supertest from 'supertest'; import { ReportingCore } from '..'; import { ExportTypesRegistry } from '../lib/export_types_registry'; -import { createMockReportingCore, createMockLevelLogger } from '../test_helpers'; +import { createMockLevelLogger, createMockReportingCore } from '../test_helpers'; +import { createMockPluginSetup } from '../test_helpers/create_mock_reportingplugin'; import { registerJobGenerationRoutes } from './generation'; import type { ReportingRequestHandlerContext } from '../types'; @@ -37,7 +38,7 @@ describe('POST /api/reporting/generate', () => { case 'index': return '.reporting'; case 'queue.pollEnabled': - return false; + return true; default: return; } @@ -56,7 +57,7 @@ describe('POST /api/reporting/generate', () => { callClusterStub = sinon.stub().resolves({}); - const mockSetupDeps = ({ + const mockSetupDeps = createMockPluginSetup({ elasticsearch: { legacy: { client: { callAsInternalUser: callClusterStub } }, }, @@ -68,7 +69,7 @@ describe('POST /api/reporting/generate', () => { }, router: httpSetup.createRouter(''), licensing: { license$: of({ isActive: true, isAvailable: true, type: 'gold' }) }, - } as unknown) as any; + }); core = await createMockReportingCore(config, mockSetupDeps); diff --git a/x-pack/plugins/reporting/server/routes/jobs.test.ts b/x-pack/plugins/reporting/server/routes/jobs.test.ts index 847d27d44ea72d..706a8d5dad7dd7 100644 --- a/x-pack/plugins/reporting/server/routes/jobs.test.ts +++ b/x-pack/plugins/reporting/server/routes/jobs.test.ts @@ -12,7 +12,12 @@ import supertest from 'supertest'; import { ReportingCore } from '..'; import { ReportingInternalSetup } from '../core'; import { ExportTypesRegistry } from '../lib/export_types_registry'; -import { createMockConfig, createMockConfigSchema, createMockReportingCore } from '../test_helpers'; +import { + createMockConfig, + createMockConfigSchema, + createMockPluginSetup, + createMockReportingCore, +} from '../test_helpers'; import { ExportTypeDefinition, ReportingRequestHandlerContext } from '../types'; import { registerJobInfoRoutes } from './jobs'; @@ -41,7 +46,7 @@ describe('GET /api/reporting/jobs/download', () => { 'reporting', () => ({}) ); - core = await createMockReportingCore(config, ({ + const mockSetupDeps = createMockPluginSetup({ elasticsearch: { legacy: { client: { callAsInternalUser: jest.fn() } }, }, @@ -65,7 +70,9 @@ describe('GET /api/reporting/jobs/download', () => { type: 'gold', }), }, - } as unknown) as ReportingInternalSetup); + }); + + core = await createMockReportingCore(config, mockSetupDeps); // @ts-ignore exportTypesRegistry = new ExportTypesRegistry(); exportTypesRegistry.register({ diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts index e9e057b6d32110..ea8480ef3493d9 100644 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts +++ b/x-pack/plugins/reporting/server/test_helpers/create_mock_reportingplugin.ts @@ -12,6 +12,7 @@ jest.mock('../lib/create_queue'); import _ from 'lodash'; import * as Rx from 'rxjs'; +import { coreMock } from 'src/core/server/mocks'; import { ReportingConfig, ReportingCore } from '../'; import { featuresPluginMock } from '../../../features/server/mocks'; import { @@ -22,7 +23,6 @@ import { import { ReportingConfigType } from '../config'; import { ReportingInternalSetup, ReportingInternalStart } from '../core'; import { ReportingStore } from '../lib'; -import { ReportingStartDeps } from '../types'; import { createMockLevelLogger } from './create_mock_levellogger'; (initializeBrowserDriverFactory as jest.Mock< @@ -31,10 +31,7 @@ import { createMockLevelLogger } from './create_mock_levellogger'; (chromium as any).createDriverFactory.mockImplementation(() => ({})); -const createMockPluginSetup = ( - mockReportingCore: ReportingCore, - setupMock?: any -): ReportingInternalSetup => { +export const createMockPluginSetup = (setupMock?: any): ReportingInternalSetup => { return { features: featuresPluginMock.createSetup(), elasticsearch: setupMock.elasticsearch || { legacy: { client: {} } }, @@ -42,6 +39,7 @@ const createMockPluginSetup = ( router: setupMock.router, security: setupMock.security, licensing: { license$: Rx.of({ isAvailable: true, isActive: true, type: 'basic' }) } as any, + ...setupMock, }; }; @@ -58,6 +56,7 @@ const createMockPluginStart = ( savedObjects: startMock.savedObjects || { getScopedClient: jest.fn() }, uiSettings: startMock.uiSettings || { asScopedToClient: () => ({ get: jest.fn() }) }, store, + ...startMock, }; }; @@ -73,7 +72,7 @@ interface ReportingConfigTestType { export const createMockConfigSchema = ( overrides: Partial = {} -): ReportingConfigTestType => { +): ReportingConfigType => { // deeply merge the defaults and the provided partial schema return { index: '.reporting', @@ -93,13 +92,16 @@ export const createMockConfigSchema = ( ...overrides.capture, }, queue: { + indexInterval: 'week', + pollEnabled: true, + pollInterval: 3000, timeout: 120000, ...overrides.queue, }, csv: { ...overrides.csv, }, - }; + } as any; }; export const createMockConfig = ( @@ -114,35 +116,28 @@ export const createMockConfig = ( }; }; -export const createMockStartDeps = (startMock?: any): ReportingStartDeps => ({ - data: startMock.data, -}); - export const createMockReportingCore = async ( config: ReportingConfig, setupDepsMock: ReportingInternalSetup | undefined = undefined, startDepsMock: ReportingInternalStart | undefined = undefined ) => { - const mockReportingCore = { - getConfig: () => config, - getElasticsearchService: () => setupDepsMock?.elasticsearch, - } as ReportingCore; + config = config || {}; if (!setupDepsMock) { - setupDepsMock = createMockPluginSetup(mockReportingCore, {}); - } - if (!startDepsMock) { - startDepsMock = createMockPluginStart(mockReportingCore, {}); + setupDepsMock = createMockPluginSetup({}); } - config = config || {}; + const context = coreMock.createPluginInitializerContext(createMockConfigSchema()); const core = new ReportingCore(logger); + core.setConfig(config); core.pluginSetup(setupDepsMock); - core.setConfig(config); await core.pluginSetsUp(); - core.pluginStart(startDepsMock); + if (!startDepsMock) { + startDepsMock = createMockPluginStart(core, context); + } + await core.pluginStart(startDepsMock); await core.pluginStartsUp(); return core; diff --git a/x-pack/plugins/reporting/server/test_helpers/create_mock_server.ts b/x-pack/plugins/reporting/server/test_helpers/create_mock_server.ts deleted file mode 100644 index 4805bf07a76a2c..00000000000000 --- a/x-pack/plugins/reporting/server/test_helpers/create_mock_server.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { createHttpServer, createCoreContext } from 'src/core/server/http/test_utils'; -import { coreMock } from 'src/core/server/mocks'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { ContextService } from 'src/core/server/context/context_service'; - -const coreId = Symbol('reporting'); - -export const createMockServer = async () => { - const coreContext = createCoreContext({ coreId }); - const contextService = new ContextService(coreContext); - - const server = createHttpServer(coreContext); - const httpSetup = await server.setup({ - context: contextService.setup({ pluginDependencies: new Map() }), - }); - const handlerContext = coreMock.createRequestHandlerContext(); - - httpSetup.registerRouteHandlerContext(coreId, 'core', async (ctx, req, res) => { - return handlerContext; - }); - - return { - server, - httpSetup, - handlerContext, - }; -}; diff --git a/x-pack/plugins/reporting/server/test_helpers/index.ts b/x-pack/plugins/reporting/server/test_helpers/index.ts index edf52fe5c2126a..fe8c92d928af57 100644 --- a/x-pack/plugins/reporting/server/test_helpers/index.ts +++ b/x-pack/plugins/reporting/server/test_helpers/index.ts @@ -11,6 +11,6 @@ export { createMockLevelLogger } from './create_mock_levellogger'; export { createMockConfig, createMockConfigSchema, + createMockPluginSetup, createMockReportingCore, } from './create_mock_reportingplugin'; -export { createMockServer } from './create_mock_server'; diff --git a/x-pack/plugins/reporting/server/types.ts b/x-pack/plugins/reporting/server/types.ts index 4b7ef68f5e70b3..b395738ad44450 100644 --- a/x-pack/plugins/reporting/server/types.ts +++ b/x-pack/plugins/reporting/server/types.ts @@ -14,11 +14,11 @@ import { LicensingPluginSetup } from '../../licensing/server'; import { AuthenticatedUser, SecurityPluginSetup } from '../../security/server'; import { SpacesPluginSetup } from '../../spaces/server'; import { CancellationToken } from '../common'; -import { BaseParams } from '../common/types'; +import { BaseParams, TaskRunResult } from '../common/types'; import { ReportingConfigType } from './config'; import { ReportingCore } from './core'; import { LevelLogger } from './lib'; -import { ReportTaskParams, TaskRunResult } from './lib/tasks'; +import { ReportTaskParams } from './lib/tasks'; /* * Plugin Contract From 40fa961d53f4d1e2b0307a24796d3febfa003791 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 1 Mar 2021 10:41:08 -0700 Subject: [PATCH 02/24] [Reporting] Remove unused priority field (#92552) * [Reporting] Remove unused priority field * fix test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/reporting/common/types.ts | 2 -- .../public/components/buttons/report_info_button.tsx | 5 ----- x-pack/plugins/reporting/server/lib/store/report.test.ts | 6 ------ x-pack/plugins/reporting/server/lib/store/report.ts | 4 ---- x-pack/plugins/reporting/server/lib/store/store.test.ts | 4 ---- x-pack/plugins/reporting/server/routes/generation.test.ts | 3 --- .../reporting_without_security/job_apis.ts | 1 - 7 files changed, 25 deletions(-) diff --git a/x-pack/plugins/reporting/common/types.ts b/x-pack/plugins/reporting/common/types.ts index e787ccec5faec7..3af329cbf03033 100644 --- a/x-pack/plugins/reporting/common/types.ts +++ b/x-pack/plugins/reporting/common/types.ts @@ -76,7 +76,6 @@ export interface ReportSource { started_at?: string; completed_at?: string; created_at: string; - priority?: number; process_expiration?: string; } @@ -113,7 +112,6 @@ export interface ReportApiJSON { kibana_id: string; browser_type: string | undefined; created_at: string; - priority?: number; jobtype: string; created_by: string | false; timeout?: number; diff --git a/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx b/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx index 84e6903fd4a28f..7f2d5b6adcc333 100644 --- a/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx +++ b/x-pack/plugins/reporting/public/components/buttons/report_info_button.tsx @@ -87,7 +87,6 @@ export class ReportInfoButton extends Component { const attempts = info.attempts ? info.attempts.toString() : NA; const maxAttempts = info.max_attempts ? info.max_attempts.toString() : NA; - const priority = info.priority ? info.priority.toString() : NA; const timeout = info.timeout ? info.timeout.toString() : NA; const warnings = info.output && info.output.warnings ? info.output.warnings.join(',') : null; @@ -153,10 +152,6 @@ export class ReportInfoButton extends Component { title: 'Max Attempts', description: maxAttempts, }, - { - title: 'Priority', - description: priority, - }, { title: 'Timeout', description: timeout, diff --git a/x-pack/plugins/reporting/server/lib/store/report.test.ts b/x-pack/plugins/reporting/server/lib/store/report.test.ts index 1d58b6f36f8261..4c5cd755f71c45 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.test.ts @@ -18,7 +18,6 @@ describe('Class Report', () => { payload: { headers: 'payload_test_field', objectType: 'testOt', title: 'cool report' }, meta: { objectType: 'test' }, timeout: 30000, - priority: 1, }); expect(report.toEsDocsJSON()).toMatchObject({ @@ -32,7 +31,6 @@ describe('Class Report', () => { max_attempts: 50, meta: { objectType: 'test' }, payload: { headers: 'payload_test_field', objectType: 'testOt' }, - priority: 1, started_at: undefined, status: 'pending', timeout: 30000, @@ -47,7 +45,6 @@ describe('Class Report', () => { max_attempts: 50, payload: { headers: 'payload_test_field', objectType: 'testOt' }, meta: { objectType: 'test' }, - priority: 1, status: 'pending', timeout: 30000, }); @@ -65,7 +62,6 @@ describe('Class Report', () => { payload: { headers: 'payload_test_field', objectType: 'testOt', title: 'hot report' }, meta: { objectType: 'stange' }, timeout: 30000, - priority: 1, }); const metadata = { @@ -88,7 +84,6 @@ describe('Class Report', () => { max_attempts: 50, meta: { objectType: 'stange' }, payload: { objectType: 'testOt' }, - priority: 1, started_at: undefined, status: 'pending', timeout: 30000, @@ -105,7 +100,6 @@ describe('Class Report', () => { max_attempts: 50, meta: { objectType: 'stange' }, payload: { headers: 'payload_test_field', objectType: 'testOt' }, - priority: 1, started_at: undefined, status: 'pending', timeout: 30000, diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts index 0f18ae3b4eac7e..735ba274322cd2 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -36,7 +36,6 @@ export class Report implements Partial { public readonly started_at?: ReportSource['started_at']; public readonly completed_at?: ReportSource['completed_at']; public readonly process_expiration?: ReportSource['process_expiration']; - public readonly priority?: ReportSource['priority']; public readonly timeout?: ReportSource['timeout']; /* @@ -63,7 +62,6 @@ export class Report implements Partial { this.created_by = opts.created_by || false; this.meta = opts.meta || { objectType: 'unknown' }; this.browser_type = opts.browser_type; - this.priority = opts.priority; this.status = opts.status || JOB_STATUSES.PENDING; this.output = opts.output || null; @@ -98,7 +96,6 @@ export class Report implements Partial { meta: this.meta, timeout: this.timeout, max_attempts: this.max_attempts, - priority: this.priority, browser_type: this.browser_type, status: this.status, attempts: this.attempts, @@ -124,7 +121,6 @@ export class Report implements Partial { meta: this.meta, timeout: this.timeout, max_attempts: this.max_attempts, - priority: this.priority, browser_type: this.browser_type, status: this.status, attempts: this.attempts, diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index e86b17d4f75ea4..4e8e113fb06983 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -203,7 +203,6 @@ describe('ReportingStore', () => { browserTimezone: 'ABC', }, timeout: 30000, - priority: 1, }); await store.setReportClaimed(report, { testDoc: 'test' } as any); @@ -244,7 +243,6 @@ describe('ReportingStore', () => { browserTimezone: 'BCD', }, timeout: 30000, - priority: 1, }); await store.setReportFailed(report, { errors: 'yes' } as any); @@ -285,7 +283,6 @@ describe('ReportingStore', () => { browserTimezone: 'CDE', }, timeout: 30000, - priority: 1, }); await store.setReportCompleted(report, { certainly_completed: 'yes' } as any); @@ -326,7 +323,6 @@ describe('ReportingStore', () => { browserTimezone: 'utc', }, timeout: 30000, - priority: 1, }); await store.setReportCompleted(report, { diff --git a/x-pack/plugins/reporting/server/routes/generation.test.ts b/x-pack/plugins/reporting/server/routes/generation.test.ts index 490b9b7e45664c..f6966a3b28ea9d 100644 --- a/x-pack/plugins/reporting/server/routes/generation.test.ts +++ b/x-pack/plugins/reporting/server/routes/generation.test.ts @@ -158,7 +158,6 @@ describe('POST /api/reporting/generate', () => { it(`returns 200 if job handler doesn't error`, async () => { callClusterStub.withArgs('index').resolves({ _id: 'foo', _index: 'foo-index' }); - registerJobGenerationRoutes(core, mockLogger); await server.start(); @@ -180,9 +179,7 @@ describe('POST /api/reporting/generate', () => { test1: 'yes', }, }, - priority: 10, status: 'pending', - timeout: 10000, }, path: 'undefined/api/reporting/jobs/download/foo', }); diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts index 98113ec965bf74..99832e3f4b1c23 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis.ts @@ -45,7 +45,6 @@ export default function ({ getService }: FtrProviderContext) { created_by: false, jobtype: 'csv', max_attempts: 1, - priority: 10, status: 'pending', timeout: 120000, browser_type: 'chromium', // TODO: remove this field from the API response From 4ea10d9a90fe7007e466aefbd8190a6647aacdff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Mon, 1 Mar 2021 17:41:23 +0000 Subject: [PATCH 03/24] [Usage Collection] Remove unused `applicationUsageTracker` from Public API (#92670) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/plugins/data/public/public.api.md | 1 - .../collectors/application_usage/README.md | 11 ----------- src/plugins/usage_collection/public/mocks.tsx | 18 +++++++++++++----- src/plugins/usage_collection/public/plugin.tsx | 7 ------- .../public/services/application_usage.ts | 5 ++++- 5 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 884873f4535ccb..2859d764ef26f2 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -10,7 +10,6 @@ import { Adapters as Adapters_2 } from 'src/plugins/inspector/common'; import { ApiResponse } from '@elastic/elasticsearch'; import { ApiResponse as ApiResponse_2 } from '@elastic/elasticsearch/lib/Transport'; import { ApplicationStart } from 'kibana/public'; -import { ApplicationUsageTracker } from '@kbn/analytics'; import { Assign } from '@kbn/utility-types'; import { BehaviorSubject } from 'rxjs'; import { BfetchPublicSetup } from 'src/plugins/bfetch/public'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md b/src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md index 3e3afe88c596a3..c66074b792bd0f 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md +++ b/src/plugins/kibana_usage_collection/server/collectors/application_usage/README.md @@ -70,17 +70,6 @@ Application Usage will automatically track the active minutes on screen and clic The prop `viewId` is used as a unique identifier for your plugin. The Application Id is automatically attached to the tracked usage, based on the ID used when registering your app via `core.application.register`. -#### Advanced Usage - -If you have a custom use case not provided by the Application Usage helpers you can use the `usageCollection.applicationUsageTracker` public api directly. - -To start tracking a view: `applicationUsageTracker.trackApplicationViewUsage(viewId)` -Calling this method will marks the specified `viewId` as active. applicationUsageTracker will start tracking clicks and screen minutes for the view. - -To stop tracking a view: `applicationUsageTracker.flushTrackedView(viewId)` -Calling this method will stop tracking the clicks and screen minutes for that view. Usually once the view is no longer active. - - ## Application Usage Telemetry Data This collector reports the number of general clicks and minutes on screen for each registered application in Kibana. diff --git a/src/plugins/usage_collection/public/mocks.tsx b/src/plugins/usage_collection/public/mocks.tsx index 268ed3316542ef..369d8009b27595 100644 --- a/src/plugins/usage_collection/public/mocks.tsx +++ b/src/plugins/usage_collection/public/mocks.tsx @@ -13,11 +13,20 @@ import { ApplicationUsageContext } from './components/track_application_view'; export type Setup = jest.Mocked; -export const createApplicationUsageTrackerMock = (): ApplicationUsageTracker => { - const applicationUsageTrackerMock: jest.Mocked = { - setCurrentAppId: jest.fn(), +// This is to avoid having to mock every private property of the class +type ApplicationUsageTrackerPublic = Pick; + +export const createApplicationUsageTrackerMock = (): ApplicationUsageTrackerPublic => { + const applicationUsageTrackerMock: jest.Mocked = { trackApplicationViewUsage: jest.fn(), - } as any; + flushTrackedView: jest.fn(), + updateViewClickCounter: jest.fn(), + setCurrentAppId: jest.fn(), + start: jest.fn(), + stop: jest.fn(), + pauseTrackingAll: jest.fn(), + resumeTrackingAll: jest.fn(), + }; return applicationUsageTrackerMock; }; @@ -32,7 +41,6 @@ const createSetupContract = (): Setup => { ), }, - applicationUsageTracker: applicationUsageTrackerMock, reportUiCounter: jest.fn(), }; diff --git a/src/plugins/usage_collection/public/plugin.tsx b/src/plugins/usage_collection/public/plugin.tsx index 79260b17e0c820..f50919eb8f6220 100644 --- a/src/plugins/usage_collection/public/plugin.tsx +++ b/src/plugins/usage_collection/public/plugin.tsx @@ -35,16 +35,11 @@ export interface UsageCollectionSetup { components: { ApplicationUsageTrackingProvider: React.FC; }; - applicationUsageTracker: IApplicationUsageTracker; reportUiCounter: Reporter['reportUiCounter']; } export interface UsageCollectionStart { reportUiCounter: Reporter['reportUiCounter']; - applicationUsageTracker: Pick< - ApplicationUsageTracker, - 'trackApplicationViewUsage' | 'flushTrackedView' | 'updateViewClickCounter' - >; } export function isUnauthenticated(http: HttpSetup) { @@ -83,7 +78,6 @@ export class UsageCollectionPlugin implements Plugin ), }, - applicationUsageTracker, reportUiCounter: this.reporter.reportUiCounter, }; } @@ -105,7 +99,6 @@ export class UsageCollectionPlugin implements Plugin, - applicationUsageTracker: ApplicationUsageTracker + applicationUsageTracker: Pick< + ApplicationUsageTracker, + 'updateViewClickCounter' | 'setCurrentAppId' | 'trackApplicationViewUsage' + > ) { const windowClickSubscrition = fromEvent(window, 'click').subscribe(() => { applicationUsageTracker.updateViewClickCounter(MAIN_APP_DEFAULT_VIEW_ID); From f44916b6aabb5ffb519a6953d2bd674415550cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Mon, 1 Mar 2021 18:30:51 +0000 Subject: [PATCH 04/24] [Telemetry] Full `schema` definition (#90273) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .telemetryrc.json | 9 - src/plugins/telemetry/schema/README.md | 17 ++ .../telemetry/schema/legacy_plugins.json | 3 - src/plugins/telemetry/schema/oss_root.json | 199 ++++++++++++++++ .../apis/telemetry/telemetry_local.ts | 214 +++++++++--------- .../apis/telemetry/utils/flat_keys.test.js | 119 ++++++++++ .../apis/telemetry/utils/flat_keys.ts | 28 +++ .../apis/telemetry/utils/index.ts | 10 + .../utils/schema_to_config_schema.test.js | 155 +++++++++++++ .../utils/schema_to_config_schema.ts | 140 ++++++++++++ test/api_integration/jest.config.js | 13 ++ .../server/usage/actions_usage_collector.ts | 2 +- .../alerts/server/usage/alerts_telemetry.ts | 19 +- .../server/usage/alerts_usage_collector.ts | 26 +-- x-pack/plugins/alerts/server/usage/types.ts | 12 +- .../__snapshots__/apm_telemetry.test.ts.snap | 13 +- .../apm/server/lib/apm_telemetry/schema.ts | 3 +- .../apm/server/lib/apm_telemetry/types.ts | 5 +- .../spaces_usage_collector.ts | 16 ++ .../schema/README.md | 17 ++ .../schema/xpack_plugins.json | 43 +++- .../schema/xpack_root.json | 51 +++++ ...{telemetry_local.js => telemetry_local.ts} | 62 ++--- 23 files changed, 988 insertions(+), 188 deletions(-) create mode 100644 src/plugins/telemetry/schema/README.md delete mode 100644 src/plugins/telemetry/schema/legacy_plugins.json create mode 100644 src/plugins/telemetry/schema/oss_root.json create mode 100644 test/api_integration/apis/telemetry/utils/flat_keys.test.js create mode 100644 test/api_integration/apis/telemetry/utils/flat_keys.ts create mode 100644 test/api_integration/apis/telemetry/utils/index.ts create mode 100644 test/api_integration/apis/telemetry/utils/schema_to_config_schema.test.js create mode 100644 test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts create mode 100644 test/api_integration/jest.config.js create mode 100644 x-pack/plugins/telemetry_collection_xpack/schema/README.md create mode 100644 x-pack/plugins/telemetry_collection_xpack/schema/xpack_root.json rename x-pack/test/api_integration/apis/telemetry/{telemetry_local.js => telemetry_local.ts} (78%) diff --git a/.telemetryrc.json b/.telemetryrc.json index 0f1530c6225d6a..a408a5e2842f90 100644 --- a/.telemetryrc.json +++ b/.telemetryrc.json @@ -2,15 +2,6 @@ { "output": "src/plugins/telemetry/schema/oss_plugins.json", "root": "src/plugins/", - "exclude": [ - "src/plugins/kibana_react/", - "src/plugins/testbed/", - "src/plugins/kibana_utils/" - ] - }, - { - "output": "src/plugins/telemetry/schema/legacy_plugins.json", - "root": "src/legacy/server/", "exclude": [] } ] diff --git a/src/plugins/telemetry/schema/README.md b/src/plugins/telemetry/schema/README.md new file mode 100644 index 00000000000000..fb02d5fc49af34 --- /dev/null +++ b/src/plugins/telemetry/schema/README.md @@ -0,0 +1,17 @@ +# Telemetry Schemas + +This list of `.json` files describes the format of the payloads sent to the Remote Telemetry Service. All the files should follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`, with the addition of the type `pass_through`. This additional `type` indicates Kibana sends the payload as-is from the output of an external ES query. + +There are currently 2 files: + +- `oss_root.json`: Defines the schema for the payload from the root keys. + Manually maintained for now because the frequency it changes should be pretty low. +- `oss_plugins.json`: The schema for the content that will be nested in `stack_stats.kibana.plugins`. + It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API. + More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs. + +NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service. + +## Testing + +Functional tests are defined at `test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files, and validates the actual output of the telemetry endpoint against the final schema. \ No newline at end of file diff --git a/src/plugins/telemetry/schema/legacy_plugins.json b/src/plugins/telemetry/schema/legacy_plugins.json deleted file mode 100644 index d5b0514b64918c..00000000000000 --- a/src/plugins/telemetry/schema/legacy_plugins.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "properties": {} -} diff --git a/src/plugins/telemetry/schema/oss_root.json b/src/plugins/telemetry/schema/oss_root.json new file mode 100644 index 00000000000000..658f5ee4e66dac --- /dev/null +++ b/src/plugins/telemetry/schema/oss_root.json @@ -0,0 +1,199 @@ +{ + "properties": { + "timestamp": { + "type": "date" + }, + "cluster_uuid": { + "type": "keyword" + }, + "cluster_name": { + "type": "keyword" + }, + "version": { + "type": "keyword" + }, + "collection": { + "type": "keyword" + }, + "collectionSource": { + "type": "keyword" + }, + "stack_stats": { + "properties": { + "data": { + "type": "array", + "items": { + "properties": { + "data_stream": { + "properties": { + "dataset": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "package": { + "properties": { + "name": { + "type": "keyword" + } + } + }, + "shipper": { + "type": "keyword" + }, + "pattern_name": { + "type": "keyword" + }, + "index_count": { + "type": "long" + }, + "ecs_index_count": { + "type": "long" + }, + "doc_count": { + "type": "long" + }, + "size_in_bytes": { + "type": "long" + } + } + } + }, + "kibana": { + "properties": { + "timelion_sheet": { + "properties": { + "total": { + "type": "long" + } + } + }, + "visualization": { + "properties": { + "total": { + "type": "long" + } + } + }, + "search": { + "properties": { + "total": { + "type": "long" + } + } + }, + "index_pattern": { + "properties": { + "total": { + "type": "long" + } + } + }, + "dashboard": { + "properties": { + "total": { + "type": "long" + } + } + }, + "graph_workspace": { + "properties": { + "total": { + "type": "long" + } + } + }, + "count": { + "type": "short" + }, + "indices": { + "type": "short" + }, + "os": { + "properties": { + "platforms": { + "type": "array", + "items": { + "properties": { + "platform": { + "type": "keyword" + }, + "count": { + "type": "short" + } + } + } + }, + "platformReleases": { + "type": "array", + "items": { + "properties": { + "platformRelease": { + "type": "keyword" + }, + "count": { + "type": "short" + } + } + } + }, + "distros": { + "type": "array", + "items": { + "properties": { + "distro": { + "type": "keyword" + }, + "count": { + "type": "short" + } + } + } + }, + "distroReleases": { + "type": "array", + "items": { + "properties": { + "distroRelease": { + "type": "keyword" + }, + "count": { + "type": "short" + } + } + } + } + } + }, + "versions": { + "type": "array", + "items": { + "properties": { + "version": { + "type": "keyword" + }, + "count": { + "type": "short" + } + } + } + }, + "plugins": { + "properties": { + "THIS_WILL_BE_REPLACED_BY_THE_PLUGINS_JSON": { + "type": "text" + } + } + } + } + } + } + }, + "cluster_stats": { + "type": "pass_through" + } + } +} diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index b424cab9ff45b2..9d3ced52245020 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -7,27 +7,12 @@ */ import expect from '@kbn/expect'; -import _ from 'lodash'; import { basicUiCounters } from './__fixtures__/ui_counters'; import { FtrProviderContext } from '../../ftr_provider_context'; import { SavedObject } from '../../../../src/core/server'; -/* - * Create a single-level array with strings for all the paths to values in the - * source object, up to 3 deep. Going deeper than 3 causes a bit too much churn - * in the tests. - */ -function flatKeys(source: Record) { - const recursivelyFlatKeys = (obj: unknown, path: string[] = [], depth = 0): string[] => { - return depth < 3 && _.isObject(obj) - ? Object.entries(obj).reduce( - (acc, [k, v]) => [...acc, ...recursivelyFlatKeys(v, [...path, k], depth + 1)], - [] as string[] - ) - : [path.join('.')]; - }; - - return _.uniq(_.flattenDeep(recursivelyFlatKeys(source))).sort((a, b) => a.localeCompare(b)); -} +import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; +import ossPluginsTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_plugins.json'; +import { assertTelemetryPayload, flatKeys } from './utils'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -46,46 +31,110 @@ export default function ({ getService }: FtrProviderContext) { await es.indices.delete({ index: 'filebeat-telemetry_tests_logs' }); }); - it('should pull local stats and validate data types', async () => { - const { body } = await supertest - .post('/api/telemetry/v2/clusters/_stats') - .set('kbn-xsrf', 'xxx') - .send({ unencrypted: true }) - .expect(200); + describe('validate data types', () => { + let stats: Record; - expect(body.length).to.be(1); - const stats = body[0]; - expect(stats.collection).to.be('local'); - expect(stats.collectionSource).to.be('local'); - expect(stats.license).to.be(undefined); // OSS cannot get the license - expect(stats.stack_stats.kibana.count).to.be.a('number'); - expect(stats.stack_stats.kibana.indices).to.be.a('number'); - expect(stats.stack_stats.kibana.os.platforms[0].platform).to.be.a('string'); - expect(stats.stack_stats.kibana.os.platforms[0].count).to.be(1); - expect(stats.stack_stats.kibana.os.platformReleases[0].platformRelease).to.be.a('string'); - expect(stats.stack_stats.kibana.os.platformReleases[0].count).to.be(1); - expect(stats.stack_stats.kibana.plugins.telemetry.opt_in_status).to.be(false); - expect(stats.stack_stats.kibana.plugins.telemetry.usage_fetcher).to.be.a('string'); - expect(stats.stack_stats.kibana.plugins.stack_management).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.ui_metric).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.ui_counters).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.application_usage).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.kql.defaultQueryLanguage).to.be.a('string'); - expect(stats.stack_stats.kibana.plugins.localization).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); - expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); - expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); + before('pull local stats', async () => { + const { body } = await supertest + .post('/api/telemetry/v2/clusters/_stats') + .set('kbn-xsrf', 'xxx') + .send({ unencrypted: true }) + .expect(200); + + expect(body.length).to.be(1); + stats = body[0]; + }); - // Testing stack_stats.data - expect(stats.stack_stats.data).to.be.an('object'); - expect(stats.stack_stats.data).to.be.an('array'); - expect(stats.stack_stats.data[0]).to.be.an('object'); - expect(stats.stack_stats.data[0].pattern_name).to.be('filebeat'); - expect(stats.stack_stats.data[0].shipper).to.be('filebeat'); - expect(stats.stack_stats.data[0].index_count).to.be(1); - expect(stats.stack_stats.data[0].doc_count).to.be(0); - expect(stats.stack_stats.data[0].ecs_index_count).to.be(0); - expect(stats.stack_stats.data[0].size_in_bytes).to.be.a('number'); + it('should pass the schema validation', () => { + try { + assertTelemetryPayload( + { root: ossRootTelemetrySchema, plugins: ossPluginsTelemetrySchema }, + stats + ); + } catch (err) { + err.message = `The telemetry schemas in 'src/plugins/telemetry/schema/' are out-of-date, please update it as required: ${err.message}`; + throw err; + } + }); + + it('should pass ad-hoc enforced validations', () => { + expect(stats.collection).to.be('local'); + expect(stats.collectionSource).to.be('local'); + expect(stats.license).to.be(undefined); // OSS cannot get the license + expect(stats.stack_stats.kibana.count).to.be.a('number'); + expect(stats.stack_stats.kibana.indices).to.be.a('number'); + expect(stats.stack_stats.kibana.os.platforms[0].platform).to.be.a('string'); + expect(stats.stack_stats.kibana.os.platforms[0].count).to.be(1); + expect(stats.stack_stats.kibana.os.platformReleases[0].platformRelease).to.be.a('string'); + expect(stats.stack_stats.kibana.os.platformReleases[0].count).to.be(1); + expect(stats.stack_stats.kibana.plugins.telemetry.opt_in_status).to.be(false); + expect(stats.stack_stats.kibana.plugins.telemetry.usage_fetcher).to.be.a('string'); + expect(stats.stack_stats.kibana.plugins.stack_management).to.be.an('object'); + expect(stats.stack_stats.kibana.plugins.ui_metric).to.be.an('object'); + expect(stats.stack_stats.kibana.plugins.ui_counters).to.be.an('object'); + expect(stats.stack_stats.kibana.plugins.application_usage).to.be.an('object'); + expect(stats.stack_stats.kibana.plugins.kql.defaultQueryLanguage).to.be.a('string'); + expect(stats.stack_stats.kibana.plugins.localization).to.be.an('object'); + expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); + expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); + expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); + + // Testing stack_stats.data + expect(stats.stack_stats.data).to.be.an('object'); + expect(stats.stack_stats.data).to.be.an('array'); + expect(stats.stack_stats.data[0]).to.be.an('object'); + expect(stats.stack_stats.data[0].pattern_name).to.be('filebeat'); + expect(stats.stack_stats.data[0].shipper).to.be('filebeat'); + expect(stats.stack_stats.data[0].index_count).to.be(1); + expect(stats.stack_stats.data[0].doc_count).to.be(0); + expect(stats.stack_stats.data[0].ecs_index_count).to.be(0); + expect(stats.stack_stats.data[0].size_in_bytes).to.be.a('number'); + }); + + it('should validate mandatory fields exist', () => { + const actual = flatKeys(stats); + expect(actual).to.be.an('array'); + const expected = [ + 'cluster_name', + 'cluster_stats.cluster_uuid', + 'cluster_stats.indices.analysis', + 'cluster_stats.indices.completion', + 'cluster_stats.indices.count', + 'cluster_stats.indices.docs', + 'cluster_stats.indices.fielddata', + 'cluster_stats.indices.mappings', + 'cluster_stats.indices.query_cache', + 'cluster_stats.indices.segments', + 'cluster_stats.indices.shards', + 'cluster_stats.indices.store', + 'cluster_stats.nodes.count', + 'cluster_stats.nodes.discovery_types', + 'cluster_stats.nodes.fs', + 'cluster_stats.nodes.ingest', + 'cluster_stats.nodes.jvm', + 'cluster_stats.nodes.network_types', + 'cluster_stats.nodes.os', + 'cluster_stats.nodes.packaging_types', + 'cluster_stats.nodes.plugins', + 'cluster_stats.nodes.process', + 'cluster_stats.nodes.versions', + 'cluster_stats.nodes.usage', + 'cluster_stats.status', + 'cluster_stats.timestamp', + 'cluster_uuid', + 'collection', + 'collectionSource', + 'stack_stats.kibana.count', + 'stack_stats.kibana.indices', + 'stack_stats.kibana.os', + 'stack_stats.kibana.plugins', + 'stack_stats.kibana.versions', + 'timestamp', + 'version', + ]; + + expect(expected.every((m) => actual.includes(m))).to.be.ok(); + }); }); describe('UI Counters telemetry', () => { @@ -104,59 +153,6 @@ export default function ({ getService }: FtrProviderContext) { }); }); - it('should pull local stats and validate fields', async () => { - const { body } = await supertest - .post('/api/telemetry/v2/clusters/_stats') - .set('kbn-xsrf', 'xxx') - .send({ unencrypted: true }) - .expect(200); - - const stats = body[0]; - - const actual = flatKeys(stats); - expect(actual).to.be.an('array'); - const expected = [ - 'cluster_name', - 'cluster_stats.cluster_uuid', - 'cluster_stats.indices.analysis', - 'cluster_stats.indices.completion', - 'cluster_stats.indices.count', - 'cluster_stats.indices.docs', - 'cluster_stats.indices.fielddata', - 'cluster_stats.indices.mappings', - 'cluster_stats.indices.query_cache', - 'cluster_stats.indices.segments', - 'cluster_stats.indices.shards', - 'cluster_stats.indices.store', - 'cluster_stats.nodes.count', - 'cluster_stats.nodes.discovery_types', - 'cluster_stats.nodes.fs', - 'cluster_stats.nodes.ingest', - 'cluster_stats.nodes.jvm', - 'cluster_stats.nodes.network_types', - 'cluster_stats.nodes.os', - 'cluster_stats.nodes.packaging_types', - 'cluster_stats.nodes.plugins', - 'cluster_stats.nodes.process', - 'cluster_stats.nodes.versions', - 'cluster_stats.nodes.usage', - 'cluster_stats.status', - 'cluster_stats.timestamp', - 'cluster_uuid', - 'collection', - 'collectionSource', - 'stack_stats.kibana.count', - 'stack_stats.kibana.indices', - 'stack_stats.kibana.os', - 'stack_stats.kibana.plugins', - 'stack_stats.kibana.versions', - 'timestamp', - 'version', - ]; - - expect(expected.every((m) => actual.includes(m))).to.be.ok(); - }); - describe('application usage limits', () => { function createSavedObject(viewId?: string) { return supertest diff --git a/test/api_integration/apis/telemetry/utils/flat_keys.test.js b/test/api_integration/apis/telemetry/utils/flat_keys.test.js new file mode 100644 index 00000000000000..778e5c38a1804e --- /dev/null +++ b/test/api_integration/apis/telemetry/utils/flat_keys.test.js @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * It's a JS file because we cannot use Jest types in here because of a clash in the `expect` types + */ + +import { flatKeys } from './flat_keys'; + +describe(`flatKeys`, () => { + test('no keys to be listed', () => { + expect(flatKeys({})).toStrictEqual([]); + }); + test('one-level list', () => { + expect( + flatKeys({ + prop1: 1, + prop2: 'a', + prop3: true, + prop4: [], + }) + ).toStrictEqual(['prop1', 'prop2', 'prop3', 'prop4']); + }); + test('two-level list', () => { + expect( + flatKeys({ + prop1: 1, + prop2: 'a', + prop3: true, + prop4: [], + prop5: [1], + prop6: { + prop6_1: 1, + }, + }) + ).toStrictEqual(['prop1', 'prop2', 'prop3', 'prop4', 'prop5.0', 'prop6.prop6_1']); + }); + test('three-level list', () => { + expect( + flatKeys({ + prop1: 1, + prop2: 'a', + prop3: true, + prop4: [], + prop5: [1], + prop6: { + prop6_1: 1, + prop6_2: { + prop6_2_1: 1, + }, + }, + prop7: [{ a: 1, b: [] }], + prop8: [1, true, { a: 1 }], + }) + ).toStrictEqual([ + 'prop1', + 'prop2', + 'prop3', + 'prop4', + 'prop5.0', + 'prop6.prop6_1', + 'prop6.prop6_2.prop6_2_1', + 'prop7.0.a', + 'prop7.0.b', + 'prop8.0', + 'prop8.1', + 'prop8.2.a', + ]); + }); + test('four-level+ list: it stays at 3 levels only', () => { + expect( + flatKeys({ + prop1: 1, + prop2: 'a', + prop3: true, + prop4: [], + prop5: [1], + prop6: { + prop6_1: 1, + prop6_2: { + prop6_2_1: 1, + prop6_2_2: { + prop6_2_2_1: 1, + }, + }, + }, + prop7: [{ a: 1, b: [], c: [1], d: [{ a: 1 }], e: [1, { a: 1 }] }], + prop8: [1, true, { a: 1 }], + }) + ).toStrictEqual([ + 'prop1', + 'prop2', + 'prop3', + 'prop4', + 'prop5.0', + 'prop6.prop6_1', + 'prop6.prop6_2.prop6_2_1', + 'prop6.prop6_2.prop6_2_2', + // 'prop6.prop6_2.prop6_2_2.prop6_2_2_1', Not reported because of the depth-limit + 'prop7.0.a', + 'prop7.0.b', + 'prop7.0.c', + // 'prop7.0.c.0', Not reported because of the depth-limit + 'prop7.0.d', + // 'prop7.0.d.0.a', Not reported because of the depth-limit + 'prop7.0.e', + // 'prop7.0.e.0', Not reported because of the depth-limit + // 'prop7.0.e.1.a', Not reported because of the depth-limit + 'prop8.0', + 'prop8.1', + 'prop8.2.a', + ]); + }); +}); diff --git a/test/api_integration/apis/telemetry/utils/flat_keys.ts b/test/api_integration/apis/telemetry/utils/flat_keys.ts new file mode 100644 index 00000000000000..d22737d01227a1 --- /dev/null +++ b/test/api_integration/apis/telemetry/utils/flat_keys.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import _ from 'lodash'; + +/** + * Create a single-level array with strings for all the paths to values in the + * source object, up to 3 deep. Going deeper than 3 causes a bit too much churn + * in the tests. + * @param source The object to extract the keys from. + */ +export function flatKeys(source: Record) { + const recursivelyFlatKeys = (obj: unknown, path: string[] = [], depth = 0): string[] => { + return depth < 3 && _.isObject(obj) && _.size(obj) > 0 + ? Object.entries(obj).reduce( + (acc, [k, v]) => [...acc, ...recursivelyFlatKeys(v, [...path, k], depth + 1)], + [] as string[] + ) + : [path.join('.')].filter(Boolean); + }; + + return _.uniq(_.flattenDeep(recursivelyFlatKeys(source))).sort((a, b) => a.localeCompare(b)); +} diff --git a/test/api_integration/apis/telemetry/utils/index.ts b/test/api_integration/apis/telemetry/utils/index.ts new file mode 100644 index 00000000000000..83bb23a665f99c --- /dev/null +++ b/test/api_integration/apis/telemetry/utils/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { flatKeys } from './flat_keys'; +export { assertTelemetryPayload } from './schema_to_config_schema'; diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.test.js b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.test.js new file mode 100644 index 00000000000000..f568a4338ebe55 --- /dev/null +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.test.js @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * It's a JS file because we cannot use Jest types in here because of a clash in the `expect` types + */ + +import { assertTelemetryPayload } from './schema_to_config_schema'; + +describe(`assertTelemetryPayload`, () => { + test('empty schemas => errors with malformed schema', () => { + // @ts-expect-error: root and plugins don't match expected types + expect(() => assertTelemetryPayload({ root: {}, plugins: {} }, {})).toThrow(/Malformed schema/); + }); + test('minimal schemas and empty stats => pass', () => { + expect(() => + // @ts-expect-error: root doesn't match expected types + assertTelemetryPayload({ root: {}, plugins: { properties: {} } }, {}) + ).not.toThrow(); + }); + test('stats has fields not defined in the schema => fail', () => { + expect(() => + // @ts-expect-error: root doesn't match expected types + assertTelemetryPayload({ root: {}, plugins: { properties: {} } }, { version: 'some-version' }) + ).toThrow('[version]: definition for this key is missing. Received `"some-version"`'); + }); + test('stats has nested-fields not defined in the schema => fail', () => { + expect(() => + assertTelemetryPayload( + // @ts-expect-error: root doesn't match expected types + { root: {}, plugins: { properties: {} } }, + { an_array: [{ docs: { missing: 1 } }] } + ) + ).toThrow( + '[an_array]: definition for this key is missing. Received `[{"docs":{"missing":1}}]`' + ); + expect(() => + assertTelemetryPayload( + { + root: { + properties: { + an_array: { + type: 'array', + items: { + properties: {}, + }, + }, + }, + }, + plugins: { properties: {} }, + }, + { an_array: [{ docs: { missing: 1 } }] } + ) + ).toThrow('[an_array.0.docs]: definition for this key is missing. Received `{"missing":1}`'); + expect(() => + assertTelemetryPayload( + { + root: { + properties: { + an_array: { + type: 'array', + items: { + properties: { + docs: { + properties: {}, + }, + }, + }, + }, + }, + }, + plugins: { properties: {} }, + }, + { an_array: [{ docs: { missing: 1 } }] } + ) + ).toThrow('[an_array.0.docs.missing]: definition for this key is missing. Received `1`'); + }); + test('stats has nested-fields defined in the schema, but with wrong type => fail', () => { + expect(() => + assertTelemetryPayload( + { + root: { + properties: { + an_array: { + type: 'array', + items: { + properties: { + docs: { + properties: { + field: { type: 'short' }, + }, + }, + }, + }, + }, + }, + }, + plugins: { properties: {} }, + }, + { an_array: [{ docs: { field: 'abc' } }] } + ) + ).toThrow(`[an_array.0.docs.field]: types that failed validation: +- [an_array.0.docs.field.0]: expected value of type [number] but got [string] +- [an_array.0.docs.field.1]: expected value to equal [null]`); + }); + test('stats has nested-fields defined in the schema => succeed', () => { + expect(() => + assertTelemetryPayload( + { + root: { + properties: { + an_array: { + type: 'array', + items: { + properties: { + docs: { + properties: { + field: { type: 'short' }, + }, + }, + }, + }, + }, + }, + }, + plugins: { properties: {} }, + }, + { an_array: [{ docs: { field: 1 } }] } + ) + ).not.toThrow(); + }); + + test('allow pass_through properties', () => { + expect(() => + assertTelemetryPayload( + { + root: { + properties: { + im_only_passing_through_data: { + type: 'pass_through', + }, + }, + }, + plugins: { properties: {} }, + }, + { im_only_passing_through_data: [{ docs: { field: 1 } }] } + ) + ).not.toThrow(); + }); +}); diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts new file mode 100644 index 00000000000000..d5b18eb4bd2026 --- /dev/null +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema, ObjectType, Type } from '@kbn/config-schema'; +import { get } from 'lodash'; +import { set } from '@elastic/safer-lodash-set'; +import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server'; + +/** + * Type that defines all the possible values that the Telemetry Schema accepts. + * These types definitions are helping to identify earlier the possible missing `properties` nesting when + * manually defining the schemas. + */ +export type TelemetrySchemaValue = + | { + type: AllowedSchemaTypes | 'pass_through' | string; + } + | { type: 'array'; items: TelemetrySchemaValue } + | TelemetrySchemaObject; + +export interface TelemetrySchemaObject { + properties: Record; +} + +function isOneOfCandidate( + schemas: Array> +): schemas is [Type | Type] { + return schemas.length === 2; +} + +/** + * Converts each telemetry schema value to the @kbn/config-schema equivalent + * @param value + */ +function valueSchemaToConfigSchema(value: TelemetrySchemaValue): Type { + if ('properties' in value) { + const { DYNAMIC_KEY, ...properties } = value.properties; + const schemas: Array> = [objectSchemaToConfigSchema({ properties })]; + if (DYNAMIC_KEY) { + schemas.push(schema.recordOf(schema.string(), valueSchemaToConfigSchema(DYNAMIC_KEY))); + } + return isOneOfCandidate(schemas) ? schema.oneOf(schemas) : schemas[0]; + } else { + const valueType = value.type; // Copied in here because of TS reasons, it's not available in the `default` case + switch (value.type) { + case 'pass_through': + return schema.any(); + case 'boolean': + return schema.boolean(); + case 'keyword': + case 'text': + case 'date': + return schema.string(); + case 'byte': + case 'double': + case 'float': + case 'integer': + case 'long': + case 'short': + // Some plugins return `null` when there is no number to report + return schema.oneOf([schema.number(), schema.literal(null)]); + case 'array': + if ('items' in value) { + return schema.arrayOf(valueSchemaToConfigSchema(value.items)); + } + default: + throw new Error( + `Unsupported schema type ${valueType}. Did you forget to wrap your object definition in a nested 'properties' field?` + ); + } + } +} + +function objectSchemaToConfigSchema(objectSchema: TelemetrySchemaObject): ObjectType { + return schema.object( + Object.fromEntries( + Object.entries(objectSchema.properties).map(([key, value]) => { + try { + return [key, schema.maybe(valueSchemaToConfigSchema(value))]; + } catch (err) { + err.failedKey = [key, ...(err.failedKey || [])]; + throw err; + } + }) + ) + ); +} + +/** + * Converts the JSON generated from the Usage Collection schema to a @kbn/config-schema object + * so it can be used for validation. All entries are considered optional. + * @param telemetrySchema JSON generated by @kbn/telemetry-tools from the Usage Collection schemas + */ +function convertSchemaToConfigSchema(telemetrySchema: { + properties: Record; +}): ObjectType { + try { + return objectSchemaToConfigSchema(telemetrySchema); + } catch (err) { + if (err.failedKey) { + err.message = `Malformed schema for key [${err.failedKey.join('.')}]: ${err.message}`; + } + throw err; + } +} + +/** + * Merges the telemetrySchema, generates a @kbn/config-schema version from it, and uses it to validate stats. + * @param telemetrySchema The JSON schema definitions for root and plugins + * @param stats The full output of the telemetry plugin + */ +export function assertTelemetryPayload( + telemetrySchema: { root: TelemetrySchemaObject; plugins: TelemetrySchemaObject }, + stats: unknown +): void { + const fullSchema = telemetrySchema.root; + set( + fullSchema, + 'properties.stack_stats.properties.kibana.properties.plugins', + telemetrySchema.plugins + ); + const ossTelemetryValidationSchema = convertSchemaToConfigSchema(fullSchema); + + // Run @kbn/config-schema validation to the entire payload + try { + ossTelemetryValidationSchema.validate(stats); + } catch (err) { + // "[path.to.key]: definition for this key is missing" + const [, pathToKey] = err.message.match(/^\[(.*)\]\: definition for this key is missing/) ?? []; + if (pathToKey) { + err.message += `. Received \`${JSON.stringify(get(stats, pathToKey))}\``; + } + throw err; + } +} diff --git a/test/api_integration/jest.config.js b/test/api_integration/jest.config.js new file mode 100644 index 00000000000000..d5e6af5d621e59 --- /dev/null +++ b/test/api_integration/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/test/api_integration'], +}; diff --git a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts index 8c200c1dd13174..f8a91e3a0a67a8 100644 --- a/x-pack/plugins/actions/server/usage/actions_usage_collector.ts +++ b/x-pack/plugins/actions/server/usage/actions_usage_collector.ts @@ -45,7 +45,7 @@ export function createActionsUsageCollector( try { const doc = await getLatestTaskState(await taskManager); // get the accumulated state from the recurring task - const state: ActionsUsage = get(doc, 'state') as ActionsUsage; + const { runs, ...state } = get(doc, 'state') as ActionsUsage & { runs: number }; return { ...state, diff --git a/x-pack/plugins/alerts/server/usage/alerts_telemetry.ts b/x-pack/plugins/alerts/server/usage/alerts_telemetry.ts index 5165307fd43e98..c66110f2647c6c 100644 --- a/x-pack/plugins/alerts/server/usage/alerts_telemetry.ts +++ b/x-pack/plugins/alerts/server/usage/alerts_telemetry.ts @@ -7,6 +7,7 @@ import { LegacyAPICaller } from 'kibana/server'; import { SearchResponse } from 'elasticsearch'; +import { AlertsUsage } from './types'; const alertTypeMetric = { scripted_metric: { @@ -34,14 +35,22 @@ const alertTypeMetric = { }, }; -export async function getTotalCountAggregations(callCluster: LegacyAPICaller, kibanaInex: string) { +export async function getTotalCountAggregations( + callCluster: LegacyAPICaller, + kibanaInex: string +): Promise< + Pick< + AlertsUsage, + 'count_total' | 'count_by_type' | 'throttle_time' | 'schedule_time' | 'connectors_per_alert' + > +> { const throttleTimeMetric = { scripted_metric: { init_script: 'state.min = 0; state.max = 0; state.totalSum = 0; state.totalCount = 0;', map_script: ` if (doc['alert.throttle'].size() > 0) { def throttle = doc['alert.throttle'].value; - + if (throttle.length() > 1) { // get last char String timeChar = throttle.substring(throttle.length() - 1); @@ -51,7 +60,7 @@ export async function getTotalCountAggregations(callCluster: LegacyAPICaller, ki if (throttle.chars().allMatch(Character::isDigit)) { // using of regex is not allowed in painless language int parsed = Integer.parseInt(throttle); - + if (timeChar.equals("s")) { parsed = parsed; } else if (timeChar.equals("m")) { @@ -107,7 +116,7 @@ export async function getTotalCountAggregations(callCluster: LegacyAPICaller, ki map_script: ` if (doc['alert.schedule.interval'].size() > 0) { def interval = doc['alert.schedule.interval'].value; - + if (interval.length() > 1) { // get last char String timeChar = interval.substring(interval.length() - 1); @@ -117,7 +126,7 @@ export async function getTotalCountAggregations(callCluster: LegacyAPICaller, ki if (interval.chars().allMatch(Character::isDigit)) { // using of regex is not allowed in painless language int parsed = Integer.parseInt(interval); - + if (timeChar.equals("s")) { parsed = parsed; } else if (timeChar.equals("m")) { diff --git a/x-pack/plugins/alerts/server/usage/alerts_usage_collector.ts b/x-pack/plugins/alerts/server/usage/alerts_usage_collector.ts index 9154ec896f9d89..884120d3d03dff 100644 --- a/x-pack/plugins/alerts/server/usage/alerts_usage_collector.ts +++ b/x-pack/plugins/alerts/server/usage/alerts_usage_collector.ts @@ -57,7 +57,7 @@ export function createAlertsUsageCollector( try { const doc = await getLatestTaskState(await taskManager); // get the accumulated state from the recurring task - const state: AlertsUsage = get(doc, 'state') as AlertsUsage; + const { runs, ...state } = get(doc, 'state') as AlertsUsage & { runs: number }; return { ...state, @@ -68,14 +68,14 @@ export function createAlertsUsageCollector( count_active_total: 0, count_disabled_total: 0, throttle_time: { - min: 0, - avg: 0, - max: 0, + min: '0s', + avg: '0s', + max: '0s', }, schedule_time: { - min: 0, - avg: 0, - max: 0, + min: '0s', + avg: '0s', + max: '0s', }, connectors_per_alert: { min: 0, @@ -92,14 +92,14 @@ export function createAlertsUsageCollector( count_active_total: { type: 'long' }, count_disabled_total: { type: 'long' }, throttle_time: { - min: { type: 'long' }, - avg: { type: 'float' }, - max: { type: 'long' }, + min: { type: 'keyword' }, + avg: { type: 'keyword' }, + max: { type: 'keyword' }, }, schedule_time: { - min: { type: 'long' }, - avg: { type: 'float' }, - max: { type: 'long' }, + min: { type: 'keyword' }, + avg: { type: 'keyword' }, + max: { type: 'keyword' }, }, connectors_per_alert: { min: { type: 'long' }, diff --git a/x-pack/plugins/alerts/server/usage/types.ts b/x-pack/plugins/alerts/server/usage/types.ts index 93ec51a2bf19a6..c3c750da73a7fa 100644 --- a/x-pack/plugins/alerts/server/usage/types.ts +++ b/x-pack/plugins/alerts/server/usage/types.ts @@ -12,14 +12,14 @@ export interface AlertsUsage { count_by_type: Record; count_active_by_type: Record; throttle_time: { - min: number; - avg: number; - max: number; + min: string; + avg: string; + max: string; }; schedule_time: { - min: number; - avg: number; - max: number; + min: string; + avg: string; + max: string; }; connectors_per_alert: { min: number; diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index ccd5f143440dae..14343bd8d52c4b 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -530,7 +530,7 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the }, "environments": { "properties": { - "services_without_environments": { + "services_without_environment": { "type": "long" }, "services_with_multiple_environments": { @@ -1008,6 +1008,17 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } } + }, + "environments": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } } } } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts index 332dd2ff26867f..565e437504ee5e 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/schema.ts @@ -116,7 +116,7 @@ export const apmSchema: MakeSchemaFrom = { }, }, environments: { - services_without_environments: long, + services_without_environment: long, services_with_multiple_environments: long, top_environments: { type: 'array', items: { type: 'keyword' } }, }, @@ -192,5 +192,6 @@ export const apmSchema: MakeSchemaFrom = { agents: { took: { ms: long } }, indices_stats: { took: { ms: long } }, cardinality: { took: { ms: long } }, + environments: { took: { ms: long } }, }, }; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index 7e194a84d1fb3f..6dc829425eadac 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -35,7 +35,7 @@ export interface APMUsage { }; }; environments: { - services_without_environments: number; + services_without_environment: number; services_with_multiple_environments: number; top_environments: string[]; }; @@ -140,7 +140,8 @@ export interface APMUsage { | 'integrations' | 'agents' | 'indices_stats' - | 'cardinality', + | 'cardinality' + | 'environments', { took: { ms: number } } >; } diff --git a/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts b/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts index 1c0bc42bc3535c..60a2acc5319df3 100644 --- a/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts +++ b/x-pack/plugins/spaces/server/usage_collection/spaces_usage_collector.ts @@ -131,6 +131,14 @@ export interface UsageData extends UsageStats { count?: number; usesFeatureControls?: boolean; disabledFeatures: { + // "feature": number; + [key: string]: number | undefined; + // Known registered features + stackAlerts?: number; + actions?: number; + enterpriseSearch?: number; + fleet?: number; + savedObjectsTagging?: number; indexPatterns?: number; discover?: number; canvas?: number; @@ -173,6 +181,14 @@ export function getSpacesUsageCollector( schema: { usesFeatureControls: { type: 'boolean' }, disabledFeatures: { + // "feature": number; + DYNAMIC_KEY: { type: 'long' }, + // Known registered features + stackAlerts: { type: 'long' }, + actions: { type: 'long' }, + enterpriseSearch: { type: 'long' }, + fleet: { type: 'long' }, + savedObjectsTagging: { type: 'long' }, indexPatterns: { type: 'long' }, discover: { type: 'long' }, canvas: { type: 'long' }, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/README.md b/x-pack/plugins/telemetry_collection_xpack/schema/README.md new file mode 100644 index 00000000000000..e6145b751e7d86 --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/schema/README.md @@ -0,0 +1,17 @@ +# X-Pack Telemetry Schemas + +This is an extension of the [OSS Telemetry Schemas](../../../../src/plugins/telemetry/schema) to add the X-Pack-related data. The payloads described in these `.json` files must be merged to the OSS ones to get the structure of the full payload sent to the Remote Telemetry Service. All the files follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`. + +There are currently 2 files: + +- `xpack_root.json`: Defines the extra fields x-pack reports over the OSS payload defined in the `oss_root.json`. + Manually maintained for now because the frequency it changes is expected to be pretty low. +- `xpack_plugins.json`: The X-Pack related schema for the content that will be nested in `stack_stats.kibana.plugins`. + It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API. + More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs. + +NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service. + +## Testing + +Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files (+ the OSS definitions), and validates the actual output of the telemetry endpoint against the final schema. \ No newline at end of file diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index d3487078fd114b..527fb0fc040ae6 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -92,26 +92,26 @@ "throttle_time": { "properties": { "min": { - "type": "long" + "type": "keyword" }, "avg": { - "type": "float" + "type": "keyword" }, "max": { - "type": "long" + "type": "keyword" } } }, "schedule_time": { "properties": { "min": { - "type": "long" + "type": "keyword" }, "avg": { - "type": "float" + "type": "keyword" }, "max": { - "type": "long" + "type": "keyword" } } }, @@ -1031,7 +1031,7 @@ }, "environments": { "properties": { - "services_without_environments": { + "services_without_environment": { "type": "long" }, "services_with_multiple_environments": { @@ -1521,6 +1521,17 @@ } } } + }, + "environments": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } } } } @@ -3454,6 +3465,24 @@ }, "disabledFeatures": { "properties": { + "DYNAMIC_KEY": { + "type": "long" + }, + "stackAlerts": { + "type": "long" + }, + "actions": { + "type": "long" + }, + "enterpriseSearch": { + "type": "long" + }, + "fleet": { + "type": "long" + }, + "savedObjectsTagging": { + "type": "long" + }, "indexPatterns": { "type": "long" }, diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_root.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_root.json new file mode 100644 index 00000000000000..afadfc1ec9e922 --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_root.json @@ -0,0 +1,51 @@ +{ + "properties": { + "license": { + "properties": { + "uid": { + "type": "keyword" + }, + "issue_date": { + "type": "date" + }, + "expiry_date": { + "type": "date" + }, + "expiry_date_in_millis": { + "type": "long" + }, + "issue_date_in_millis": { + "type": "long" + }, + "start_date_in_millis": { + "type": "long" + }, + "issued_to": { + "type": "keyword" + }, + "issuer": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "max_nodes": { + "type": "long" + }, + "max_resource_units": { + "type": "long" + } + } + }, + "stack_stats": { + "properties": { + "xpack": { + "type": "pass_through" + } + } + } + } +} diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.js b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts similarity index 78% rename from x-pack/test/api_integration/apis/telemetry/telemetry_local.js rename to x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index 3be24b273ae4c1..7055c60b6cd348 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.js +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -6,22 +6,16 @@ */ import expect from '@kbn/expect'; -import _ from 'lodash'; - -/* - * Create a single-level array with strings for all the paths to values in the - * source object, up to 3 deep. Going deeper than 3 causes a bit too much churn - * in the tests. - */ -function flatKeys(source) { - const recursivelyFlatKeys = (obj, path = [], depth = 0) => { - return depth < 3 && _.isObject(obj) - ? _.map(obj, (v, k) => recursivelyFlatKeys(v, [...path, k], depth + 1)) - : path.join('.'); - }; - - return _.uniq(_.flattenDeep(recursivelyFlatKeys(source))).sort((a, b) => a.localeCompare(b)); -} +import deepmerge from 'deepmerge'; +import type { FtrProviderContext } from '../../ftr_provider_context'; +import { + assertTelemetryPayload, + flatKeys, +} from '../../../../../test/api_integration/apis/telemetry/utils'; +import ossRootTelemetrySchema from '../../../../../src/plugins/telemetry/schema/oss_root.json'; +import ossPluginsTelemetrySchema from '../../../../../src/plugins/telemetry/schema/oss_plugins.json'; +import xpackRootTelemetrySchema from '../../../../plugins/telemetry_collection_xpack/schema/xpack_root.json'; +import xpackPluginsTelemetrySchema from '../../../../plugins/telemetry_collection_xpack/schema/xpack_plugins.json'; const disableCollection = { persistent: { @@ -35,17 +29,17 @@ const disableCollection = { }, }; -export default function ({ getService }) { +export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const esSupertest = getService('esSupertest'); + const es = getService('es'); describe('/api/telemetry/v2/clusters/_stats with monitoring disabled', () => { - before('', async () => { - await esSupertest.put('/_cluster/settings').send(disableCollection).expect(200); + let stats: Record; + + before('disable monitoring and pull local stats', async () => { + await es.cluster.put_settings({ body: disableCollection }); await new Promise((r) => setTimeout(r, 1000)); - }); - it('should pull local stats and validate data types', async () => { const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') @@ -53,8 +47,21 @@ export default function ({ getService }) { .expect(200); expect(body.length).to.be(1); - const stats = body[0]; + stats = body[0]; + }); + it('should pass the schema validation', () => { + const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); + const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); + try { + assertTelemetryPayload({ root, plugins }, stats); + } catch (err) { + err.message = `The telemetry schemas in 'x-pack/plugins/telemetry_collection_xpack/schema/' are out-of-date, please update it as required: ${err.message}`; + throw err; + } + }); + + it('should pass ad-hoc enforced validations', () => { expect(stats.collection).to.be('local'); expect(stats.collectionSource).to.be('local_xpack'); @@ -103,14 +110,7 @@ export default function ({ getService }) { expect(stats.stack_stats.xpack.rollup).to.be.an('object'); }); - it('should pull local stats and validate fields', async () => { - const { body } = await supertest - .post('/api/telemetry/v2/clusters/_stats') - .set('kbn-xsrf', 'xxx') - .send({ unencrypted: true }) - .expect(200); - - const stats = body[0]; + it('should validate mandatory fields exist', () => { const actual = flatKeys(stats); const expected = [ From 1a7709541cfc5f5e5471ee7fb046447f60f941ea Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Mon, 1 Mar 2021 12:09:39 -0700 Subject: [PATCH 05/24] Changes out the default arrays and adds types (#93063) ## Summary Follow up from: https://github.com/elastic/kibana/pull/92928 Removes the default arrays and adds typing to the rule schema in order to see which ones require default arrays vs. which ones can/should be defaulted as `undefined`. Updates unit tests. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../signals/build_bulk_body.test.ts | 21 ------------------- .../signals/build_rule.test.ts | 12 ----------- .../detection_engine/signals/build_rule.ts | 15 ++++++++----- 3 files changed, 10 insertions(+), 38 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 08e33351708970..362c368881b37f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -131,9 +131,6 @@ describe('buildBulkBody', () => { created_at: fakeSignalSourceHit.signal.rule?.created_at, updated_at: fakeSignalSourceHit.signal.rule?.updated_at, exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, @@ -256,9 +253,6 @@ describe('buildBulkBody', () => { created_at: fakeSignalSourceHit.signal.rule?.created_at, updated_at: fakeSignalSourceHit.signal.rule?.updated_at, exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, threshold_result: { terms: [ @@ -380,9 +374,6 @@ describe('buildBulkBody', () => { throttle: 'no_actions', threat: [], exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, @@ -494,9 +485,6 @@ describe('buildBulkBody', () => { updated_at: fakeSignalSourceHit.signal.rule?.updated_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, @@ -601,9 +589,6 @@ describe('buildBulkBody', () => { created_at: fakeSignalSourceHit.signal.rule?.created_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, @@ -707,9 +692,6 @@ describe('buildBulkBody', () => { created_at: fakeSignalSourceHit.signal.rule?.created_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, @@ -813,9 +795,6 @@ describe('buildBulkBody', () => { created_at: fakeSignalSourceHit.signal.rule?.created_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }, depth: 1, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts index 40cc15786392c1..48e04df3704ab1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.test.ts @@ -105,9 +105,6 @@ describe('buildRule', () => { ], exceptions_list: getListArrayMock(), version: 1, - threat_filters: [], - threat_index: [], - threat_mapping: [], }; expect(rule).toEqual(expected); }); @@ -166,9 +163,6 @@ describe('buildRule', () => { created_at: rule.created_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }; expect(rule).toEqual(expected); }); @@ -227,9 +221,6 @@ describe('buildRule', () => { created_at: rule.created_at, throttle: 'no_actions', exceptions_list: getListArrayMock(), - threat_filters: [], - threat_index: [], - threat_mapping: [], }; expect(rule).toEqual(expected); }); @@ -292,9 +283,6 @@ describe('buildRule', () => { throttle: 'no_actions', exceptions_list: getListArrayMock(), version: 1, - threat_filters: [], - threat_index: [], - threat_mapping: [], }; expect(rule).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts index 167724836e01c1..0681a5dddb127a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_rule.ts @@ -64,9 +64,14 @@ export const buildRule = ({ ruleNameMapping: ruleParams.ruleNameOverride, }); - const meta = { ...ruleParams.meta, ...riskScoreMeta, ...severityMeta, ...ruleNameMeta }; + const meta: RulesSchema['meta'] = { + ...ruleParams.meta, + ...riskScoreMeta, + ...severityMeta, + ...ruleNameMeta, + }; - const rule = { + const rule: RulesSchema = { id, rule_id: ruleParams.ruleId ?? '(unknown rule_id)', actions, @@ -103,11 +108,11 @@ export const buildRule = ({ created_by: createdBy, updated_by: updatedBy, threat: ruleParams.threat ?? [], - threat_mapping: ruleParams.threatMapping ?? [], - threat_filters: ruleParams.threatFilters ?? [], + threat_mapping: ruleParams.threatMapping, + threat_filters: ruleParams.threatFilters, threat_indicator_path: ruleParams.threatIndicatorPath, threat_query: ruleParams.threatQuery, - threat_index: ruleParams.threatIndex ?? [], + threat_index: ruleParams.threatIndex, threat_language: ruleParams.threatLanguage, timestamp_override: ruleParams.timestampOverride, throttle, From d9043c1c46adafba3e39c9c1a199fc20033cc3d6 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 1 Mar 2021 14:47:38 -0500 Subject: [PATCH 06/24] [Security Solution][Case][Bug] Removing empty collections when filtering on status (#92048) * Removing empty collections when not filtering on status * Fixing add comment response Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/case/server/services/index.ts | 21 +++++-- .../basic/tests/cases/find_cases.ts | 57 ++++++++++++++++++- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/case/server/services/index.ts b/x-pack/plugins/case/server/services/index.ts index f74e91ca102243..11ceb48d11e9fc 100644 --- a/x-pack/plugins/case/server/services/index.ts +++ b/x-pack/plugins/case/server/services/index.ts @@ -32,6 +32,7 @@ import { CaseType, CaseResponse, caseTypeField, + CasesFindRequest, } from '../../common/api'; import { combineFilters, defaultSortField, groupTotalAlertsByID } from '../common'; import { defaultPage, defaultPerPage } from '../routes/api'; @@ -194,6 +195,8 @@ interface CasesMapWithPageInfo { perPage: number; } +type FindCaseOptions = CasesFindRequest & SavedObjectFindOptions; + export interface CaseServiceSetup { deleteCase(args: GetCaseArgs): Promise<{}>; deleteComment(args: GetCommentArgs): Promise<{}>; @@ -271,7 +274,7 @@ export class CaseService implements CaseServiceSetup { subCaseOptions, }: { client: SavedObjectsClientContract; - caseOptions: SavedObjectFindOptions; + caseOptions: FindCaseOptions; subCaseOptions?: SavedObjectFindOptions; }): Promise { const cases = await this.findCases({ @@ -291,10 +294,20 @@ export class CaseService implements CaseServiceSetup { const subCasesForCase = subCasesResp.subCasesMap.get(caseInfo.id); /** - * This will include empty collections unless the query explicitly requested type === CaseType.individual, in which - * case we'd not have any collections anyway. + * If this case is an individual add it to the return map + * If it is a collection and it has sub cases add it to the return map + * If it is a collection and it does not have sub cases, check and see if we're filtering on a status, + * if we're filtering on a status then exclude the empty collection from the results + * if we're not filtering on a status then include the empty collection (that way we can display all the collections + * when the UI isn't doing any filtering) */ - accMap.set(caseInfo.id, { case: caseInfo, subCases: subCasesForCase }); + if ( + caseInfo.attributes.type === CaseType.individual || + subCasesForCase !== undefined || + !caseOptions.status + ) { + accMap.set(caseInfo.id, { case: caseInfo, subCases: subCasesForCase }); + } return accMap; }, new Map()); diff --git a/x-pack/test/case_api_integration/basic/tests/cases/find_cases.ts b/x-pack/test/case_api_integration/basic/tests/cases/find_cases.ts index 7514044d376ca4..6791c9d1c9e71f 100644 --- a/x-pack/test/case_api_integration/basic/tests/cases/find_cases.ts +++ b/x-pack/test/case_api_integration/basic/tests/cases/find_cases.ts @@ -304,7 +304,9 @@ export default ({ getService }: FtrProviderContext): void => { .get(`${CASES_URL}/_find?sortOrder=asc&status=open`) .expect(200); - expect(body.total).to.eql(2); + // since we're filtering on status and the collection only has an in-progress case, it should only return the + // individual case that has the open status and no collections + expect(body.total).to.eql(1); expect(body.count_closed_cases).to.eql(1); expect(body.count_open_cases).to.eql(1); expect(body.count_in_progress_cases).to.eql(1); @@ -353,7 +355,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(body.count_in_progress_cases).to.eql(0); }); - it('correctly counts stats including a collection without sub cases', async () => { + it('correctly counts stats including a collection without sub cases when not filtering on status', async () => { // delete the sub case on the collection so that it doesn't have any sub cases await supertest .delete(`${SUB_CASES_PATCH_DEL_URL}?ids=["${collection.newSubCaseInfo.subCases![0].id}"]`) @@ -365,11 +367,62 @@ export default ({ getService }: FtrProviderContext): void => { .get(`${CASES_URL}/_find?sortOrder=asc`) .expect(200); + // it should include the collection without sub cases because we did not pass in a filter on status + expect(body.total).to.eql(3); + expect(body.count_closed_cases).to.eql(1); + expect(body.count_open_cases).to.eql(1); + expect(body.count_in_progress_cases).to.eql(0); + }); + + it('correctly counts stats including a collection without sub cases when filtering on tags', async () => { + // delete the sub case on the collection so that it doesn't have any sub cases + await supertest + .delete(`${SUB_CASES_PATCH_DEL_URL}?ids=["${collection.newSubCaseInfo.subCases![0].id}"]`) + .set('kbn-xsrf', 'true') + .send() + .expect(204); + + const { body }: { body: CasesFindResponse } = await supertest + .get(`${CASES_URL}/_find?sortOrder=asc&tags=defacement`) + .expect(200); + + // it should include the collection without sub cases because we did not pass in a filter on status expect(body.total).to.eql(3); expect(body.count_closed_cases).to.eql(1); expect(body.count_open_cases).to.eql(1); expect(body.count_in_progress_cases).to.eql(0); }); + + it('does not return collections without sub cases matching the requested status', async () => { + const { body }: { body: CasesFindResponse } = await supertest + .get(`${CASES_URL}/_find?sortOrder=asc&status=closed`) + .expect(200); + + // it should not include the collection that has a sub case as in-progress + expect(body.total).to.eql(1); + expect(body.count_closed_cases).to.eql(1); + expect(body.count_open_cases).to.eql(1); + expect(body.count_in_progress_cases).to.eql(1); + }); + + it('does not return empty collections when filtering on status', async () => { + // delete the sub case on the collection so that it doesn't have any sub cases + await supertest + .delete(`${SUB_CASES_PATCH_DEL_URL}?ids=["${collection.newSubCaseInfo.subCases![0].id}"]`) + .set('kbn-xsrf', 'true') + .send() + .expect(204); + + const { body }: { body: CasesFindResponse } = await supertest + .get(`${CASES_URL}/_find?sortOrder=asc&status=closed`) + .expect(200); + + // it should not include the collection that has a sub case as in-progress + expect(body.total).to.eql(1); + expect(body.count_closed_cases).to.eql(1); + expect(body.count_open_cases).to.eql(1); + expect(body.count_in_progress_cases).to.eql(0); + }); }); it('unhappy path - 400s when bad query supplied', async () => { From 8552bec23c70fdb2664d84af3d5f1e022883a5a1 Mon Sep 17 00:00:00 2001 From: Yuliia Naumenko Date: Mon, 1 Mar 2021 12:05:19 -0800 Subject: [PATCH 07/24] [Alerting][Docs] Fixed Kibana API does not consistently refer to Kibana Spaces Requests for Alarms and Connectors. (#92948) * [Alerting][Docs] Fixed Kibana API does not consistently refer to Kibana Spaces Requests for Alarms and Connectors. * Apply suggestions from code review Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> --- docs/api/actions-and-connectors/create.asciidoc | 10 +++++++++- docs/api/actions-and-connectors/delete.asciidoc | 6 +++++- docs/api/actions-and-connectors/execute.asciidoc | 7 ++++++- docs/api/actions-and-connectors/get.asciidoc | 5 +++++ docs/api/actions-and-connectors/get_all.asciidoc | 8 ++++++++ docs/api/actions-and-connectors/list.asciidoc | 8 ++++++++ docs/api/actions-and-connectors/update.asciidoc | 5 +++++ docs/api/alerts/create.asciidoc | 5 +++++ docs/api/alerts/delete.asciidoc | 5 +++++ docs/api/alerts/disable.asciidoc | 5 +++++ docs/api/alerts/enable.asciidoc | 5 +++++ docs/api/alerts/find.asciidoc | 8 ++++++++ docs/api/alerts/get.asciidoc | 5 +++++ docs/api/alerts/health.asciidoc | 8 ++++++++ docs/api/alerts/list.asciidoc | 8 ++++++++ docs/api/alerts/mute.asciidoc | 5 +++++ docs/api/alerts/mute_all.asciidoc | 5 +++++ docs/api/alerts/unmute.asciidoc | 5 +++++ docs/api/alerts/unmute_all.asciidoc | 5 +++++ docs/api/alerts/update.asciidoc | 5 +++++ 20 files changed, 120 insertions(+), 3 deletions(-) diff --git a/docs/api/actions-and-connectors/create.asciidoc b/docs/api/actions-and-connectors/create.asciidoc index 22f360fe63eb39..230dad22d3bedc 100644 --- a/docs/api/actions-and-connectors/create.asciidoc +++ b/docs/api/actions-and-connectors/create.asciidoc @@ -11,6 +11,14 @@ Creates an action. `POST :/api/actions/action` +`POST :/s//api/actions/action` + +[[actions-and-connectors-api-create-path-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-create-request-body]] ==== Request body @@ -67,4 +75,4 @@ The API returns the following: }, "isPreconfigured": false } --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- diff --git a/docs/api/actions-and-connectors/delete.asciidoc b/docs/api/actions-and-connectors/delete.asciidoc index e90b9ae44c5bd3..b1270ce822b74c 100644 --- a/docs/api/actions-and-connectors/delete.asciidoc +++ b/docs/api/actions-and-connectors/delete.asciidoc @@ -13,12 +13,17 @@ WARNING: When you delete an action, _it cannot be recovered_. `DELETE :/api/actions/action/` +`DELETE :/s//api/actions/action/` + [[actions-and-connectors-api-delete-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the action. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-delete-response-codes]] ==== Response code @@ -32,4 +37,3 @@ WARNING: When you delete an action, _it cannot be recovered_. $ curl -X DELETE api/actions/action/c55b6eb0-6bad-11eb-9f3b-611eebc6c3ad -------------------------------------------------- // KIBANA - diff --git a/docs/api/actions-and-connectors/execute.asciidoc b/docs/api/actions-and-connectors/execute.asciidoc index 12f1405eb44560..05a27988578ff1 100644 --- a/docs/api/actions-and-connectors/execute.asciidoc +++ b/docs/api/actions-and-connectors/execute.asciidoc @@ -11,12 +11,17 @@ Executes an action by ID. `POST :/api/actions/action//_execute` +`POST :/s//api/actions/action//_execute` + [[actions-and-connectors-api-execute-params]] ==== Path parameters `id`:: (Required, string) The ID of the action. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-execute-request-body]] ==== Request body @@ -80,4 +85,4 @@ The API returns the following: }, "actionId": "c55b6eb0-6bad-11eb-9f3b-611eebc6c3ad" } --------------------------------------------------- \ No newline at end of file +-------------------------------------------------- diff --git a/docs/api/actions-and-connectors/get.asciidoc b/docs/api/actions-and-connectors/get.asciidoc index 6be554e65db049..51af187257d421 100644 --- a/docs/api/actions-and-connectors/get.asciidoc +++ b/docs/api/actions-and-connectors/get.asciidoc @@ -11,12 +11,17 @@ Retrieves an action by ID. `GET :/api/actions/action/` +`GET :/s//api/actions/action/` + [[actions-and-connectors-api-get-params]] ==== Path parameters `id`:: (Required, string) The ID of the action. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-get-codes]] ==== Response code diff --git a/docs/api/actions-and-connectors/get_all.asciidoc b/docs/api/actions-and-connectors/get_all.asciidoc index 9863963c8395e6..7a8025d0d215e1 100644 --- a/docs/api/actions-and-connectors/get_all.asciidoc +++ b/docs/api/actions-and-connectors/get_all.asciidoc @@ -11,6 +11,14 @@ Retrieves all actions. `GET :/api/actions` +`GET :/s//api/actions` + +[[actions-and-connectors-api-get-all-path-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-get-all-codes]] ==== Response code diff --git a/docs/api/actions-and-connectors/list.asciidoc b/docs/api/actions-and-connectors/list.asciidoc index b800b7ff3b4f2f..3647bf06c98e0d 100644 --- a/docs/api/actions-and-connectors/list.asciidoc +++ b/docs/api/actions-and-connectors/list.asciidoc @@ -11,6 +11,14 @@ Retrieves a list of all action types. `GET :/api/actions/list_action_types` +`GET :/s//api/actions/list_action_types` + +[[actions-and-connectors-api-list-path-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-list-codes]] ==== Response code diff --git a/docs/api/actions-and-connectors/update.asciidoc b/docs/api/actions-and-connectors/update.asciidoc index e08ec2f8da1b67..46e6d91cf9e971 100644 --- a/docs/api/actions-and-connectors/update.asciidoc +++ b/docs/api/actions-and-connectors/update.asciidoc @@ -11,12 +11,17 @@ Updates the attributes for an existing action. `PUT :/api/actions/action/` +`PUT :/s//api/actions/action/` + [[actions-and-connectors-api-update-params]] ==== Path parameters `id`:: (Required, string) The ID of the action. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[actions-and-connectors-api-update-request-body]] ==== Request body diff --git a/docs/api/alerts/create.asciidoc b/docs/api/alerts/create.asciidoc index 996503bc591487..c3e6d36813972b 100644 --- a/docs/api/alerts/create.asciidoc +++ b/docs/api/alerts/create.asciidoc @@ -11,12 +11,17 @@ Create {kib} alerts. `POST :/api/alerts/alert/` +`POST :/s//api/alerts/alert/` + [[alerts-api-create-path-params]] ==== Path parameters ``:: (Optional, string) Specifies a UUID v1 or v4 to use instead of a randomly generated ID. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-create-request-body]] ==== Request body diff --git a/docs/api/alerts/delete.asciidoc b/docs/api/alerts/delete.asciidoc index b51005daae658d..72dfd5e87336cb 100644 --- a/docs/api/alerts/delete.asciidoc +++ b/docs/api/alerts/delete.asciidoc @@ -13,12 +13,17 @@ WARNING: Once you delete an alert, you cannot recover it. `DELETE :/api/alerts/alert/` +`DELETE :/s//api/alerts/alert/` + [[alerts-api-delete-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert that you want to remove. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-delete-response-codes]] ==== Response code diff --git a/docs/api/alerts/disable.asciidoc b/docs/api/alerts/disable.asciidoc index 5f74c333794095..86c58c37c2ecd1 100644 --- a/docs/api/alerts/disable.asciidoc +++ b/docs/api/alerts/disable.asciidoc @@ -11,12 +11,17 @@ Disable an alert. `POST :/api/alerts/alert//_disable` +`POST :/s//api/alerts/alert//_disable` + [[alerts-api-disable-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert that you want to disable. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-disable-response-codes]] ==== Response code diff --git a/docs/api/alerts/enable.asciidoc b/docs/api/alerts/enable.asciidoc index a10383f2a440d1..de1a5f7985a38a 100644 --- a/docs/api/alerts/enable.asciidoc +++ b/docs/api/alerts/enable.asciidoc @@ -11,12 +11,17 @@ Enable an alert. `POST :/api/alerts/alert//_enable` +`POST :/s//api/alerts/alert//_enable` + [[alerts-api-enable-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert that you want to enable. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-enable-response-codes]] ==== Response code diff --git a/docs/api/alerts/find.asciidoc b/docs/api/alerts/find.asciidoc index 5af01efbc77873..cc66d4e0f41834 100644 --- a/docs/api/alerts/find.asciidoc +++ b/docs/api/alerts/find.asciidoc @@ -14,6 +14,14 @@ change. Use the find API for traditional paginated results, but avoid using it t `GET :/api/alerts/_find` +`GET :/s//api/alerts/_find` + +[[alerts-api-find-path-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-find-query-params]] ==== Query Parameters diff --git a/docs/api/alerts/get.asciidoc b/docs/api/alerts/get.asciidoc index 934d7466dec3d7..433605e8573325 100644 --- a/docs/api/alerts/get.asciidoc +++ b/docs/api/alerts/get.asciidoc @@ -11,12 +11,17 @@ Retrieve an alert by ID. `GET :/api/alerts/alert/` +`GET :/s//api/alerts/alert/` + [[alerts-api-get-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert to retrieve. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-get-codes]] ==== Response code diff --git a/docs/api/alerts/health.asciidoc b/docs/api/alerts/health.asciidoc index 3710ccf4249454..b29e5def533849 100644 --- a/docs/api/alerts/health.asciidoc +++ b/docs/api/alerts/health.asciidoc @@ -11,6 +11,14 @@ Retrieve the health status of the Alerting framework. `GET :/api/alerts/_health` +`GET :/s//api/alerts/_health` + +[[alerts-api-health-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-health-codes]] ==== Response code diff --git a/docs/api/alerts/list.asciidoc b/docs/api/alerts/list.asciidoc index 0bc3e158ec2634..e180945accfd3e 100644 --- a/docs/api/alerts/list.asciidoc +++ b/docs/api/alerts/list.asciidoc @@ -11,6 +11,14 @@ Retrieve a list of all alert types. `GET :/api/alerts/list_alert_types` +`GET :/s//api/alerts/list_alert_types` + +[[alerts-api-list-params]] +==== Path parameters + +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-list-codes]] ==== Response code diff --git a/docs/api/alerts/mute.asciidoc b/docs/api/alerts/mute.asciidoc index 9279786deae4cf..84a2996b658386 100644 --- a/docs/api/alerts/mute.asciidoc +++ b/docs/api/alerts/mute.asciidoc @@ -11,6 +11,8 @@ Mute an alert instance. `POST :/api/alerts/alert//alert_instance//_mute` +`POST :/s//api/alerts/alert//alert_instance//_mute` + [[alerts-api-mute-path-params]] ==== Path parameters @@ -20,6 +22,9 @@ Mute an alert instance. `alert_instance_id`:: (Required, string) The ID of the alert instance that you want to mute. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-mute-response-codes]] ==== Response code diff --git a/docs/api/alerts/mute_all.asciidoc b/docs/api/alerts/mute_all.asciidoc index f8a8c137240c6b..02f41eb3b768ec 100644 --- a/docs/api/alerts/mute_all.asciidoc +++ b/docs/api/alerts/mute_all.asciidoc @@ -11,12 +11,17 @@ Mute all alert instances. `POST :/api/alerts/alert//_mute_all` +`POST :/s//api/alerts/alert//_mute_all` + [[alerts-api-mute-all-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert whose instances you want to mute. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-mute-all-response-codes]] ==== Response code diff --git a/docs/api/alerts/unmute.asciidoc b/docs/api/alerts/unmute.asciidoc index f091ae3f453257..eb73bb539154f8 100644 --- a/docs/api/alerts/unmute.asciidoc +++ b/docs/api/alerts/unmute.asciidoc @@ -11,6 +11,8 @@ Unmute an alert instance. `POST :/api/alerts/alert//alert_instance//_unmute` +`POST :/s//api/alerts/alert//alert_instance//_unmute` + [[alerts-api-unmute-path-params]] ==== Path parameters @@ -20,6 +22,9 @@ Unmute an alert instance. `alert_instance_id`:: (Required, string) The ID of the alert instance that you want to unmute. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-unmute-response-codes]] ==== Response code diff --git a/docs/api/alerts/unmute_all.asciidoc b/docs/api/alerts/unmute_all.asciidoc index 2359d120cf2602..a20a20fd8204a4 100644 --- a/docs/api/alerts/unmute_all.asciidoc +++ b/docs/api/alerts/unmute_all.asciidoc @@ -11,12 +11,17 @@ Unmute all alert instances. `POST :/api/alerts/alert//_unmute_all` +`POST :/s//api/alerts/alert//_unmute_all` + [[alerts-api-unmute-all-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert whose instances you want to unmute. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-unmute-all-response-codes]] ==== Response code diff --git a/docs/api/alerts/update.asciidoc b/docs/api/alerts/update.asciidoc index aee2dd049a66f7..a0b147ed4a15d1 100644 --- a/docs/api/alerts/update.asciidoc +++ b/docs/api/alerts/update.asciidoc @@ -11,12 +11,17 @@ Update the attributes for an existing alert. `PUT :/api/alerts/alert/` +`PUT :/s//api/alerts/alert/` + [[alerts-api-update-path-params]] ==== Path parameters `id`:: (Required, string) The ID of the alert that you want to update. +`space_id`:: + (Optional, string) An identifier for the space. If `space_id` is not provided in the URL, the default space is used. + [[alerts-api-update-request-body]] ==== Request body From c502e4cd8dfd0bac2047678ad4af5b49f21e1b4e Mon Sep 17 00:00:00 2001 From: Devon Thomson Date: Mon, 1 Mar 2021 15:14:17 -0500 Subject: [PATCH 08/24] [Dashboard] Refactor Initial View Mode (#92747) * refactored dashboard initial view mode --- .../public/application/dashboard_app.tsx | 14 +++- .../application/dashboard_state.test.ts | 70 +++++++++++++++++++ .../application/dashboard_state_manager.ts | 35 +++++++--- .../hooks/use_dashboard_container.test.tsx | 50 +++++++++---- .../hooks/use_dashboard_container.ts | 12 ++-- .../hooks/use_dashboard_state_manager.ts | 11 ++- .../application/lib/get_app_state_defaults.ts | 4 +- .../lib/session_restoration.test.ts | 3 +- .../application/top_nav/dashboard_top_nav.tsx | 1 - .../application/top_nav/get_top_nav_config.ts | 2 +- 10 files changed, 162 insertions(+), 40 deletions(-) diff --git a/src/plugins/dashboard/public/application/dashboard_app.tsx b/src/plugins/dashboard/public/application/dashboard_app.tsx index 8466cf009db9df..fd73741cef8cbd 100644 --- a/src/plugins/dashboard/public/application/dashboard_app.tsx +++ b/src/plugins/dashboard/public/application/dashboard_app.tsx @@ -63,14 +63,26 @@ export function DashboardApp({ const [indexPatterns, setIndexPatterns] = useState([]); const savedDashboard = useSavedDashboard(savedDashboardId, history); + + const getIncomingEmbeddable = useCallback( + (removeAfterFetch?: boolean) => { + return embeddable + .getStateTransfer() + .getIncomingEmbeddablePackage(DashboardConstants.DASHBOARDS_ID, removeAfterFetch); + }, + [embeddable] + ); + const { dashboardStateManager, viewMode, setViewMode } = useDashboardStateManager( savedDashboard, - history + history, + getIncomingEmbeddable ); const [unsavedChanges, setUnsavedChanges] = useState(false); const dashboardContainer = useDashboardContainer({ timeFilter: data.query.timefilter.timefilter, dashboardStateManager, + getIncomingEmbeddable, setUnsavedChanges, history, }); diff --git a/src/plugins/dashboard/public/application/dashboard_state.test.ts b/src/plugins/dashboard/public/application/dashboard_state.test.ts index c5bda98c31b700..ffe5c80febe026 100644 --- a/src/plugins/dashboard/public/application/dashboard_state.test.ts +++ b/src/plugins/dashboard/public/application/dashboard_state.test.ts @@ -43,6 +43,7 @@ describe('DashboardState', function () { savedDashboard, hideWriteControls: false, allowByValueEmbeddables: false, + hasPendingEmbeddable: () => false, kibanaVersion: '7.0.0', kbnUrlStateStorage: createKbnUrlStateStorage(), history: createBrowserHistory(), @@ -199,4 +200,73 @@ describe('DashboardState', function () { expect(dashboardState.getIsDirty()).toBeFalsy(); }); }); + + describe('initial view mode', () => { + test('initial view mode set to view when hideWriteControls is true', () => { + const initialViewModeDashboardState = new DashboardStateManager({ + savedDashboard, + hideWriteControls: true, + allowByValueEmbeddables: false, + hasPendingEmbeddable: () => false, + kibanaVersion: '7.0.0', + kbnUrlStateStorage: createKbnUrlStateStorage(), + history: createBrowserHistory(), + toasts: coreMock.createStart().notifications.toasts, + hasTaggingCapabilities: mockHasTaggingCapabilities, + }); + expect(initialViewModeDashboardState.getViewMode()).toBe(ViewMode.VIEW); + }); + + test('initial view mode set to edit if edit mode specified in URL', () => { + const kbnUrlStateStorage = createKbnUrlStateStorage(); + kbnUrlStateStorage.set('_a', { viewMode: ViewMode.EDIT }); + + const initialViewModeDashboardState = new DashboardStateManager({ + savedDashboard, + kbnUrlStateStorage, + kibanaVersion: '7.0.0', + hideWriteControls: false, + allowByValueEmbeddables: false, + history: createBrowserHistory(), + hasPendingEmbeddable: () => false, + toasts: coreMock.createStart().notifications.toasts, + hasTaggingCapabilities: mockHasTaggingCapabilities, + }); + expect(initialViewModeDashboardState.getViewMode()).toBe(ViewMode.EDIT); + }); + + test('initial view mode set to edit if the dashboard is new', () => { + const newDashboard = getSavedDashboardMock(); + newDashboard.id = undefined; + const initialViewModeDashboardState = new DashboardStateManager({ + savedDashboard: newDashboard, + kibanaVersion: '7.0.0', + hideWriteControls: false, + allowByValueEmbeddables: false, + history: createBrowserHistory(), + hasPendingEmbeddable: () => false, + kbnUrlStateStorage: createKbnUrlStateStorage(), + toasts: coreMock.createStart().notifications.toasts, + hasTaggingCapabilities: mockHasTaggingCapabilities, + }); + expect(initialViewModeDashboardState.getViewMode()).toBe(ViewMode.EDIT); + }); + + test('initial view mode set to edit if there is a pending embeddable', () => { + const newDashboard = getSavedDashboardMock(); + newDashboard.id = undefined; + const initialViewModeDashboardState = new DashboardStateManager({ + savedDashboard: newDashboard, + kibanaVersion: '7.0.0', + hideWriteControls: false, + allowByValueEmbeddables: false, + history: createBrowserHistory(), + hasPendingEmbeddable: () => true, + kbnUrlStateStorage: createKbnUrlStateStorage(), + toasts: coreMock.createStart().notifications.toasts, + hasTaggingCapabilities: mockHasTaggingCapabilities, + }); + expect(initialViewModeDashboardState.getViewMode()).toBe(ViewMode.EDIT); + }); + }); }); diff --git a/src/plugins/dashboard/public/application/dashboard_state_manager.ts b/src/plugins/dashboard/public/application/dashboard_state_manager.ts index 51b12baad47692..d11bdd0399d411 100644 --- a/src/plugins/dashboard/public/application/dashboard_state_manager.ts +++ b/src/plugins/dashboard/public/application/dashboard_state_manager.ts @@ -88,6 +88,7 @@ export class DashboardStateManager { private readonly usageCollection: UsageCollectionSetup | undefined; public readonly hasTaggingCapabilities: SavedObjectTagDecoratorTypeGuard; + private hasPendingEmbeddable: () => boolean; /** * @@ -104,6 +105,7 @@ export class DashboardStateManager { usageCollection, hideWriteControls, kbnUrlStateStorage, + hasPendingEmbeddable, dashboardPanelStorage, hasTaggingCapabilities, allowByValueEmbeddables, @@ -111,6 +113,7 @@ export class DashboardStateManager { history: History; kibanaVersion: string; hideWriteControls: boolean; + hasPendingEmbeddable: () => boolean; allowByValueEmbeddables: boolean; savedDashboard: DashboardSavedObject; toasts: NotificationsStart['toasts']; @@ -126,15 +129,17 @@ export class DashboardStateManager { this.usageCollection = usageCollection; this.hasTaggingCapabilities = hasTaggingCapabilities; this.allowByValueEmbeddables = allowByValueEmbeddables; + this.hasPendingEmbeddable = hasPendingEmbeddable; + this.dashboardPanelStorage = dashboardPanelStorage; + this.kbnUrlStateStorage = kbnUrlStateStorage; // get state defaults from saved dashboard, make sure it is migrated + const viewMode = this.getInitialViewMode(); this.stateDefaults = migrateAppState( - getAppStateDefaults(this.savedDashboard, this.hideWriteControls, this.hasTaggingCapabilities), + getAppStateDefaults(viewMode, this.savedDashboard, this.hasTaggingCapabilities), kibanaVersion, usageCollection ); - this.dashboardPanelStorage = dashboardPanelStorage; - this.kbnUrlStateStorage = kbnUrlStateStorage; // setup initial state by merging defaults with state from url & panels storage // also run migration, as state in url could be of older version @@ -357,8 +362,9 @@ export class DashboardStateManager { // The right way to fix this might be to ensure the defaults object stored on state is a deep // clone, but given how much code uses the state object, I determined that to be too risky of a change for // now. TODO: revisit this! + const currentViewMode = this.stateContainer.get().viewMode; this.stateDefaults = migrateAppState( - getAppStateDefaults(this.savedDashboard, this.hideWriteControls, this.hasTaggingCapabilities), + getAppStateDefaults(currentViewMode, this.savedDashboard, this.hasTaggingCapabilities), this.kibanaVersion, this.usageCollection ); @@ -369,8 +375,7 @@ export class DashboardStateManager { this.stateDefaults.filters = [...this.getLastSavedFilterBars()]; this.isDirty = false; - const currentViewMode = this.stateContainer.get().viewMode; - this.stateContainer.set({ ...this.stateDefaults, viewMode: currentViewMode }); + this.stateContainer.set(this.stateDefaults); } /** @@ -534,9 +539,7 @@ export class DashboardStateManager { return this.appState.viewMode; } // get viewMode should work properly even before the state container is created - return this.savedDashboard.id - ? this.kbnUrlStateStorage.get(STATE_STORAGE_KEY)?.viewMode ?? ViewMode.VIEW - : ViewMode.EDIT; + return this.getInitialViewMode(); } public getIsViewMode() { @@ -668,6 +671,7 @@ export class DashboardStateManager { public switchViewMode(newMode: ViewMode) { this.stateContainer.transitions.set('viewMode', newMode); + this.restorePanels(); } /** @@ -692,6 +696,7 @@ export class DashboardStateManager { ...this.stateDefaults, ...unsavedState, ...this.kbnUrlStateStorage.get(STATE_STORAGE_KEY), + viewMode: this.getViewMode(), }, this.kibanaVersion, this.usageCollection @@ -739,6 +744,18 @@ export class DashboardStateManager { return stateWithoutPanels; } + private getInitialViewMode() { + if (this.hideWriteControls) { + return ViewMode.VIEW; + } + const viewModeFromUrl = this.kbnUrlStateStorage.get(STATE_STORAGE_KEY) + ?.viewMode; + if (viewModeFromUrl) { + return viewModeFromUrl; + } + return !this.savedDashboard.id || this.hasPendingEmbeddable() ? ViewMode.EDIT : ViewMode.VIEW; + } + private checkIsDirty() { // Filters need to be compared manually because they sometimes have a $$hashkey stored on the object. // Query needs to be compared manually because saved legacy queries get migrated in app state automatically diff --git a/src/plugins/dashboard/public/application/hooks/use_dashboard_container.test.tsx b/src/plugins/dashboard/public/application/hooks/use_dashboard_container.test.tsx index 6a6dc58db78157..b2fc7c917d285e 100644 --- a/src/plugins/dashboard/public/application/hooks/use_dashboard_container.test.tsx +++ b/src/plugins/dashboard/public/application/hooks/use_dashboard_container.test.tsx @@ -37,6 +37,7 @@ const createDashboardState = () => hideWriteControls: false, allowByValueEmbeddables: false, history: createBrowserHistory(), + hasPendingEmbeddable: () => false, kbnUrlStateStorage: createKbnUrlStateStorage(), hasTaggingCapabilities: mockHasTaggingCapabilities, toasts: coreMock.createStart().notifications.toasts, @@ -53,6 +54,8 @@ const defaultCapabilities: DashboardCapabilities = { storeSearchSession: true, }; +const getIncomingEmbeddable = () => undefined; + const services = { dashboardCapabilities: defaultCapabilities, data: dataPluginMock.createStartContract(), @@ -87,7 +90,12 @@ test('container is destroyed on unmount', async () => { const dashboardStateManager = createDashboardState(); const { result, unmount, waitForNextUpdate } = renderHook( - () => useDashboardContainer({ dashboardStateManager, history }), + () => + useDashboardContainer({ + getIncomingEmbeddable, + dashboardStateManager, + history, + }), { wrapper: ({ children }) => ( {children} @@ -115,12 +123,20 @@ test('old container is destroyed on new dashboardStateManager', async () => { const { result, waitForNextUpdate, rerender } = renderHook< DashboardStateManager, DashboardContainer | null - >((dashboardStateManager) => useDashboardContainer({ dashboardStateManager, history }), { - wrapper: ({ children }) => ( - {children} - ), - initialProps: createDashboardState(), - }); + >( + (dashboardStateManager) => + useDashboardContainer({ + getIncomingEmbeddable, + dashboardStateManager, + history, + }), + { + wrapper: ({ children }) => ( + {children} + ), + initialProps: createDashboardState(), + } + ); expect(result.current).toBeNull(); // null on initial render @@ -150,12 +166,20 @@ test('destroyed if rerendered before resolved', async () => { const { result, waitForNextUpdate, rerender } = renderHook< DashboardStateManager, DashboardContainer | null - >((dashboardStateManager) => useDashboardContainer({ dashboardStateManager, history }), { - wrapper: ({ children }) => ( - {children} - ), - initialProps: createDashboardState(), - }); + >( + (dashboardStateManager) => + useDashboardContainer({ + getIncomingEmbeddable, + dashboardStateManager, + history, + }), + { + wrapper: ({ children }) => ( + {children} + ), + initialProps: createDashboardState(), + } + ); expect(result.current).toBeNull(); // null on initial render diff --git a/src/plugins/dashboard/public/application/hooks/use_dashboard_container.ts b/src/plugins/dashboard/public/application/hooks/use_dashboard_container.ts index f4fe55f8774004..109380ed0d89bf 100644 --- a/src/plugins/dashboard/public/application/hooks/use_dashboard_container.ts +++ b/src/plugins/dashboard/public/application/hooks/use_dashboard_container.ts @@ -14,6 +14,7 @@ import { ContainerOutput, EmbeddableFactoryNotFoundError, EmbeddableInput, + EmbeddablePackageState, ErrorEmbeddable, isErrorEmbeddable, ViewMode, @@ -21,7 +22,7 @@ import { import { DashboardStateManager } from '../dashboard_state_manager'; import { getDashboardContainerInput, getSearchSessionIdFromURL } from '../dashboard_app_functions'; -import { DashboardConstants, DashboardContainer, DashboardContainerInput } from '../..'; +import { DashboardContainer, DashboardContainerInput } from '../..'; import { DashboardAppServices } from '../types'; import { DASHBOARD_CONTAINER_TYPE } from '..'; import { TimefilterContract } from '../../services/data'; @@ -30,6 +31,7 @@ export const useDashboardContainer = ({ history, timeFilter, setUnsavedChanges, + getIncomingEmbeddable, dashboardStateManager, isEmbeddedExternally, }: { @@ -38,6 +40,7 @@ export const useDashboardContainer = ({ timeFilter?: TimefilterContract; setUnsavedChanges?: (dirty: boolean) => void; dashboardStateManager: DashboardStateManager | null; + getIncomingEmbeddable: (removeAfterFetch?: boolean) => EmbeddablePackageState | undefined; }) => { const { dashboardCapabilities, @@ -77,11 +80,8 @@ export const useDashboardContainer = ({ searchSession.restore(searchSessionIdFromURL); } - const incomingEmbeddable = embeddable - .getStateTransfer() - .getIncomingEmbeddablePackage(DashboardConstants.DASHBOARDS_ID, true); - // when dashboard state manager initially loads, determine whether or not there are unsaved changes + const incomingEmbeddable = getIncomingEmbeddable(true); setUnsavedChanges?.( Boolean(incomingEmbeddable) || dashboardStateManager.hasUnsavedPanelState() ); @@ -131,7 +131,6 @@ export const useDashboardContainer = ({ (incomingEmbeddable.embeddableId && !pendingContainer.getInput().panels[incomingEmbeddable.embeddableId])) ) { - dashboardStateManager.switchViewMode(ViewMode.EDIT); pendingContainer.addNewEmbeddable( incomingEmbeddable.type, incomingEmbeddable.input @@ -154,6 +153,7 @@ export const useDashboardContainer = ({ }, [ dashboardCapabilities, dashboardStateManager, + getIncomingEmbeddable, isEmbeddedExternally, setUnsavedChanges, searchSession, diff --git a/src/plugins/dashboard/public/application/hooks/use_dashboard_state_manager.ts b/src/plugins/dashboard/public/application/hooks/use_dashboard_state_manager.ts index effd598cc3ee87..72b43723f07fbf 100644 --- a/src/plugins/dashboard/public/application/hooks/use_dashboard_state_manager.ts +++ b/src/plugins/dashboard/public/application/hooks/use_dashboard_state_manager.ts @@ -29,7 +29,7 @@ import { createSessionRestorationDataProvider } from '../lib/session_restoration import { DashboardStateManager } from '../dashboard_state_manager'; import { getDashboardTitle } from '../../dashboard_strings'; import { DashboardAppServices } from '../types'; -import { ViewMode } from '../../services/embeddable'; +import { EmbeddablePackageState, ViewMode } from '../../services/embeddable'; // TS is picky with type guards, we can't just inline `() => false` function defaultTaggingGuard(_obj: SavedObject): _obj is TagDecoratedSavedObject { @@ -44,7 +44,8 @@ interface DashboardStateManagerReturn { export const useDashboardStateManager = ( savedDashboard: DashboardSavedObject | null, - history: History + history: History, + getIncomingEmbeddable: () => EmbeddablePackageState | undefined ): DashboardStateManagerReturn => { const { data: dataPlugin, @@ -87,6 +88,7 @@ export const useDashboardStateManager = ( }); const stateManager = new DashboardStateManager({ + hasPendingEmbeddable: () => Boolean(getIncomingEmbeddable()), toasts: core.notifications.toasts, hasTaggingCapabilities, dashboardPanelStorage, @@ -182,10 +184,6 @@ export const useDashboardStateManager = ( } ); - if (stateManager.getIsEditMode()) { - stateManager.restorePanels(); - } - setDashboardStateManager(stateManager); setViewMode(stateManager.getViewMode()); @@ -201,6 +199,7 @@ export const useDashboardStateManager = ( hasTaggingCapabilities, initializerContext.config, dashboardPanelStorage, + getIncomingEmbeddable, hideWriteControls, history, kibanaVersion, diff --git a/src/plugins/dashboard/public/application/lib/get_app_state_defaults.ts b/src/plugins/dashboard/public/application/lib/get_app_state_defaults.ts index a184ad527f9a31..d8d335317c2b23 100644 --- a/src/plugins/dashboard/public/application/lib/get_app_state_defaults.ts +++ b/src/plugins/dashboard/public/application/lib/get_app_state_defaults.ts @@ -12,8 +12,8 @@ import { DashboardSavedObject } from '../../saved_dashboards'; import { DashboardAppStateDefaults } from '../../types'; export function getAppStateDefaults( + viewMode: ViewMode, savedDashboard: DashboardSavedObject, - hideWriteControls: boolean, hasTaggingCapabilities: SavedObjectTagDecoratorTypeGuard ): DashboardAppStateDefaults { return { @@ -26,6 +26,6 @@ export function getAppStateDefaults( options: savedDashboard.optionsJSON ? JSON.parse(savedDashboard.optionsJSON) : {}, query: savedDashboard.getQuery(), filters: savedDashboard.getFilters(), - viewMode: savedDashboard.id || hideWriteControls ? ViewMode.VIEW : ViewMode.EDIT, + viewMode, }; } diff --git a/src/plugins/dashboard/public/application/lib/session_restoration.test.ts b/src/plugins/dashboard/public/application/lib/session_restoration.test.ts index e74108ed5559fb..a6740d8647825c 100644 --- a/src/plugins/dashboard/public/application/lib/session_restoration.test.ts +++ b/src/plugins/dashboard/public/application/lib/session_restoration.test.ts @@ -11,6 +11,7 @@ import { createSessionRestorationDataProvider } from './session_restoration'; import { getAppStateDefaults } from './get_app_state_defaults'; import { getSavedDashboardMock } from '../test_helpers'; import { SavedObjectTagDecoratorTypeGuard } from '../../../../saved_objects_tagging_oss/public'; +import { ViewMode } from '../../services/embeddable'; describe('createSessionRestorationDataProvider', () => { const mockDataPlugin = dataPluginMock.createStartContract(); @@ -18,8 +19,8 @@ describe('createSessionRestorationDataProvider', () => { data: mockDataPlugin, getAppState: () => getAppStateDefaults( + ViewMode.VIEW, getSavedDashboardMock(), - false, ((() => false) as unknown) as SavedObjectTagDecoratorTypeGuard ), getDashboardTitle: () => 'Dashboard', diff --git a/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx b/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx index b51382347b952e..65cb4db5ad543d 100644 --- a/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx +++ b/src/plugins/dashboard/public/application/top_nav/dashboard_top_nav.tsx @@ -166,7 +166,6 @@ export function DashboardTopNav({ function switchViewMode() { dashboardStateManager.switchViewMode(newMode); - dashboardStateManager.restorePanels(); if (savedDashboard?.id && allowByValueEmbeddables) { const { getFullEditPath, title, id } = savedDashboard; diff --git a/src/plugins/dashboard/public/application/top_nav/get_top_nav_config.ts b/src/plugins/dashboard/public/application/top_nav/get_top_nav_config.ts index 801ab54eb9839c..b1b1386c00b198 100644 --- a/src/plugins/dashboard/public/application/top_nav/get_top_nav_config.ts +++ b/src/plugins/dashboard/public/application/top_nav/get_top_nav_config.ts @@ -151,7 +151,7 @@ function getViewConfig(action: NavAction, disableButton?: boolean) { disableButton, id: 'cancel', label: i18n.translate('dashboard.topNave.cancelButtonAriaLabel', { - defaultMessage: 'Return', + defaultMessage: 'Cancel', }), description: i18n.translate('dashboard.topNave.viewConfigDescription', { defaultMessage: 'Switch to view-only mode', From 33b24c5d89b7a755b3690ee025a7aa6715ea548a Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Mon, 1 Mar 2021 12:27:02 -0800 Subject: [PATCH 09/24] [Security Solution][Endpoint][Admin] Fixes 7.12 ransomware migration and mac bug (#92639) --- .../common/endpoint/models/policy_config.ts | 14 ----------- .../policy/migrations/to_v7_12_0.test.ts | 24 ++----------------- .../endpoint/policy/migrations/to_v7_12_0.ts | 9 +++---- .../common/endpoint/types/index.ts | 7 +----- .../common/license/policy_config.test.ts | 19 --------------- .../common/license/policy_config.ts | 16 ++++--------- .../policy/store/policy_details/index.test.ts | 5 ---- .../policy/store/policy_details/middleware.ts | 1 - .../policy/store/policy_details/selectors.ts | 1 - .../policy_forms/protections/ransomware.tsx | 6 ++--- .../apps/endpoint/policy_details.ts | 15 ------------ 11 files changed, 14 insertions(+), 103 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index 04fa8e08785edf..cbac2d03cfb97b 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -54,18 +54,11 @@ export const policyFactory = (): PolicyConfig => { malware: { mode: ProtectionModes.prevent, }, - ransomware: { - mode: ProtectionModes.prevent, - }, popup: { malware: { message: '', enabled: true, }, - ransomware: { - message: '', - enabled: true, - }, }, logging: { file: 'info', @@ -111,19 +104,12 @@ export const policyFactoryWithoutPaidFeatures = ( }, mac: { ...policy.mac, - ransomware: { - mode: ProtectionModes.off, - }, popup: { ...policy.mac.popup, malware: { message: '', enabled: true, }, - ransomware: { - message: '', - enabled: false, - }, }, }, }; diff --git a/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.test.ts b/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.test.ts index 6c583d60b7b15c..936d90cc1aa9c3 100644 --- a/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.test.ts @@ -49,15 +49,6 @@ describe('7.12.0 Endpoint Package Policy migration', () => { }, }, }, - mac: { - // @ts-expect-error - popup: { - malware: { - message: '', - enabled: false, - }, - }, - }, }, }, }, @@ -96,20 +87,9 @@ describe('7.12.0 Endpoint Package Policy migration', () => { policy: { value: { windows: { - ransomware: ProtectionModes.off, - popup: { - malware: { - message: '', - enabled: false, - }, - ransomware: { - message: '', - enabled: false, - }, + ransomware: { + mode: ProtectionModes.off, }, - }, - mac: { - ransomware: ProtectionModes.off, popup: { malware: { message: '', diff --git a/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.ts b/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.ts index 5c0bf8e9650263..06d505a71025f2 100644 --- a/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.ts +++ b/x-pack/plugins/security_solution/common/endpoint/policy/migrations/to_v7_12_0.ts @@ -19,16 +19,17 @@ export const migratePackagePolicyToV7120: SavedObjectMigrationFn; + mac: Pick; /** * Linux-specific policy configuration that is supported via the UI */ diff --git a/x-pack/plugins/security_solution/common/license/policy_config.test.ts b/x-pack/plugins/security_solution/common/license/policy_config.test.ts index fcfb1acfa85b45..e8637e43ce1c73 100644 --- a/x-pack/plugins/security_solution/common/license/policy_config.test.ts +++ b/x-pack/plugins/security_solution/common/license/policy_config.test.ts @@ -77,7 +77,6 @@ describe('policy_config and licenses', () => { it('allows ransomware to be turned on for Platinum licenses', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.ransomware.mode = ProtectionModes.prevent; - policy.mac.ransomware.mode = ProtectionModes.prevent; const valid = isEndpointPolicyValidForLicense(policy, Platinum); expect(valid).toBeTruthy(); @@ -85,7 +84,6 @@ describe('policy_config and licenses', () => { it('blocks ransomware to be turned on for Gold and below licenses', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.ransomware.mode = ProtectionModes.prevent; - policy.mac.ransomware.mode = ProtectionModes.prevent; let valid = isEndpointPolicyValidForLicense(policy, Gold); expect(valid).toBeFalsy(); @@ -96,14 +94,12 @@ describe('policy_config and licenses', () => { it('allows ransomware notification to be turned on with a Platinum license', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.popup.ransomware.enabled = true; - policy.mac.popup.ransomware.enabled = true; const valid = isEndpointPolicyValidForLicense(policy, Platinum); expect(valid).toBeTruthy(); }); it('blocks ransomware notification to be turned on for Gold and below licenses', () => { const policy = policyFactoryWithoutPaidFeatures(); policy.windows.popup.ransomware.enabled = true; - policy.mac.popup.ransomware.enabled = true; let valid = isEndpointPolicyValidForLicense(policy, Gold); expect(valid).toBeFalsy(); @@ -114,14 +110,12 @@ describe('policy_config and licenses', () => { it('allows ransomware notification message changes with a Platinum license', () => { const policy = policyFactory(); policy.windows.popup.ransomware.message = 'BOOM'; - policy.mac.popup.ransomware.message = 'BOOM'; const valid = isEndpointPolicyValidForLicense(policy, Platinum); expect(valid).toBeTruthy(); }); it('blocks ransomware notification message changes for Gold and below licenses', () => { const policy = policyFactory(); policy.windows.popup.ransomware.message = 'BOOM'; - policy.mac.popup.ransomware.message = 'BOOM'; let valid = isEndpointPolicyValidForLicense(policy, Gold); expect(valid).toBeFalsy(); @@ -154,19 +148,13 @@ describe('policy_config and licenses', () => { const policy = policyFactory(); const popupMessage = 'WOOP WOOP'; policy.windows.ransomware.mode = ProtectionModes.detect; - policy.mac.ransomware.mode = ProtectionModes.detect; policy.windows.popup.ransomware.enabled = false; - policy.mac.popup.ransomware.enabled = false; policy.windows.popup.ransomware.message = popupMessage; - policy.mac.popup.ransomware.message = popupMessage; const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Platinum); expect(retPolicy.windows.ransomware.mode).toEqual(ProtectionModes.detect); - expect(retPolicy.mac.ransomware.mode).toEqual(ProtectionModes.detect); expect(retPolicy.windows.popup.ransomware.enabled).toBeFalsy(); - expect(retPolicy.mac.popup.ransomware.enabled).toBeFalsy(); expect(retPolicy.windows.popup.ransomware.message).toEqual(popupMessage); - expect(retPolicy.mac.popup.ransomware.message).toEqual(popupMessage); }); it('resets Platinum-paid malware fields for lower license tiers', () => { @@ -178,14 +166,12 @@ describe('policy_config and licenses', () => { policy.windows.popup.malware.enabled = false; policy.windows.popup.ransomware.message = popupMessage; - policy.mac.popup.ransomware.message = popupMessage; policy.windows.popup.ransomware.enabled = false; const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Gold); expect(retPolicy.windows.popup.malware.enabled).toEqual( defaults.windows.popup.malware.enabled ); expect(retPolicy.windows.popup.malware.message).not.toEqual(popupMessage); - expect(retPolicy.mac.popup.malware.message).not.toEqual(popupMessage); // need to invert the test, since it could be either value expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.malware.message); @@ -196,22 +182,17 @@ describe('policy_config and licenses', () => { const policy = policyFactory(); // what we will modify, and should be reset const popupMessage = 'WOOP WOOP'; policy.windows.popup.ransomware.message = popupMessage; - policy.mac.popup.ransomware.message = popupMessage; const retPolicy = unsetPolicyFeaturesAboveLicenseLevel(policy, Gold); expect(retPolicy.windows.ransomware.mode).toEqual(defaults.windows.ransomware.mode); - expect(retPolicy.mac.ransomware.mode).toEqual(defaults.mac.ransomware.mode); expect(retPolicy.windows.popup.ransomware.enabled).toEqual( defaults.windows.popup.ransomware.enabled ); - expect(retPolicy.mac.popup.ransomware.enabled).toEqual(defaults.mac.popup.ransomware.enabled); expect(retPolicy.windows.popup.ransomware.message).not.toEqual(popupMessage); - expect(retPolicy.mac.popup.ransomware.message).not.toEqual(popupMessage); // need to invert the test, since it could be either value expect(['', DefaultMalwareMessage]).toContain(retPolicy.windows.popup.ransomware.message); - expect(['', DefaultMalwareMessage]).toContain(retPolicy.mac.popup.ransomware.message); }); }); diff --git a/x-pack/plugins/security_solution/common/license/policy_config.ts b/x-pack/plugins/security_solution/common/license/policy_config.ts index fd5105ca03502b..903e241b1b490d 100644 --- a/x-pack/plugins/security_solution/common/license/policy_config.ts +++ b/x-pack/plugins/security_solution/common/license/policy_config.ts @@ -45,27 +45,19 @@ export const isEndpointPolicyValidForLicense = ( } // only platinum or higher may enable ransomware - if ( - policy.windows.ransomware.mode !== defaults.windows.ransomware.mode || - policy.mac.ransomware.mode !== defaults.mac.ransomware.mode - ) { + if (policy.windows.ransomware.mode !== defaults.windows.ransomware.mode) { return false; } // only platinum or higher may enable ransomware notification - if ( - policy.windows.popup.ransomware.enabled !== defaults.windows.popup.ransomware.enabled || - policy.mac.popup.ransomware.enabled !== defaults.mac.popup.ransomware.enabled - ) { + if (policy.windows.popup.ransomware.enabled !== defaults.windows.popup.ransomware.enabled) { return false; } // Only Platinum or higher may change the ransomware message (which can be blank or what Endpoint defaults) if ( - [policy.windows, policy.mac].some( - (p) => - p.popup.ransomware.message !== '' && p.popup.ransomware.message !== DefaultMalwareMessage - ) + policy.windows.popup.ransomware.message !== '' && + policy.windows.popup.ransomware.message !== DefaultMalwareMessage ) { return false; } diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 14c3ecd591631f..74dfbe4dec3bab 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -303,16 +303,11 @@ describe('policy details: ', () => { mac: { events: { process: true, file: true, network: true }, malware: { mode: 'prevent' }, - ransomware: { mode: 'off' }, popup: { malware: { enabled: true, message: '', }, - ransomware: { - enabled: false, - message: '', - }, }, logging: { file: 'info' }, }, diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts index 8cd712cdb21fd6..2e44021a791266 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/middleware.ts @@ -44,7 +44,6 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory UIPolicyConfig = createSel advanced: mac.advanced, events: mac.events, malware: mac.malware, - ransomware: mac.ransomware, popup: mac.popup, }, linux: { diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/ransomware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/ransomware.tsx index 7a672474029e3f..ccc056d81e9125 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/ransomware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/ransomware.tsx @@ -42,7 +42,7 @@ import { AppAction } from '../../../../../../common/store/actions'; import { SupportedVersionNotice } from './supported_version'; import { RadioFlexGroup } from './malware'; -const OSes: Immutable = [OS.windows, OS.mac]; +const OSes: Immutable = [OS.windows]; const protection = 'ransomware'; const ProtectionRadio = React.memo( @@ -50,7 +50,6 @@ const ProtectionRadio = React.memo( const policyDetailsConfig = usePolicyDetailsSelector(policyConfig); const dispatch = useDispatch<(action: AppAction) => void>(); const radioButtonId = useMemo(() => htmlIdGenerator()(), []); - // currently just taking windows.ransomware, but both windows.ransomware and mac.ransomware should be the same value const selected = policyDetailsConfig && policyDetailsConfig.windows.ransomware.mode; const handleRadioChange: EuiRadioProps['onChange'] = useCallback(() => { @@ -97,7 +96,6 @@ ProtectionRadio.displayName = 'ProtectionRadio'; export const Ransomware = React.memo(() => { const policyDetailsConfig = usePolicyDetailsSelector(policyConfig); const dispatch = useDispatch<(action: AppAction) => void>(); - // currently just taking windows.ransomware, but both windows.ransomware and mac.ransomware should be the same value const selected = policyDetailsConfig && policyDetailsConfig.windows.ransomware.mode; const userNotificationSelected = policyDetailsConfig && policyDetailsConfig.windows.popup.ransomware.enabled; @@ -318,7 +316,7 @@ export const Ransomware = React.memo(() => { type={i18n.translate('xpack.securitySolution.endpoint.policy.details.ransomware', { defaultMessage: 'Ransomware', })} - supportedOss={[OperatingSystem.WINDOWS, OperatingSystem.MAC]} + supportedOss={[OperatingSystem.WINDOWS]} dataTestSubj="ransomwareProtectionsForm" rightCorner={protectionSwitch} > diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index ba4687e497230d..0d19d9c28c31ba 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -257,16 +257,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { events: { file: false, network: true, process: true }, logging: { file: 'info' }, malware: { mode: 'prevent' }, - ransomware: { mode: 'prevent' }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, - ransomware: { - enabled: true, - message: 'Elastic Security {action} {filename}', - }, }, }, windows: { @@ -411,16 +406,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { events: { file: true, network: true, process: true }, logging: { file: 'info' }, malware: { mode: 'prevent' }, - ransomware: { mode: 'prevent' }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, - ransomware: { - enabled: true, - message: 'Elastic Security {action} {filename}', - }, }, }, windows: { @@ -558,16 +548,11 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { events: { file: true, network: true, process: true }, logging: { file: 'info' }, malware: { mode: 'prevent' }, - ransomware: { mode: 'prevent' }, popup: { malware: { enabled: true, message: 'Elastic Security {action} {filename}', }, - ransomware: { - enabled: true, - message: 'Elastic Security {action} {filename}', - }, }, }, windows: { From be0797670f7a22672c5e2e26ad9dacd99f516da1 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Mon, 1 Mar 2021 12:29:14 -0800 Subject: [PATCH 10/24] [DOCS] Fixes documentation version (#93101) --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index b91af2ab51ebf8..eb6f794434f8a1 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -7,7 +7,7 @@ :blog-ref: https://www.elastic.co/blog/ :wikipedia: https://en.wikipedia.org/wiki -include::{docs-root}/shared/versions/stack/7.10.asciidoc[] +include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[] :docker-repo: docker.elastic.co/kibana/kibana :docker-image: docker.elastic.co/kibana/kibana:{version} From 1a3bbbf917b6c1c9c1ab31bf098a63c76e260925 Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Mon, 1 Mar 2021 12:31:05 -0800 Subject: [PATCH 11/24] [Security Solution][Endpoint][Admin] Fixes policy sticky footer save test (#92919) * commented code to close out toast --- .../apps/endpoint/policy_details.ts | 6 +++++- .../security_solution_endpoint/page_objects/policy_page.ts | 2 -- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 0d19d9c28c31ba..710099dfca8e9c 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -22,7 +22,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const policyTestResources = getService('policyTestResources'); // Failing: See https://github.com/elastic/kibana/issues/92567 - describe.skip('When on the Endpoint Policy Details Page', function () { + describe('When on the Endpoint Policy Details Page', function () { this.tags(['ciGroup7']); describe('with an invalid policy id', () => { @@ -449,6 +449,10 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { // Clear the value await advancedPolicyField.click(); await advancedPolicyField.clearValueWithKeyboard(); + + // Make sure the toast button closes so the save button on the sticky footer is visible + await (await testSubjects.find('toastCloseButton')).click(); + await testSubjects.waitForHidden('toastCloseButton'); await pageObjects.policy.confirmAndSave(); await testSubjects.existOrFail('policyDetailsSuccessMessage'); diff --git a/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts b/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts index 719e8d345bf441..d1a037a47ff08f 100644 --- a/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts +++ b/x-pack/test/security_solution_endpoint/page_objects/policy_page.ts @@ -10,7 +10,6 @@ import { FtrProviderContext } from '../ftr_provider_context'; export function EndpointPolicyPageProvider({ getService, getPageObjects }: FtrProviderContext) { const pageObjects = getPageObjects(['common', 'header']); const testSubjects = getService('testSubjects'); - const browser = getService('browser'); return { /** @@ -70,7 +69,6 @@ export function EndpointPolicyPageProvider({ getService, getPageObjects }: FtrPr */ async confirmAndSave() { await this.ensureIsOnDetailsPage(); - await browser.scrollTop(); await (await this.findSaveButton()).click(); await testSubjects.existOrFail('policyDetailsConfirmModal'); await pageObjects.common.clickConfirmOnModal(); From d02294cb987fe663b9421b4cfd3fac7165c9bba4 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 1 Mar 2021 13:42:14 -0700 Subject: [PATCH 12/24] [Maps] fix fit to data on heatmap not working (#92697) * [Maps] fix fit to data on heatmap not working * tslint Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../layers/heatmap_layer/heatmap_layer.ts | 33 +++++++++++- .../classes/layers/vector_layer/index.ts | 2 +- .../classes/layers/vector_layer/utils.tsx | 44 +++++++++++++++- .../layers/vector_layer/vector_layer.tsx | 52 ++++--------------- 4 files changed, 86 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.ts b/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.ts index 8eebd7c57afd7f..96c7fcedaf3d9c 100644 --- a/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.ts +++ b/x-pack/plugins/maps/public/classes/layers/heatmap_layer/heatmap_layer.ts @@ -12,7 +12,7 @@ import { HeatmapStyle } from '../../styles/heatmap/heatmap_style'; import { EMPTY_FEATURE_COLLECTION, LAYER_TYPE } from '../../../../common/constants'; import { HeatmapLayerDescriptor, MapQuery } from '../../../../common/descriptor_types'; import { ESGeoGridSource } from '../../sources/es_geo_grid_source'; -import { addGeoJsonMbSource, syncVectorSource } from '../vector_layer'; +import { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from '../vector_layer'; import { DataRequestContext } from '../../../actions'; import { DataRequestAbortError } from '../../util/data_request'; @@ -46,6 +46,12 @@ export class HeatmapLayer extends AbstractLayer { } } + destroy() { + if (this.getSource()) { + this.getSource().destroy(); + } + } + getSource(): ESGeoGridSource { return super.getSource() as ESGeoGridSource; } @@ -179,4 +185,29 @@ export class HeatmapLayer extends AbstractLayer { const metricFields = this.getSource().getMetricFields(); return this.getCurrentStyle().renderLegendDetails(metricFields[0]); } + + async getBounds(syncContext: DataRequestContext) { + return await getVectorSourceBounds({ + layerId: this.getId(), + syncContext, + source: this.getSource(), + sourceQuery: this.getQuery() as MapQuery, + }); + } + + async isFilteredByGlobalTime(): Promise { + return this.getSource().getApplyGlobalTime() && (await this.getSource().isTimeAware()); + } + + getIndexPatternIds() { + return this.getSource().getIndexPatternIds(); + } + + getQueryableIndexPatternIds() { + return this.getSource().getQueryableIndexPatternIds(); + } + + async getLicensedFeatures() { + return await this.getSource().getLicensedFeatures(); + } } diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts index 4b509ba5dff000..b6777f8a5e4544 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/index.ts @@ -5,5 +5,5 @@ * 2.0. */ -export { addGeoJsonMbSource, syncVectorSource } from './utils'; +export { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from './utils'; export { IVectorLayer, VectorLayer, VectorLayerArguments } from './vector_layer'; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/utils.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/utils.tsx index a3754b20de8185..91bdd74c158f9d 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/utils.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/utils.tsx @@ -9,10 +9,11 @@ import { FeatureCollection } from 'geojson'; import { Map as MbMap } from 'mapbox-gl'; import { EMPTY_FEATURE_COLLECTION, + SOURCE_BOUNDS_DATA_REQUEST_ID, SOURCE_DATA_REQUEST_ID, VECTOR_SHAPE_TYPE, } from '../../../../common/constants'; -import { VectorSourceRequestMeta } from '../../../../common/descriptor_types'; +import { MapExtent, MapQuery, VectorSourceRequestMeta } from '../../../../common/descriptor_types'; import { DataRequestContext } from '../../../actions'; import { IVectorSource } from '../../sources/vector_source'; import { DataRequestAbortError } from '../../util/data_request'; @@ -112,3 +113,44 @@ export async function syncVectorSource({ throw error; } } + +export async function getVectorSourceBounds({ + layerId, + syncContext, + source, + sourceQuery, +}: { + layerId: string; + syncContext: DataRequestContext; + source: IVectorSource; + sourceQuery: MapQuery | null; +}): Promise { + const { startLoading, stopLoading, registerCancelCallback, dataFilters } = syncContext; + + const requestToken = Symbol(`${SOURCE_BOUNDS_DATA_REQUEST_ID}-${layerId}`); + + // Do not pass all searchFilters to source.getBoundsForFilters(). + // For example, do not want to filter bounds request by extent and buffer. + const boundsFilters = { + sourceQuery: sourceQuery ? sourceQuery : undefined, + query: dataFilters.query, + timeFilters: dataFilters.timeFilters, + filters: dataFilters.filters, + applyGlobalQuery: source.getApplyGlobalQuery(), + applyGlobalTime: source.getApplyGlobalTime(), + }; + + let bounds = null; + try { + startLoading(SOURCE_BOUNDS_DATA_REQUEST_ID, requestToken, boundsFilters); + bounds = await source.getBoundsForFilters( + boundsFilters, + registerCancelCallback.bind(null, requestToken) + ); + } finally { + // Use stopLoading callback instead of onLoadError callback. + // Function is loading bounds and not feature data. + stopLoading(SOURCE_BOUNDS_DATA_REQUEST_ID, requestToken, bounds ? bounds : {}); + } + return bounds; +} diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx index 2373ed3ba2062f..b21bff99226717 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx @@ -17,7 +17,6 @@ import { FEATURE_ID_PROPERTY_NAME, SOURCE_META_DATA_REQUEST_ID, SOURCE_FORMATTERS_DATA_REQUEST_ID, - SOURCE_BOUNDS_DATA_REQUEST_ID, FEATURE_VISIBLE_PROPERTY_NAME, EMPTY_FEATURE_COLLECTION, KBN_TOO_MANY_FEATURES_PROPERTY, @@ -60,7 +59,7 @@ import { IDynamicStyleProperty } from '../../styles/vector/properties/dynamic_st import { IESSource } from '../../sources/es_source'; import { PropertiesMap } from '../../../../common/elasticsearch_util'; import { ITermJoinSource } from '../../sources/term_join_source'; -import { addGeoJsonMbSource, syncVectorSource } from './utils'; +import { addGeoJsonMbSource, getVectorSourceBounds, syncVectorSource } from './utils'; interface SourceResult { refreshed: boolean; @@ -241,47 +240,16 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { return this.getCurrentStyle().renderLegendDetails(); } - async getBounds({ - startLoading, - stopLoading, - registerCancelCallback, - dataFilters, - }: DataRequestContext) { + async getBounds(syncContext: DataRequestContext) { const isStaticLayer = !this.getSource().isBoundsAware(); - if (isStaticLayer || this.hasJoins()) { - return getFeatureCollectionBounds(this._getSourceFeatureCollection(), this.hasJoins()); - } - - const requestToken = Symbol(`${SOURCE_BOUNDS_DATA_REQUEST_ID}-${this.getId()}`); - const searchFilters: VectorSourceRequestMeta = this._getSearchFilters( - dataFilters, - this.getSource(), - this.getCurrentStyle() - ); - // Do not pass all searchFilters to source.getBoundsForFilters(). - // For example, do not want to filter bounds request by extent and buffer. - const boundsFilters = { - sourceQuery: searchFilters.sourceQuery, - query: searchFilters.query, - timeFilters: searchFilters.timeFilters, - filters: searchFilters.filters, - applyGlobalQuery: searchFilters.applyGlobalQuery, - applyGlobalTime: searchFilters.applyGlobalTime, - }; - - let bounds = null; - try { - startLoading(SOURCE_BOUNDS_DATA_REQUEST_ID, requestToken, boundsFilters); - bounds = await this.getSource().getBoundsForFilters( - boundsFilters, - registerCancelCallback.bind(null, requestToken) - ); - } finally { - // Use stopLoading callback instead of onLoadError callback. - // Function is loading bounds and not feature data. - stopLoading(SOURCE_BOUNDS_DATA_REQUEST_ID, requestToken, bounds ? bounds : {}, boundsFilters); - } - return bounds; + return isStaticLayer || this.hasJoins() + ? getFeatureCollectionBounds(this._getSourceFeatureCollection(), this.hasJoins()) + : getVectorSourceBounds({ + layerId: this.getId(), + syncContext, + source: this.getSource(), + sourceQuery: this.getQuery() as MapQuery, + }); } async getLeftJoinFields() { From a55d8b60eaea4f8b4b394567a8cf3e7ccff68787 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 1 Mar 2021 14:57:35 -0600 Subject: [PATCH 13/24] [Security Solution][Detecttions] Indicator enrichment tweaks (#92989) * Update copy of rule config * Encode threat index as part of our named query * Add index to named query, and enrich both id and index We still need mappings and to fix integration tests, but this generates the correct data. * Update integration tests with new enrichment fields Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../rules/step_about_rule/schema.tsx | 4 +- .../build_threat_mapping_filter.ts | 1 + .../enrich_signal_threat_matches.mock.ts | 1 + .../enrich_signal_threat_matches.test.ts | 94 +++++++++++++++++-- .../enrich_signal_threat_matches.ts | 2 +- .../signals/threat_mapping/types.ts | 1 + .../signals/threat_mapping/utils.test.ts | 5 +- .../signals/threat_mapping/utils.ts | 11 ++- .../tests/create_threat_matching.ts | 46 ++++++--- 9 files changed, 136 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx index 07012af17e7343..3467b34d47135c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/schema.tsx @@ -198,14 +198,14 @@ export const schema: FormSchema = { label: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathLabel', { - defaultMessage: 'Threat Indicator Path', + defaultMessage: 'Indicator prefix override', } ), helpText: i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThreatIndicatorPathHelpText', { defaultMessage: - 'Specify the document path containing your threat indicator fields. Used for enrichment of indicator match alerts.', + 'Specify the document prefix containing your indicator fields. Used for enrichment of indicator match alerts.', } ), labelAppend: OptionalFieldLabel, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 0a2789ec2f1d0d..18204bb678a477 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -84,6 +84,7 @@ export const createInnerAndClauses = ({ query: value[0], _name: encodeThreatMatchNamedQuery({ id: threatListItem._id, + index: threatListItem._index, field: threatMappingEntry.field, value: threatMappingEntry.value, }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts index a3ff932e97886e..f66dc461c6d408 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts @@ -12,6 +12,7 @@ export const getNamedQueryMock = ( overrides: Partial = {} ): ThreatMatchNamedQuery => ({ id: 'id', + index: 'index', field: 'field', value: 'value', ...overrides, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index b77e8228e72d89..7b3ca099cc93c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -88,7 +88,12 @@ describe('buildMatchedIndicator', () => { }), ]; queries = [ - getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }), + getNamedQueryMock({ + id: '123', + index: 'threat-index', + field: 'event.field', + value: 'threat.indicator.domain', + }), ]; }); @@ -112,6 +117,26 @@ describe('buildMatchedIndicator', () => { expect(get(indicator, 'matched.atomic')).toEqual('domain_1'); }); + it('returns the _id of the matched indicator as matched.id', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + indicatorPath, + }); + + expect(get(indicator, 'matched.id')).toEqual('123'); + }); + + it('returns the _index of the matched indicator as matched.index', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + indicatorPath, + }); + + expect(get(indicator, 'matched.index')).toEqual('threat-index'); + }); + it('returns the field of the matched indicator as matched.field', () => { const [indicator] = buildMatchedIndicator({ queries, @@ -173,6 +198,8 @@ describe('buildMatchedIndicator', () => { domain: 'domain_1', matched: { atomic: 'domain_1', + id: '123', + index: 'threat-index', field: 'event.field', type: 'type_1', }, @@ -211,6 +238,8 @@ describe('buildMatchedIndicator', () => { indicator_field: 'indicator_field_1', matched: { atomic: 'domain_1', + id: '123', + index: 'threat-index', field: 'event.field', type: 'indicator_type', }, @@ -237,6 +266,8 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: undefined, }, @@ -262,6 +293,8 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: undefined, }, @@ -295,6 +328,8 @@ describe('buildMatchedIndicator', () => { domain: 'foo', matched: { atomic: undefined, + id: '123', + index: 'threat-index', field: 'event.field', type: 'first', }, @@ -362,7 +397,12 @@ describe('enrichSignalThreatMatches', () => { }), ]; matchedQuery = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }) + getNamedQueryMock({ + id: '123', + index: 'indicator_index', + field: 'event.field', + value: 'threat.indicator.domain', + }) ); }); @@ -395,7 +435,13 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -418,7 +464,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { - matched: { atomic: undefined, field: 'event.field', type: undefined }, + matched: { + atomic: undefined, + id: '123', + index: 'indicator_index', + field: 'event.field', + type: undefined, + }, }, ]); }); @@ -441,7 +493,13 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -477,6 +535,7 @@ describe('enrichSignalThreatMatches', () => { matchedQuery = encodeThreatMatchNamedQuery( getNamedQueryMock({ id: '123', + index: 'custom_index', field: 'event.field', value: 'custom_threat.custom_indicator.domain', }) @@ -496,7 +555,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { domain: 'custom_domain', - matched: { atomic: 'custom_domain', field: 'event.field', type: 'custom_type' }, + matched: { + atomic: 'custom_domain', + id: '123', + index: 'custom_index', + field: 'event.field', + type: 'custom_type', + }, other: 'custom_other', type: 'custom_type', }, @@ -526,7 +591,12 @@ describe('enrichSignalThreatMatches', () => { _id: 'signal123', matched_queries: [ encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '456', field: 'event.other', value: 'threat.indicator.domain' }) + getNamedQueryMock({ + id: '456', + index: 'other_custom_index', + field: 'event.other', + value: 'threat.indicator.domain', + }) ), ], }); @@ -545,7 +615,13 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, + matched: { + atomic: 'domain_1', + id: '123', + index: 'indicator_index', + field: 'event.field', + type: 'type_1', + }, other: 'other_1', type: 'type_1', }, @@ -553,6 +629,8 @@ describe('enrichSignalThreatMatches', () => { domain: 'domain_2', matched: { atomic: 'domain_2', + id: '456', + index: 'other_custom_index', field: 'event.other', type: 'type_2', }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 3c8b80886cabeb..bc2be4ecaab329 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -60,7 +60,7 @@ export const buildMatchedIndicator = ({ return { ...indicator, - matched: { atomic, field: query.field, type }, + matched: { atomic, field: query.field, id: query.id, index: query.index, type }, }; }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 1c35a5af09b38b..14763f3493c149 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -199,6 +199,7 @@ export interface SortWithTieBreaker { export interface ThreatMatchNamedQuery { id: string; + index: string; field: string; value: string; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 897143f9ae5744..6219da93036ee8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -589,6 +589,7 @@ describe('utils', () => { it('generates a string that can be later decoded', () => { const encoded = encodeThreatMatchNamedQuery({ id: 'id', + index: 'index', field: 'field', value: 'value', }); @@ -601,6 +602,7 @@ describe('utils', () => { it('can decode an encoded query', () => { const query: ThreatMatchNamedQuery = { id: 'my_id', + index: 'index', field: 'threat.indicator.domain', value: 'host.name', }; @@ -623,6 +625,7 @@ describe('utils', () => { it('raises an error if the query is missing a value', () => { const badQuery: ThreatMatchNamedQuery = { id: 'my_id', + index: 'index', // @ts-expect-error field intentionally undefined field: undefined, value: 'host.name', @@ -630,7 +633,7 @@ describe('utils', () => { const badInput = encodeThreatMatchNamedQuery(badQuery); expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( - 'Decoded query is invalid. Decoded value: {"id":"my_id","field":"","value":"host.name"}' + 'Decoded query is invalid. Decoded value: {"id":"my_id","index":"index","field":"","value":"host.name"}' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 72d9257798e1c9..805aca563701c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -115,21 +115,22 @@ export const combineConcurrentResults = ( return combineResults(currentResult, maxedNewResult); }; -const separator = '___SEPARATOR___'; +const separator = '__SEP__'; export const encodeThreatMatchNamedQuery = ({ id, + index, field, value, }: ThreatMatchNamedQuery): string => { - return [id, field, value].join(separator); + return [id, index, field, value].join(separator); }; export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQuery => { const queryValues = encoded.split(separator); - const [id, field, value] = queryValues; - const query = { id, field, value }; + const [id, index, field, value] = queryValues; + const query = { id, index, field, value }; - if (queryValues.length !== 3 || !queryValues.every(Boolean)) { + if (queryValues.length !== 4 || !queryValues.every(Boolean)) { const queryString = JSON.stringify(query); throw new Error(`Decoded query is invalid. Decoded value: ${queryString}`); } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 20d2b107dc2cc1..9da98e316315db 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -307,6 +307,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -327,6 +329,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -388,6 +392,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -401,6 +407,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'ip', }, @@ -468,6 +476,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -475,18 +485,6 @@ export default ({ getService }: FtrProviderContext) => { provider: 'geenensp', type: 'url', }, - { - description: 'this should match auditbeat/hosts on ip', - first_seen: '2021-01-26T11:06:03.000Z', - ip: '45.115.45.3', - matched: { - atomic: '45.115.45.3', - field: 'source.ip', - type: 'ip', - }, - provider: 'other_provider', - type: 'ip', - }, // We do not merge matched indicators during enrichment, so in // certain circumstances a given indicator document could appear // multiple times in an enriched alert (albeit with different @@ -498,6 +496,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.port', type: 'url', }, @@ -505,6 +505,20 @@ export default ({ getService }: FtrProviderContext) => { provider: 'geenensp', type: 'url', }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + id: '978787', + index: 'filebeat-8.0.0-2021.01.26-000001', + field: 'source.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, ], }, ]); @@ -570,6 +584,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -590,6 +606,8 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', + id: '978783', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'destination.ip', type: 'url', }, @@ -606,6 +624,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.ip', type: 'url', }, @@ -619,6 +639,8 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, + id: '978785', + index: 'filebeat-8.0.0-2021.01.26-000001', field: 'source.port', type: 'url', }, From 08c40955a41abb263a65f59c34fb2ce2e1905b0f Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 1 Mar 2021 14:11:18 -0700 Subject: [PATCH 14/24] [Maps] fix results trimmed tooltip message doubles feature count for line and polygon features (#92932) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../data_request_descriptor_types.ts | 1 + .../maps/public/actions/data_request_actions.ts | 12 ++++++++++-- .../sources/es_search_source/es_search_source.tsx | 12 +++++------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts index 5394d00ba16eb1..dd01b7b596c302 100644 --- a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts @@ -75,6 +75,7 @@ export type VectorStyleRequestMeta = MapFilters & { export type ESSearchSourceResponseMeta = { areResultsTrimmed?: boolean; + resultsCount?: number; // top hits meta areEntitiesTrimmed?: boolean; diff --git a/x-pack/plugins/maps/public/actions/data_request_actions.ts b/x-pack/plugins/maps/public/actions/data_request_actions.ts index f5a09fdc8fffce..6b57da132e895e 100644 --- a/x-pack/plugins/maps/public/actions/data_request_actions.ts +++ b/x-pack/plugins/maps/public/actions/data_request_actions.ts @@ -14,7 +14,12 @@ import uuid from 'uuid/v4'; import { multiPoint } from '@turf/helpers'; import { FeatureCollection } from 'geojson'; import { MapStoreState } from '../reducers/store'; -import { LAYER_STYLE_TYPE, LAYER_TYPE, SOURCE_DATA_REQUEST_ID } from '../../common/constants'; +import { + KBN_IS_CENTROID_FEATURE, + LAYER_STYLE_TYPE, + LAYER_TYPE, + SOURCE_DATA_REQUEST_ID, +} from '../../common/constants'; import { getDataFilters, getDataRequestDescriptor, @@ -246,7 +251,10 @@ function endDataLoad( const layer = getLayerById(layerId, getState()); const resultMeta: ResultMeta = {}; if (layer && layer.getType() === LAYER_TYPE.VECTOR) { - resultMeta.featuresCount = features.length; + const featuresWithoutCentroids = features.filter((feature) => { + return feature.properties ? !feature.properties[KBN_IS_CENTROID_FEATURE] : true; + }); + resultMeta.featuresCount = featuresWithoutCentroids.length; } eventHandlers.onDataLoadEnd({ diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index d5087664b399c7..785b00c06dd54d 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -11,7 +11,7 @@ import rison from 'rison-node'; import { i18n } from '@kbn/i18n'; import { IFieldType, IndexPattern } from 'src/plugins/data/public'; -import { FeatureCollection, GeoJsonProperties } from 'geojson'; +import { GeoJsonProperties } from 'geojson'; import { AbstractESSource } from '../es_source'; import { getHttp, getSearchService } from '../../../kibana_services'; import { addFieldToDSL, getField, hitsToGeoJson } from '../../../../common/elasticsearch_util'; @@ -399,6 +399,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye return { hits: resp.hits.hits.reverse(), // Reverse hits so top documents by sort are drawn on top meta: { + resultsCount: resp.hits.hits.length, areResultsTrimmed: resp.hits.total > resp.hits.hits.length, }, }; @@ -589,11 +590,8 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye } getSourceTooltipContent(sourceDataRequest?: DataRequest): SourceTooltipConfig { - const featureCollection: FeatureCollection | null = sourceDataRequest - ? (sourceDataRequest.getData() as FeatureCollection) - : null; const meta = sourceDataRequest ? sourceDataRequest.getMeta() : null; - if (!featureCollection || !meta) { + if (!meta) { // no tooltip content needed when there is no feature collection or meta return { tooltipContent: null, @@ -631,7 +629,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye return { tooltipContent: i18n.translate('xpack.maps.esSearch.resultsTrimmedMsg', { defaultMessage: `Results limited to first {count} documents.`, - values: { count: featureCollection.features.length }, + values: { count: meta.resultsCount }, }), areResultsTrimmed: true, }; @@ -640,7 +638,7 @@ export class ESSearchSource extends AbstractESSource implements ITiledSingleLaye return { tooltipContent: i18n.translate('xpack.maps.esSearch.featureCountMsg', { defaultMessage: `Found {count} documents.`, - values: { count: featureCollection.features.length }, + values: { count: meta.resultsCount }, }), areResultsTrimmed: false, }; From 59c0380de0fec9df7d15a8d71bc92a86f9e33939 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Mon, 1 Mar 2021 14:11:39 -0700 Subject: [PATCH 15/24] [Maps] fix MapboxDraw import from pointing to dist just pointing to folder (#93087) --- .../connected_components/mb_map/draw_control/draw_control.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js index 4734234f4ec856..aaec7ecb2b34fd 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js +++ b/x-pack/plugins/maps/public/connected_components/mb_map/draw_control/draw_control.js @@ -8,7 +8,7 @@ import _ from 'lodash'; import React from 'react'; import { DRAW_TYPE } from '../../../../common/constants'; -import MapboxDraw from '@mapbox/mapbox-gl-draw/dist/mapbox-gl-draw-unminified'; +import MapboxDraw from '@mapbox/mapbox-gl-draw'; import DrawRectangle from 'mapbox-gl-draw-rectangle-mode'; import { DrawCircle } from './draw_circle'; import { From 0320f1afb7e21078ffe1f59926ed6c196f47c518 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Mon, 1 Mar 2021 15:16:29 -0600 Subject: [PATCH 16/24] Hide instances latency distribution chart (#92869) ...until #88852 and #92631 are resolved. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../service_overview_instances_chart_and_table.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx index b01529a86e88fb..435def8bb9a91c 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_instances_chart_and_table.tsx @@ -10,9 +10,15 @@ import React from 'react'; import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context'; import { useUrlParams } from '../../../context/url_params_context/use_url_params'; import { useFetcher } from '../../../hooks/use_fetcher'; -import { InstancesLatencyDistributionChart } from '../../shared/charts/instances_latency_distribution_chart'; import { ServiceOverviewInstancesTable } from './service_overview_instances_table'; +// We're hiding this chart until these issues are resolved in the 7.13 timeframe: +// +// * [[APM] Tooltips for instances latency distribution chart](https://github.com/elastic/kibana/issues/88852) +// * [[APM] x-axis on the instance bubble chart is broken](https://github.com/elastic/kibana/issues/92631) +// +// import { InstancesLatencyDistributionChart } from '../../shared/charts/instances_latency_distribution_chart'; + interface ServiceOverviewInstancesChartAndTableProps { chartHeight: number; serviceName: string; @@ -66,13 +72,13 @@ export function ServiceOverviewInstancesChartAndTable({ return ( <> - + {/* - + */} Date: Mon, 1 Mar 2021 23:21:00 +0200 Subject: [PATCH 17/24] [Security Solution][Case] Migrate category & subcategory fields of ServiceNow ITSM connector (#93092) --- .../server/saved_object_types/migrations.ts | 22 +++++++++++++--- .../basic/tests/cases/migrations.ts | 24 +++++++++++++++++- .../cases/migrations/7.11.1/data.json.gz | Bin 679 -> 1039 bytes 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/case/server/saved_object_types/migrations.ts b/x-pack/plugins/case/server/saved_object_types/migrations.ts index 117e2eaeea4d82..bf9694d7e6bb0d 100644 --- a/x-pack/plugins/case/server/saved_object_types/migrations.ts +++ b/x-pack/plugins/case/server/saved_object_types/migrations.ts @@ -8,7 +8,13 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { SavedObjectUnsanitizedDoc, SavedObjectSanitizedDoc } from '../../../../../src/core/server'; -import { ConnectorTypes, CommentType, CaseType, AssociationType } from '../../common/api'; +import { + ConnectorTypes, + CommentType, + CaseType, + AssociationType, + ESConnectorFields, +} from '../../common/api'; interface UnsanitizedCaseConnector { connector_id: string; @@ -24,7 +30,7 @@ interface SanitizedCaseConnector { id: string; name: string | null; type: string | null; - fields: null; + fields: null | ESConnectorFields; }; } @@ -88,13 +94,21 @@ export const caseMigrations = { }; }, '7.12.0': ( - doc: SavedObjectUnsanitizedDoc> - ): SavedObjectSanitizedDoc => { + doc: SavedObjectUnsanitizedDoc + ): SavedObjectSanitizedDoc => { + const { fields, type } = doc.attributes.connector; return { ...doc, attributes: { ...doc.attributes, type: CaseType.individual, + connector: { + ...doc.attributes.connector, + fields: + Array.isArray(fields) && fields.length > 0 && type === ConnectorTypes.serviceNowITSM + ? [...fields, { key: 'category', value: null }, { key: 'subcategory', value: null }] + : fields, + }, }, references: doc.references || [], }; diff --git a/x-pack/test/case_api_integration/basic/tests/cases/migrations.ts b/x-pack/test/case_api_integration/basic/tests/cases/migrations.ts index e75ea0f8a89c9f..e66b623138e605 100644 --- a/x-pack/test/case_api_integration/basic/tests/cases/migrations.ts +++ b/x-pack/test/case_api_integration/basic/tests/cases/migrations.ts @@ -58,7 +58,6 @@ export default function createGetTests({ getService }: FtrProviderContext) { // tests upgrading a 7.11.1 saved object to the latest version describe('7.11.1 -> latest stack version', () => { - const caseID = '2ea28c10-7855-11eb-9ca6-83ec5acb735f'; before(async () => { await esArchiver.load('cases/migrations/7.11.1'); }); @@ -68,6 +67,7 @@ export default function createGetTests({ getService }: FtrProviderContext) { }); it('adds rule info to only alert comments for 7.12', async () => { + const caseID = '2ea28c10-7855-11eb-9ca6-83ec5acb735f'; // user comment let { body } = await supertest .get(`${CASES_URL}/${caseID}/comments/34a20a00-7855-11eb-9ca6-83ec5acb735f`) @@ -84,6 +84,28 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(body).key('rule'); expect(body.rule).to.eql({ id: null, name: null }); }); + + it('adds category and subcategory to the ITSM connector', async () => { + const { body } = await supertest + .get(`${CASES_URL}/6f973440-7abd-11eb-9ca6-83ec5acb735f`) + .set('kbn-xsrf', 'true') + .send() + .expect(200); + + expect(body).key('connector'); + expect(body.connector).to.eql({ + id: '444ebab0-7abd-11eb-9ca6-83ec5acb735f', + name: 'SN', + type: '.servicenow', + fields: { + impact: '2', + severity: '2', + urgency: '2', + category: null, + subcategory: null, + }, + }); + }); }); }); } diff --git a/x-pack/test/functional/es_archives/cases/migrations/7.11.1/data.json.gz b/x-pack/test/functional/es_archives/cases/migrations/7.11.1/data.json.gz index 219f58b7e9d216faa6b20e2ea4d7e6a5a5c0a47c..efb62cba5675de208dc0af5a77b2f07566218af8 100644 GIT binary patch literal 1039 zcmV+q1n~PGiwFq7JUw6l17u-zVJ>QOZ*BnXSWR=AJP^M7S1_Cn1LD)1-pqUIC6{!1 zNS*Nrv37Wig$IeGo{ax{B>}b|xW#?rw4G^AVOP7-emsw~ewi7D$@e=j6T{3(V$P-D z2^Y1Lyoe9+nP*Z1wNg8>!O&yQlc~M z9nVWDmt#X#Og}Yz<B6TRoy{p7;v7`4I4(q_LSTdP;sf@RSlyDU>x&Wt}x3T zP<>~PH^aJ`_!(Ha)k^Z98|1uDK}WXh+OF~MAe4HSoh=zc2?}iFd-r1&zrJL%LFUYD zzNxS*+b5`4Q#~DIE$*6B937p^D_BDXlz^V%V=|8bX#5<-6EYYC_swqYUPo6e$wM<4 z%3(HpxsaeNQf{|EIZHerBL_RzVXEUcgKRcZ>FLI3o2!#ervzaZ1b+Y}>}vLShM-&3 z7(sW%lslaT-N2>?FRA;PF4t>zS-TtklsD~m!A;oOb$P{%l_Zw|S$){_^3$kZ0s9Q} zPQgvWVB2NkwK3>M@fxKnN6|ZB5X+n7gF1~*Qg4AFK;RKTB*Zw1aTE~;1yQ(IK#;Dp zzzHMAbNn>Q+|(7NByoV<5Ef1tPsQx`o$y;?_LOCo!`5Q?MqJs<%3zJ_f?JOKbImsc zxhlqAl>Z`>cQ-A%2FjCgy@)*D7pcK%_UDfR{qn1IUtq6ps%j&8W#22H(s#)*k=aun z=9By~c!DbDZ|^wY?l9pOxYfSKCL-VWA;syGO1E|$2<~pr%uDT%K+EUpQ+xMckq}x|YomE@roirKUsL9B9&iSKpcWa!Q@b{dRel!;u*4Zeo&gJhjmkin} zyb4=dvYjk}N)ahbMSqiLe%F6sn_gId+( zuJLkWH1RDN?Qqb$tSqu)Du4P-I6#D~g7g30g%>=2x}Lb^yRaAFWVVw(Jx-YbzX3QQ JuzMyJ008kR`@;YL literal 679 zcmV;Y0$BYYiwFokH924a17u-zVJ>QOZ*BnXRZVZ4Fc7`(uRxpwvdtRUJvHj7_o|0f z(P}WW8@FIq*rbV~{P$wM76>bCs-|hBa%0Bx@#guBpH`0Jn#Uu$TgOcc;a*w8J!F-& zJk>w-uS~61fJVJ#2rL$qcu~x`M=2)WMnLGr0SXQxi2}ZFlGr$v;)4zKzsm#^>}c=N zMOBJZ1*gkCGFVeqXiRrtYO|`c?COOpd0cQ6N}(`CIZ0MK_7bzymiRid-d>H5v_Ms% z(4E*7!(0aMu%ZgE64u592i=hcND zsEd3=$?Z5mnydjG`IP$9`8b189n-UvF}7vgXY1f*DdPPpTg@it=JHU2rR@urx@B)H zvQBsHD27C*@Dle}q7pbB@i5Fm0BxSZe8B`cziF#=wT4tXyN6a5%JXXV^vVV0i6YNY z8NChGzzC4H*;GTB*)&_L&2;T_m#ep3z7py%&i{dxaIEy*nFZasmKJnRrmb{VbhDU= zysYn6ZuywA*IjpKI`a0qz2YV8;`+IQ)`gH3kT;dhWInCyg|N>^?*#554R*IId}|HT zDBiQAnNjqIGzj)3`P6i?7D=!H@t0055?PPza9 From ff546a1af4ca0f7a372f122589c94df0a03413b0 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Mon, 1 Mar 2021 16:30:21 -0500 Subject: [PATCH 18/24] [actions] for simplistic email servers, set rejectUnauthorized to false (#91760) resolves https://github.com/elastic/kibana/issues/91686 The poor email action has not had great success in setting TLS options correctly. Prior to 7.11, it was basically always setting `rejectUnauthorized` to false, so was never validating certificates. Starting in 7.11.0, it started respecting TLS certificates, but there are some simple/test servers in use that use self-signed certificates. The real fix for this will be the resolution of issue https://github.com/elastic/kibana/issues/80120 , but until then, this PR does a special-case check if the `secure` option is off (so the email client connects with a plain socket and then upgrades to TLS via STARTTLS) and both the user and password for the server are not set, then it will use `rejectUnauthorized: false`. Otherwise, it uses the global configured value of this setting. This also changes some other cases, where `secure: true` often did not set any `rejectUnauthorized` property at all, and so did not get verified. Now in all cases, `rejectUnauthorized` will be set, and the value will correspond to the globally configured value, except for the special case checked here, and when a proxy is in use (that logic did not change). So it is possible this would break customers, who were using insecure servers and email action worked, but with this fix the connections will be rejected. They should have been rejected all this time though. The work-around for this problem, if we don't implement a fix like this, is that customers will need to set the global `rejectUnauthorized` to `false`, which means NONE of their TLS connections for any actions will be verified. Which seems extreme. --- .../builtin_action_types/lib/send_email.test.ts | 5 ++++- .../server/builtin_action_types/lib/send_email.ts | 11 +++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.test.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.test.ts index c6317a6a980bb7..cc3f03f50c36fe 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.test.ts @@ -138,7 +138,7 @@ describe('send_email module', () => { "port": 1025, "secure": false, "tls": Object { - "rejectUnauthorized": true, + "rejectUnauthorized": false, }, }, ] @@ -187,6 +187,9 @@ describe('send_email module', () => { "host": "example.com", "port": 1025, "secure": true, + "tls": Object { + "rejectUnauthorized": true, + }, }, ] `); diff --git a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts index 79842f4aec02bb..d4905015f7663b 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/lib/send_email.ts @@ -80,10 +80,13 @@ export async function sendEmail(logger: Logger, options: SendEmailOptions): Prom }; transportConfig.proxy = proxySettings.proxyUrl; transportConfig.headers = proxySettings.proxyHeaders; - } else if (!transportConfig.secure) { - transportConfig.tls = { - rejectUnauthorized, - }; + } else if (!transportConfig.secure && user == null && password == null) { + // special case - if secure:false && user:null && password:null set + // rejectUnauthorized false, because simple/test servers that don't even + // authenticate rarely have valid certs; eg cloud proxy, and npm maildev + transportConfig.tls = { rejectUnauthorized: false }; + } else { + transportConfig.tls = { rejectUnauthorized }; } } From 892d44cafd597340a41c5bd06604baa0b9a25f2b Mon Sep 17 00:00:00 2001 From: Jason Stoltzfus Date: Mon, 1 Mar 2021 16:49:57 -0500 Subject: [PATCH 19/24] [App Search] Implement various Relevance Tuning states and form actions (#92644) --- .../app_search/__mocks__/engine_logic.mock.ts | 12 +- .../app_search/__mocks__/index.ts | 2 +- .../relevance_tuning/boosts/boost_item.tsx | 18 +-- .../boost_item_content.test.tsx | 2 +- .../boost_item_content/boost_item_content.tsx | 2 +- .../boost_item_content/value_boost_form.tsx | 2 +- .../relevance_tuning.test.tsx | 60 ++++++++- .../relevance_tuning/relevance_tuning.tsx | 104 +++++++++------ .../relevance_tuning_callouts.test.tsx | 86 ++++++++++++ .../relevance_tuning_callouts.tsx | 123 ++++++++++++++++++ .../relevance_tuning_form.test.tsx | 22 ++++ .../relevance_tuning_form.tsx | 35 ++++- .../relevance_tuning_layout.test.tsx | 61 +++++++++ .../relevance_tuning_layout.tsx | 85 ++++++++++++ .../relevance_tuning_logic.test.ts | 45 ++++--- .../relevance_tuning_logic.ts | 15 +-- 16 files changed, 584 insertions(+), 90 deletions(-) create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.test.tsx create mode 100644 x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.tsx diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts index c4c177f3a955ab..485ac19f2eb820 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/engine_logic.mock.ts @@ -5,11 +5,16 @@ * 2.0. */ +import { EngineDetails } from '../components/engine/types'; import { generateEncodedPath } from '../utils/encode_path_params'; export const mockEngineValues = { engineName: 'some-engine', - engine: {}, + engine: {} as EngineDetails, +}; + +export const mockEngineActions = { + initializeEngine: jest.fn(), }; export const mockGenerateEnginePath = jest.fn((path, pathParams = {}) => @@ -17,6 +22,9 @@ export const mockGenerateEnginePath = jest.fn((path, pathParams = {}) => ); jest.mock('../components/engine', () => ({ - EngineLogic: { values: mockEngineValues }, + EngineLogic: { + values: mockEngineValues, + actions: mockEngineActions, + }, generateEnginePath: mockGenerateEnginePath, })); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/index.ts index d89c09d8e78ce8..271a09849cba73 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/__mocks__/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export { mockEngineValues } from './engine_logic.mock'; +export { mockEngineValues, mockEngineActions } from './engine_logic.mock'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item.tsx index 641628c32659c7..1dea62b2fd4785 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item.tsx @@ -33,18 +33,14 @@ export const BoostItem: React.FC = ({ id, boost, index, name }) => { className="boosts__item" buttonContentClassName="boosts__itemButton" buttonContent={ - - - - - - - {BOOST_TYPE_TO_DISPLAY_MAP[boost.type]} - - {summary} - - + + + + {BOOST_TYPE_TO_DISPLAY_MAP[boost.type]} + + {summary} + {boost.factor} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.test.tsx index 3296155fdce5d0..a16620e75412d1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.test.tsx @@ -85,7 +85,7 @@ describe('BoostItemContent', () => { expect(actions.updateBoostFactor).toHaveBeenCalledWith('foo', 3, 2); }); - it("will delete the current boost if the 'Delete Boost' button is clicked", () => { + it("will delete the current boost if the 'Delete boost' button is clicked", () => { const boost = { factor: 8, type: 'proximity' as BoostType, diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.tsx index 7a19564543c81f..f83ec99acb1acb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/boost_item_content.tsx @@ -74,7 +74,7 @@ export const BoostItemContent: React.FC = ({ boost, index, name }) => { {i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.boosts.deleteBoostButtonLabel', { - defaultMessage: 'Delete Boost', + defaultMessage: 'Delete boost', } )} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/value_boost_form.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/value_boost_form.tsx index 15d19a9741d0a2..7fcd07d9a07aad 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/value_boost_form.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/boosts/boost_item_content/value_boost_form.tsx @@ -70,7 +70,7 @@ export const ValueBoostForm: React.FC = ({ boost, index, name }) => { {i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.boosts.value.addValueButtonLabel', { - defaultMessage: 'Add Value', + defaultMessage: 'Add value', } )} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.test.tsx index 85cf3dd8a68c97..e2adce7dd76876 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.test.tsx @@ -5,33 +5,85 @@ * 2.0. */ import '../../../__mocks__/shallow_useeffect.mock'; -import { setMockActions } from '../../../__mocks__/kea.mock'; +import { setMockActions, setMockValues } from '../../../__mocks__/kea.mock'; import React from 'react'; -import { shallow, ShallowWrapper } from 'enzyme'; +import { shallow } from 'enzyme'; + +import { EuiEmptyPrompt } from '@elastic/eui'; + +import { Loading } from '../../../shared/loading'; +import { UnsavedChangesPrompt } from '../../../shared/unsaved_changes_prompt'; import { RelevanceTuning } from './relevance_tuning'; import { RelevanceTuningForm } from './relevance_tuning_form'; describe('RelevanceTuning', () => { - let wrapper: ShallowWrapper; + const values = { + engineHasSchemaFields: true, + engine: { + invalidBoosts: false, + unsearchedUnconfirmedFields: false, + }, + schemaFieldsWithConflicts: [], + unsavedChanges: false, + dataLoading: false, + }; const actions = { initializeRelevanceTuning: jest.fn(), + updateSearchSettings: jest.fn(), + resetSearchSettings: jest.fn(), }; + const subject = () => shallow(); + beforeEach(() => { jest.clearAllMocks(); + setMockValues(values); setMockActions(actions); - wrapper = shallow(); }); it('renders', () => { + const wrapper = subject(); expect(wrapper.find(RelevanceTuningForm).exists()).toBe(true); + expect(wrapper.find(Loading).exists()).toBe(false); + expect(wrapper.find('EmptyCallout').exists()).toBe(false); }); it('initializes relevance tuning data', () => { + subject(); expect(actions.initializeRelevanceTuning).toHaveBeenCalled(); }); + + it('will render an empty message when the engine has no schema', () => { + setMockValues({ + ...values, + engineHasSchemaFields: false, + }); + const wrapper = subject(); + expect(wrapper.find('EmptyCallout').dive().find(EuiEmptyPrompt).exists()).toBe(true); + expect(wrapper.find(Loading).exists()).toBe(false); + expect(wrapper.find(RelevanceTuningForm).exists()).toBe(false); + }); + + it('will show a loading message if data is loading', () => { + setMockValues({ + ...values, + dataLoading: true, + }); + const wrapper = subject(); + expect(wrapper.find(Loading).exists()).toBe(true); + expect(wrapper.find('EmptyCallout').exists()).toBe(false); + expect(wrapper.find(RelevanceTuningForm).exists()).toBe(false); + }); + + it('will prevent user from leaving the page if there are unsaved changes', () => { + setMockValues({ + ...values, + unsavedChanges: true, + }); + expect(subject().find(UnsavedChangesPrompt).prop('hasUnsavedChanges')).toBe(true); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.tsx index f65a86b1e02f0a..0ae3c8fd3b5dce 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning.tsx @@ -7,67 +7,93 @@ import React, { useEffect } from 'react'; -import { useActions } from 'kea'; - -import { - EuiPageHeader, - EuiPageHeaderSection, - EuiTitle, - EuiText, - EuiSpacer, - EuiFlexGroup, - EuiFlexItem, - EuiTextColor, -} from '@elastic/eui'; +import { useActions, useValues } from 'kea'; +import { EuiButton, EuiEmptyPrompt, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; -import { FlashMessages } from '../../../shared/flash_messages'; -import { SetAppSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; +import { Loading } from '../../../shared/loading'; +import { UnsavedChangesPrompt } from '../../../shared/unsaved_changes_prompt'; +import { DOCS_PREFIX } from '../../routes'; -import { RELEVANCE_TUNING_TITLE } from './constants'; import { RelevanceTuningForm } from './relevance_tuning_form'; -import { RelevanceTuningLogic } from './relevance_tuning_logic'; +import { RelevanceTuningLayout } from './relevance_tuning_layout'; + +import { RelevanceTuningLogic } from '.'; interface Props { engineBreadcrumb: string[]; } +const EmptyCallout: React.FC = () => { + return ( + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.emptyErrorMessageTitle', + { + defaultMessage: 'Tuning requires schema fields', + } + )} + + } + body={i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.emptyErrorMessage', + { + defaultMessage: 'Index documents to tune relevance.', + } + )} + actions={ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.emptyButtonLabel', + { + defaultMessage: 'Read the relevance tuning guide', + } + )} + + } + /> + ); +}; + export const RelevanceTuning: React.FC = ({ engineBreadcrumb }) => { + const { dataLoading, engineHasSchemaFields, unsavedChanges } = useValues(RelevanceTuningLogic); const { initializeRelevanceTuning } = useActions(RelevanceTuningLogic); useEffect(() => { initializeRelevanceTuning(); }, []); - return ( - <> - - - - -

{RELEVANCE_TUNING_TITLE}

- - - - {i18n.translate( - 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.description', - { - defaultMessage: 'Set field weights and boosts', - } - )} - - - - - - + const body = () => { + if (dataLoading) { + return ; + } + + if (!engineHasSchemaFields) { + return ; + } + + return ( - + ); + }; + + return ( + + + {body()} + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.test.tsx new file mode 100644 index 00000000000000..8ab706f5953fbd --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.test.tsx @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import '../../__mocks__/engine_logic.mock'; +import { setMockValues } from '../../../__mocks__/kea.mock'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { RelevanceTuningCallouts } from './relevance_tuning_callouts'; + +describe('RelevanceTuningCallouts', () => { + const values = { + engineHasSchemaFields: true, + engine: { + invalidBoosts: false, + unsearchedUnconfirmedFields: false, + }, + schemaFieldsWithConflicts: [], + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + }); + + const subject = () => shallow(); + + it('renders', () => { + const wrapper = subject(); + expect(wrapper.find('[data-test-subj="RelevanceTuningInvalidBoostsCallout"]').exists()).toBe( + false + ); + expect(wrapper.find('[data-test-subj="RelevanceTuningUnsearchedFieldsCallout"]').exists()).toBe( + false + ); + expect(subject().find('[data-test-subj="SchemaConflictsCallout"]').exists()).toBe(false); + }); + + it('shows a message when there are invalid boosts', () => { + // An invalid boost would be if a user creats a functional boost on a number field, then that + // field later changes to text. At this point, the boost still exists but is invalid for + // a text field. + setMockValues({ + ...values, + engine: { + invalidBoosts: true, + unsearchedUnconfirmedFields: false, + }, + }); + expect(subject().find('[data-test-subj="RelevanceTuningInvalidBoostsCallout"]').exists()).toBe( + true + ); + }); + + it('shows a message when there are unconfirmed fields', () => { + // An invalid boost would be if a user creats a functional boost on a number field, then that + // field later changes to text. At this point, the boost still exists but is invalid for + // a text field. + setMockValues({ + ...values, + engine: { + invalidBoosts: false, + unsearchedUnconfirmedFields: true, + }, + }); + expect( + subject().find('[data-test-subj="RelevanceTuningUnsearchedFieldsCallout"]').exists() + ).toBe(true); + }); + + it('shows a message when there are schema field conflicts', () => { + // Schema conflicts occur when a meta engine has fields in source engines with have differing types, + // hence relevance tuning cannot be applied as we don't know the actual type + setMockValues({ + ...values, + schemaFieldsWithConflicts: ['fe', 'fi', 'fo'], + }); + expect(subject().find('[data-test-subj="SchemaConflictsCallout"]').exists()).toBe(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.tsx new file mode 100644 index 00000000000000..c981d35ff20cba --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_callouts.tsx @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { FormattedMessage } from 'react-intl'; + +import { useValues } from 'kea'; + +import { EuiCallOut, EuiLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { EuiLinkTo } from '../../../shared/react_router_helpers'; +import { DOCS_PREFIX, ENGINE_SCHEMA_PATH } from '../../routes'; +import { EngineLogic, generateEnginePath } from '../engine'; + +import { RelevanceTuningLogic } from '.'; + +export const RelevanceTuningCallouts: React.FC = () => { + const { schemaFieldsWithConflicts } = useValues(RelevanceTuningLogic); + const { + engine: { invalidBoosts, unsearchedUnconfirmedFields }, + } = useValues(EngineLogic); + + const schemaFieldsWithConflictsCount = schemaFieldsWithConflicts.length; + + const invalidBoostsCallout = () => ( + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.invalidBoostsErrorMessage', + { + defaultMessage: + 'One or more of your boosts is no longer valid, possibly due to a schema type change. Delete any old or invalid boosts to dismiss this alert.', + } + )} + + ); + + const unsearchedUnconfirmedFieldsCallout = () => ( + + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.schemaFieldsLinkLabel', + { + defaultMessage: 'schema fields', + } + )} + + ), + }} + /> + + ); + + const schemaFieldsWithConflictsCallout = () => ( + + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.whatsThisLinkLabel', + { + defaultMessage: "What's this?", + } + )} + + ), + }} + /> + + ); + + return ( + <> + {invalidBoosts && invalidBoostsCallout()} + {unsearchedUnconfirmedFields && unsearchedUnconfirmedFieldsCallout()} + {schemaFieldsWithConflictsCount > 0 && schemaFieldsWithConflictsCallout()} + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.test.tsx index 3965e9e81d1ba3..2857b227749449 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.test.tsx @@ -23,6 +23,7 @@ describe('RelevanceTuningForm', () => { filterInputValue: '', schemaFields: ['foo', 'bar', 'baz'], filteredSchemaFields: ['foo', 'bar'], + filteredSchemaFieldsWithConflicts: [], schema: { foo: 'text', bar: 'number', @@ -95,6 +96,27 @@ describe('RelevanceTuningForm', () => { weight: 1, }); }); + + it('wont show disabled fields section if there are no fields with schema conflicts', () => { + expect(wrapper.find('[data-test-subj="DisabledFieldsSection"]').exists()).toBe(false); + }); + }); + + it('will show a disabled fields section if there are fields that have schema conflicts', () => { + // There will only ever be fields with schema conflicts if this is the relevance tuning + // page for a meta engine + setMockValues({ + ...values, + filteredSchemaFieldsWithConflicts: ['fe', 'fi', 'fo'], + }); + + const wrapper = mount(); + expect(wrapper.find('[data-test-subj="DisabledFieldsSection"]').exists()).toBe(true); + expect(wrapper.find('[data-test-subj="DisabledField"]').map((f) => f.text())).toEqual([ + 'fe', + 'fi', + 'fo', + ]); }); describe('field filtering', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx index e39c93fd5de3cb..87b9e1615774f5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_form/relevance_tuning_form.tsx @@ -17,6 +17,7 @@ import { EuiSpacer, EuiAccordion, EuiPanel, + EuiHealth, } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; @@ -34,6 +35,7 @@ export const RelevanceTuningForm: React.FC = () => { filterInputValue, schemaFields, filteredSchemaFields, + filteredSchemaFieldsWithConflicts, schema, searchSettings, } = useValues(RelevanceTuningLogic); @@ -42,8 +44,6 @@ export const RelevanceTuningForm: React.FC = () => { return (
- {/* TODO SchemaConflictCallout */} - @@ -100,6 +100,37 @@ export const RelevanceTuningForm: React.FC = () => { ))} + + {filteredSchemaFieldsWithConflicts.length > 0 && ( + <> + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.disabledFields.title', + { + defaultMessage: 'Disabled fields', + } + )} +

+ + + {filteredSchemaFieldsWithConflicts.map((fieldName) => ( + + +

{fieldName}

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.disabledFieldsExplanationMessage', + { + defaultMessage: 'Inactive due to field-type conflict', + } + )} + + + ))} + + )}
); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.test.tsx new file mode 100644 index 00000000000000..edd417cc1ffe88 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.test.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setMockActions, setMockValues } from '../../../__mocks__/kea.mock'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { EuiPageHeader } from '@elastic/eui'; + +import { RelevanceTuningLayout } from './relevance_tuning_layout'; + +describe('RelevanceTuningLayout', () => { + const values = { + engineHasSchemaFields: true, + schemaFieldsWithConflicts: [], + }; + + const actions = { + updateSearchSettings: jest.fn(), + resetSearchSettings: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + const subject = () => shallow(); + + it('renders a Save button that will save the current changes', () => { + const buttons = subject().find(EuiPageHeader).prop('rightSideItems') as React.ReactElement[]; + expect(buttons.length).toBe(2); + const saveButton = shallow(buttons[0]); + saveButton.simulate('click'); + expect(actions.updateSearchSettings).toHaveBeenCalled(); + }); + + it('renders a Reset button that will remove all weights and boosts', () => { + const buttons = subject().find(EuiPageHeader).prop('rightSideItems') as React.ReactElement[]; + expect(buttons.length).toBe(2); + const resetButton = shallow(buttons[1]); + resetButton.simulate('click'); + expect(actions.resetSearchSettings).toHaveBeenCalled(); + }); + + it('will not render buttons if the engine has no schema', () => { + setMockValues({ + ...values, + engineHasSchemaFields: false, + }); + const buttons = subject().find(EuiPageHeader).prop('rightSideItems') as React.ReactElement[]; + expect(buttons.length).toBe(0); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.tsx new file mode 100644 index 00000000000000..d6644d21e9df35 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_layout.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useActions, useValues } from 'kea'; + +import { EuiPageHeader, EuiSpacer, EuiButton } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import { FlashMessages } from '../../../shared/flash_messages'; +import { SetAppSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; + +import { RELEVANCE_TUNING_TITLE } from './constants'; +import { RelevanceTuningCallouts } from './relevance_tuning_callouts'; +import { RelevanceTuningLogic } from './relevance_tuning_logic'; + +interface Props { + engineBreadcrumb: string[]; +} + +export const RelevanceTuningLayout: React.FC = ({ engineBreadcrumb, children }) => { + const { resetSearchSettings, updateSearchSettings } = useActions(RelevanceTuningLogic); + const { engineHasSchemaFields } = useValues(RelevanceTuningLogic); + + const pageHeader = () => ( + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.saveButtonLabel', + { + defaultMessage: 'Save', + } + )} + , + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.relevanceTuning.resetButtonLabel', + { + defaultMessage: 'Restore defaults', + } + )} + , + ] + : [] + } + /> + ); + + return ( + <> + + {pageHeader()} + + + + {children} + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts index 8ce07dc699cbbe..86e1f679a1636d 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.test.ts @@ -6,6 +6,7 @@ */ import { LogicMounter, mockFlashMessageHelpers, mockHttpValues } from '../../../__mocks__'; +import { mockEngineValues, mockEngineActions } from '../../__mocks__'; import { nextTick } from '@kbn/test/jest'; @@ -13,10 +14,6 @@ import { Boost, BoostOperation, BoostType, FunctionalBoostFunction } from './typ import { RelevanceTuningLogic } from './'; -jest.mock('../engine', () => ({ - EngineLogic: { values: { engineName: 'test-engine' } }, -})); - describe('RelevanceTuningLogic', () => { const { mount } = new LogicMounter(RelevanceTuningLogic); @@ -64,7 +61,6 @@ describe('RelevanceTuningLogic', () => { query: '', resultsLoading: false, searchResults: null, - showSchemaConflictCallout: true, engineHasSchemaFields: false, schemaFields: [], schemaFieldsWithConflicts: [], @@ -74,6 +70,9 @@ describe('RelevanceTuningLogic', () => { beforeEach(() => { jest.clearAllMocks(); + mockEngineValues.engineName = 'test-engine'; + mockEngineValues.engine.invalidBoosts = false; + mockEngineValues.engine.unsearchedUnconfirmedFields = false; }); it('has expected default values', () => { @@ -207,20 +206,6 @@ describe('RelevanceTuningLogic', () => { }); }); - describe('dismissSchemaConflictCallout', () => { - it('should set showSchemaConflictCallout to false', () => { - mount({ - showSchemaConflictCallout: true, - }); - RelevanceTuningLogic.actions.dismissSchemaConflictCallout(); - - expect(RelevanceTuningLogic.values).toEqual({ - ...DEFAULT_VALUES, - showSchemaConflictCallout: false, - }); - }); - }); - describe('setSearchSettingsResponse', () => { it('should set searchSettings state and unsavedChanges to false', () => { mount({ @@ -545,6 +530,28 @@ describe('RelevanceTuningLogic', () => { expect(flashAPIErrors).toHaveBeenCalledWith('error'); expect(RelevanceTuningLogic.actions.onSearchSettingsError).toHaveBeenCalled(); }); + + it('will re-fetch the current engine after settings are updated if there were invalid boosts', async () => { + mockEngineValues.engine.invalidBoosts = true; + mount({}); + http.put.mockReturnValueOnce(Promise.resolve(searchSettings)); + + RelevanceTuningLogic.actions.updateSearchSettings(); + await nextTick(); + + expect(mockEngineActions.initializeEngine).toHaveBeenCalled(); + }); + + it('will re-fetch the current engine after settings are updated if there were unconfirmed search fieldds', async () => { + mockEngineValues.engine.unsearchedUnconfirmedFields = true; + mount({}); + http.put.mockReturnValueOnce(Promise.resolve(searchSettings)); + + RelevanceTuningLogic.actions.updateSearchSettings(); + await nextTick(); + + expect(mockEngineActions.initializeEngine).toHaveBeenCalled(); + }); }); describe('resetSearchSettings', () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts index d567afee9d0627..0d30296de285ff 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/relevance_tuning/relevance_tuning_logic.ts @@ -51,7 +51,6 @@ interface RelevanceTuningActions { setResultsLoading(resultsLoading: boolean): boolean; clearSearchResults(): void; resetSearchSettingsState(): void; - dismissSchemaConflictCallout(): void; initializeRelevanceTuning(): void; getSearchResults(): void; setSearchSettingsResponse(searchSettings: SearchSettings): { searchSettings: SearchSettings }; @@ -107,7 +106,6 @@ interface RelevanceTuningValues { filteredSchemaFields: string[]; filteredSchemaFieldsWithConflicts: string[]; schemaConflicts: SchemaConflicts; - showSchemaConflictCallout: boolean; engineHasSchemaFields: boolean; filterInputValue: string; query: string; @@ -130,7 +128,6 @@ export const RelevanceTuningLogic = kea< setResultsLoading: (resultsLoading) => resultsLoading, clearSearchResults: true, resetSearchSettingsState: true, - dismissSchemaConflictCallout: true, initializeRelevanceTuning: true, getSearchResults: true, setSearchSettingsResponse: (searchSettings) => ({ @@ -186,12 +183,6 @@ export const RelevanceTuningLogic = kea< onInitializeRelevanceTuning: (_, { schemaConflicts }) => schemaConflicts || {}, }, ], - showSchemaConflictCallout: [ - true, - { - dismissSchemaConflictCallout: () => false, - }, - ], filterInputValue: [ '', { @@ -330,6 +321,12 @@ export const RelevanceTuningLogic = kea< } catch (e) { flashAPIErrors(e); actions.onSearchSettingsError(); + } finally { + const { invalidBoosts, unsearchedUnconfirmedFields } = EngineLogic.values.engine; + if (invalidBoosts || unsearchedUnconfirmedFields) { + // Re-fetch engine data so that any navigation flags are updated dynamically + EngineLogic.actions.initializeEngine(); + } } }, resetSearchSettings: async () => { From cd38671565fb55d4085e6671d8f4b0c0a7d4a859 Mon Sep 17 00:00:00 2001 From: Nick Peihl Date: Mon, 1 Mar 2021 14:05:08 -0800 Subject: [PATCH 20/24] Bump ems landing page to 7.12 (#93065) --- src/plugins/maps_legacy/common/ems_defaults.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/maps_legacy/common/ems_defaults.ts b/src/plugins/maps_legacy/common/ems_defaults.ts index 7d9be52eec5650..6d99f2041484c9 100644 --- a/src/plugins/maps_legacy/common/ems_defaults.ts +++ b/src/plugins/maps_legacy/common/ems_defaults.ts @@ -9,6 +9,6 @@ // Default config for the elastic hosted EMS endpoints export const DEFAULT_EMS_FILE_API_URL = 'https://vector.maps.elastic.co'; export const DEFAULT_EMS_TILE_API_URL = 'https://tiles.maps.elastic.co'; -export const DEFAULT_EMS_LANDING_PAGE_URL = 'https://maps.elastic.co/v7.11'; +export const DEFAULT_EMS_LANDING_PAGE_URL = 'https://maps.elastic.co/v7.12'; export const DEFAULT_EMS_FONT_LIBRARY_URL = 'https://tiles.maps.elastic.co/fonts/{fontstack}/{range}.pbf'; From cb053f4672ae0a1f3eb134f7865222c7fba339ff Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 1 Mar 2021 17:10:22 -0500 Subject: [PATCH 21/24] [Security Solution][Detections][7.12] Critical Threshold Rule Fixes (#92667) * Threshold cardinality validation * Remove comments * Fix legacy threshold signal dupe mitigation * Add find_threshold_signals tests * remove comment * bug fixes * Fix edit form value initialization for cardinality_value * Fix test * Type and test fixes * Tests/types * Reenable threshold cypress test * Schema fixes * Types and tests, normalize threshold field util * Continue cleaning up types * Some more pre-7.12 tests * Limit cardinality_field to length 1 for now * Cardinality to array * Cardinality to array * Tests/types * cardinality can be null * Handle empty threshold field in bulk_create_threshold_signals * Remove cardinality_field, cardinality_value --- .../schemas/common/schemas.ts | 48 +- .../common/detection_engine/utils.test.ts | 26 +- .../common/detection_engine/utils.ts | 12 + .../matrix_histogram/index.ts | 6 +- .../detection_rules/threshold_rule.spec.ts | 177 ++++--- .../components/matrix_histogram/types.ts | 6 +- .../rules/query_preview/index.test.tsx | 24 +- .../components/rules/query_preview/index.tsx | 6 +- .../rules/query_preview/reducer.test.ts | 36 +- .../rules/step_define_rule/index.tsx | 35 +- .../rules/step_define_rule/schema.tsx | 123 +++-- .../rules/threshold_input/index.tsx | 6 +- .../rules/all/__mocks__/mock.ts | 16 +- .../detection_engine/rules/create/helpers.ts | 12 +- .../detection_engine/rules/helpers.test.tsx | 10 +- .../pages/detection_engine/rules/helpers.tsx | 21 +- .../pages/detection_engine/rules/types.ts | 6 +- .../schemas/rule_converters.ts | 6 +- .../signals/__mocks__/es_results.ts | 35 +- .../bulk_create_threshold_signals.test.ts | 250 +++++++++- .../signals/bulk_create_threshold_signals.ts | 83 +--- .../signals/find_threshold_signals.test.ts | 445 ++++++++++++++++++ .../signals/find_threshold_signals.ts | 145 +++--- .../signals/signal_params_schema.test.ts | 52 ++ .../signals/signal_params_schema.ts | 13 +- .../signals/signal_rule_alert_type.ts | 7 +- .../threshold_get_bucket_filters.test.ts | 133 +++++- .../signals/threshold_get_bucket_filters.ts | 5 +- 28 files changed, 1392 insertions(+), 352 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts index bfe450d240b08b..a5398f291627cc 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts @@ -465,26 +465,56 @@ export type Threats = t.TypeOf; export const threatsOrUndefined = t.union([threats, t.undefined]); export type ThreatsOrUndefined = t.TypeOf; +export const thresholdField = t.exact( + t.type({ + field: t.union([t.string, t.array(t.string)]), // Covers pre- and post-7.12 + value: PositiveIntegerGreaterThanZero, + }) +); +export type ThresholdField = t.TypeOf; + +export const thresholdFieldNormalized = t.exact( + t.type({ + field: t.array(t.string), + value: PositiveIntegerGreaterThanZero, + }) +); +export type ThresholdFieldNormalized = t.TypeOf; + +export const thresholdCardinalityField = t.exact( + t.type({ + field: t.string, + value: PositiveInteger, + }) +); +export type ThresholdCardinalityField = t.TypeOf; + export const threshold = t.intersection([ - t.exact( - t.type({ - field: t.union([t.string, t.array(t.string)]), - value: PositiveIntegerGreaterThanZero, - }) - ), + thresholdField, t.exact( t.partial({ - cardinality_field: t.union([t.string, t.array(t.string), t.undefined, t.null]), - cardinality_value: t.union([PositiveInteger, t.undefined, t.null]), // TODO: cardinality_value should be set if cardinality_field is set + cardinality: t.union([t.array(thresholdCardinalityField), t.null]), }) ), ]); -// TODO: codec to transform threshold field string to string[] ? export type Threshold = t.TypeOf; export const thresholdOrUndefined = t.union([threshold, t.undefined]); export type ThresholdOrUndefined = t.TypeOf; +export const thresholdNormalized = t.intersection([ + thresholdFieldNormalized, + t.exact( + t.partial({ + cardinality: t.union([t.array(thresholdCardinalityField), t.null]), + }) + ), +]); +export type ThresholdNormalized = t.TypeOf; + +export const thresholdNormalizedOrUndefined = t.union([thresholdNormalized, t.undefined]); +export type ThresholdNormalizedOrUndefined = t.TypeOf; + export const created_at = IsoDateString; export const updated_at = IsoDateString; export const updated_by = t.string; diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts index 72c3a49bb66b8f..9377255dc85d51 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -5,7 +5,13 @@ * 2.0. */ -import { hasEqlSequenceQuery, hasLargeValueList, hasNestedEntry, isThreatMatchRule } from './utils'; +import { + hasEqlSequenceQuery, + hasLargeValueList, + hasNestedEntry, + isThreatMatchRule, + normalizeThresholdField, +} from './utils'; import { EntriesArray } from '../shared_imports'; describe('#hasLargeValueList', () => { @@ -151,3 +157,21 @@ describe('#hasEqlSequenceQuery', () => { }); }); }); + +describe('normalizeThresholdField', () => { + it('converts a string to a string array', () => { + expect(normalizeThresholdField('host.name')).toEqual(['host.name']); + }); + it('returns a string array when a string array is passed in', () => { + expect(normalizeThresholdField(['host.name'])).toEqual(['host.name']); + }); + it('converts undefined to an empty array', () => { + expect(normalizeThresholdField(undefined)).toEqual([]); + }); + it('converts null to an empty array', () => { + expect(normalizeThresholdField(null)).toEqual([]); + }); + it('converts an empty string to an empty array', () => { + expect(normalizeThresholdField('')).toEqual([]); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts index 79b912e082fdb1..838bac542bb875 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/utils.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { isEmpty } from 'lodash'; + import { CreateExceptionListItemSchema, EntriesArray, @@ -42,3 +44,13 @@ export const isQueryRule = (ruleType: Type | undefined): boolean => ruleType === 'query' || ruleType === 'saved_query'; export const isThreatMatchRule = (ruleType: Type | undefined): boolean => ruleType === 'threat_match'; + +export const normalizeThresholdField = ( + thresholdField: string | string[] | null | undefined +): string[] => { + return Array.isArray(thresholdField) + ? thresholdField + : isEmpty(thresholdField) + ? [] + : [thresholdField!]; +}; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index c71108d58d980f..0b59170dfc778a 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -40,8 +40,10 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions { | { field: string | string[] | undefined; value: number; - cardinality_field?: string | undefined; - cardinality_value?: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; inspect?: Maybe; diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts index 572422a4936df4..56f316087ae8a8 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts @@ -79,103 +79,100 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; import { DETECTIONS_URL } from '../../urls/navigation'; -// Skipped until post-FF for 7.12 -describe.skip('Threshold Rules', () => { - describe('Detection rules, threshold', () => { - const expectedUrls = newThresholdRule.referenceUrls.join(''); - const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); - const expectedTags = newThresholdRule.tags.join(''); - const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); - - const rule = { ...newThresholdRule }; - - beforeEach(() => { - cleanKibana(); - createTimeline(newThresholdRule.timeline).then((response) => { - rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; - }); +describe('Detection rules, threshold', () => { + const expectedUrls = newThresholdRule.referenceUrls.join(''); + const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join(''); + const expectedTags = newThresholdRule.tags.join(''); + const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre); + + const rule = { ...newThresholdRule }; + + beforeEach(() => { + cleanKibana(); + createTimeline(newThresholdRule.timeline).then((response) => { + rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; }); + }); - it('Creates and activates a new threshold rule', () => { - loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); - waitForAlertsPanelToBeLoaded(); - waitForAlertsIndexToBeCreated(); - goToManageAlertsDetectionRules(); - waitForRulesTableToBeLoaded(); - goToCreateNewRule(); - selectThresholdRuleType(); - fillDefineThresholdRuleAndContinue(rule); - fillAboutRuleAndContinue(rule); - fillScheduleRuleAndContinue(rule); - createAndActivateRule(); - - cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); - - changeRowsPerPageTo300(); - - const expectedNumberOfRules = 1; - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); - }); + it('Creates and activates a new threshold rule', () => { + loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); + goToManageAlertsDetectionRules(); + waitForRulesTableToBeLoaded(); + goToCreateNewRule(); + selectThresholdRuleType(); + fillDefineThresholdRuleAndContinue(rule); + fillAboutRuleAndContinue(rule); + fillScheduleRuleAndContinue(rule); + createAndActivateRule(); + + cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)'); + + changeRowsPerPageTo300(); + + const expectedNumberOfRules = 1; + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules); + }); - filterByCustomRules(); + filterByCustomRules(); - cy.get(RULES_TABLE).then(($table) => { - cy.wrap($table.find(RULES_ROW).length).should('eql', 1); - }); - cy.get(RULE_NAME).should('have.text', rule.name); - cy.get(RISK_SCORE).should('have.text', rule.riskScore); - cy.get(SEVERITY).should('have.text', rule.severity); - cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); - - goToRuleDetails(); - - cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); - cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); - cy.get(ABOUT_DETAILS).within(() => { - getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); - getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); - getDetails(REFERENCE_URLS_DETAILS).should((details) => { - expect(removeExternalLinkText(details.text())).equal(expectedUrls); - }); - getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); - getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { - expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); - }); - getDetails(TAGS_DETAILS).should('have.text', expectedTags); - }); - cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); - cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); - cy.get(DEFINITION_DETAILS).within(() => { - getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); - getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); - getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); - getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); - getDetails(THRESHOLD_DETAILS).should( - 'have.text', - `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` - ); + cy.get(RULES_TABLE).then(($table) => { + cy.wrap($table.find(RULES_ROW).length).should('eql', 1); + }); + cy.get(RULE_NAME).should('have.text', rule.name); + cy.get(RISK_SCORE).should('have.text', rule.riskScore); + cy.get(SEVERITY).should('have.text', rule.severity); + cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true'); + + goToRuleDetails(); + + cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`); + cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description); + cy.get(ABOUT_DETAILS).within(() => { + getDetails(SEVERITY_DETAILS).should('have.text', rule.severity); + getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore); + getDetails(REFERENCE_URLS_DETAILS).should((details) => { + expect(removeExternalLinkText(details.text())).equal(expectedUrls); }); - cy.get(SCHEDULE_DETAILS).within(() => { - getDetails(RUNS_EVERY_DETAILS).should( - 'have.text', - `${rule.runsEvery.interval}${rule.runsEvery.type}` - ); - getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( - 'have.text', - `${rule.lookBack.interval}${rule.lookBack.type}` - ); + getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives); + getDetails(MITRE_ATTACK_DETAILS).should((mitre) => { + expect(removeExternalLinkText(mitre.text())).equal(expectedMitre); }); + getDetails(TAGS_DETAILS).should('have.text', expectedTags); + }); + cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true }); + cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN); + cy.get(DEFINITION_DETAILS).within(() => { + getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join('')); + getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery); + getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold'); + getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None'); + getDetails(THRESHOLD_DETAILS).should( + 'have.text', + `Results aggregated by ${rule.thresholdField} >= ${rule.threshold}` + ); + }); + cy.get(SCHEDULE_DETAILS).within(() => { + getDetails(RUNS_EVERY_DETAILS).should( + 'have.text', + `${rule.runsEvery.interval}${rule.runsEvery.type}` + ); + getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should( + 'have.text', + `${rule.lookBack.interval}${rule.lookBack.type}` + ); + }); - waitForTheRuleToBeExecuted(); - waitForAlertsToPopulate(); + waitForTheRuleToBeExecuted(); + waitForAlertsToPopulate(); - cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); - cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); - cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); - cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); - cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); - cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); - }); + cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100)); + cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name); + cy.get(ALERT_RULE_VERSION).first().should('have.text', '1'); + cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold'); + cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); + cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts index b993bcda56b8ed..d846d887cb6817 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts @@ -78,8 +78,10 @@ export interface MatrixHistogramQueryProps { | { field: string | string[] | undefined; value: number; - cardinality_field?: string | undefined; - cardinality_value?: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; skip?: boolean; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx index 700c2d516b995d..4d43040880ae15 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx @@ -296,8 +296,10 @@ describe('PreviewQuery', () => { threshold={{ field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -343,8 +345,10 @@ describe('PreviewQuery', () => { threshold={{ field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -387,8 +391,10 @@ describe('PreviewQuery', () => { threshold={{ field: undefined, value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> @@ -419,8 +425,10 @@ describe('PreviewQuery', () => { threshold={{ field: ' ', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }} isDisabled={false} /> diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx index 377259fc9b212a..e3920856ea19e0 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx @@ -60,8 +60,10 @@ export type Threshold = | { field: string | string[] | undefined; value: number; - cardinality_field: string | undefined; - cardinality_value: number | undefined; + cardinality?: { + field: string[]; + value: number; + }; } | undefined; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts index d1a9e5c5f768f4..b0728cd8cc827e 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts @@ -337,8 +337,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -355,8 +357,10 @@ describe('queryPreviewReducer', () => { threshold: { field: undefined, value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -373,8 +377,10 @@ describe('queryPreviewReducer', () => { threshold: { field: ' ', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'threshold', }); @@ -391,8 +397,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'eql', }); @@ -408,8 +416,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'query', }); @@ -425,8 +435,10 @@ describe('queryPreviewReducer', () => { threshold: { field: 'agent.hostname', value: 200, - cardinality_field: 'user.name', - cardinality_value: 2, + cardinality: { + field: ['user.name'], + value: 2, + }, }, ruleType: 'saved_query', }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx index 4c7a34dbdf0807..c6bb35fdb72668 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx @@ -82,8 +82,10 @@ const stepDefineDefaultValue: DefineStepRule = { threshold: { field: [], value: '200', - cardinality_field: [], - cardinality_value: '2', + cardinality: { + field: [], + value: '', + }, }, timeline: { id: null, @@ -154,15 +156,15 @@ const StepDefineRuleComponent: FC = ({ threatIndex: formThreatIndex, 'threshold.field': formThresholdField, 'threshold.value': formThresholdValue, - 'threshold.cardinality_field': formThresholdCardinalityField, - 'threshold.cardinality_value': formThresholdCardinalityValue, + 'threshold.cardinality.field': formThresholdCardinalityField, + 'threshold.cardinality.value': formThresholdCardinalityValue, }, ] = useFormData< DefineStepRule & { 'threshold.field': string[] | undefined; 'threshold.value': number | undefined; - 'threshold.cardinality_field': string[] | undefined; - 'threshold.cardinality_value': number | undefined; + 'threshold.cardinality.field': string[] | undefined; + 'threshold.cardinality.value': number | undefined; } >({ form, @@ -172,8 +174,8 @@ const StepDefineRuleComponent: FC = ({ 'queryBar', 'threshold.field', 'threshold.value', - 'threshold.cardinality_field', - 'threshold.cardinality_value', + 'threshold.cardinality.field', + 'threshold.cardinality.value', 'threatIndex', ], }); @@ -289,15 +291,14 @@ const StepDefineRuleComponent: FC = ({ }, []); const thresholdFormValue = useMemo((): Threshold | undefined => { - return formThresholdValue != null && - formThresholdField != null && - formThresholdCardinalityField != null && - formThresholdCardinalityValue != null + return formThresholdValue != null ? { - field: formThresholdField[0], + field: formThresholdField ?? [], value: formThresholdValue, - cardinality_field: formThresholdCardinalityField[0], - cardinality_value: formThresholdCardinalityValue, + cardinality: { + field: formThresholdCardinalityField ?? [], + value: formThresholdCardinalityValue ?? 0, // FIXME + }, } : undefined; }, [ @@ -460,10 +461,10 @@ const StepDefineRuleComponent: FC = ({ path: 'threshold.value', }, thresholdCardinalityField: { - path: 'threshold.cardinality_field', + path: 'threshold.cardinality.field', }, thresholdCardinalityValue: { - path: 'threshold.cardinality_value', + path: 'threshold.cardinality.value', }, }} > diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx index a5352ede83d51d..194584ec8eb87f 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx @@ -239,52 +239,85 @@ export const schema: FormSchema = { }, ], }, - cardinality_field: { - type: FIELD_TYPES.COMBO_BOX, - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel', - { - defaultMessage: 'Count', - } - ), - helpText: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText', - { - defaultMessage: 'Select a field to check cardinality', - } - ), - }, - cardinality_value: { - type: FIELD_TYPES.NUMBER, - label: i18n.translate( - 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel', - { - defaultMessage: 'Unique values', - } - ), - validations: [ - { - validator: ( - ...args: Parameters - ): ReturnType> | undefined => { - const [{ formData }] = args; - const needsValidation = isThresholdRule(formData.ruleType); - if (!needsValidation) { - return; - } - return fieldValidators.numberGreaterThanField({ - than: 1, - message: i18n.translate( - 'xpack.securitySolution.detectionEngine.validations.thresholdValueFieldData.numberGreaterThanOrEqualOneErrorMessage', - { - defaultMessage: 'Value must be greater than or equal to one.', - } - ), - allowEquality: true, - })(...args); + cardinality: { + field: { + defaultValue: [], + fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'], + type: FIELD_TYPES.COMBO_BOX, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel', + { + defaultMessage: 'Count', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isThresholdRule(formData.ruleType); + if (!needsValidation) { + return; + } + if ( + isEmpty(formData['threshold.cardinality.field']) && + !isEmpty(formData['threshold.cardinality.value']) + ) { + return fieldValidators.emptyField( + i18n.translate( + 'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityFieldFieldData.thresholdCardinalityFieldNotSuppliedMessage', + { + defaultMessage: 'A Cardinality Field is required.', + } + ) + )(...args); + } + }, }, - }, - ], + ], + helpText: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText', + { + defaultMessage: 'Select a field to check cardinality', + } + ), + }, + value: { + fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'], + type: FIELD_TYPES.NUMBER, + label: i18n.translate( + 'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel', + { + defaultMessage: 'Unique values', + } + ), + validations: [ + { + validator: ( + ...args: Parameters + ): ReturnType> | undefined => { + const [{ formData }] = args; + const needsValidation = isThresholdRule(formData.ruleType); + if (!needsValidation) { + return; + } + if (!isEmpty(formData['threshold.cardinality.field'])) { + return fieldValidators.numberGreaterThanField({ + than: 1, + message: i18n.translate( + 'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityValueFieldData.numberGreaterThanOrEqualOneErrorMessage', + { + defaultMessage: 'Value must be greater than or equal to one.', + } + ), + allowEquality: true, + })(...args); + } + }, + }, + ], + }, }, }, threatIndex: { diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx index 287c99dce3e60a..77c88918abf9c7 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx @@ -19,8 +19,10 @@ const FIELD_COMBO_BOX_WIDTH = 410; export interface FieldValueThreshold { field: string[]; value: string; - cardinality_field: string[]; - cardinality_value: string; + cardinality?: { + field: string[]; + value: string; + }; } interface ThresholdInputProps { diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts index 8bdbb1a74c73a1..ee2c2c48d22ee0 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts @@ -143,8 +143,12 @@ export const mockRuleWithEverything = (id: string): Rule => ({ threshold: { field: ['host.name'], value: 50, - cardinality_field: ['process.name'], - cardinality_value: 2, + cardinality: [ + { + field: 'process.name', + value: 2, + }, + ], }, throttle: 'no_actions', timestamp_override: 'event.ingested', @@ -192,10 +196,12 @@ export const mockDefineStepRule = (): DefineStepRule => ({ }, threatIndex: [], threshold: { - field: [''], + field: [], value: '100', - cardinality_field: [''], - cardinality_value: '2', + cardinality: { + field: ['process.name'], + value: '2', + }, }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts index b8824d2b8798e1..64dfac5787f231 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts @@ -221,8 +221,16 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep threshold: { field: ruleFields.threshold?.field ?? [], value: parseInt(ruleFields.threshold?.value, 10) ?? 0, - cardinality_field: ruleFields.threshold.cardinality_field[0] ?? '', - cardinality_value: parseInt(ruleFields.threshold?.cardinality_value, 10) ?? 0, + cardinality: + !isEmpty(ruleFields.threshold.cardinality?.field) && + ruleFields.threshold.cardinality?.value != null + ? [ + { + field: ruleFields.threshold.cardinality.field[0], + value: parseInt(ruleFields.threshold.cardinality.value, 10), + }, + ] + : [], }, }), } diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx index 29d1512030e74f..9c2e7751753ee3 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx @@ -84,8 +84,10 @@ describe('rule helpers', () => { threshold: { field: ['host.name'], value: '50', - cardinality_field: ['process.name'], - cardinality_value: '2', + cardinality: { + field: ['process.name'], + value: '2', + }, }, threatIndex: [], threatMapping: [], @@ -215,8 +217,6 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', - cardinality_field: [], - cardinality_value: '0', }, threatIndex: [], threatMapping: [], @@ -259,8 +259,6 @@ describe('rule helpers', () => { threshold: { field: [], value: '100', - cardinality_field: [], - cardinality_value: '0', }, threatIndex: [], threatMapping: [], diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx index 7c3930bb21d9a4..9bc3ab9103b425 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx @@ -13,6 +13,7 @@ import { useLocation } from 'react-router-dom'; import styled from 'styled-components'; import { EuiFlexItem } from '@elastic/eui'; import { ActionVariables } from '../../../../../../triggers_actions_ui/public'; +import { normalizeThresholdField } from '../../../../../common/detection_engine/utils'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { assertUnreachable } from '../../../../../common/utility_types'; import { transformRuleToAlertAction } from '../../../../../common/detection_engine/transform_actions'; @@ -99,18 +100,16 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({ title: rule.timeline_title ?? null, }, threshold: { - field: rule.threshold?.field - ? Array.isArray(rule.threshold.field) - ? rule.threshold.field - : [rule.threshold.field] - : [], + field: normalizeThresholdField(rule.threshold?.field), value: `${rule.threshold?.value || 100}`, - cardinality_field: Array.isArray(rule.threshold?.cardinality_field) - ? rule.threshold!.cardinality_field - : rule.threshold?.cardinality_field != null - ? [rule.threshold!.cardinality_field] - : [], - cardinality_value: `${rule.threshold?.cardinality_value ?? 0}`, + ...(rule.threshold?.cardinality?.length + ? { + cardinality: { + field: [`${rule.threshold.cardinality[0].field}`], + value: `${rule.threshold.cardinality[0].value}`, + }, + } + : {}), }, }); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts index 668ca556539ad0..b6ea18d0494e59 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts @@ -160,8 +160,10 @@ export interface DefineStepRuleJson { threshold?: { field: string[]; value: number; - cardinality_field: string; - cardinality_value: number; + cardinality: Array<{ + field: string; + value: number; + }>; }; threat_query?: string; threat_mapping?: ThreatMapping; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts index e9a75af14310e9..58ce1e7e144602 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts @@ -7,6 +7,7 @@ import uuid from 'uuid'; import { InternalRuleCreate, InternalRuleResponse, TypeSpecificRuleParams } from './rule_schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { assertUnreachable } from '../../../../common/utility_types'; import { CreateRulesSchema, @@ -207,7 +208,10 @@ export const typeSpecificCamelToSnake = (params: TypeSpecificRuleParams): Respon query: params.query, filters: params.filters, saved_id: params.savedId, - threshold: params.threshold, + threshold: { + ...params.threshold, + field: normalizeThresholdField(params.threshold.field), + }, }; } case 'machine_learning': { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 977b38e59f8561..2d6e90443c00f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -434,8 +434,12 @@ export const sampleThresholdSignalHit = (): SignalHit => ({ threshold: { field: ['host.name'], value: 5, - cardinality_field: 'process.name', - cardinality_value: 2, + cardinality: [ + { + field: 'process.name', + value: 2, + }, + ], }, updated_by: 'elastic_kibana', tags: ['some fake tag 1', 'some fake tag 2'], @@ -460,6 +464,25 @@ export const sampleThresholdSignalHit = (): SignalHit => ({ }, }); +const sampleThresholdHit = sampleThresholdSignalHit(); +export const sampleLegacyThresholdSignalHit = (): unknown => ({ + ...sampleThresholdHit, + signal: { + ...sampleThresholdHit.signal, + rule: { + ...sampleThresholdHit.signal.rule, + threshold: { + field: 'host.name', + value: 5, + }, + }, + threshold_result: { + count: 72, + value: 'a hostname', + }, + }, +}); + export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => { return { _index: 'myFakeSignalIndex', @@ -468,6 +491,14 @@ export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => { }; }; +export const sampleWrappedLegacyThresholdSignalHit = (): WrappedSignalHit => { + return { + _index: 'myFakeSignalIndex', + _id: 'adb9d636-fbbe-4962-ac1c-e282f3ec5879', + _source: sampleLegacyThresholdSignalHit() as SignalHit, + }; +}; + export const sampleBulkCreateDuplicateResult = { took: 60, errors: true, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index 56d71048bb81b6..130077d2fdf2b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -9,15 +9,172 @@ import { loggingSystemMock } from '../../../../../../../src/core/server/mocks'; import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results'; import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals'; import { calculateThresholdSignalUuid } from './utils'; -import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; +import { + Threshold, + ThresholdNormalized, +} from '../../../../common/detection_engine/schemas/common/schemas'; -describe('transformThresholdResultsToEcs', () => { - it('should return transformed threshold results', () => { +describe('transformThresholdNormalizedResultsToEcs', () => { + it('should return transformed threshold results for pre-7.12 rules', () => { const threshold: Threshold = { + field: 'source.ip', + value: 1, + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + 'threshold_0:source.ip': { + buckets: [ + { + key: '127.0.0.1', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, ['source.ip'], '127.0.0.1'); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [ + { + field: 'source.ip', + value: '127.0.0.1', + }, + ], + cardinality: undefined, + count: 15, + }, + }, + }, + ], + }, + }); + }); + + it('should return transformed threshold results for pre-7.12 rules without threshold field', () => { + const threshold: Threshold = { + field: '', + value: 1, + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold_0: { + buckets: [ + { + key: '', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, [], ''); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [], + cardinality: undefined, + count: 15, + }, + }, + }, + ], + }, + }); + }); + + it('should return transformed threshold results', () => { + const threshold: ThresholdNormalized = { field: ['source.ip', 'host.name'], value: 1, - cardinality_field: 'destination.ip', - cardinality_value: 5, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], }; const startedAt = new Date('2020-12-17T16:27:00Z'); const transformedResults = transformThresholdResultsToEcs( @@ -112,4 +269,87 @@ describe('transformThresholdResultsToEcs', () => { }, }); }); + + it('should return transformed threshold results without threshold fields', () => { + const threshold: ThresholdNormalized = { + field: [], + value: 1, + cardinality: [ + { + field: 'destination.ip', + value: 5, + }, + ], + }; + const startedAt = new Date('2020-12-17T16:27:00Z'); + const transformedResults = transformThresholdResultsToEcs( + { + ...sampleDocSearchResultsNoSortId('abcd'), + aggregations: { + threshold_0: { + buckets: [ + { + key: '', + doc_count: 15, + top_threshold_hits: { + hits: { + hits: [sampleDocNoSortId('abcd')], + }, + }, + cardinality_count: { + value: 7, + }, + }, + ], + }, + }, + }, + 'test', + startedAt, + undefined, + loggingSystemMock.createLogger(), + threshold, + '1234', + undefined + ); + const _id = calculateThresholdSignalUuid('1234', startedAt, [], ''); + expect(transformedResults).toEqual({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + results: { + hits: { + total: 1, + }, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + _id, + _index: 'test', + _source: { + '@timestamp': '2020-04-20T21:27:45+0000', + threshold_result: { + terms: [], + cardinality: [ + { + field: 'destination.ip', + value: 7, + }, + ], + count: 15, + }, + }, + }, + ], + }, + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 29fd189bb34f34..1c1b2bb18900a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { get, isEmpty } from 'lodash/fp'; +import { get } from 'lodash/fp'; import set from 'set-value'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { - Threshold, + ThresholdNormalized, TimestampOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../src/core/server'; @@ -56,59 +57,19 @@ const getTransformedHits = ( inputIndex: string, startedAt: Date, logger: Logger, - threshold: Threshold, + threshold: ThresholdNormalized, ruleId: string, filter: unknown, timestampOverride: TimestampOverrideOrUndefined ) => { - if (isEmpty(threshold.field)) { - const totalResults = - typeof results.hits.total === 'number' ? results.hits.total : results.hits.total.value; - - if (totalResults < threshold.value) { - return []; - } - - const hit = results.hits.hits[0]; - if (hit == null) { - logger.warn(`No hits returned, but totalResults >= threshold.value (${threshold.value})`); - return []; - } - const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); - if (timestampArray == null) { - return []; - } - const timestamp = timestampArray[0]; - if (typeof timestamp !== 'string') { - return []; - } - - const source = { - '@timestamp': timestamp, - threshold_result: { - terms: [ - { - value: ruleId, - }, - ], - count: totalResults, - }, - }; - - return [ - { - _index: inputIndex, - _id: calculateThresholdSignalUuid( - ruleId, - startedAt, - Array.isArray(threshold.field) ? threshold.field : [threshold.field] - ), - _source: source, - }, - ]; - } + const aggParts = threshold.field.length + ? results.aggregations && getThresholdAggregationParts(results.aggregations) + : { + field: null, + index: 0, + name: 'threshold_0', + }; - const aggParts = results.aggregations && getThresholdAggregationParts(results.aggregations); if (!aggParts) { return []; } @@ -119,7 +80,7 @@ const getTransformedHits = ( const nextLevelIdx = i + 1; const nextLevelAggParts = getThresholdAggregationParts(bucket, nextLevelIdx); if (nextLevelAggParts == null) { - throw new Error('Something went horribly wrong'); + throw new Error('Unable to parse aggregation.'); } const nextLevelPath = `['${nextLevelAggParts.name}']['buckets']`; const nextBuckets = get(nextLevelPath, bucket); @@ -132,7 +93,7 @@ const getTransformedHits = ( value: bucket.key, }, ...val.terms, - ], + ].filter((term) => term.field != null), cardinality: val.cardinality, topThresholdHits: val.topThresholdHits, docCount: val.docCount, @@ -146,13 +107,11 @@ const getTransformedHits = ( field, value: bucket.key, }, - ], - cardinality: !isEmpty(threshold.cardinality_field) + ].filter((term) => term.field != null), + cardinality: threshold.cardinality?.length ? [ { - field: Array.isArray(threshold.cardinality_field) - ? threshold.cardinality_field[0] - : threshold.cardinality_field!, + field: threshold.cardinality[0].field, value: bucket.cardinality_count!.value, }, ] @@ -208,7 +167,7 @@ const getTransformedHits = ( _id: calculateThresholdSignalUuid( ruleId, startedAt, - Array.isArray(threshold.field) ? threshold.field : [threshold.field], + threshold.field, bucket.terms.map((term) => term.value).join(',') ), _source: source, @@ -226,7 +185,7 @@ export const transformThresholdResultsToEcs = ( startedAt: Date, filter: unknown, logger: Logger, - threshold: Threshold, + threshold: ThresholdNormalized, ruleId: string, timestampOverride: TimestampOverrideOrUndefined ): SignalSearchResponse => { @@ -259,13 +218,17 @@ export const bulkCreateThresholdSignals = async ( params: BulkCreateThresholdSignalsParams ): Promise => { const thresholdResults = params.someResult; + const threshold = params.ruleParams.threshold!; const ecsResults = transformThresholdResultsToEcs( thresholdResults, params.inputIndexPattern.join(','), params.startedAt, params.filter, params.logger, - params.ruleParams.threshold!, + { + ...threshold, + field: normalizeThresholdField(threshold.field), + }, params.ruleParams.ruleId, params.timestampOverride ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts new file mode 100644 index 00000000000000..6f7985fe52ecf0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts @@ -0,0 +1,445 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; +import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter'; +import { mockLogger } from './__mocks__/es_results'; +import { findThresholdSignals } from './find_threshold_signals'; +import { buildRuleMessageFactory } from './rule_messages'; +import * as single_search_after from './single_search_after'; + +const buildRuleMessage = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); + +const queryFilter = getQueryFilter('', 'kuery', [], ['*'], []); +const mockSingleSearchAfter = jest.fn(); + +describe('findThresholdSignals', () => { + let mockService: AlertServicesMock; + + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(single_search_after, 'singleSearchAfter').mockImplementation(mockSingleSearchAfter); + mockService = alertsMock.createAlertServices(); + }); + + it('should generate a threshold signal for pre-7.12 rules', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: 'host.name', + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a signal for pre-7.12 rules with no threshold field', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: '', + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 100, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when only a value is provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: [], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 100, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when a field and value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name'], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when multiple fields and a value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name', 'user.name'], + value: 100, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + 'threshold_1:user.name': { + terms: { + field: 'user.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when multiple fields, a value, and cardinality field/value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + field: ['host.name', 'user.name'], + value: 100, + cardinality: [ + { + field: 'destination.ip', + value: 2, + }, + ], + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + 'threshold_0:host.name': { + terms: { + field: 'host.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + 'threshold_1:user.name': { + terms: { + field: 'user.name', + min_doc_count: 100, + size: 10000, + }, + aggs: { + cardinality_count: { + cardinality: { + field: 'destination.ip', + }, + }, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: 'params.cardinalityCount >= 2', + }, + }, + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }, + }, + }) + ); + }); + + it('should generate a threshold signal query when only a value and a cardinality field/value are provided', async () => { + await findThresholdSignals({ + from: 'now-6m', + to: 'now', + inputIndexPattern: ['*'], + services: mockService, + logger: mockLogger, + filter: queryFilter, + threshold: { + cardinality: [ + { + field: 'source.ip', + value: 5, + }, + ], + field: [], + value: 200, + }, + buildRuleMessage, + timestampOverride: undefined, + }); + expect(mockSingleSearchAfter).toHaveBeenCalledWith( + expect.objectContaining({ + aggregations: { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: 200, + }, + aggs: { + cardinality_count: { + cardinality: { + field: 'source.ip', + }, + }, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: 'params.cardinalityCount >= 5', + }, + }, + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }, + }, + }, + }, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 7796346e9876d0..8b446bba90f658 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -6,12 +6,12 @@ */ import { set } from '@elastic/safer-lodash-set'; -import { isEmpty } from 'lodash/fp'; import { Threshold, TimestampOverrideOrUndefined, } from '../../../../common/detection_engine/schemas/common/schemas'; +import { normalizeThresholdField } from '../../../../common/detection_engine/utils'; import { singleSearchAfter } from './single_search_after'; import { @@ -50,69 +50,98 @@ export const findThresholdSignals = async ({ searchDuration: string; searchErrors: string[]; }> => { - const thresholdFields = Array.isArray(threshold.field) ? threshold.field : [threshold.field]; + const topHitsAgg = { + top_hits: { + sort: [ + { + [timestampOverride ?? '@timestamp']: { + order: 'desc', + }, + }, + ], + fields: [ + { + field: '*', + include_unmapped: true, + }, + ], + size: 1, + }, + }; - const aggregations = - threshold && !isEmpty(threshold.field) - ? thresholdFields.reduce((acc, field, i) => { - const aggPath = [...Array(i + 1).keys()] - .map((j) => { - return `['threshold_${j}:${thresholdFields[j]}']`; - }) - .join(`['aggs']`); - set(acc, aggPath, { - terms: { - field, - min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set - size: 10000, // max 10k buckets - }, - }); - if (i === threshold.field.length - 1) { - const topHitsAgg = { - top_hits: { - sort: [ - { - [timestampOverride ?? '@timestamp']: { - order: 'desc', - }, - }, - ], - fields: [ - { - field: '*', - include_unmapped: true, - }, - ], - size: 1, + const thresholdFields = normalizeThresholdField(threshold.field); + + const aggregations = thresholdFields.length + ? thresholdFields.reduce((acc, field, i) => { + const aggPath = [...Array(i + 1).keys()] + .map((j) => { + return `['threshold_${j}:${thresholdFields[j]}']`; + }) + .join(`['aggs']`); + set(acc, aggPath, { + terms: { + field, + min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set + size: 10000, // max 10k buckets + }, + }); + if (i === (thresholdFields.length ?? 0) - 1) { + if (threshold.cardinality?.length) { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + cardinality_count: { + cardinality: { + field: threshold.cardinality[0].field, + }, }, - }; - // TODO: support case where threshold fields are not supplied, but cardinality is? - if (!isEmpty(threshold.cardinality_field)) { - set(acc, `${aggPath}['aggs']`, { - top_threshold_hits: topHitsAgg, - cardinality_count: { - cardinality: { - field: threshold.cardinality_field, + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', }, + script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator }, - cardinality_check: { - bucket_selector: { - buckets_path: { - cardinalityCount: 'cardinality_count', + }, + }); + } else { + set(acc, `${aggPath}['aggs']`, { + top_threshold_hits: topHitsAgg, + }); + } + } + return acc; + }, {}) + : { + threshold_0: { + terms: { + script: { + source: '""', + lang: 'painless', + }, + min_doc_count: threshold.value, + }, + aggs: { + top_threshold_hits: topHitsAgg, + ...(threshold.cardinality?.length + ? { + cardinality_count: { + cardinality: { + field: threshold.cardinality[0].field, }, - script: `params.cardinalityCount >= ${threshold.cardinality_value}`, // TODO: cardinality operator }, - }, - }); - } else { - set(acc, `${aggPath}['aggs']`, { - top_threshold_hits: topHitsAgg, - }); - } - } - return acc; - }, {}) - : {}; + cardinality_check: { + bucket_selector: { + buckets_path: { + cardinalityCount: 'cardinality_count', + }, + script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator + }, + }, + } + : {}), + }, + }, + }; return singleSearchAfter({ aggregations, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts index 06c5fd38099bcd..21db1e55b9810e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts @@ -103,4 +103,56 @@ describe('signal_params_schema', () => { const { falsePositives, ...withoutFalsePositives } = getSignalParamsSchemaMock(); expect(schema.validate(withoutFalsePositives).falsePositives).toEqual([]); }); + + test('threshold validates with `value` only', () => { + const schema = signalParamsSchema(); + const threshold = { + value: 200, + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(schema.validate(mock).threshold?.value).toEqual(200); + }); + + test('threshold does not validate without `value`', () => { + const schema = signalParamsSchema(); + const threshold = { + field: 'agent.id', + cardinality: [ + { + field: ['host.name'], + value: 5, + }, + ], + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(() => schema.validate(mock)).toThrow(); + }); + + test('threshold `cardinality` cannot currently be greater than length 1', () => { + const schema = signalParamsSchema(); + const threshold = { + value: 100, + cardinality: [ + { + field: 'host.name', + value: 5, + }, + { + field: 'user.name', + value: 5, + }, + ], + }; + const mock = { + ...getSignalParamsSchemaMock(), + threshold, + }; + expect(() => schema.validate(mock)).toThrow(); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index 710a925fe315b9..bfa452af0f3e95 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -41,10 +41,19 @@ export const signalSchema = schema.object({ threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threshold: schema.maybe( schema.object({ + // Can be an empty string or empty array field: schema.nullable(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])), + // Always required value: schema.number(), - cardinality_field: schema.nullable(schema.string()), // TODO: depends on `field` being defined? - cardinality_value: schema.nullable(schema.number()), + cardinality: schema.nullable( + schema.arrayOf( + schema.object({ + field: schema.string(), + value: schema.number(), + }), + { maxSize: 1 } + ) + ), }) ), timestampOverride: schema.nullable(schema.string()), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 14a65bc1eeb7ba..4bded347c32ead 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -25,6 +25,7 @@ import { isEqlRule, isThreatMatchRule, hasLargeValueItem, + normalizeThresholdField, } from '../../../../common/detection_engine/utils'; import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates'; import { SetupPlugins } from '../../../plugin'; @@ -374,10 +375,6 @@ export const signalRulesAlertType = ({ } const inputIndex = await getInputIndex(services, version, index); - const thresholdFields = Array.isArray(threshold.field) - ? threshold.field - : [threshold.field]; - const { filters: bucketFilters, searchErrors: previousSearchErrors, @@ -388,7 +385,7 @@ export const signalRulesAlertType = ({ services, logger, ruleId, - bucketByFields: thresholdFields, + bucketByFields: normalizeThresholdField(threshold.field), timestampOverride, buildRuleMessage, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts index ed9aa9a5ba698a..97d7d96a50e12d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts @@ -6,7 +6,11 @@ */ import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; -import { mockLogger, sampleWrappedThresholdSignalHit } from './__mocks__/es_results'; +import { + mockLogger, + sampleWrappedThresholdSignalHit, + sampleWrappedLegacyThresholdSignalHit, +} from './__mocks__/es_results'; import { getThresholdBucketFilters } from './threshold_get_bucket_filters'; import { buildRuleMessageFactory } from './rule_messages'; @@ -82,4 +86,131 @@ describe('thresholdGetBucketFilters', () => { searchErrors: [], }); }); + + it('should generate filters for threshold signal detection based on pre-7.12 signals', async () => { + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [sampleWrappedLegacyThresholdSignalHit()], + }, + }); + const result = await getThresholdBucketFilters({ + from: 'now-6m', + to: 'now', + indexPattern: ['*'], + services: mockService, + logger: mockLogger, + ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + bucketByFields: ['host.name'], + timestampOverride: undefined, + buildRuleMessage, + }); + expect(result).toEqual({ + filters: [ + { + bool: { + must_not: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2021-02-16T17:37:34.275Z', + }, + }, + }, + { + term: { + 'host.name': 'a hostname', + }, + }, + ], + }, + }, + ], + }, + }, + ], + searchErrors: [], + }); + }); + + it('should generate filters for threshold signal detection with mixed pre-7.12 and post-7.12 signals', async () => { + const signalHit = sampleWrappedThresholdSignalHit(); + const wrappedSignalHit = { + ...signalHit, + _source: { + ...signalHit._source, + signal: { + ...signalHit._source.signal, + original_time: '2021-02-16T18:37:34.275Z', + }, + }, + }; + mockService.callCluster.mockResolvedValue({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [sampleWrappedLegacyThresholdSignalHit(), wrappedSignalHit], + }, + }); + const result = await getThresholdBucketFilters({ + from: 'now-6m', + to: 'now', + indexPattern: ['*'], + services: mockService, + logger: mockLogger, + ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9', + bucketByFields: ['host.name'], + timestampOverride: undefined, + buildRuleMessage, + }); + expect(result).toEqual({ + filters: [ + { + bool: { + must_not: [ + { + bool: { + filter: [ + { + range: { + '@timestamp': { + lte: '2021-02-16T18:37:34.275Z', + }, + }, + }, + { + term: { + 'host.name': 'a hostname', + }, + }, + ], + }, + }, + ], + }, + }, + ], + searchErrors: [], + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts index e1727c0361afc8..3091a8d5350346 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts @@ -74,8 +74,9 @@ export const getThresholdBucketFilters = async ({ if (signalTerms == null) { signalTerms = [ { - field: (((hit._source.rule as RulesSchema).threshold as unknown) as { field: string }) - .field, + field: (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as { + field: string; + }).field, value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value, }, ]; From 4739eab490c99c971eead4c48c48dee276bb520b Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 1 Mar 2021 17:11:47 -0500 Subject: [PATCH 22/24] [Security Solution][Case][Bug] Prevent closing collection when pushing (#93095) * Prevent closing collection when pushing * Fixing translations --- .../plugins/case/server/client/cases/push.ts | 28 ++++++- .../configure_cases/closure_options.tsx | 7 +- .../configure_cases/translations.ts | 7 ++ .../translations/translations/ja-JP.json | 82 +++++++++---------- .../translations/translations/zh-CN.json | 82 +++++++++---------- .../basic/tests/cases/push_case.ts | 61 +++++++++++++- 6 files changed, 179 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/case/server/client/cases/push.ts b/x-pack/plugins/case/server/client/cases/push.ts index 352328ed1dd40d..80dcc7a0e018c3 100644 --- a/x-pack/plugins/case/server/client/cases/push.ts +++ b/x-pack/plugins/case/server/client/cases/push.ts @@ -11,6 +11,8 @@ import { SavedObjectsClientContract, SavedObjectsUpdateResponse, Logger, + SavedObjectsFindResponse, + SavedObject, } from 'kibana/server'; import { ActionResult, ActionsClient } from '../../../../actions/server'; import { flattenCaseSavedObject, getAlertIndicesAndIDs } from '../../routes/api/utils'; @@ -25,6 +27,8 @@ import { CommentAttributes, CaseUserActionsResponse, User, + ESCasesConfigureAttributes, + CaseType, } from '../../../common/api'; import { buildCaseUserActionItem } from '../../services/user_actions/helpers'; @@ -37,6 +41,22 @@ import { import { CaseClientHandler } from '../client'; import { createCaseError } from '../../common/error'; +/** + * Returns true if the case should be closed based on the configuration settings and whether the case + * is a collection. Collections are not closable because we aren't allowing their status to be changed. + * In the future we could allow push to close all the sub cases of a collection but that's not currently supported. + */ +function shouldCloseByPush( + configureSettings: SavedObjectsFindResponse, + caseInfo: SavedObject +): boolean { + return ( + configureSettings.total > 0 && + configureSettings.saved_objects[0].attributes.closure_type === 'close-by-pushing' && + caseInfo.attributes.type !== CaseType.collection + ); +} + interface PushParams { savedObjectsClient: SavedObjectsClientContract; caseService: CaseServiceSetup; @@ -190,14 +210,15 @@ export const push = async ({ let updatedCase: SavedObjectsUpdateResponse; let updatedComments: SavedObjectsBulkUpdateResponse; + const shouldMarkAsClosed = shouldCloseByPush(myCaseConfigure, myCase); + try { [updatedCase, updatedComments] = await Promise.all([ caseService.patchCase({ client: savedObjectsClient, caseId, updatedAttributes: { - ...(myCaseConfigure.total > 0 && - myCaseConfigure.saved_objects[0].attributes.closure_type === 'close-by-pushing' + ...(shouldMarkAsClosed ? { status: CaseStatuses.closed, closed_at: pushedDate, @@ -228,8 +249,7 @@ export const push = async ({ userActionService.postUserActions({ client: savedObjectsClient, actions: [ - ...(myCaseConfigure.total > 0 && - myCaseConfigure.saved_objects[0].attributes.closure_type === 'close-by-pushing' + ...(shouldMarkAsClosed ? [ buildCaseUserActionItem({ action: 'update', diff --git a/x-pack/plugins/security_solution/public/cases/components/configure_cases/closure_options.tsx b/x-pack/plugins/security_solution/public/cases/components/configure_cases/closure_options.tsx index 9417877e58f759..ba892116320ce2 100644 --- a/x-pack/plugins/security_solution/public/cases/components/configure_cases/closure_options.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/configure_cases/closure_options.tsx @@ -27,7 +27,12 @@ const ClosureOptionsComponent: React.FC = ({ {i18n.CASE_CLOSURE_OPTIONS_TITLE}} - description={i18n.CASE_CLOSURE_OPTIONS_DESC} + description={ + <> +

{i18n.CASE_CLOSURE_OPTIONS_DESC}

+

{i18n.CASE_COLSURE_OPTIONS_SUB_CASES}

+ + } data-test-subj="case-closure-options-form-group" > { @@ -228,6 +234,59 @@ export default ({ getService }: FtrProviderContext): void => { expect(body.status).to.eql('closed'); }); + it('should push a collection case but not close it when closure_type: close-by-pushing', async () => { + const { body: connector } = await supertest + .post('/api/actions/action') + .set('kbn-xsrf', 'true') + .send({ + ...getServiceNowConnector(), + config: { apiUrl: servicenowSimulatorURL }, + }) + .expect(200); + + actionsRemover.add('default', connector.id, 'action', 'actions'); + await supertest + .post(CASE_CONFIGURE_URL) + .set('kbn-xsrf', 'true') + .send({ + ...getConfiguration({ + id: connector.id, + name: connector.name, + type: connector.actionTypeId, + }), + closure_type: 'close-by-pushing', + }) + .expect(200); + + const { body: postedCase } = await supertest + .post(CASES_URL) + .set('kbn-xsrf', 'true') + .send({ + ...postCollectionReq, + connector: getConfiguration({ + id: connector.id, + name: connector.name, + type: connector.actionTypeId, + fields: { + urgency: '2', + impact: '2', + severity: '2', + category: 'software', + subcategory: 'os', + }, + }).connector, + }) + .expect(200); + + const { body } = await supertest + .post(`${CASES_URL}/${postedCase.id}/connector/${connector.id}/_push`) + .set('kbn-xsrf', 'true') + .send({}) + .expect(200); + + expect(body.status).to.eql(CaseStatuses.open); + }); + it('unhappy path - 404s when case does not exist', async () => { await supertest .post(`${CASES_URL}/fake-id/connector/fake-connector/_push`) From aa62a130eec855513bfc68fa863123a746879057 Mon Sep 17 00:00:00 2001 From: Christos Nasikas Date: Tue, 2 Mar 2021 00:38:49 +0200 Subject: [PATCH 23/24] [SecuritySolution][Case] Disable cases on detections in read-only mode (#93010) * Disable cases on detetions on read-only mode * Add cypress tests --- .../detection_alerts/attach_to_case.spec.ts | 60 ++++++++++++++ .../cypress/screens/alerts_detection_rules.ts | 2 + .../add_to_case_action.test.tsx | 80 +++++++++++++++++-- .../timeline_actions/add_to_case_action.tsx | 30 ++++--- .../timeline_actions/translations.ts | 15 ++++ .../body/events/event_column_view.tsx | 1 - 6 files changed, 170 insertions(+), 18 deletions(-) create mode 100644 x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts new file mode 100644 index 00000000000000..e63ef513cc6382 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/integration/detection_alerts/attach_to_case.spec.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { newRule } from '../../objects/rule'; +import { ROLES } from '../../../common/test'; + +import { waitForAlertsIndexToBeCreated, waitForAlertsPanelToBeLoaded } from '../../tasks/alerts'; +import { createCustomRuleActivated } from '../../tasks/api_calls/rules'; +import { cleanKibana } from '../../tasks/common'; +import { waitForAlertsToPopulate } from '../../tasks/create_new_rule'; +import { login, loginAndWaitForPage, waitForPageWithoutDateRange } from '../../tasks/login'; +import { refreshPage } from '../../tasks/security_header'; + +import { DETECTIONS_URL } from '../../urls/navigation'; +import { ATTACH_ALERT_TO_CASE_BUTTON } from '../../screens/alerts_detection_rules'; + +const loadDetectionsPage = (role: ROLES) => { + waitForPageWithoutDateRange(DETECTIONS_URL, role); + waitForAlertsToPopulate(); +}; + +describe('Alerts timeline', () => { + before(() => { + // First we login as a privileged user to create alerts. + cleanKibana(); + loginAndWaitForPage(DETECTIONS_URL, ROLES.platform_engineer); + waitForAlertsPanelToBeLoaded(); + waitForAlertsIndexToBeCreated(); + createCustomRuleActivated(newRule); + refreshPage(); + waitForAlertsToPopulate(); + + // Then we login as read-only user to test. + login(ROLES.reader); + }); + + context('Privileges: read only', () => { + beforeEach(() => { + loadDetectionsPage(ROLES.reader); + }); + + it('should not allow user with read only privileges to attach alerts to cases', () => { + cy.get(ATTACH_ALERT_TO_CASE_BUTTON).first().should('be.disabled'); + }); + }); + + context('Privileges: can crud', () => { + beforeEach(() => { + loadDetectionsPage(ROLES.platform_engineer); + }); + + it('should allow a user with crud privileges to attach alerts to cases', () => { + cy.get(ATTACH_ALERT_TO_CASE_BUTTON).first().should('not.be.disabled'); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts b/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts index 30365c9bd4c708..c74284eee15e41 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts_detection_rules.ts @@ -5,6 +5,8 @@ * 2.0. */ +export const ATTACH_ALERT_TO_CASE_BUTTON = '[data-test-subj="attach-alert-to-case-button"]'; + export const BULK_ACTIONS_BTN = '[data-test-subj="bulkActions"] span'; export const CREATE_NEW_RULE_BTN = '[data-test-subj="create-new-rule"]'; diff --git a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.test.tsx b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.test.tsx index b3302a05cfcb21..c99cabb50e3dc2 100644 --- a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.test.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.test.tsx @@ -10,7 +10,7 @@ import React, { ReactNode } from 'react'; import { mount } from 'enzyme'; import { EuiGlobalToastList } from '@elastic/eui'; -import { useKibana } from '../../../common/lib/kibana'; +import { useKibana, useGetUserSavedObjectPermissions } from '../../../common/lib/kibana'; import { useStateToaster } from '../../../common/components/toasters'; import { TestProviders } from '../../../common/mock'; import { usePostComment } from '../../containers/use_post_comment'; @@ -113,8 +113,8 @@ describe('AddToCaseAction', () => { ecsRowData: { _id: 'test-id', _index: 'test-index', + signal: { rule: { id: ['rule-id'], name: ['rule-name'], false_positives: [] } }, }, - disabled: false, }; const mockDispatchToaster = jest.fn(); @@ -127,6 +127,10 @@ describe('AddToCaseAction', () => { (useKibana as jest.Mock).mockReturnValue({ services: { application: { navigateToApp: mockNavigateToApp } }, }); + (useGetUserSavedObjectPermissions as jest.Mock).mockReturnValue({ + crud: true, + read: true, + }); }); it('it renders', async () => { @@ -181,8 +185,8 @@ describe('AddToCaseAction', () => { alertId: 'test-id', index: 'test-index', rule: { - id: null, - name: null, + id: 'rule-id', + name: 'rule-name', }, type: 'alert', }); @@ -218,7 +222,38 @@ describe('AddToCaseAction', () => { alertId: 'test-id', index: 'test-index', rule: { - id: null, + id: 'rule-id', + name: 'rule-name', + }, + type: 'alert', + }); + }); + + it('it set rule information as null when missing', async () => { + const wrapper = mount( + + + + ); + + wrapper.find(`[data-test-subj="attach-alert-to-case-button"]`).first().simulate('click'); + wrapper.find(`[data-test-subj="add-new-case-item"]`).first().simulate('click'); + + wrapper.find(`[data-test-subj="form-context-on-success"]`).first().simulate('click'); + + expect(postComment.mock.calls[0][0].caseId).toBe('new-case'); + expect(postComment.mock.calls[0][0].data).toEqual({ + alertId: 'test-id', + index: 'test-index', + rule: { + id: 'rule-id', name: null, }, type: 'alert', @@ -291,4 +326,39 @@ describe('AddToCaseAction', () => { path: '/selected-case', }); }); + + it('disabled when event type is not supported', async () => { + const wrapper = mount( + + + + ); + + expect( + wrapper.find(`[data-test-subj="attach-alert-to-case-button"]`).first().prop('disabled') + ).toBeTruthy(); + }); + + it('disabled when user does not have crud permissions', async () => { + (useGetUserSavedObjectPermissions as jest.Mock).mockReturnValue({ + crud: false, + read: true, + }); + + const wrapper = mount( + + + + ); + + expect( + wrapper.find(`[data-test-subj="attach-alert-to-case-button"]`).first().prop('disabled') + ).toBeTruthy(); + }); }); diff --git a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.tsx b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.tsx index 3000551dd3c07f..4a4420a164d0cb 100644 --- a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.tsx +++ b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/add_to_case_action.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { isEmpty } from 'lodash'; import React, { memo, useState, useCallback, useMemo } from 'react'; import { EuiPopover, @@ -22,7 +23,7 @@ import { usePostComment } from '../../containers/use_post_comment'; import { Case } from '../../containers/types'; import { useStateToaster } from '../../../common/components/toasters'; import { APP_ID } from '../../../../common/constants'; -import { useKibana } from '../../../common/lib/kibana'; +import { useGetUserSavedObjectPermissions, useKibana } from '../../../common/lib/kibana'; import { getCaseDetailsUrl } from '../../../common/components/link_to'; import { SecurityPageName } from '../../../app/types'; import { useAllCasesModal } from '../use_all_cases_modal'; @@ -34,13 +35,11 @@ import { CreateCaseFlyout } from '../create/flyout'; interface AddToCaseActionProps { ariaLabel?: string; ecsRowData: Ecs; - disabled: boolean; } const AddToCaseActionComponent: React.FC = ({ ariaLabel = i18n.ACTION_ADD_TO_CASE_ARIA_LABEL, ecsRowData, - disabled, }) => { const eventId = ecsRowData._id; const eventIndex = ecsRowData._index; @@ -51,6 +50,16 @@ const AddToCaseActionComponent: React.FC = ({ const [isPopoverOpen, setIsPopoverOpen] = useState(false); const openPopover = useCallback(() => setIsPopoverOpen(true), []); const closePopover = useCallback(() => setIsPopoverOpen(false), []); + const userPermissions = useGetUserSavedObjectPermissions(); + + const isEventSupported = !isEmpty(ecsRowData.signal?.rule?.id); + const userCanCrud = userPermissions?.crud ?? false; + const isDisabled = !userCanCrud || !isEventSupported; + const tooltipContext = userCanCrud + ? isEventSupported + ? i18n.ACTION_ADD_TO_CASE_TOOLTIP + : i18n.UNSUPPORTED_EVENTS_MSG + : i18n.PERMISSIONS_MSG; const { postComment } = usePostComment(); @@ -137,7 +146,7 @@ const AddToCaseActionComponent: React.FC = ({ onClick={addNewCaseClick} aria-label={i18n.ACTION_ADD_NEW_CASE} data-test-subj="add-new-case-item" - disabled={disabled} + disabled={isDisabled} > {i18n.ACTION_ADD_NEW_CASE} , @@ -146,31 +155,28 @@ const AddToCaseActionComponent: React.FC = ({ onClick={addExistingCaseClick} aria-label={i18n.ACTION_ADD_EXISTING_CASE} data-test-subj="add-existing-case-menu-item" - disabled={disabled} + disabled={isDisabled} > {i18n.ACTION_ADD_EXISTING_CASE} , ], - [addExistingCaseClick, addNewCaseClick, disabled] + [addExistingCaseClick, addNewCaseClick, isDisabled] ); const button = useMemo( () => ( - + ), - [ariaLabel, disabled, openPopover] + [ariaLabel, isDisabled, openPopover, tooltipContext] ); return ( diff --git a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/translations.ts b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/translations.ts index da07b1b79cee9d..7c1437ede9ce50 100644 --- a/x-pack/plugins/security_solution/public/cases/components/timeline_actions/translations.ts +++ b/x-pack/plugins/security_solution/public/cases/components/timeline_actions/translations.ts @@ -61,3 +61,18 @@ export const VIEW_CASE = i18n.translate( defaultMessage: 'View Case', } ); + +export const PERMISSIONS_MSG = i18n.translate( + 'xpack.securitySolution.case.timeline.actions.permissionsMessage', + { + defaultMessage: + 'You are currently missing the required permissions to attach alerts to cases. Please contact your administrator for further assistance.', + } +); + +export const UNSUPPORTED_EVENTS_MSG = i18n.translate( + 'xpack.securitySolution.case.timeline.actions.unsupportedEventsMessage', + { + defaultMessage: 'This event cannot be attached to a case', + } +); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx index 9d7b76af25a59d..c6caf0a7b5b155 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/body/events/event_column_view.tsx @@ -175,7 +175,6 @@ export const EventColumnView = React.memo( ariaLabel={i18n.ATTACH_ALERT_TO_CASE_FOR_ROW({ ariaRowindex, columnValues })} key="attach-to-case" ecsRowData={ecsData} - disabled={eventType !== 'signal'} />, ] : []), From fb1394812d88d91bde2cdbabfac95487b73b64bf Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Mon, 1 Mar 2021 17:35:21 -0800 Subject: [PATCH 24/24] [Security Solution][Detections] -Fixes rule edit flow bug with max_signals (#92748) ### Summary Fixes a bug where max_signals was being reverted to it's default value when the rule was edited via the UI. --- .../detection_rules/custom_query_rule.spec.ts | 25 +++++++++++++++++++ .../security_solution/cypress/objects/rule.ts | 10 ++++++++ .../cypress/tasks/api_calls/rules.ts | 1 + .../cypress/tasks/rule_details.ts | 5 ---- .../detection_engine/rules/edit/index.tsx | 1 + .../rules/queries/query_with_max_signals.json | 9 +++++++ 6 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_max_signals.json diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts index ecfa96d59170fa..201a3c3a5563e8 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/custom_query_rule.spec.ts @@ -108,6 +108,7 @@ import { } from '../../tasks/create_new_rule'; import { saveEditedRule, waitForKibana } from '../../tasks/edit_rule'; import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; +import { activatesRule } from '../../tasks/rule_details'; import { DETECTIONS_URL } from '../../urls/navigation'; @@ -308,6 +309,21 @@ describe('Custom detection rules deletion and edition', () => { reload(); }); + it('Only modifies rule active status on enable/disable', () => { + activatesRule(); + + cy.intercept('GET', `/api/detection_engine/rules?id=`).as('fetchRuleDetails'); + + goToRuleDetails(); + + cy.wait('@fetchRuleDetails').then(({ response }) => { + cy.wrap(response!.statusCode).should('eql', 200); + + cy.wrap(response!.body.max_signals).should('eql', existingRule.maxSignals); + cy.wrap(response!.body.enabled).should('eql', false); + }); + }); + it('Allows a rule to be edited', () => { editFirstRule(); waitForKibana(); @@ -347,8 +363,17 @@ describe('Custom detection rules deletion and edition', () => { goToAboutStepTab(); cy.get(TAGS_CLEAR_BUTTON).click({ force: true }); fillAboutRule(editedRule); + + cy.intercept('GET', '/api/detection_engine/rules?id').as('getRule'); + saveEditedRule(); + cy.wait('@getRule').then(({ response }) => { + cy.wrap(response!.statusCode).should('eql', 200); + // ensure that editing rule does not modify max_signals + cy.wrap(response!.body.max_signals).should('eql', existingRule.maxSignals); + }); + cy.get(RULE_NAME_HEADER).should('have.text', `${editedRule.name}`); cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', editedRule.description); cy.get(ABOUT_DETAILS).within(() => { diff --git a/x-pack/plugins/security_solution/cypress/objects/rule.ts b/x-pack/plugins/security_solution/cypress/objects/rule.ts index dadcb98cade8d8..88dcd998fc06d2 100644 --- a/x-pack/plugins/security_solution/cypress/objects/rule.ts +++ b/x-pack/plugins/security_solution/cypress/objects/rule.ts @@ -54,6 +54,7 @@ export interface CustomRule { runsEvery: Interval; lookBack: Interval; timeline: CompleteTimeline; + maxSignals: number; } export interface ThresholdRule extends CustomRule { @@ -174,6 +175,7 @@ export const newRule: CustomRule = { runsEvery, lookBack, timeline, + maxSignals: 100, }; export const existingRule: CustomRule = { @@ -192,6 +194,9 @@ export const existingRule: CustomRule = { runsEvery, lookBack, timeline, + // Please do not change, or if you do, needs + // to be any number other than default value + maxSignals: 500, }; export const newOverrideRule: OverrideRule = { @@ -213,6 +218,7 @@ export const newOverrideRule: OverrideRule = { runsEvery, lookBack, timeline, + maxSignals: 100, }; export const newThresholdRule: ThresholdRule = { @@ -232,6 +238,7 @@ export const newThresholdRule: ThresholdRule = { runsEvery, lookBack, timeline, + maxSignals: 100, }; export const machineLearningRule: MachineLearningRule = { @@ -265,6 +272,7 @@ export const eqlRule: CustomRule = { runsEvery, lookBack, timeline, + maxSignals: 100, }; export const eqlSequenceRule: CustomRule = { @@ -285,6 +293,7 @@ export const eqlSequenceRule: CustomRule = { runsEvery, lookBack, timeline, + maxSignals: 100, }; export const newThreatIndicatorRule: ThreatIndicatorRule = { @@ -304,6 +313,7 @@ export const newThreatIndicatorRule: ThreatIndicatorRule = { indicatorMapping: 'agent.id', indicatorIndexField: 'agent.threat', timeline, + maxSignals: 100, }; export const severitiesOverride = ['Low', 'Medium', 'High', 'Critical']; diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts index 99f5bd9c20230c..4bf5508c19aa94 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls/rules.ts @@ -85,6 +85,7 @@ export const createCustomRuleActivated = (rule: CustomRule, ruleId = '1') => language: 'kuery', enabled: true, tags: ['rule1'], + max_signals: 500, }, headers: { 'kbn-xsrf': 'cypress-creds' }, failOnStatusCode: false, diff --git a/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts b/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts index 411f326a0ace6b..21a27453954192 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/rule_details.ts @@ -34,11 +34,6 @@ export const activatesRule = () => { }); }; -export const deactivatesRule = () => { - cy.get(RULE_SWITCH).should('be.visible'); - cy.get(RULE_SWITCH).click(); -}; - export const addsException = (exception: Exception) => { cy.get(LOADING_SPINNER).should('exist'); cy.get(LOADING_SPINNER).should('not.exist'); diff --git a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/edit/index.tsx b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/edit/index.tsx index 74fe97d0c7210e..da5cf720d53154 100644 --- a/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/edit/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/edit/index.tsx @@ -251,6 +251,7 @@ const EditRulePageComponent: FC = () => { rule ), ...(ruleId ? { id: ruleId } : {}), + ...(rule != null ? { max_signals: rule.max_signals } : {}), }); } }, [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_max_signals.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_max_signals.json new file mode 100644 index 00000000000000..d03eb8e2366aea --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_max_signals.json @@ -0,0 +1,9 @@ +{ + "name": "Query With Max Signals", + "description": "Simplest query with max signals set to something other than default", + "risk_score": 1, + "severity": "high", + "type": "query", + "query": "user.name: root or user.name: admin", + "max_signals": 500 +}