From 595aedea9784fd33a5c985844ea1040b8215f4a8 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Fri, 1 Nov 2019 18:16:24 -0400 Subject: [PATCH 01/11] tests for detection engine get/put utils --- .../alerts/__mocks__/es_results.ts | 93 ++++++ .../alerts/build_events_query.test.ts | 144 ++++++++++ .../alerts/build_events_query.ts | 2 +- .../lib/detection_engine/alerts/utils.test.ts | 269 ++++++++++++++++++ .../lib/detection_engine/alerts/utils.ts | 23 +- 5 files changed, 527 insertions(+), 4 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts new file mode 100644 index 00000000000000..a2a8805eb9c24e --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from '../types'; + +export const sampleSignalAlertParams = (): SignalAlertParams => ({ + id: 'rule-1', + description: 'Detecting root and admin users', + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + name: 'Detect Root/Admin Users', + type: 'query', + from: 'now-6m', + to: 'now', + severity: 'high', + query: 'user.name: root or user.name: admin', + language: 'kuery', + references: ['http://google.com'], + maxSignals: 100, + enabled: true, + filter: undefined, + filters: undefined, + savedId: undefined, + size: undefined, +}); + +export const sampleDocNoSortId: SignalSourceHit = { + _index: 'myFakeSignalIndex', + _type: 'doc', + _score: 100, + _version: 1, + _id: 'someFakeId', + _source: { + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + }, +}; + +export const sampleDocWithSortId: SignalSourceHit = { + _index: 'myFakeSignalIndex', + _type: 'doc', + _score: 100, + _version: 1, + _id: 'someFakeId', + _source: { + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + }, + sort: ['1234567891111'], +}; + +export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 100, + max_score: 100, + hits: [ + { + ...sampleDocNoSortId, + }, + ], + }, +}; + +export const sampleDocSearchResultsWithSortId: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: [ + { + ...sampleDocWithSortId, + }, + ], + }, +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts index 6c95e82485e452..d20c92bd2698a9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts @@ -147,4 +147,148 @@ describe('create_signals', () => { }, }); }); + test('if searchAfterSortId is a valid sortId string', () => { + const fakeSortId = '123456789012'; + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: fakeSortId, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + track_total_hits: true, + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + search_after: [fakeSortId], + }, + }); + }); + test('if searchAfterSortId is a valid sortId number', () => { + const fakeSortIdNumber = 123456789012; + const query = buildEventsSearchQuery({ + index: ['auditbeat-*'], + from: 'now-5m', + to: 'today', + filter: {}, + size: 100, + searchAfterSortId: fakeSortIdNumber, + }); + expect(query).toEqual({ + allowNoIndices: true, + index: ['auditbeat-*'], + size: 100, + ignoreUnavailable: true, + body: { + query: { + bool: { + filter: [ + {}, + { + bool: { + filter: [ + { + bool: { + should: [ + { + range: { + '@timestamp': { + gte: 'now-5m', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + should: [ + { + range: { + '@timestamp': { + lte: 'today', + }, + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + }, + }, + { + match_all: {}, + }, + ], + }, + }, + track_total_hits: true, + sort: [ + { + '@timestamp': { + order: 'asc', + }, + }, + ], + search_after: [fakeSortIdNumber], + }, + }); + }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts index d2fb7c21f66f54..541650d0ebe388 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts @@ -10,7 +10,7 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; - searchAfterSortId?: string; + searchAfterSortId?: string | number; } export const buildEventsSearchQuery = ({ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts new file mode 100644 index 00000000000000..6fa8bc039d4a4b --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -0,0 +1,269 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { BulkIndexDocumentsParams, SearchParams } from 'elasticsearch'; +import { savedObjectsClientMock } from 'src/core/server/mocks'; + +import { AlertServices } from '../../../../../alerting/server/types'; +import { Logger } from '../../../../../../../../src/core/server'; +import { + buildBulkBody, + singleBulkIndex, + singleSearchAfter, + searchAfterAndBulkIndex, +} from './utils'; +import { SignalHit } from '../../types'; +import { + sampleDocNoSortId, + sampleSignalAlertParams, + sampleDocSearchResultsNoSortId, + sampleDocSearchResultsWithSortId, +} from './__mocks__/es_results'; +import { SignalAlertParams } from './types'; +import { buildEventsSearchQuery } from './build_events_query'; + +describe('utils', () => { + describe('buildBulkBody', () => { + test('if bulk body builds well-defined body', () => { + const sampleParams: SignalAlertParams = sampleSignalAlertParams(); + const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams); + expect(fakeSignalSourceHit).toEqual({ + someKey: 'someValue', + '@timestamp': 'someTimeStamp', + signal: { + '@timestamp': fakeSignalSourceHit.signal['@timestamp'], // timestamp generated in the body + rule_revision: 1, + rule_id: sampleParams.id, + rule_type: sampleParams.type, + parent: { + id: sampleDocNoSortId._id, + type: 'event', + index: sampleDocNoSortId._index, + depth: 1, + }, + name: sampleParams.name, + severity: sampleParams.severity, + description: sampleParams.description, + original_time: sampleDocNoSortId._source['@timestamp'], + index_patterns: sampleParams.index, + references: sampleParams.references, + }, + }); + }); + }); + describe('singleBulkIndex', () => { + test('create successful bulk index', async () => { + // need a sample search result, sample signal params, mock service, mock logger + const sampleParams = sampleSignalAlertParams(); + const sampleSearchResult = sampleDocSearchResultsNoSortId; + const savedObjectsClient = savedObjectsClientMock.create(); + const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ + { + index: { + _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _id: doc._id, + }, + }, + buildBulkBody(doc, sampleParams), + ]); + const mockService: AlertServices = { + callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + + // timestamps are annoying... + (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; + expect(params.body).toEqual(bulkBody); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.info).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(0); + expect(successfulSingleBulkIndex).toEqual(true); + }); + }); + describe('singleSearchAfter', () => { + test('if singleSearchAfter works without a given sort id', async () => { + let searchAfterSortId; + const sampleParams = sampleSignalAlertParams(); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsNoSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + try { + const searchAfterResult = await singleSearchAfter( + searchAfterSortId, + sampleParams, + mockService, + mockLogger + ); + expect(searchAfterResult).toBeEmpty(); + } catch (exc) { + expect(exc.message).toEqual('Attempted to search after with empty sort id'); + } + }); + test('if singleSearchAfter works with a given sort id', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const searchAfterResult = await singleSearchAfter( + searchAfterSortId, + sampleParams, + mockService, + mockLogger + ); + expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId); + }); + }); + describe('searchAfterAndBulkIndex', () => { + test('if one successful iteration of searchAfterAndBulkIndex', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsWithSortId, + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index a514baa186fd26..e3e2cef94c8cab 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -60,6 +60,20 @@ export const singleBulkIndex = async ( refresh: false, body: bulkBody, }); + // return service + // .callCluster('bulk', { + // index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + // refresh: false, + // body: bulkBody, + // }) + // .then(result => { + // if (result.errors) { + // logger.error(result.errors); + // return Promise.reject(false); // eslint-disable-line prefer-promise-reject-errors + // } + // logger.info('finished'); + // return Promise.resolve(true); + // }); const time2 = performance.now(); logger.debug(`individual bulk process time took: ${time2 - time1} milliseconds`); logger.debug(`took property says bulk took: ${firstResult.took} milliseconds`); @@ -86,10 +100,13 @@ export const singleSearchAfter = async ( from: params.from, to: params.to, filter: params.filter, - size: params.size ? params.size : 1000, + size: params.size ? params.size : 1, searchAfterSortId, }); - const nextSearchAfterResult = await service.callCluster('search', searchAfterQuery); + const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( + 'search', + searchAfterQuery + ); return nextSearchAfterResult; } catch (exc) { logger.error(`[-] nextSearchAfter threw an error ${exc}`); @@ -130,7 +147,7 @@ export const searchAfterAndBulkIndex = async ( if (sortIds != null) { sortId = sortIds[0]; } - while (size < totalHits) { + while (size < totalHits && size !== 0) { // utilize track_total_hits instead of true try { logger.debug(`sortIds: ${sortIds}`); From 4228bd151cdddc6733d1099561c30b4be2fba159 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Sun, 10 Nov 2019 23:30:36 -0500 Subject: [PATCH 02/11] increases unit test code statement coverage to 100% for search_after / bulk index reindexer --- .../alerts/__mocks__/es_results.ts | 60 +- .../alerts/build_events_query.test.ts | 4 + .../alerts/build_events_query.ts | 7 +- .../alerts/signals_alert_type.ts | 4 +- .../lib/detection_engine/alerts/types.ts | 2 +- .../lib/detection_engine/alerts/utils.test.ts | 571 +++++++++++++++++- .../lib/detection_engine/alerts/utils.ts | 42 +- 7 files changed, 654 insertions(+), 36 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index a2a8805eb9c24e..642df5f28f32e5 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from '../types'; +import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../types'; -export const sampleSignalAlertParams = (): SignalAlertParams => ({ +export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({ id: 'rule-1', description: 'Detecting root and admin users', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], @@ -19,7 +19,7 @@ export const sampleSignalAlertParams = (): SignalAlertParams => ({ query: 'user.name: root or user.name: admin', language: 'kuery', references: ['http://google.com'], - maxSignals: 100, + maxSignals, enabled: true, filter: undefined, filters: undefined, @@ -52,6 +52,22 @@ export const sampleDocWithSortId: SignalSourceHit = { sort: ['1234567891111'], }; +export const sampleEmptyDocSearchResults: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [], + }, +}; + export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { took: 10, timed_out: false, @@ -72,6 +88,44 @@ export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { }, }; +export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [ + { + ...sampleDocNoSortId, + }, + ], + }, +}; + +export const repeatedSearchResultsWithSortId = (repeat: number) => ({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: Array.from({ length: repeat }).map(x => ({ + ...sampleDocWithSortId, + })), + }, +}); + export const sampleDocSearchResultsWithSortId: SignalSearchResponse = { took: 10, timed_out: false, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts index d20c92bd2698a9..3886b82df1f1d1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts @@ -14,6 +14,7 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, + maxDocs: undefined, searchAfterSortId: undefined, }); expect(query).toEqual({ @@ -84,6 +85,7 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, + maxDocs: undefined, searchAfterSortId: '', }); expect(query).toEqual({ @@ -156,6 +158,7 @@ describe('create_signals', () => { filter: {}, size: 100, searchAfterSortId: fakeSortId, + maxDocs: undefined, }); expect(query).toEqual({ allowNoIndices: true, @@ -227,6 +230,7 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, + maxDocs: undefined, searchAfterSortId: fakeSortIdNumber, }); expect(query).toEqual({ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts index 541650d0ebe388..b57d7769045f33 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts @@ -10,7 +10,8 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; - searchAfterSortId?: string | number; + maxDocs: number | undefined; + searchAfterSortId: string | number | undefined; } export const buildEventsSearchQuery = ({ @@ -19,6 +20,7 @@ export const buildEventsSearchQuery = ({ to, filter, size, + maxDocs, searchAfterSortId, }: BuildEventsSearchQuery) => { const filterWithTime = [ @@ -74,7 +76,8 @@ export const buildEventsSearchQuery = ({ ], }, }, - track_total_hits: true, + // if we have maxDocs, don't utilize track total hits. + track_total_hits: maxDocs != null ? false : true, sort: [ { '@timestamp': { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 44175360f80b32..46a2e22ee1eed2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -82,6 +82,8 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, filter: esFilter, size: searchAfterSize, + maxDocs: maxSignals, + searchAfterSortId: undefined, }); try { @@ -99,7 +101,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp name, timeDetected: new Date().toISOString(), filter: esFilter, - maxDocs: maxSignals, + maxDocs: typeof maxSignals === 'number' ? maxSignals : 1000, ruleRevision: 1, id, type, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 7edc60618c2516..59011ba09f2fe7 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -33,7 +33,7 @@ export interface SignalAlertParams { interval: string; id: string; language: string | undefined; - maxSignals: number; + maxSignals: number | undefined; name: string; query: string | undefined; references: string[]; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 6fa8bc039d4a4b..6ea2379c3dc65b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -19,7 +19,10 @@ import { sampleDocNoSortId, sampleSignalAlertParams, sampleDocSearchResultsNoSortId, + sampleDocSearchResultsNoSortIdNoHits, sampleDocSearchResultsWithSortId, + sampleEmptyDocSearchResults, + repeatedSearchResultsWithSortId, } from './__mocks__/es_results'; import { SignalAlertParams } from './types'; import { buildEventsSearchQuery } from './build_events_query'; @@ -27,7 +30,7 @@ import { buildEventsSearchQuery } from './build_events_query'; describe('utils', () => { describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { - const sampleParams: SignalAlertParams = sampleSignalAlertParams(); + const sampleParams: SignalAlertParams = sampleSignalAlertParams(undefined); const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams); expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', @@ -56,7 +59,7 @@ describe('utils', () => { describe('singleBulkIndex', () => { test('create successful bulk index', async () => { // need a sample search result, sample signal params, mock service, mock logger - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; const savedObjectsClient = savedObjectsClientMock.create(); const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ @@ -112,11 +115,101 @@ describe('utils', () => { expect(mockLogger.warn).toHaveBeenCalledTimes(0); expect(successfulSingleBulkIndex).toEqual(true); }); + test('create unsuccessful bulk index due to empty search results', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleEmptyDocSearchResults; + const savedObjectsClient = savedObjectsClientMock.create(); + const mockService: AlertServices = { + callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return false; // value is irrelevant + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.info).toHaveBeenCalledTimes(0); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(successfulSingleBulkIndex).toEqual(false); + }); + test('create unsuccessful bulk index due to bulk index errors', async () => { + // need a sample search result, sample signal params, mock service, mock logger + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleDocSearchResultsNoSortId; + const savedObjectsClient = savedObjectsClientMock.create(); + const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ + { + index: { + _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _id: doc._id, + }, + }, + buildBulkBody(doc, sampleParams), + ]); + const mockService: AlertServices = { + callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + + // timestamps are annoying... + (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; + expect(params.body).toEqual(bulkBody); + return { + took: 100, + errors: true, + }; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.info).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(0); + expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(successfulSingleBulkIndex).toEqual(false); + }); }); describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -124,6 +217,8 @@ describe('utils', () => { to: sampleParams.to, filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, + maxDocs: undefined, + searchAfterSortId: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -162,7 +257,7 @@ describe('utils', () => { }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -171,6 +266,7 @@ describe('utils', () => { filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, searchAfterSortId, + maxDocs: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -203,11 +299,54 @@ describe('utils', () => { ); expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId); }); + test('if singleSearchAfter throws error', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + throw Error('Fake Error'); + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + try { + await singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger); + } catch (exc) { + expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(exc.message).toEqual('Fake Error'); + } + }); }); describe('searchAfterAndBulkIndex', () => { test('if one successful iteration of searchAfterAndBulkIndex', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -216,6 +355,7 @@ describe('utils', () => { filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, searchAfterSortId, + maxDocs: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -265,5 +405,426 @@ describe('utils', () => { ); expect(result).toEqual(true); }); + test('if successful iteration of while loop with maxDocs', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + const toReturn = repeatedSearchResultsWithSortId(4); + return toReturn; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if unsuccessful first bulk index', async () => { + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: true, // will cause singleBulkIndex to return false + }; + } + return {}; // nothing + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + expect(logString).toEqual('[-] bulkResponse had errors: true'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on first search but expected more'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortId, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on first search but expected more'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortIdNoHits, + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsNoSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on search'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(true); + }); + test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(5); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + + const mockService: AlertServices = { + callCluster: jest.fn(async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + if (params.body.length > 3) { + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } else { + return { + took: 100, + errors: true, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }), + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + // expect(logString).toEqual('something'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + // expect(logString).toEqual('[-] bulk index failed'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalledTimes(2); + expect(result).toEqual(true); + }); + test('if returns false when singleSearchAfter throws an exception', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + throw Error('Fake Error'); + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(false); + }); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index e3e2cef94c8cab..d95b00b417482e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -60,25 +60,11 @@ export const singleBulkIndex = async ( refresh: false, body: bulkBody, }); - // return service - // .callCluster('bulk', { - // index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', - // refresh: false, - // body: bulkBody, - // }) - // .then(result => { - // if (result.errors) { - // logger.error(result.errors); - // return Promise.reject(false); // eslint-disable-line prefer-promise-reject-errors - // } - // logger.info('finished'); - // return Promise.resolve(true); - // }); const time2 = performance.now(); logger.debug(`individual bulk process time took: ${time2 - time1} milliseconds`); logger.debug(`took property says bulk took: ${firstResult.took} milliseconds`); if (firstResult.errors) { - logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}}`); + logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}`); return false; } return true; @@ -101,6 +87,7 @@ export const singleSearchAfter = async ( to: params.to, filter: params.filter, size: params.size ? params.size : 1, + maxDocs: params.maxSignals, searchAfterSortId, }); const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( @@ -134,11 +121,18 @@ export const searchAfterAndBulkIndex = async ( const totalHits = typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; - let size = someResult.hits.hits.length - 1; - logger.debug(`first size: ${size}`); + // maxTotalHitsSize represents the total number of docs to + // query for. If maxSignals is present we will only query + // up to max signals - otherwise use the value + // from track_total_hits. + const maxTotalHitsSize = params.maxSignals ? params.maxSignals : totalHits; + + // number of docs in the current search result + let hitsSize = someResult.hits.hits.length; + logger.debug(`first size: ${hitsSize}`); let sortIds = someResult.hits.hits[0].sort; if (sortIds == null && totalHits > 0) { - logger.error(`sortIds was empty on first search when encountering ${totalHits}`); + logger.warn('sortIds was empty on first search but expected more'); return false; } else if (sortIds == null && totalHits === 0) { return true; @@ -147,8 +141,7 @@ export const searchAfterAndBulkIndex = async ( if (sortIds != null) { sortId = sortIds[0]; } - while (size < totalHits && size !== 0) { - // utilize track_total_hits instead of true + while (hitsSize < maxTotalHitsSize && hitsSize !== 0) { try { logger.debug(`sortIds: ${sortIds}`); const searchAfterResult: SignalSearchResponse = await singleSearchAfter( @@ -157,12 +150,13 @@ export const searchAfterAndBulkIndex = async ( service, logger ); - size += searchAfterResult.hits.hits.length - 1; - logger.debug(`size adjusted: ${size}`); + sortIds = searchAfterResult.hits.hits[0].sort; + hitsSize += searchAfterResult.hits.hits.length; + logger.debug(`size adjusted: ${hitsSize}`); sortIds = searchAfterResult.hits.hits[0].sort; if (sortIds == null) { - logger.error('sortIds was empty search when running a signal rule'); - return false; + logger.error('sortIds was empty on search'); + return true; // no more search results } sortId = sortIds[0]; logger.debug('next bulk index'); From 3cbad0f3b8c42cae75addcf0be8ae0945d7f781e Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 11 Nov 2019 18:15:09 -0500 Subject: [PATCH 03/11] removes mockLogger declaration from individual test cases - clears mock counts before each test case runs so as to not accumulate method calls after each test case --- .../lib/detection_engine/alerts/utils.test.ts | 227 ++---------------- 1 file changed, 18 insertions(+), 209 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 6ea2379c3dc65b..6e2d0b61a0a263 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -27,7 +27,25 @@ import { import { SignalAlertParams } from './types'; import { buildEventsSearchQuery } from './build_events_query'; +const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), +}; describe('utils', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { const sampleParams: SignalAlertParams = sampleSignalAlertParams(undefined); @@ -58,7 +76,6 @@ describe('utils', () => { }); describe('singleBulkIndex', () => { test('create successful bulk index', async () => { - // need a sample search result, sample signal params, mock service, mock logger const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; const savedObjectsClient = savedObjectsClientMock.create(); @@ -92,19 +109,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -128,19 +132,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -181,19 +172,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -230,19 +208,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; try { const searchAfterResult = await singleSearchAfter( searchAfterSortId, @@ -278,19 +243,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; const searchAfterResult = await singleSearchAfter( searchAfterSortId, sampleParams, @@ -322,19 +274,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - }; try { await singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger); } catch (exc) { @@ -380,23 +319,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - expect(logString).toEqual('something'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - expect(logString).toEqual('something'); - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( sampleDocSearchResultsWithSortId, sampleParams, @@ -442,23 +364,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - expect(logString).toEqual('something'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - expect(logString).toEqual('something'); - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -485,22 +390,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - expect(logString).toEqual('[-] bulkResponse had errors: true'); - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -546,22 +435,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - expect(logString).toEqual('sortIds was empty on first search but expected more'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( sampleDocSearchResultsNoSortId, sampleParams, @@ -607,22 +480,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - expect(logString).toEqual('sortIds was empty on first search but expected more'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( sampleDocSearchResultsNoSortIdNoHits, sampleParams, @@ -667,22 +524,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - expect(logString).toEqual('sortIds was empty on search'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -741,23 +582,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - // expect(logString).toEqual('something'); - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - // expect(logString).toEqual('[-] bulk index failed'); - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -803,21 +627,6 @@ describe('utils', () => { alertInstanceFactory: jest.fn(), savedObjectsClient, }; - const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), - trace: jest.fn(), - debug: jest.fn(), - error: jest.fn((logString: string) => { - return logString; - }), - fatal: jest.fn(), - log: jest.fn(), - }; const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, From a11b11bcf721e223a588ec93c88489a3f55cb137 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 11 Nov 2019 18:23:46 -0500 Subject: [PATCH 04/11] resets default paging size to 1000 - typo from when I was working through my tests --- .../server/lib/detection_engine/alerts/__mocks__/es_results.ts | 2 +- .../plugins/siem/server/lib/detection_engine/alerts/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 642df5f28f32e5..37516e6cc1648b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -24,7 +24,7 @@ export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalA filter: undefined, filters: undefined, savedId: undefined, - size: undefined, + size: 1000, }); export const sampleDocNoSortId: SignalSourceHit = { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index d95b00b417482e..ead3faca2d3096 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -86,7 +86,7 @@ export const singleSearchAfter = async ( from: params.from, to: params.to, filter: params.filter, - size: params.size ? params.size : 1, + size: params.size ? params.size : 1000, maxDocs: params.maxSignals, searchAfterSortId, }); From 7eb873418fad2de4224bdbcf1c0768d3154d5334 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 11 Nov 2019 18:40:52 -0500 Subject: [PATCH 05/11] updates tests after rebase with master --- .../lib/detection_engine/alerts/utils.test.ts | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 6e2d0b61a0a263..453018a5efce39 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -35,7 +35,9 @@ const mockLogger: Logger = { return logString; }), trace: jest.fn(), - debug: jest.fn(), + debug: jest.fn((logString: string) => { + return logString; + }), error: jest.fn((logString: string) => { return logString; }), @@ -115,7 +117,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.info).toHaveBeenCalledTimes(2); + expect(mockLogger.debug).toHaveBeenCalledTimes(2); expect(mockLogger.warn).toHaveBeenCalledTimes(0); expect(successfulSingleBulkIndex).toEqual(true); }); @@ -138,9 +140,9 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.info).toHaveBeenCalledTimes(0); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); - expect(successfulSingleBulkIndex).toEqual(false); + expect(mockLogger.debug).toHaveBeenCalledTimes(0); + expect(mockLogger.error).toHaveBeenCalledTimes(0); + expect(successfulSingleBulkIndex).toEqual(true); }); test('create unsuccessful bulk index due to bulk index errors', async () => { // need a sample search result, sample signal params, mock service, mock logger @@ -178,7 +180,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.info).toHaveBeenCalledTimes(2); + expect(mockLogger.debug).toHaveBeenCalledTimes(2); expect(mockLogger.warn).toHaveBeenCalledTimes(0); expect(mockLogger.error).toHaveBeenCalledTimes(1); expect(successfulSingleBulkIndex).toEqual(false); @@ -283,6 +285,33 @@ describe('utils', () => { }); }); describe('searchAfterAndBulkIndex', () => { + test('if successful with empty search results', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const result = await searchAfterAndBulkIndex( + sampleEmptyDocSearchResults, + sampleParams, + mockService, + mockLogger + ); + expect(mockService.callCluster).toHaveBeenCalledTimes(0); + expect(result).toEqual(true); + }); test('if one successful iteration of searchAfterAndBulkIndex', async () => { const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); @@ -396,7 +425,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.error).toHaveBeenCalledTimes(2); expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { @@ -530,7 +559,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.error).toHaveBeenCalledTimes(1); expect(result).toEqual(true); }); test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { From 3eb4b5ee3cb8089b1cc083952c88ba6b154dfd78 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Mon, 11 Nov 2019 23:53:55 -0500 Subject: [PATCH 06/11] fixes type check after fixing test from rebase with master --- .../server/lib/detection_engine/alerts/utils.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 453018a5efce39..b715b1f2edf54e 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -286,18 +286,8 @@ describe('utils', () => { }); describe('searchAfterAndBulkIndex', () => { test('if successful with empty search results', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, - }); const mockService: AlertServices = { callCluster: jest.fn(), alertInstanceFactory: jest.fn(), From ca57e9c7c7d750bfe2bef718b1668de3f5220578 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2019 17:12:16 -0500 Subject: [PATCH 07/11] removes undefined from maxSignals in type definition, updates tests with pure jest function implementations of logger and services - modifying only the return values or creating a mock implementation when necessary, removed some overlapping test cases --- .../legacy/plugins/siem/common/constants.ts | 1 + .../alerts/__mocks__/es_results.ts | 4 +- .../alerts/build_events_query.test.ts | 12 +- .../alerts/build_events_query.ts | 4 - .../alerts/signals_alert_type.ts | 7 +- .../lib/detection_engine/alerts/utils.test.ts | 547 ++++-------------- .../lib/detection_engine/alerts/utils.ts | 10 +- 7 files changed, 139 insertions(+), 446 deletions(-) diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts index 6845648ee921dd..5c3bc8ab5b309c 100644 --- a/x-pack/legacy/plugins/siem/common/constants.ts +++ b/x-pack/legacy/plugins/siem/common/constants.ts @@ -15,6 +15,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults'; export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults'; export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults'; export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults'; +export const DEFAULT_SIGNALS_INDEX = '.siem-signals'; export const DEFAULT_ANOMALY_SCORE = 'siem:defaultAnomalyScore'; export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000; export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled'; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 37516e6cc1648b..5608a81c19896c 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -19,7 +19,7 @@ export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalA query: 'user.name: root or user.name: admin', language: 'kuery', references: ['http://google.com'], - maxSignals, + maxSignals: maxSignals ? maxSignals : 10000, enabled: true, filter: undefined, filters: undefined, @@ -118,7 +118,7 @@ export const repeatedSearchResultsWithSortId = (repeat: number) => ({ skipped: 0, }, hits: { - total: 1, + total: repeat, max_score: 100, hits: Array.from({ length: repeat }).map(x => ({ ...sampleDocWithSortId, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts index 3886b82df1f1d1..b368c8fe360542 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.test.ts @@ -14,7 +14,6 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - maxDocs: undefined, searchAfterSortId: undefined, }); expect(query).toEqual({ @@ -67,7 +66,7 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + sort: [ { '@timestamp': { @@ -85,7 +84,6 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - maxDocs: undefined, searchAfterSortId: '', }); expect(query).toEqual({ @@ -138,7 +136,7 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + sort: [ { '@timestamp': { @@ -158,7 +156,6 @@ describe('create_signals', () => { filter: {}, size: 100, searchAfterSortId: fakeSortId, - maxDocs: undefined, }); expect(query).toEqual({ allowNoIndices: true, @@ -210,7 +207,7 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + sort: [ { '@timestamp': { @@ -230,7 +227,6 @@ describe('create_signals', () => { to: 'today', filter: {}, size: 100, - maxDocs: undefined, searchAfterSortId: fakeSortIdNumber, }); expect(query).toEqual({ @@ -283,7 +279,7 @@ describe('create_signals', () => { ], }, }, - track_total_hits: true, + sort: [ { '@timestamp': { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts index b57d7769045f33..c75dddf896fd17 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts @@ -10,7 +10,6 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; - maxDocs: number | undefined; searchAfterSortId: string | number | undefined; } @@ -20,7 +19,6 @@ export const buildEventsSearchQuery = ({ to, filter, size, - maxDocs, searchAfterSortId, }: BuildEventsSearchQuery) => { const filterWithTime = [ @@ -76,8 +74,6 @@ export const buildEventsSearchQuery = ({ ], }, }, - // if we have maxDocs, don't utilize track total hits. - track_total_hits: maxDocs != null ? false : true, sort: [ { '@timestamp': { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 46a2e22ee1eed2..0d32bac59dac2a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -5,9 +5,8 @@ */ import { schema } from '@kbn/config-schema'; -import { SIGNALS_ID } from '../../../../common/constants'; +import { SIGNALS_ID, DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { Logger } from '../../../../../../../../src/core/server'; - // TODO: Remove this for the build_events_query call eventually import { buildEventsReIndex } from './build_events_reindex'; @@ -95,13 +94,13 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, // TODO: Change this out once we have solved // https://github.com/elastic/kibana/issues/47002 - signalsIndex: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + signalsIndex: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, severity, description, name, timeDetected: new Date().toISOString(), filter: esFilter, - maxDocs: typeof maxSignals === 'number' ? maxSignals : 1000, + maxDocs: maxSignals, ruleRevision: 1, id, type, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index b715b1f2edf54e..1518b3f9fd0091 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -3,10 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { BulkIndexDocumentsParams, SearchParams } from 'elasticsearch'; +import { BulkIndexDocumentsParams } from 'elasticsearch'; import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { AlertServices } from '../../../../../alerting/server/types'; +import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; + import { Logger } from '../../../../../../../../src/core/server'; import { buildBulkBody, @@ -24,33 +25,30 @@ import { sampleEmptyDocSearchResults, repeatedSearchResultsWithSortId, } from './__mocks__/es_results'; -import { SignalAlertParams } from './types'; -import { buildEventsSearchQuery } from './build_events_query'; const mockLogger: Logger = { - info: jest.fn((logString: string) => { - return logString; - }), - warn: jest.fn((logString: string) => { - return logString; - }), + log: jest.fn(), trace: jest.fn(), - debug: jest.fn((logString: string) => { - return logString; - }), - error: jest.fn((logString: string) => { - return logString; - }), + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), fatal: jest.fn(), - log: jest.fn(), }; + +const mockService = { + callCluster: jest.fn(), + alertInstanceFactory: jest.fn(), + savedObjectsClient: savedObjectsClientMock.create(), +}; + describe('utils', () => { beforeEach(() => { jest.clearAllMocks(); }); describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { - const sampleParams: SignalAlertParams = sampleSignalAlertParams(undefined); + const sampleParams = sampleSignalAlertParams(undefined); const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams); expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', @@ -80,20 +78,19 @@ describe('utils', () => { test('create successful bulk index', async () => { const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; - const savedObjectsClient = savedObjectsClientMock.create(); const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ { index: { - _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, _id: doc._id, }, }, buildBulkBody(doc, sampleParams), ]); - const mockService: AlertServices = { - callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + mockService.callCluster.mockImplementation( + async (action: string, params: BulkIndexDocumentsParams) => { expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); + expect(params.index).toEqual(DEFAULT_SIGNALS_INDEX); // timestamps are annoying... (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; @@ -107,10 +104,8 @@ describe('utils', () => { }, ], }; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + } + ); const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -124,16 +119,7 @@ describe('utils', () => { test('create unsuccessful bulk index due to empty search results', async () => { const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleEmptyDocSearchResults; - const savedObjectsClient = savedObjectsClientMock.create(); - const mockService: AlertServices = { - callCluster: async (action: string, params: BulkIndexDocumentsParams) => { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return false; // value is irrelevant - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster.mockReturnValue(false); const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -148,32 +134,10 @@ describe('utils', () => { // need a sample search result, sample signal params, mock service, mock logger const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; - const savedObjectsClient = savedObjectsClientMock.create(); - const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ - { - index: { - _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', - _id: doc._id, - }, - }, - buildBulkBody(doc, sampleParams), - ]); - const mockService: AlertServices = { - callCluster: async (action: string, params: BulkIndexDocumentsParams) => { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - - // timestamps are annoying... - (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; - expect(params.body).toEqual(bulkBody); - return { - took: 100, - errors: true, - }; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster.mockReturnValue({ + took: 100, + errors: true, + }); const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, @@ -190,61 +154,15 @@ describe('utils', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - maxDocs: undefined, - searchAfterSortId: undefined, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsNoSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; - try { - const searchAfterResult = await singleSearchAfter( - searchAfterSortId, - sampleParams, - mockService, - mockLogger - ); - expect(searchAfterResult).toBeEmpty(); - } catch (exc) { - expect(exc.message).toEqual('Attempted to search after with empty sort id'); - } + mockService.callCluster.mockReturnValue(sampleDocSearchResultsNoSortId); + await expect( + singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + ).rejects.toThrow('Attempted to search after with empty sort id'); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsWithSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster.mockReturnValue(sampleDocSearchResultsWithSortId); const searchAfterResult = await singleSearchAfter( searchAfterSortId, sampleParams, @@ -256,43 +174,17 @@ describe('utils', () => { test('if singleSearchAfter throws error', async () => { const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, + mockService.callCluster.mockImplementation(async () => { + throw Error('Fake Error'); }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - throw Error('Fake Error'); - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; - try { - await singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger); - } catch (exc) { - expect(mockLogger.error).toHaveBeenCalledTimes(1); - expect(exc.message).toEqual('Fake Error'); - } + await expect( + singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger) + ).rejects.toThrow('Fake Error'); }); }); describe('searchAfterAndBulkIndex', () => { test('if successful with empty search results', async () => { const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const mockService: AlertServices = { - callCluster: jest.fn(), - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; const result = await searchAfterAndBulkIndex( sampleEmptyDocSearchResults, sampleParams, @@ -302,113 +194,53 @@ describe('utils', () => { expect(mockService.callCluster).toHaveBeenCalledTimes(0); expect(result).toEqual(true); }); - test('if one successful iteration of searchAfterAndBulkIndex', async () => { - const searchAfterSortId = '1234567891111'; - const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsWithSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; - const result = await searchAfterAndBulkIndex( - sampleDocSearchResultsWithSortId, - sampleParams, - mockService, - mockLogger - ); - expect(result).toEqual(true); - }); test('if successful iteration of while loop with maxDocs', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(10); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: sampleParams.maxSignals, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - const toReturn = repeatedSearchResultsWithSortId(4); - return toReturn; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(repeatedSearchResultsWithSortId(4)) + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(repeatedSearchResultsWithSortId(4)) + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }); const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, mockService, mockLogger ); + expect(mockService.callCluster).toHaveBeenCalledTimes(5); expect(result).toEqual(true); }); test('if unsuccessful first bulk index', async () => { const sampleParams = sampleSignalAlertParams(10); - const savedObjectsClient = savedObjectsClientMock.create(); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: true, // will cause singleBulkIndex to return false - }; - } - return {}; // nothing - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster.mockReturnValue({ + took: 100, + errors: true, // will cause singleBulkIndex to return false + }); const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -419,86 +251,37 @@ describe('utils', () => { expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, + + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsWithSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; const result = await searchAfterAndBulkIndex( sampleDocSearchResultsNoSortId, sampleParams, mockService, mockLogger ); - expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.error).toHaveBeenCalledTimes(1); expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(undefined); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: undefined, + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsWithSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; const result = await searchAfterAndBulkIndex( sampleDocSearchResultsNoSortIdNoHits, sampleParams, @@ -508,99 +291,40 @@ describe('utils', () => { expect(result).toEqual(true); }); test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(10); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: sampleParams.maxSignals, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsNoSortId; - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(sampleDocSearchResultsNoSortId); const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, mockService, mockLogger ); - expect(mockLogger.error).toHaveBeenCalledTimes(1); expect(result).toEqual(true); }); test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(5); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: sampleParams.maxSignals, - }); - - const mockService: AlertServices = { - callCluster: jest.fn(async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - if (params.body.length > 3) { - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } else { - return { - took: 100, - errors: true, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - return sampleDocSearchResultsWithSortId; - }), - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster + .mockReturnValueOnce({ + // first bulk insert + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockReturnValueOnce(sampleDocSearchResultsWithSortId); // get some more docs const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, @@ -611,41 +335,18 @@ describe('utils', () => { expect(result).toEqual(true); }); test('if returns false when singleSearchAfter throws an exception', async () => { - const searchAfterSortId = '1234567891111'; const sampleParams = sampleSignalAlertParams(10); - const savedObjectsClient = savedObjectsClientMock.create(); - const expectedSearchAfterQuery = buildEventsSearchQuery({ - index: sampleParams.index, - from: sampleParams.from, - to: sampleParams.to, - filter: sampleParams.filter, - size: sampleParams.size ? sampleParams.size : 1, - searchAfterSortId, - maxDocs: sampleParams.maxSignals, - }); - const mockService: AlertServices = { - callCluster: async (action: string, params: SearchParams) => { - if (action === 'bulk') { - expect(action).toEqual('bulk'); - expect(params.index).toEqual('.siem-signals-10-01-2019'); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - expect(action).toEqual('search'); - expect(params.index).toEqual(sampleParams.index); - expect(params).toEqual(expectedSearchAfterQuery); - throw Error('Fake Error'); - }, - alertInstanceFactory: jest.fn(), - savedObjectsClient, - }; + mockService.callCluster + .mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }) + .mockRejectedValueOnce(Error('Fake Error')); const result = await searchAfterAndBulkIndex( repeatedSearchResultsWithSortId(4), sampleParams, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index ead3faca2d3096..509181f915f554 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -5,6 +5,7 @@ */ import { performance } from 'perf_hooks'; import { SignalHit } from '../../types'; +import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { Logger } from '../../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerting/server/types'; import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from './types'; @@ -48,7 +49,7 @@ export const singleBulkIndex = async ( const bulkBody = sr.hits.hits.flatMap(doc => [ { index: { - _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, _id: doc._id, }, }, @@ -56,7 +57,7 @@ export const singleBulkIndex = async ( ]); const time1 = performance.now(); const firstResult: BulkResponse = await service.callCluster('bulk', { - index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, refresh: false, body: bulkBody, }); @@ -87,7 +88,6 @@ export const singleSearchAfter = async ( to: params.to, filter: params.filter, size: params.size ? params.size : 1000, - maxDocs: params.maxSignals, searchAfterSortId, }); const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( @@ -132,7 +132,7 @@ export const searchAfterAndBulkIndex = async ( logger.debug(`first size: ${hitsSize}`); let sortIds = someResult.hits.hits[0].sort; if (sortIds == null && totalHits > 0) { - logger.warn('sortIds was empty on first search but expected more'); + logger.error('sortIds was empty on first search but expected more'); return false; } else if (sortIds == null && totalHits === 0) { return true; @@ -155,7 +155,7 @@ export const searchAfterAndBulkIndex = async ( logger.debug(`size adjusted: ${hitsSize}`); sortIds = searchAfterResult.hits.hits[0].sort; if (sortIds == null) { - logger.error('sortIds was empty on search'); + logger.debug('sortIds was empty on search'); return true; // no more search results } sortId = sortIds[0]; From c7c809f2eac73bafb534f1b57a49afdd5445b163 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2019 17:17:20 -0500 Subject: [PATCH 08/11] fixes type issue --- .../server/lib/detection_engine/alerts/signals_alert_type.ts | 3 +-- .../plugins/siem/server/lib/detection_engine/alerts/types.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 0d32bac59dac2a..be763f2b5386de 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -33,7 +33,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp savedId: schema.nullable(schema.string()), query: schema.nullable(schema.string()), filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))), - maxSignals: schema.number({ defaultValue: 100 }), + maxSignals: schema.number({ defaultValue: 10000 }), severity: schema.string(), tags: schema.arrayOf(schema.string(), { defaultValue: [] }), to: schema.string(), @@ -81,7 +81,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, filter: esFilter, size: searchAfterSize, - maxDocs: maxSignals, searchAfterSortId: undefined, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 59011ba09f2fe7..7edc60618c2516 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -33,7 +33,7 @@ export interface SignalAlertParams { interval: string; id: string; language: string | undefined; - maxSignals: number | undefined; + maxSignals: number; name: string; query: string | undefined; references: string[]; From f8fb9165468bda198fd314b200b686dfe6188608 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2019 17:25:50 -0500 Subject: [PATCH 09/11] replaces mock implementation with mock return value for unit test --- .../lib/detection_engine/alerts/utils.test.ts | 35 +++++-------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 1518b3f9fd0091..af0f02079e94ab 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -78,34 +78,15 @@ describe('utils', () => { test('create successful bulk index', async () => { const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; - const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ - { - index: { - _index: process.env.SIGNALS_INDEX || DEFAULT_SIGNALS_INDEX, - _id: doc._id, + mockService.callCluster.mockReturnValueOnce({ + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', }, - }, - buildBulkBody(doc, sampleParams), - ]); - mockService.callCluster.mockImplementation( - async (action: string, params: BulkIndexDocumentsParams) => { - expect(action).toEqual('bulk'); - expect(params.index).toEqual(DEFAULT_SIGNALS_INDEX); - - // timestamps are annoying... - (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; - expect(params.body).toEqual(bulkBody); - return { - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }; - } - ); + ], + }); const successfulSingleBulkIndex = await singleBulkIndex( sampleSearchResult, sampleParams, From 9bb8e14b99243017a1c3e65b490d6214711057d9 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2019 17:34:35 -0500 Subject: [PATCH 10/11] removes mock logger expected counts, just check if error logs are called, don't care about debug / warn etc. --- .../lib/detection_engine/alerts/utils.test.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index af0f02079e94ab..714d373f7aabe0 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -93,8 +93,6 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.debug).toHaveBeenCalledTimes(2); - expect(mockLogger.warn).toHaveBeenCalledTimes(0); expect(successfulSingleBulkIndex).toEqual(true); }); test('create unsuccessful bulk index due to empty search results', async () => { @@ -107,8 +105,6 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.debug).toHaveBeenCalledTimes(0); - expect(mockLogger.error).toHaveBeenCalledTimes(0); expect(successfulSingleBulkIndex).toEqual(true); }); test('create unsuccessful bulk index due to bulk index errors', async () => { @@ -125,9 +121,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.debug).toHaveBeenCalledTimes(2); - expect(mockLogger.warn).toHaveBeenCalledTimes(0); - expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(mockLogger.error).toHaveBeenCalled(); expect(successfulSingleBulkIndex).toEqual(false); }); }); @@ -228,7 +222,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.error).toHaveBeenCalledTimes(2); + expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { @@ -249,7 +243,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(false); }); test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => { @@ -312,7 +306,7 @@ describe('utils', () => { mockService, mockLogger ); - expect(mockLogger.error).toHaveBeenCalledTimes(2); + expect(mockLogger.error).toHaveBeenCalled(); expect(result).toEqual(true); }); test('if returns false when singleSearchAfter throws an exception', async () => { From f0da0ae97c442d676f33f01f7b2b94995e184fb8 Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Thu, 14 Nov 2019 18:18:27 -0500 Subject: [PATCH 11/11] fixes more type checks after rebase with master --- .../lib/detection_engine/alerts/__mocks__/es_results.ts | 3 +++ .../siem/server/lib/detection_engine/alerts/utils.test.ts | 4 ---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index 5608a81c19896c..895af32cc7af33 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -9,11 +9,14 @@ import { SignalSourceHit, SignalSearchResponse, SignalAlertParams } from '../typ export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({ id: 'rule-1', description: 'Detecting root and admin users', + falsePositives: [], + immutable: false, index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], interval: '5m', name: 'Detect Root/Admin Users', type: 'query', from: 'now-6m', + tags: ['some fake tag'], to: 'now', severity: 'high', query: 'user.name: root or user.name: admin', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 714d373f7aabe0..c3ffb6e8c230af 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -3,11 +3,8 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { BulkIndexDocumentsParams } from 'elasticsearch'; import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; - import { Logger } from '../../../../../../../../src/core/server'; import { buildBulkBody, @@ -15,7 +12,6 @@ import { singleSearchAfter, searchAfterAndBulkIndex, } from './utils'; -import { SignalHit } from '../../types'; import { sampleDocNoSortId, sampleSignalAlertParams,