From ac05ba8af93de0ee883796223ede5eacb527a43d Mon Sep 17 00:00:00 2001 From: Devin Hurley Date: Sun, 10 Nov 2019 19:24:17 -0500 Subject: [PATCH] increases unit-test code statement coverage to 100% for search_after / bulk index reindexer --- .../alerts/__mocks__/es_results.ts | 58 +- .../alerts/build_events_query.ts | 5 +- .../alerts/signals_alert_type.ts | 1 + .../lib/detection_engine/alerts/types.ts | 2 +- .../lib/detection_engine/alerts/utils.test.ts | 580 +++++++++++++++++- .../lib/detection_engine/alerts/utils.ts | 19 +- 6 files changed, 649 insertions(+), 16 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts index a2a8805eb9c24e6..adb715648697060 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts @@ -6,7 +6,7 @@ import { SignalSourceHit, SignalSearchResponse, SignalAlertParams, BulkResponse } from '../types'; -export const sampleSignalAlertParams = (): SignalAlertParams => ({ +export const sampleSignalAlertParams = (maxSignals: number | undefined): SignalAlertParams => ({ id: 'rule-1', description: 'Detecting root and admin users', index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], @@ -19,7 +19,7 @@ export const sampleSignalAlertParams = (): SignalAlertParams => ({ query: 'user.name: root or user.name: admin', language: 'kuery', references: ['http://google.com'], - maxSignals: 100, + maxSignals, enabled: true, filter: undefined, filters: undefined, @@ -52,6 +52,22 @@ export const sampleDocWithSortId: SignalSourceHit = { sort: ['1234567891111'], }; +export const sampleEmptyDocSearchResults: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [], + }, +}; + export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { took: 10, timed_out: false, @@ -72,6 +88,44 @@ export const sampleDocSearchResultsNoSortId: SignalSearchResponse = { }, }; +export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = { + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 0, + max_score: 100, + hits: [ + { + ...sampleDocNoSortId, + }, + ], + }, +}; + +export const repeatedSearchResultsWithSortId = (repeat: number) => ({ + took: 10, + timed_out: false, + _shards: { + total: 10, + successful: 10, + failed: 0, + skipped: 0, + }, + hits: { + total: 1, + max_score: 100, + hits: Array.from({ length: repeat }).map(x => ({ + ...sampleDocWithSortId, + })), + }, +}); + export const sampleDocSearchResultsWithSortId: SignalSearchResponse = { took: 10, timed_out: false, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts index 541650d0ebe3887..c0e801d6864760d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/build_events_query.ts @@ -10,6 +10,7 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; + maxDocs: number | undefined; searchAfterSortId?: string | number; } @@ -19,6 +20,7 @@ export const buildEventsSearchQuery = ({ to, filter, size, + maxDocs, searchAfterSortId, }: BuildEventsSearchQuery) => { const filterWithTime = [ @@ -74,7 +76,8 @@ export const buildEventsSearchQuery = ({ ], }, }, - track_total_hits: true, + // if we have maxDocs, don't utilize track total hits. + track_total_hits: maxDocs != null ? false : true, sort: [ { '@timestamp': { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index fc18c1b552198ea..601275d052d223d 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -87,6 +87,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp to, filter: esFilter, size: searchAfterSize, + maxDocs: maxSignals, }); try { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index b8d7af5c453033b..696dd32716001a9 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -31,7 +31,7 @@ export interface SignalAlertParams { interval: string; id: string; language: string | undefined; - maxSignals: number; + maxSignals: number | undefined; name: string; query: string | undefined; references: string[]; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts index 6fa8bc039d4a4bd..7390fd7bda08894 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts @@ -19,7 +19,10 @@ import { sampleDocNoSortId, sampleSignalAlertParams, sampleDocSearchResultsNoSortId, + sampleDocSearchResultsNoSortIdNoHits, sampleDocSearchResultsWithSortId, + sampleEmptyDocSearchResults, + repeatedSearchResultsWithSortId, } from './__mocks__/es_results'; import { SignalAlertParams } from './types'; import { buildEventsSearchQuery } from './build_events_query'; @@ -27,7 +30,7 @@ import { buildEventsSearchQuery } from './build_events_query'; describe('utils', () => { describe('buildBulkBody', () => { test('if bulk body builds well-defined body', () => { - const sampleParams: SignalAlertParams = sampleSignalAlertParams(); + const sampleParams: SignalAlertParams = sampleSignalAlertParams(undefined); const fakeSignalSourceHit = buildBulkBody(sampleDocNoSortId, sampleParams); expect(fakeSignalSourceHit).toEqual({ someKey: 'someValue', @@ -56,7 +59,7 @@ describe('utils', () => { describe('singleBulkIndex', () => { test('create successful bulk index', async () => { // need a sample search result, sample signal params, mock service, mock logger - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const sampleSearchResult = sampleDocSearchResultsNoSortId; const savedObjectsClient = savedObjectsClientMock.create(); const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ @@ -112,11 +115,101 @@ describe('utils', () => { expect(mockLogger.warn).toHaveBeenCalledTimes(0); expect(successfulSingleBulkIndex).toEqual(true); }); + test('create unsuccessful bulk index due to empty search results', async () => { + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleEmptyDocSearchResults; + const savedObjectsClient = savedObjectsClientMock.create(); + const mockService: AlertServices = { + callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return false; // value is irrelevant + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.info).toHaveBeenCalledTimes(0); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(successfulSingleBulkIndex).toEqual(false); + }); + test('create unsuccessful bulk index due to bulk index errors', async () => { + // need a sample search result, sample signal params, mock service, mock logger + const sampleParams = sampleSignalAlertParams(undefined); + const sampleSearchResult = sampleDocSearchResultsNoSortId; + const savedObjectsClient = savedObjectsClientMock.create(); + const bulkBody = sampleDocSearchResultsNoSortId.hits.hits.flatMap(doc => [ + { + index: { + _index: process.env.SIGNALS_INDEX || '.siem-signals-10-01-2019', + _id: doc._id, + }, + }, + buildBulkBody(doc, sampleParams), + ]); + const mockService: AlertServices = { + callCluster: async (action: string, params: BulkIndexDocumentsParams) => { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + + // timestamps are annoying... + (bulkBody[1] as SignalHit).signal['@timestamp'] = params.body[1].signal['@timestamp']; + expect(params.body).toEqual(bulkBody); + return { + took: 100, + errors: true, + }; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + const successfulSingleBulkIndex = await singleBulkIndex( + sampleSearchResult, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.info).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(0); + expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(successfulSingleBulkIndex).toEqual(false); + }); }); describe('singleSearchAfter', () => { test('if singleSearchAfter works without a given sort id', async () => { let searchAfterSortId; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -124,6 +217,7 @@ describe('utils', () => { to: sampleParams.to, filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, + maxDocs: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -162,7 +256,7 @@ describe('utils', () => { }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -171,6 +265,7 @@ describe('utils', () => { filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, searchAfterSortId, + maxDocs: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -203,11 +298,54 @@ describe('utils', () => { ); expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId); }); + test('if singleSearchAfter throws error', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + throw Error('Fake Error'); + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn(), + fatal: jest.fn(), + log: jest.fn(), + }; + try { + await singleSearchAfter(searchAfterSortId, sampleParams, mockService, mockLogger); + } catch (exc) { + expect(mockLogger.error).toHaveBeenCalledTimes(1); + expect(exc.message).toEqual('Fake Error'); + } + }); }); describe('searchAfterAndBulkIndex', () => { test('if one successful iteration of searchAfterAndBulkIndex', async () => { const searchAfterSortId = '1234567891111'; - const sampleParams = sampleSignalAlertParams(); + const sampleParams = sampleSignalAlertParams(undefined); const savedObjectsClient = savedObjectsClientMock.create(); const expectedSearchAfterQuery = buildEventsSearchQuery({ index: sampleParams.index, @@ -216,6 +354,7 @@ describe('utils', () => { filter: sampleParams.filter, size: sampleParams.size ? sampleParams.size : 1, searchAfterSortId, + maxDocs: undefined, }); const mockService: AlertServices = { callCluster: async (action: string, params: SearchParams) => { @@ -265,5 +404,436 @@ describe('utils', () => { ); expect(result).toEqual(true); }); + test('if successful iteration of while loop with maxDocs', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + const toReturn = repeatedSearchResultsWithSortId(4); + return toReturn; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + expect(logString).toEqual('something'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if unsuccessful first bulk index', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: true, // will cause singleBulkIndex to return false + }; + } + return {}; // nothing + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + expect(logString).toEqual('[-] bulkResponse had errors: true'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on first search but expected more'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortId, + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(false); + }); + test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(undefined); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: undefined, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on first search but expected more'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + sampleDocSearchResultsNoSortIdNoHits, + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(true); + }); + test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsNoSortId; + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + expect(logString).toEqual('sortIds was empty on search'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(result).toEqual(true); + }); + test('if successful iteration of while loop with maxDocs but one unsuccessful with bulk index failure', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(4); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + + const mockService: AlertServices = { + callCluster: jest.fn(async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + if (params.body.length > 3) { + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } else { + return { + took: 100, + errors: true, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + return sampleDocSearchResultsWithSortId; + }), + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + // expect(logString).toEqual('something'); + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + // expect(logString).toEqual('[-] bulk index failed'); + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(mockLogger.error).toHaveBeenCalledTimes(2); + expect(result).toEqual(true); + }); + test('if returns false when singleSearchAfter throws an exception', async () => { + const searchAfterSortId = '1234567891111'; + const sampleParams = sampleSignalAlertParams(10); + const savedObjectsClient = savedObjectsClientMock.create(); + const expectedSearchAfterQuery = buildEventsSearchQuery({ + index: sampleParams.index, + from: sampleParams.from, + to: sampleParams.to, + filter: sampleParams.filter, + size: sampleParams.size ? sampleParams.size : 1, + searchAfterSortId, + maxDocs: sampleParams.maxSignals, + }); + const mockService: AlertServices = { + callCluster: async (action: string, params: SearchParams) => { + if (action === 'bulk') { + expect(action).toEqual('bulk'); + expect(params.index).toEqual('.siem-signals-10-01-2019'); + return { + took: 100, + errors: false, + items: [ + { + fakeItemValue: 'fakeItemKey', + }, + ], + }; + } + expect(action).toEqual('search'); + expect(params.index).toEqual(sampleParams.index); + expect(params).toEqual(expectedSearchAfterQuery); + throw Error('Fake Error'); + }, + alertInstanceFactory: jest.fn(), + savedObjectsClient, + }; + const mockLogger: Logger = { + info: jest.fn((logString: string) => { + return logString; + }), + warn: jest.fn((logString: string) => { + return logString; + }), + trace: jest.fn(), + debug: jest.fn(), + error: jest.fn((logString: string) => { + return logString; + }), + fatal: jest.fn(), + log: jest.fn(), + }; + const result = await searchAfterAndBulkIndex( + repeatedSearchResultsWithSortId(4), + sampleParams, + mockService, + mockLogger + ); + expect(result).toEqual(false); + }); }); }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts index 7d864b0141543c7..5a182c8862b1867 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts @@ -79,7 +79,7 @@ export const singleBulkIndex = async ( logger.info(`individual bulk process time took: ${time2 - time1} milliseconds`); logger.info(`took property says bulk took: ${firstResult.took} milliseconds`); if (firstResult.errors) { - logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}}`); + logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}`); return false; } return true; @@ -102,6 +102,7 @@ export const singleSearchAfter = async ( to: params.to, filter: params.filter, size: params.size ? params.size : 1, + maxDocs: params.maxSignals, searchAfterSortId, }); const nextSearchAfterResult: SignalSearchResponse = await service.callCluster( @@ -131,11 +132,16 @@ export const searchAfterAndBulkIndex = async ( const totalHits = typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value; + // maxIterations represents the total number of docs to + // query for. If maxSignals is present we will only query + // up to max signals - otherwise use the value + // from track_total_hits. + const maxIterations = params.maxSignals ? params.maxSignals : totalHits; let size = someResult.hits.hits.length - 1; logger.info(`first size: ${size}`); let sortIds = someResult.hits.hits[0].sort; if (sortIds == null && totalHits > 0) { - logger.warn('sortIds was empty on first search but expected more '); + logger.warn('sortIds was empty on first search but expected more'); return false; } else if (sortIds == null && totalHits === 0) { return true; @@ -144,8 +150,7 @@ export const searchAfterAndBulkIndex = async ( if (sortIds != null) { sortId = sortIds[0]; } - while (size < totalHits && size !== 0) { - // utilize track_total_hits instead of true + while (size < maxIterations && size !== 0) { try { logger.info(`sortIds: ${sortIds}`); const searchAfterResult: SignalSearchResponse = await singleSearchAfter( @@ -154,12 +159,12 @@ export const searchAfterAndBulkIndex = async ( service, logger ); - size += searchAfterResult.hits.hits.length - 1; + size += searchAfterResult.hits.hits.length; logger.info(`size: ${size}`); sortIds = searchAfterResult.hits.hits[0].sort; if (sortIds == null) { - logger.warn('sortIds was empty search'); - return false; + logger.warn('sortIds was empty on search'); + return true; // no more search results } sortId = sortIds[0]; logger.info('next bulk index');