From 80b720da11637f600baba770c3d115966d3b6c08 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Thu, 28 Jan 2021 09:18:59 -0700 Subject: [PATCH] [Maps] Geo containment latency and concurrent containment fix (#86980) --- .../alert_types/geo_containment/types.ts | 1 - .../alert_types/geo_containment/alert_type.ts | 2 - .../geo_containment/geo_containment.ts | 105 +++--- .../geo_containment/tests/alert_type.test.ts | 1 - .../tests/geo_containment.test.ts | 325 +++++++++++++----- 5 files changed, 289 insertions(+), 145 deletions(-) diff --git a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts index d1f64c9298f15d..c67043106c0d1b 100644 --- a/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts +++ b/x-pack/plugins/stack_alerts/public/alert_types/geo_containment/types.ts @@ -18,7 +18,6 @@ export interface GeoContainmentAlertParams extends AlertTypeParams { boundaryIndexId: string; boundaryGeoField: string; boundaryNameField?: string; - delayOffsetWithUnits?: string; indexQuery?: Query; boundaryIndexQuery?: Query; } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts index 85e02b21cc78cd..6b6f17bf4ba2a2 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/alert_type.ts @@ -98,7 +98,6 @@ export const ParamsSchema = schema.object({ boundaryIndexId: schema.string({ minLength: 1 }), boundaryGeoField: schema.string({ minLength: 1 }), boundaryNameField: schema.maybe(schema.string({ minLength: 1 })), - delayOffsetWithUnits: schema.maybe(schema.string({ minLength: 1 })), indexQuery: schema.maybe(schema.any({})), boundaryIndexQuery: schema.maybe(schema.any({})), }); @@ -114,7 +113,6 @@ export interface GeoContainmentParams extends AlertTypeParams { boundaryIndexId: string; boundaryGeoField: string; boundaryNameField?: string; - delayOffsetWithUnits?: string; indexQuery?: Query; boundaryIndexQuery?: Query; } diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts index 24232e47225f01..1648ad9ad2a623 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/geo_containment.ts @@ -24,7 +24,7 @@ export function transformResults( results: SearchResponse | undefined, dateField: string, geoField: string -): Map { +): Map { if (!results) { return new Map(); } @@ -64,12 +64,15 @@ export function transformResults( // Get unique .reduce( ( - accu: Map, + accu: Map, el: LatestEntityLocation & { entityName: string } ) => { const { entityName, ...locationData } = el; - if (!accu.has(entityName)) { - accu.set(entityName, locationData); + if (entityName) { + if (!accu.has(entityName)) { + accu.set(entityName, []); + } + accu.get(entityName)!.push(locationData); } return accu; }, @@ -78,26 +81,9 @@ export function transformResults( return orderedResults; } -function getOffsetTime(delayOffsetWithUnits: string, oldTime: Date): Date { - const timeUnit = delayOffsetWithUnits.slice(-1); - const time: number = +delayOffsetWithUnits.slice(0, -1); - - const adjustedDate = new Date(oldTime.getTime()); - if (timeUnit === 's') { - adjustedDate.setSeconds(adjustedDate.getSeconds() - time); - } else if (timeUnit === 'm') { - adjustedDate.setMinutes(adjustedDate.getMinutes() - time); - } else if (timeUnit === 'h') { - adjustedDate.setHours(adjustedDate.getHours() - time); - } else if (timeUnit === 'd') { - adjustedDate.setDate(adjustedDate.getDate() - time); - } - return adjustedDate; -} - export function getActiveEntriesAndGenerateAlerts( - prevLocationMap: Record, - currLocationMap: Map, + prevLocationMap: Map, + currLocationMap: Map, alertInstanceFactory: AlertServices< GeoContainmentInstanceState, GeoContainmentInstanceContext, @@ -106,32 +92,55 @@ export function getActiveEntriesAndGenerateAlerts( shapesIdsNamesMap: Record, currIntervalEndTime: Date ) { - const allActiveEntriesMap: Map = new Map([ - ...Object.entries(prevLocationMap || {}), + const allActiveEntriesMap: Map = new Map([ + ...prevLocationMap, ...currLocationMap, ]); - allActiveEntriesMap.forEach(({ location, shapeLocationId, dateInShape, docId }, entityName) => { - const containingBoundaryName = shapesIdsNamesMap[shapeLocationId] || shapeLocationId; - const context = { - entityId: entityName, - entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null, - entityDocumentId: docId, - detectionDateTime: new Date(currIntervalEndTime).toISOString(), - entityLocation: `POINT (${location[0]} ${location[1]})`, - containingBoundaryId: shapeLocationId, - containingBoundaryName, - }; - const alertInstanceId = `${entityName}-${containingBoundaryName}`; - if (shapeLocationId === OTHER_CATEGORY) { + allActiveEntriesMap.forEach((locationsArr, entityName) => { + // Generate alerts + locationsArr.forEach(({ location, shapeLocationId, dateInShape, docId }) => { + const context = { + entityId: entityName, + entityDateTime: dateInShape ? new Date(dateInShape).toISOString() : null, + entityDocumentId: docId, + detectionDateTime: new Date(currIntervalEndTime).toISOString(), + entityLocation: `POINT (${location[0]} ${location[1]})`, + containingBoundaryId: shapeLocationId, + containingBoundaryName: shapesIdsNamesMap[shapeLocationId] || shapeLocationId, + }; + const alertInstanceId = `${entityName}-${context.containingBoundaryName}`; + if (shapeLocationId !== OTHER_CATEGORY) { + alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); + } + }); + + if (locationsArr[0].shapeLocationId === OTHER_CATEGORY) { allActiveEntriesMap.delete(entityName); + return; + } + + const otherCatIndex = locationsArr.findIndex( + ({ shapeLocationId }) => shapeLocationId === OTHER_CATEGORY + ); + if (otherCatIndex >= 0) { + const afterOtherLocationsArr = locationsArr.slice(0, otherCatIndex); + allActiveEntriesMap.set(entityName, afterOtherLocationsArr); } else { - alertInstanceFactory(alertInstanceId).scheduleActions(ActionGroupId, context); + allActiveEntriesMap.set(entityName, locationsArr); } }); return allActiveEntriesMap; } + export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType['executor'] => - async function ({ previousStartedAt, startedAt, services, params, alertId, state }) { + async function ({ + previousStartedAt: currIntervalStartTime, + startedAt: currIntervalEndTime, + services, + params, + alertId, + state, + }) { const { shapesFilters, shapesIdsNamesMap } = state.shapesFilters ? state : await getShapesFilters( @@ -147,15 +156,6 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[ const executeEsQuery = await executeEsQueryFactory(params, services, log, shapesFilters); - let currIntervalStartTime = previousStartedAt; - let currIntervalEndTime = startedAt; - if (params.delayOffsetWithUnits) { - if (currIntervalStartTime) { - currIntervalStartTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalStartTime); - } - currIntervalEndTime = getOffsetTime(params.delayOffsetWithUnits, currIntervalEndTime); - } - // Start collecting data only on the first cycle let currentIntervalResults: SearchResponse | undefined; if (!currIntervalStartTime) { @@ -169,14 +169,17 @@ export const getGeoContainmentExecutor = (log: Logger): GeoContainmentAlertType[ currentIntervalResults = await executeEsQuery(currIntervalStartTime, currIntervalEndTime); } - const currLocationMap: Map = transformResults( + const currLocationMap: Map = transformResults( currentIntervalResults, params.dateField, params.geoField ); + const prevLocationMap: Map = new Map([ + ...Object.entries((state.prevLocationMap as Record) || {}), + ]); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( - state.prevLocationMap as Record, + prevLocationMap, currLocationMap, services.alertInstanceFactory, shapesIdsNamesMap, diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts index 98842c8cc2cba2..1aba7d6fb10109 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/alert_type.test.ts @@ -38,7 +38,6 @@ describe('alertType', () => { boundaryIndexId: 'testIndex', boundaryGeoField: 'testField', boundaryNameField: 'testField', - delayOffsetWithUnits: 'testOffset', }; expect(alertType.validate?.params?.validate(params)).toBeTruthy(); diff --git a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts index 26b51060c2e734..40ad6454c36731 100644 --- a/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts +++ b/x-pack/plugins/stack_alerts/server/alert_types/geo_containment/tests/geo_containment.test.ts @@ -27,39 +27,47 @@ describe('geo_containment', () => { new Map([ [ '936', - { - dateInShape: '2020-09-28T18:01:41.190Z', - docId: 'N-ng1XQB6yyY-xQxnGSM', - location: [-82.8814151789993, 40.62806099653244], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.190Z', + docId: 'N-ng1XQB6yyY-xQxnGSM', + location: [-82.8814151789993, 40.62806099653244], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2019', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'iOng1XQB6yyY-xQxnGSM', - location: [-82.22068064846098, 39.006176185794175], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'iOng1XQB6yyY-xQxnGSM', + location: [-82.22068064846098, 39.006176185794175], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2323', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'n-ng1XQB6yyY-xQxnGSM', - location: [-84.71324851736426, 41.6677269525826], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'n-ng1XQB6yyY-xQxnGSM', + location: [-84.71324851736426, 41.6677269525826], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'ABD5250', - { - dateInShape: '2020-09-28T18:01:41.192Z', - docId: 'GOng1XQB6yyY-xQxnGWM', - location: [6.073727197945118, 39.07997465226799], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.192Z', + docId: 'GOng1XQB6yyY-xQxnGWM', + location: [6.073727197945118, 39.07997465226799], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], ]) ); @@ -77,39 +85,47 @@ describe('geo_containment', () => { new Map([ [ '936', - { - dateInShape: '2020-09-28T18:01:41.190Z', - docId: 'N-ng1XQB6yyY-xQxnGSM', - location: [-82.8814151789993, 40.62806099653244], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.190Z', + docId: 'N-ng1XQB6yyY-xQxnGSM', + location: [-82.8814151789993, 40.62806099653244], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2019', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'iOng1XQB6yyY-xQxnGSM', - location: [-82.22068064846098, 39.006176185794175], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'iOng1XQB6yyY-xQxnGSM', + location: [-82.22068064846098, 39.006176185794175], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'AAL2323', - { - dateInShape: '2020-09-28T18:01:41.191Z', - docId: 'n-ng1XQB6yyY-xQxnGSM', - location: [-84.71324851736426, 41.6677269525826], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.191Z', + docId: 'n-ng1XQB6yyY-xQxnGSM', + location: [-84.71324851736426, 41.6677269525826], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], [ 'ABD5250', - { - dateInShape: '2020-09-28T18:01:41.192Z', - docId: 'GOng1XQB6yyY-xQxnGWM', - location: [6.073727197945118, 39.07997465226799], - shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', - }, + [ + { + dateInShape: '2020-09-28T18:01:41.192Z', + docId: 'GOng1XQB6yyY-xQxnGWM', + location: [6.073727197945118, 39.07997465226799], + shapeLocationId: '0DrJu3QB6yyY-xQxv6Ip', + }, + ], ], ]) ); @@ -131,30 +147,36 @@ describe('geo_containment', () => { const currLocationMap = new Map([ [ 'a', - { - location: [0, 0], - shapeLocationId: '123', - dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId1', - }, + [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ], ], [ 'b', - { - location: [0, 0], - shapeLocationId: '456', - dateInShape: 'Wed Dec 09 2020 15:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId2', - }, + [ + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 16 2020 15:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId2', + }, + ], ], [ 'c', - { - location: [0, 0], - shapeLocationId: '789', - dateInShape: 'Wed Dec 09 2020 16:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId3', - }, + [ + { + location: [0, 0], + shapeLocationId: '789', + dateInShape: 'Wed Dec 23 2020 16:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId3', + }, + ], ], ]); @@ -215,7 +237,7 @@ describe('geo_containment', () => { const currentDateTime = new Date(); it('should use currently active entities if no older entity entries', () => { - const emptyPrevLocationMap = {}; + const emptyPrevLocationMap = new Map(); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( emptyPrevLocationMap, currLocationMap, @@ -227,14 +249,19 @@ describe('geo_containment', () => { expect(testAlertActionArr).toMatchObject(expectedContext); }); it('should overwrite older identical entity entries', () => { - const prevLocationMapWithIdenticalEntityEntry = { - a: { - location: [0, 0], - shapeLocationId: '999', - dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId7', - }, - }; + const prevLocationMapWithIdenticalEntityEntry = new Map([ + [ + 'a', + [ + { + location: [0, 0], + shapeLocationId: '999', + dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId7', + }, + ], + ], + ]); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( prevLocationMapWithIdenticalEntityEntry, currLocationMap, @@ -246,14 +273,19 @@ describe('geo_containment', () => { expect(testAlertActionArr).toMatchObject(expectedContext); }); it('should preserve older non-identical entity entries', () => { - const prevLocationMapWithNonIdenticalEntityEntry = { - d: { - location: [0, 0], - shapeLocationId: '999', - dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId7', - }, - }; + const prevLocationMapWithNonIdenticalEntityEntry = new Map([ + [ + 'd', + [ + { + location: [0, 0], + shapeLocationId: '999', + dateInShape: 'Wed Dec 09 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId7', + }, + ], + ], + ]); const expectedContextPlusD = [ { actionGroupId: 'Tracked entity contained', @@ -279,14 +311,17 @@ describe('geo_containment', () => { expect(allActiveEntriesMap.has('d')).toBeTruthy(); expect(testAlertActionArr).toMatchObject(expectedContextPlusD); }); + it('should remove "other" entries and schedule the expected number of actions', () => { - const emptyPrevLocationMap = {}; - const currLocationMapWithOther = new Map(currLocationMap).set('d', { - location: [0, 0], - shapeLocationId: OTHER_CATEGORY, - dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', - docId: 'docId1', - }); + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 09 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); expect(currLocationMapWithOther).not.toEqual(currLocationMap); const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( emptyPrevLocationMap, @@ -298,5 +333,115 @@ describe('geo_containment', () => { expect(allActiveEntriesMap).toEqual(currLocationMap); expect(testAlertActionArr).toMatchObject(expectedContext); }); + + it('should generate multiple alerts per entity if found in multiple shapes in interval', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithThreeMore = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '789', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId2', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId3', + }, + ]); + getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithThreeMore, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + let numEntitiesInShapes = 0; + currLocationMapWithThreeMore.forEach((v) => { + numEntitiesInShapes += v.length; + }); + expect(testAlertActionArr.length).toEqual(numEntitiesInShapes); + }); + + it('should not return entity as active entry if most recent location is "other"', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); + expect(currLocationMapWithOther).not.toEqual(currLocationMap); + const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithOther, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + expect(allActiveEntriesMap).toEqual(currLocationMap); + }); + + it('should return entity as active entry if "other" not the latest location but remove "other" and earlier entries', () => { + const emptyPrevLocationMap = new Map(); + const currLocationMapWithOther = new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: OTHER_CATEGORY, + dateInShape: 'Wed Dec 08 2020 12:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + { + location: [0, 0], + shapeLocationId: '456', + dateInShape: 'Wed Dec 07 2020 10:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]); + const allActiveEntriesMap = getActiveEntriesAndGenerateAlerts( + emptyPrevLocationMap, + currLocationMapWithOther, + alertInstanceFactory, + emptyShapesIdsNamesMap, + currentDateTime + ); + expect(allActiveEntriesMap).toEqual( + new Map([...currLocationMap]).set('d', [ + { + location: [0, 0], + shapeLocationId: '123', + dateInShape: 'Wed Dec 10 2020 14:31:31 GMT-0700 (Mountain Standard Time)', + docId: 'docId1', + }, + ]) + ); + }); }); });