From af399c11774ee64be08b350fc352e3fe0ef04f68 Mon Sep 17 00:00:00 2001 From: Khristinin Nikita Date: Mon, 9 Sep 2024 21:45:08 +0200 Subject: [PATCH] Add intended timestamp (#191717) ## Add new field to alert Add optional `kibana.alert.intended_timestamp`. For scheduled rules it has the same values as ALERT_RULE_EXECUTION_TIMESTAMP (`kibana.alert.rule.execution.timestamp`) for manual rule runs (backfill) it - will get the startedAtOverridden For example if i have event at 14:30 And if we run manual rule run from 14:00-15:00, then alert will have `kibana.alert.intended_timestamp` at 15:00 --------- Co-authored-by: Elastic Machine --- .../src/field_maps/alert_field_map.ts | 6 ++ .../src/schemas/generated/alert_schema.ts | 1 + .../src/schemas/generated/security_schema.ts | 1 + .../src/default_alerts_as_data.ts | 5 ++ .../field_maps/mapping_from_field_map.test.ts | 3 + .../alert_as_data_fields.test.ts.snap | 40 ++++++++++++ .../technical_rule_field_map.test.ts | 5 ++ .../create_persistence_rule_type_wrapper.ts | 19 ++++++ .../execution_logic/custom_query.ts | 65 +++++++++++++++++++ .../execution_logic/indicator_match.ts | 1 + ...ove_random_valued_properties_from_alert.ts | 3 +- 11 files changed, 148 insertions(+), 1 deletion(-) diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts index fdc6f8290897aa..ed6a7211c7a904 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts @@ -47,6 +47,7 @@ import { EVENT_KIND, EVENT_ORIGINAL, TAGS, + ALERT_INTENDED_TIMESTAMP, } from '@kbn/rule-data-utils'; import { MultiField } from './types'; @@ -133,6 +134,11 @@ export const alertFieldMap = { array: false, required: false, }, + [ALERT_INTENDED_TIMESTAMP]: { + type: 'date', + array: false, + required: false, + }, [ALERT_RULE_EXECUTION_UUID]: { type: 'keyword', array: false, diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts index ad2c0c0ad22ed5..4a4117a8f2197a 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/alert_schema.ts @@ -93,6 +93,7 @@ const AlertOptional = rt.partial({ 'kibana.alert.end': schemaDate, 'kibana.alert.flapping': schemaBoolean, 'kibana.alert.flapping_history': schemaBooleanArray, + 'kibana.alert.intended_timestamp': schemaDate, 'kibana.alert.last_detected': schemaDate, 'kibana.alert.maintenance_window_ids': schemaStringArray, 'kibana.alert.previous_action_group': schemaString, diff --git a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts index 809db444714466..3573efa5535e4a 100644 --- a/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts +++ b/packages/kbn-alerts-as-data-utils/src/schemas/generated/security_schema.ts @@ -137,6 +137,7 @@ const SecurityAlertOptional = rt.partial({ 'kibana.alert.group.id': schemaString, 'kibana.alert.group.index': schemaNumber, 'kibana.alert.host.criticality_level': schemaString, + 'kibana.alert.intended_timestamp': schemaDate, 'kibana.alert.last_detected': schemaDate, 'kibana.alert.maintenance_window_ids': schemaStringArray, 'kibana.alert.new_terms': schemaStringArray, diff --git a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts index c0d00d16ae79c3..d56510e51465d7 100644 --- a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts +++ b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts @@ -59,6 +59,9 @@ const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; // kibana.alert.last_detected - timestamp when the alert was last seen const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected` as const; +// kiana.alert.intended_timestamp - timestamp when the alert was intended to be detected, useful for backfilling +const ALERT_INTENDED_TIMESTAMP = `${ALERT_NAMESPACE}.intended_timestamp` as const; + // kibana.alert.reason - human readable reason that this alert is active const ALERT_REASON = `${ALERT_NAMESPACE}.reason` as const; @@ -141,6 +144,7 @@ const fields = { ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_INTENDED_TIMESTAMP, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_NAME, ALERT_RULE_PARAMETERS, @@ -185,6 +189,7 @@ export { ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_INTENDED_TIMESTAMP, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_NAME, ALERT_RULE_PARAMETERS, diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts index d775c38117e4c2..e450cdd1e6f94b 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -260,6 +260,9 @@ describe('mappingFromFieldMap', () => { }, }, }, + intended_timestamp: { + type: 'date', + }, rule: { properties: { category: { diff --git a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap index c1a9a0a77a9f77..2ce82a1b3925fd 100644 --- a/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap +++ b/x-pack/plugins/alerting/server/integration_tests/__snapshots__/alert_as_data_fields.test.ts.snap @@ -837,6 +837,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -1930,6 +1935,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -3023,6 +3033,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -4116,6 +4131,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -5209,6 +5229,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -6308,6 +6333,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -7401,6 +7431,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, @@ -8494,6 +8529,11 @@ Object { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts index e0fc5d9317fc9b..c2963de50419cf 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts @@ -80,6 +80,11 @@ it('matches snapshot', () => { "required": true, "type": "keyword", }, + "kibana.alert.intended_timestamp": Object { + "array": false, + "required": false, + "type": "date", + }, "kibana.alert.last_detected": Object { "array": false, "required": false, diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts index c3c0f5c2480cbe..78043a961a1fd2 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_wrapper.ts @@ -25,6 +25,7 @@ import { TIMESTAMP, VERSION, ALERT_RULE_EXECUTION_TIMESTAMP, + ALERT_INTENDED_TIMESTAMP, } from '@kbn/rule-data-utils'; import { mapKeys, snakeCase } from 'lodash/fp'; import type { IRuleDataClient } from '..'; @@ -55,11 +56,13 @@ const augmentAlerts = ({ options, kibanaVersion, currentTimeOverride, + intendedTimestamp, }: { alerts: Array<{ _id: string; _source: T }>; options: RuleExecutorOptions; kibanaVersion: string; currentTimeOverride: Date | undefined; + intendedTimestamp: Date | undefined; }) => { const commonRuleFields = getCommonAlertFields(options); return alerts.map((alert) => { @@ -69,6 +72,9 @@ const augmentAlerts = ({ [ALERT_RULE_EXECUTION_TIMESTAMP]: new Date(), [ALERT_START]: currentTimeOverride ?? new Date(), [ALERT_LAST_DETECTED]: currentTimeOverride ?? new Date(), + [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp + ? intendedTimestamp + : currentTimeOverride ?? new Date(), [VERSION]: kibanaVersion, ...(options?.maintenanceWindowIds?.length ? { [ALERT_MAINTENANCE_WINDOW_IDS]: options.maintenanceWindowIds } @@ -251,6 +257,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper ...options.services, alertWithPersistence: async (alerts, refresh, maxAlerts = undefined, enrichAlerts) => { const numAlerts = alerts.length; + logger.debug(`Found ${numAlerts} alerts.`); const ruleDataClientWriter = await ruleDataClient.getWriter({ @@ -297,11 +304,17 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper alertsWereTruncated = true; } + let intendedTimestamp; + if (options.startedAtOverridden) { + intendedTimestamp = options.startedAt; + } + const augmentedAlerts = augmentAlerts({ alerts: enrichedAlerts, options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride: undefined, + intendedTimestamp, }); const response = await ruleDataClientWriter.bulk({ @@ -381,6 +394,11 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper let alertsWereTruncated = false; + let intendedTimestamp; + if (options.startedAtOverridden) { + intendedTimestamp = options.startedAt; + } + if (writeAlerts && alerts.length > 0) { const suppressionWindowStart = dateMath.parse(suppressionWindow, { forceNow: currentTimeOverride, @@ -560,6 +578,7 @@ export const createPersistenceRuleTypeWrapper: CreatePersistenceRuleTypeWrapper options, kibanaVersion: ruleDataClient.kibanaVersion, currentTimeOverride, + intendedTimestamp, }); const bulkResponse = await ruleDataClientWriter.bulk({ diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts index 0c432f02b78081..172d48b57d7e12 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/custom_query.ts @@ -18,6 +18,8 @@ import { ALERT_SUPPRESSION_TERMS, TIMESTAMP, ALERT_LAST_DETECTED, + ALERT_INTENDED_TIMESTAMP, + ALERT_RULE_EXECUTION_TIMESTAMP, } from '@kbn/rule-data-utils'; import { flattenWithPrefix } from '@kbn/securitysolution-rules'; import { Rule } from '@kbn/alerting-plugin/common'; @@ -538,6 +540,19 @@ export default ({ getService }: FtrProviderContext) => { expect(previewAlerts.length).toEqual(1); }); + it('should generate alerts with the correct intended timestamp fields', async () => { + const rule: QueryRuleCreateProps = { + ...getRuleForAlertTesting(['auditbeat-*']), + query: `_id:${ID}`, + }; + + const { previewId } = await previewRule({ supertest, rule }); + const previewAlerts = await getPreviewAlerts({ es, previewId }); + const alert = previewAlerts[0]._source; + + expect(alert?.[ALERT_INTENDED_TIMESTAMP]).toEqual(alert?.[TIMESTAMP]); + }); + describe('with suppression enabled', () => { before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/security_solution/suppression'); @@ -2447,6 +2462,56 @@ export default ({ getService }: FtrProviderContext) => { ); }); + it('alerts has intended_timestamp set to the time of the manual run', async () => { + const id = uuidv4(); + const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString(); + const secondTimestamp = new Date().toISOString(); + const firstDocument = { + id, + '@timestamp': firstTimestamp, + agent: { + name: 'agent-1', + }, + }; + const secondDocument = { + id, + '@timestamp': secondTimestamp, + agent: { + name: 'agent-2', + }, + }; + await indexListOfDocuments([firstDocument, secondDocument]); + + const rule: QueryRuleCreateProps = { + ...getRuleForAlertTesting(['ecs_compliant']), + rule_id: 'rule-1', + query: `id:${id}`, + from: 'now-1h', + interval: '1h', + }; + const createdRule = await createRule(supertest, log, rule); + const alerts = await getAlerts(supertest, log, es, createdRule); + + expect(alerts.hits.hits).toHaveLength(1); + + expect(alerts.hits.hits[0]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual( + alerts.hits.hits[0]?._source?.[ALERT_RULE_EXECUTION_TIMESTAMP] + ); + + const backfillStartDate = moment(firstTimestamp).startOf('hour'); + const backfillEndDate = moment(backfillStartDate).add(1, 'h'); + const backfill = await scheduleRuleRun(supertest, [createdRule.id], { + startDate: backfillStartDate, + endDate: backfillEndDate, + }); + + await waitForBackfillExecuted(backfill, [createdRule.id], { supertest, log }); + const allNewAlerts = await getAlerts(supertest, log, es, createdRule); + expect(allNewAlerts.hits.hits[1]?._source?.[ALERT_INTENDED_TIMESTAMP]).toEqual( + backfillEndDate.toISOString() + ); + }); + it('alerts when run on a time range that the rule has not previously seen, and deduplicates if run there more than once', async () => { const id = uuidv4(); const firstTimestamp = moment(new Date()).subtract(3, 'h').toISOString(); diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts index dc4443cffc5f6c..b688f53e288b10 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/detection_engine/rule_execution_logic/trial_license_complete_tier/execution_logic/indicator_match.ts @@ -153,6 +153,7 @@ function alertsAreTheSame(alertsA: any[], alertsB: any[]): void { 'kibana.alert.rule.uuid', 'kibana.alert.rule.execution.uuid', 'kibana.alert.rule.execution.timestamp', + 'kibana.alert.intended_timestamp', 'kibana.alert.start', 'kibana.alert.reason', 'kibana.alert.uuid', diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/remove_random_valued_properties_from_alert.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/remove_random_valued_properties_from_alert.ts index 84659d80f76981..4386474956563c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/remove_random_valued_properties_from_alert.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/utils/alerts/remove_random_valued_properties_from_alert.ts @@ -6,7 +6,7 @@ */ import { DetectionAlert } from '@kbn/security-solution-plugin/common/api/detection_engine'; -import { ALERT_LAST_DETECTED, ALERT_START } from '@kbn/rule-data-utils'; +import { ALERT_LAST_DETECTED, ALERT_START, ALERT_INTENDED_TIMESTAMP } from '@kbn/rule-data-utils'; export const removeRandomValuedPropertiesFromAlert = (alert: DetectionAlert | undefined) => { if (!alert) { @@ -24,6 +24,7 @@ export const removeRandomValuedPropertiesFromAlert = (alert: DetectionAlert | un 'kibana.alert.url': alertURL, [ALERT_START]: alertStart, [ALERT_LAST_DETECTED]: lastDetected, + [ALERT_INTENDED_TIMESTAMP]: intendedTimestamp, ...restOfAlert } = alert; return restOfAlert;