diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap index 1abe55b782c328..4f060746b92b0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/__snapshots__/get_signals_template.test.ts.snap @@ -7,7 +7,7 @@ Object { ], "mappings": Object { "_meta": Object { - "version": 35, + "version": 45, }, "dynamic": false, "properties": Object { @@ -365,6 +365,19 @@ Object { }, }, }, + "data_stream": Object { + "properties": Object { + "dataset": Object { + "type": "keyword", + }, + "namespace": Object { + "type": "keyword", + }, + "type": Object { + "type": "keyword", + }, + }, + }, "destination": Object { "properties": Object { "address": Object { @@ -1907,6 +1920,54 @@ Object { }, }, }, + "orchestrator": Object { + "properties": Object { + "api_version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "cluster": Object { + "properties": Object { + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "url": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "namespace": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "organization": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "resource": Object { + "properties": Object { + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "type": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "type": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, "organization": Object { "properties": Object { "id": Object { @@ -4467,6 +4528,6 @@ Object { }, }, }, - "version": 35, + "version": 45, } `; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json index 2967f4cb725e74..3d24384680f575 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -1,10 +1,8 @@ { - "index_patterns": [ - "try-ecs-*" - ], + "index_patterns": ["try-ecs-*"], "mappings": { "_meta": { - "version": "1.9.0" + "version": "1.10.0" }, "date_detection": false, "dynamic_templates": [ @@ -331,6 +329,19 @@ } } }, + "data_stream": { + "properties": { + "dataset": { + "type": "keyword" + }, + "namespace": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, "destination": { "properties": { "address": { @@ -1802,6 +1813,54 @@ } } }, + "orchestrator": { + "properties": { + "api_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "cluster": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "organization": { "properties": { "id": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts index 9c39ad4ee35982..4691db1b19595e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.test.ts @@ -43,6 +43,49 @@ describe('get_signals_template', () => { expect(template.settings.mapping.total_fields.limit).toBeGreaterThanOrEqual(10000); }); + // If you see this test fail, you should track down any and all "constant_keyword" in your ecs_mapping.json and replace + // those with "keyword". The paths that fail in the array below will be something like: + // - Expected - 1 + // + Received + 5 + // + // - Array [] + // + Array [ + // + "mappings.properties.data_stream.properties.dataset", + // + "mappings.properties.data_stream.properties.namespace", + // + "mappings.properties.data_stream.properties.type", + // + ] + // which means that in your ecs_mapping you have paths such as "mappings.properties.data_stream.properties.dataset" which + // contain a constant_keyword which needs to be replaced with a normal keyword instead. + // + // The reason why we deviate from ECS standards here is because when you have a many to 1 relationship where you have + // several different indexes with different "constant_keyword" values you cannot copy them over into a single "constant_keyword". + // Instead you have to use "keyword". This test was first introduced when ECS 1.10 came out and data_stream.* values which had + // "constant_keyword" fields and we needed to change those to be "keyword" instead. + test('it should NOT have any "constant_keyword" and instead those should be replaced with regular "keyword" in the mapping', () => { + const template = getSignalsTemplate('test-index'); + + // Small recursive function to find any values of "constant_keyword" and mark which fields it was found on and then error on those fields + // The matchers from jest such as jest.toMatchObject do not support recursion, so I have to write it here: + // https://github.com/facebook/jest/issues/2506 + const recursiveConstantKeywordFound = (path: string, inputTemplate: object): string[] => + Object.entries(inputTemplate).reduce((accum, [key, innerValue]) => { + if (typeof innerValue === 'object') { + return [ + ...accum, + ...recursiveConstantKeywordFound(path !== '' ? `${path}.${key}` : key, innerValue), + ]; + } else { + if (key === 'type' && innerValue === 'constant_keyword') { + return [...accum, path]; + } else { + return accum; + } + } + }, []); + const constantKeywordsFound = recursiveConstantKeywordFound('', template); + expect(constantKeywordsFound).toEqual([]); + }); + test('it should match snapshot', () => { const template = getSignalsTemplate('test-index'); expect(template).toMatchSnapshot(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index 0318218ed59001..53035ebf28cd78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -22,7 +22,7 @@ import otherMapping from './other_mappings.json'; incremented by 10 in order to add "room" for the aforementioned patch release */ -export const SIGNALS_TEMPLATE_VERSION = 35; +export const SIGNALS_TEMPLATE_VERSION = 45; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts index 491f7bb9c417e5..2294d51537fb13 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts @@ -23,6 +23,7 @@ import { deleteListsIndex, importFile, } from '../../../lists_api_integration/utils'; +import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { @@ -127,7 +128,7 @@ export default ({ getService }: FtrProviderContext) => { host: { name: ['mothra'] }, event: { kind: 'signal' }, signal: { - _meta: { version: 35 }, + _meta: { version: SIGNALS_TEMPLATE_VERSION }, parents: [ { id: diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 755847e8b645d8..e6a835462619c1 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -27,6 +27,7 @@ import { import { getCreateThreatMatchRulesSchemaMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/request/rule_schemas.mock'; import { getThreatMatchingSchemaPartialMock } from '../../../../plugins/security_solution/common/detection_engine/schemas/response/rules_schema.mocks'; +import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template'; const format = (value: unknown): string => JSON.stringify(value, null, 2); @@ -201,7 +202,7 @@ export default ({ getService }: FtrProviderContext) => { }, signal: { _meta: { - version: 35, + version: SIGNALS_TEMPLATE_VERSION, }, ancestors: [ { diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 0ba0749e75b08e..c3dbd24ae9f046 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -853,7 +853,7 @@ export default ({ getService }: FtrProviderContext) => { 'host.id': '8cc95778cce5407c809480e8e32ad76b', event: { kind: 'signal' }, signal: { - _meta: { version: 35 }, + _meta: { version: SIGNALS_TEMPLATE_VERSION }, parents: [ { depth: 0, @@ -1011,7 +1011,7 @@ export default ({ getService }: FtrProviderContext) => { 'host.id': '8cc95778cce5407c809480e8e32ad76b', event: { kind: 'signal' }, signal: { - _meta: { version: 35 }, + _meta: { version: SIGNALS_TEMPLATE_VERSION }, parents: [ { depth: 0, @@ -1101,7 +1101,7 @@ export default ({ getService }: FtrProviderContext) => { 'process.name': 'sshd', event: { kind: 'signal' }, signal: { - _meta: { version: 35 }, + _meta: { version: SIGNALS_TEMPLATE_VERSION }, parents: [ { depth: 0,