From 321fb871cce91241313f42115e09f95c5e1ce8a9 Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Mon, 6 Jul 2020 15:09:35 -0400 Subject: [PATCH] [EPM][Security Solution] Implementing dataset component templates (#70517) * Implementing dataset component templates * Fixing test * Temporary fix to include timestamp with any component template created * Update package registry docker image for CI. * Adapt to new registry filesystem layout. * Adjust tests to changed registry behavior. * Adding a test for mappings and settings overrides * Wrap all the tests in the docker check Co-authored-by: Elastic Machine Co-authored-by: Sonja Krause-Harder --- .../ingest_manager/common/types/models/epm.ts | 7 ++ .../__snapshots__/template.test.ts.snap | 3 + .../epm/elasticsearch/template/install.ts | 109 +++++++++++++++++- .../elasticsearch/template/template.test.ts | 30 +++++ .../epm/elasticsearch/template/template.ts | 8 +- .../ingest_manager/server/types/index.tsx | 1 + .../0.1.0/dataset/test/fields/fields.yml | 16 +++ .../overrides/0.1.0/dataset/test/manifest.yml | 9 ++ .../overrides/0.1.0/docs/README.md | 3 + .../0.1.0/img/logo_overrides_64_color.svg | 7 ++ .../overrides/0.1.0/manifest.yml | 20 ++++ .../apis/index.js | 1 + .../apis/install.ts | 85 ++++++++++++++ .../apis/list.ts | 2 +- .../apis/template.ts | 1 + 15 files changed, 298 insertions(+), 4 deletions(-) create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg create mode 100644 x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml create mode 100644 x-pack/test/ingest_manager_api_integration/apis/install.ts diff --git a/x-pack/plugins/ingest_manager/common/types/models/epm.ts b/x-pack/plugins/ingest_manager/common/types/models/epm.ts index 3ee3039e9e1c49..0d2825f0aa80dd 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/epm.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/epm.ts @@ -175,6 +175,12 @@ export interface Dataset { package: string; path: string; ingest_pipeline: string; + elasticsearch?: RegistryElasticsearch; +} + +export interface RegistryElasticsearch { + 'index_template.settings'?: object; + 'index_template.mappings'?: object; } // EPR types this as `[]map[string]interface{}` @@ -272,6 +278,7 @@ export interface IndexTemplate { data_stream: { timestamp_field: string; }; + composed_of: string[]; _meta: object; } diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap index f5fec020bf5b4d..848e65b7931ebc 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/__snapshots__/template.test.ts.snap @@ -94,6 +94,7 @@ exports[`tests loading base.yml: base.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "nginx" @@ -197,6 +198,7 @@ exports[`tests loading coredns.logs.yml: coredns.logs.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "coredns" @@ -1684,6 +1686,7 @@ exports[`tests loading system.yml: system.yml 1`] = ` "data_stream": { "timestamp_field": "@timestamp" }, + "composed_of": [], "_meta": { "package": { "name": "system" diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts index a318aecf347d62..e14645bbbf5fb0 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts @@ -5,7 +5,13 @@ */ import Boom from 'boom'; -import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types'; +import { + Dataset, + RegistryPackage, + ElasticsearchAssetType, + TemplateRef, + RegistryElasticsearch, +} from '../../../../types'; import { CallESAsCurrentUser } from '../../../../types'; import { Field, loadFieldsFromYaml, processFields } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; @@ -157,6 +163,98 @@ export async function installTemplateForDataset({ }); } +function putComponentTemplate( + body: object | undefined, + name: string, + callCluster: CallESAsCurrentUser +): { clusterPromise: Promise; name: string } | undefined { + if (body) { + const callClusterParams: { + method: string; + path: string; + ignore: number[]; + body: any; + } = { + method: 'PUT', + path: `/_component_template/${name}`, + ignore: [404], + body, + }; + + return { clusterPromise: callCluster('transport.request', callClusterParams), name }; + } +} + +function buildComponentTemplates(registryElasticsearch: RegistryElasticsearch | undefined) { + let mappingsTemplate; + let settingsTemplate; + + if (registryElasticsearch && registryElasticsearch['index_template.mappings']) { + mappingsTemplate = { + template: { + mappings: { + ...registryElasticsearch['index_template.mappings'], + // temporary change until https://github.com/elastic/elasticsearch/issues/58956 is resolved + // hopefully we'll be able to remove the entire properties section once that issue is resolved + properties: { + // if the timestamp_field changes here: https://github.com/elastic/kibana/blob/master/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts#L309 + // we'll need to update this as well + '@timestamp': { + type: 'date', + }, + }, + }, + }, + }; + } + + if (registryElasticsearch && registryElasticsearch['index_template.settings']) { + settingsTemplate = { + template: { + settings: registryElasticsearch['index_template.settings'], + }, + }; + } + return { settingsTemplate, mappingsTemplate }; +} + +async function installDatasetComponentTemplates( + templateName: string, + registryElasticsearch: RegistryElasticsearch | undefined, + callCluster: CallESAsCurrentUser +) { + const templates: string[] = []; + const componentPromises: Array> = []; + + const compTemplates = buildComponentTemplates(registryElasticsearch); + + const mappings = putComponentTemplate( + compTemplates.mappingsTemplate, + `${templateName}-mappings`, + callCluster + ); + + const settings = putComponentTemplate( + compTemplates.settingsTemplate, + `${templateName}-settings`, + callCluster + ); + + if (mappings) { + templates.push(mappings.name); + componentPromises.push(mappings.clusterPromise); + } + + if (settings) { + templates.push(settings.name); + componentPromises.push(settings.clusterPromise); + } + + // TODO: Check return values for errors + await Promise.all(componentPromises); + return templates; +} + export async function installTemplate({ callCluster, fields, @@ -180,13 +278,22 @@ export async function installTemplate({ packageVersion, }); } + + const composedOfTemplates = await installDatasetComponentTemplates( + templateName, + dataset.elasticsearch, + callCluster + ); + const template = getTemplate({ type: dataset.type, templateName, mappings, pipelineName, packageName, + composedOfTemplates, }); + // TODO: Check return values for errors const callClusterParams: { method: string; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts index 73a6767f6b947c..99e568bf771f83 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts @@ -29,10 +29,37 @@ test('get template', () => { templateName, packageName: 'nginx', mappings: { properties: {} }, + composedOfTemplates: [], }); expect(template.index_patterns).toStrictEqual([`${templateName}-*`]); }); +test('adds composed_of correctly', () => { + const composedOfTemplates = ['component1', 'component2']; + + const template = getTemplate({ + type: 'logs', + templateName: 'name', + packageName: 'nginx', + mappings: { properties: {} }, + composedOfTemplates, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); +}); + +test('adds empty composed_of correctly', () => { + const composedOfTemplates: string[] = []; + + const template = getTemplate({ + type: 'logs', + templateName: 'name', + packageName: 'nginx', + mappings: { properties: {} }, + composedOfTemplates, + }); + expect(template.composed_of).toStrictEqual(composedOfTemplates); +}); + test('tests loading base.yml', () => { const ymlPath = path.join(__dirname, '../../fields/tests/base.yml'); const fieldsYML = readFileSync(ymlPath, 'utf-8'); @@ -45,6 +72,7 @@ test('tests loading base.yml', () => { templateName: 'foo', packageName: 'nginx', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -62,6 +90,7 @@ test('tests loading coredns.logs.yml', () => { templateName: 'foo', packageName: 'coredns', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); @@ -79,6 +108,7 @@ test('tests loading system.yml', () => { templateName: 'whatsthis', packageName: 'system', mappings, + composedOfTemplates: [], }); expect(template).toMatchSnapshot(path.basename(ymlPath)); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts index 2de378f7175348..e7867532ed1762 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts @@ -43,14 +43,16 @@ export function getTemplate({ mappings, pipelineName, packageName, + composedOfTemplates, }: { type: string; templateName: string; mappings: IndexTemplateMappings; pipelineName?: string | undefined; packageName: string; + composedOfTemplates: string[]; }): IndexTemplate { - const template = getBaseTemplate(type, templateName, mappings, packageName); + const template = getBaseTemplate(type, templateName, mappings, packageName, composedOfTemplates); if (pipelineName) { template.template.settings.index.default_pipeline = pipelineName; } @@ -244,7 +246,8 @@ function getBaseTemplate( type: string, templateName: string, mappings: IndexTemplateMappings, - packageName: string + packageName: string, + composedOfTemplates: string[] ): IndexTemplate { return { // This takes precedence over all index templates installed by ES by default (logs-*-* and metrics-*-*) @@ -308,6 +311,7 @@ function getBaseTemplate( data_stream: { timestamp_field: '@timestamp', }, + composed_of: composedOfTemplates, _meta: { package: { name: packageName, diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 8239302a97832d..a559ca18cfedef 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -41,6 +41,7 @@ export { PackageInfo, RegistryVarsEntry, Dataset, + RegistryElasticsearch, AssetReference, ElasticsearchAssetType, IngestAssetType, diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml new file mode 100644 index 00000000000000..12a9a03c1337b4 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/fields/fields.yml @@ -0,0 +1,16 @@ +- name: dataset.type + type: constant_keyword + description: > + Dataset type. +- name: dataset.name + type: constant_keyword + description: > + Dataset name. +- name: dataset.namespace + type: constant_keyword + description: > + Dataset namespace. +- name: '@timestamp' + type: date + description: > + Event timestamp. diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml new file mode 100644 index 00000000000000..9ac3c68a0be9ec --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/dataset/test/manifest.yml @@ -0,0 +1,9 @@ +title: Test Dataset + +type: logs + +elasticsearch: + index_template.mappings: + dynamic: false + index_template.settings: + index.lifecycle.name: reference diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md new file mode 100644 index 00000000000000..17fb41ceae242d --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/docs/README.md @@ -0,0 +1,3 @@ +# Test package + +For testing the that the settings and mappings section get used diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg new file mode 100644 index 00000000000000..b03007a76ffcc5 --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/img/logo_overrides_64_color.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml new file mode 100644 index 00000000000000..ba9fd0fada006d --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/overrides/0.1.0/manifest.yml @@ -0,0 +1,20 @@ +format_version: 1.0.0 +name: overrides +title: Mappings Settings Test +description: This is a test package for testing that the mappings and settings sections in the dataset manifest are applied. +version: 0.1.0 +categories: ['security'] +release: beta +type: integration +license: basic + +requirement: + elasticsearch: + versions: '>7.7.0' + kibana: + versions: '>7.7.0' + +icons: + - src: '/img/logo_overrides_64_color.svg' + size: '16x16' + type: 'image/svg+xml' diff --git a/x-pack/test/ingest_manager_api_integration/apis/index.js b/x-pack/test/ingest_manager_api_integration/apis/index.js index ef8880f86078b3..3f8df8379e743a 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/index.js +++ b/x-pack/test/ingest_manager_api_integration/apis/index.js @@ -11,5 +11,6 @@ export default function ({ loadTestFile }) { loadTestFile(require.resolve('./file')); //loadTestFile(require.resolve('./template')); loadTestFile(require.resolve('./ilm')); + loadTestFile(require.resolve('./install')); }); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/install.ts b/x-pack/test/ingest_manager_api_integration/apis/install.ts new file mode 100644 index 00000000000000..92078c25419dfd --- /dev/null +++ b/x-pack/test/ingest_manager_api_integration/apis/install.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { warnAndSkipTest } from '../helpers'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const es = getService('es'); + const dockerServers = getService('dockerServers'); + const log = getService('log'); + + const deletePackage = async (pkgkey: string) => { + await supertest.delete(`/api/ingest_manager/epm/packages/${pkgkey}`).set('kbn-xsrf', 'xxxx'); + }; + + const mappingsPackage = 'overrides-0.1.0'; + const server = dockerServers.get('registry'); + + describe('installs packages that include settings and mappings overrides', async () => { + after(async () => { + if (server.enabled) { + // remove the package just in case it being installed will affect other tests + await deletePackage(mappingsPackage); + } + }); + + it('should install the overrides package correctly', async function () { + if (server.enabled) { + let { body } = await supertest + .post(`/api/ingest_manager/epm/packages/${mappingsPackage}`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + const templateName = body.response[0].id; + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_index_template/${templateName}`, + })); + + // make sure it has the right composed_of array, the contents should be the component templates + // that were installed + expect(body.index_templates[0].index_template.composed_of).to.contain( + `${templateName}-mappings` + ); + expect(body.index_templates[0].index_template.composed_of).to.contain( + `${templateName}-settings` + ); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}-mappings`, + })); + + // Make sure that the `dynamic` field exists and is set to false (as it is in the package) + expect(body.component_templates[0].component_template.template.mappings.dynamic).to.be( + false + ); + // Make sure that the `@timestamp` field exists and is set to date + // this can be removed once https://github.com/elastic/elasticsearch/issues/58956 is resolved + expect( + body.component_templates[0].component_template.template.mappings.properties['@timestamp'] + .type + ).to.be('date'); + + ({ body } = await es.transport.request({ + method: 'GET', + path: `/_component_template/${templateName}-settings`, + })); + + // Make sure that the lifecycle name gets set correct in the settings + expect( + body.component_templates[0].component_template.template.settings.index.lifecycle.name + ).to.be('reference'); + } else { + warnAndSkipTest(this, log); + } + }); + }); +} diff --git a/x-pack/test/ingest_manager_api_integration/apis/list.ts b/x-pack/test/ingest_manager_api_integration/apis/list.ts index 200358cb6f8f03..abed9a7b859599 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/list.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/list.ts @@ -29,7 +29,7 @@ export default function ({ getService }: FtrProviderContext) { return response.body; }; const listResponse = await fetchPackageList(); - expect(listResponse.response.length).to.be(11); + expect(listResponse.response.length).to.be(12); } else { warnAndSkipTest(this, log); } diff --git a/x-pack/test/ingest_manager_api_integration/apis/template.ts b/x-pack/test/ingest_manager_api_integration/apis/template.ts index 8911dd28dc2437..f7e5a894b83ff1 100644 --- a/x-pack/test/ingest_manager_api_integration/apis/template.ts +++ b/x-pack/test/ingest_manager_api_integration/apis/template.ts @@ -27,6 +27,7 @@ export default function ({ getService }: FtrProviderContext) { templateName, mappings, packageName: 'system', + composedOfTemplates: [], }); // This test is not an API integration test with Kibana