From cb507a5db92ea1b7b7b72b700b190308ef327a78 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Wed, 16 Jun 2021 13:41:00 -0400 Subject: [PATCH] Code & tests for correct component templates in SO on reinstall --- .../plugins/fleet/common/types/models/epm.ts | 2 +- .../epm/elasticsearch/template/install.ts | 62 ++++++++++--------- .../epm/elasticsearch/template/template.ts | 8 +-- .../services/epm/packages/_install_package.ts | 7 +-- .../server/services/epm/packages/install.ts | 16 +++-- x-pack/plugins/fleet/server/types/index.tsx | 2 +- .../apis/epm/install_remove_assets.ts | 45 +++++++++++++- 7 files changed, 91 insertions(+), 51 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 046d18c464bf1f..9582709e171823 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -432,7 +432,7 @@ export interface IndexTemplate { _meta: object; } -export interface TemplateRef { +export interface IndexTemplateEntry { templateName: string; indexTemplate: IndexTemplate; } diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts index 36d361db187e7e..12020a2751bd85 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts @@ -11,7 +11,7 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s import { ElasticsearchAssetType } from '../../../../types'; import type { RegistryDataStream, - TemplateRef, + IndexTemplateEntry, RegistryElasticsearch, InstallablePackage, } from '../../../../types'; @@ -19,7 +19,7 @@ import { loadFieldsFromYaml, processFields } from '../../fields/field'; import type { Field } from '../../fields/field'; import { getPipelineNameForInstallation } from '../ingest_pipeline/install'; import { getAsset, getPathParts } from '../../archive'; -import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install'; +import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install'; import { generateMappings, @@ -34,7 +34,7 @@ export const installTemplates = async ( esClient: ElasticsearchClient, paths: string[], savedObjectsClient: SavedObjectsClientContract -): Promise => { +): Promise => { // install any pre-built index template assets, // atm, this is only the base package's global index templates // Install component templates first, as they are used by the index templates @@ -42,16 +42,15 @@ export const installTemplates = async ( await installPreBuiltTemplates(paths, esClient); // remove package installation's references to index templates - await removeAssetsFromInstalledEsByType( - savedObjectsClient, - installablePackage.name, - ElasticsearchAssetType.indexTemplate - ); + await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [ + ElasticsearchAssetType.indexTemplate, + ElasticsearchAssetType.componentTemplate, + ]); // build templates per data stream from yml files const dataStreams = installablePackage.data_streams; if (!dataStreams) return []; - const installTemplatePromises = dataStreams.reduce>>( + const installTemplatePromises = dataStreams.reduce>>( (acc, dataStream) => { acc.push( installTemplateForDataStream({ @@ -69,25 +68,14 @@ export const installTemplates = async ( const installedTemplates = res.flat(); // get template refs to save - const templateRefs = installedTemplates.flatMap((installedTemplate) => { - const indexTemplates = [ - { - id: installedTemplate.templateName, - type: ElasticsearchAssetType.indexTemplate, - }, - ]; - const componentTemplates = installedTemplate.indexTemplate.composed_of.map( - (componentTemplateId) => ({ - id: componentTemplateId, - type: ElasticsearchAssetType.componentTemplate, - }) - ); - return indexTemplates.concat(componentTemplates); - }); + const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates); // add package installation's references to index templates - // await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs); - await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, templateRefs); + await saveInstalledEsRefs( + savedObjectsClient, + installablePackage.name, + installedIndexTemplateRefs + ); return installedTemplates; }; @@ -170,7 +158,7 @@ export async function installTemplateForDataStream({ pkg: InstallablePackage; esClient: ElasticsearchClient; dataStream: RegistryDataStream; -}): Promise { +}): Promise { const fields = await loadFieldsFromYaml(pkg, dataStream.path); return installTemplate({ esClient, @@ -298,7 +286,7 @@ export async function installTemplate({ dataStream: RegistryDataStream; packageVersion: string; packageName: string; -}): Promise { +}): Promise { const validFields = processFields(fields); const mappings = generateMappings(validFields); const templateName = generateTemplateName(dataStream); @@ -377,3 +365,21 @@ export async function installTemplate({ indexTemplate: template, }; } + +export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) { + return installedTemplates.flatMap((installedTemplate) => { + const indexTemplates = [ + { + id: installedTemplate.templateName, + type: ElasticsearchAssetType.indexTemplate, + }, + ]; + const componentTemplates = installedTemplate.indexTemplate.composed_of.map( + (componentTemplateId) => ({ + id: componentTemplateId, + type: ElasticsearchAssetType.componentTemplate, + }) + ); + return indexTemplates.concat(componentTemplates); + }); +} diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index 07d0df021c827b..158996cc574d7a 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -10,7 +10,7 @@ import type { ElasticsearchClient } from 'kibana/server'; import type { Field, Fields } from '../../fields/field'; import type { RegistryDataStream, - TemplateRef, + IndexTemplateEntry, IndexTemplate, IndexTemplateMappings, } from '../../../../types'; @@ -456,7 +456,7 @@ function getBaseTemplate( export const updateCurrentWriteIndices = async ( esClient: ElasticsearchClient, - templates: TemplateRef[] + templates: IndexTemplateEntry[] ): Promise => { if (!templates.length) return; @@ -471,7 +471,7 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur const queryDataStreamsFromTemplates = async ( esClient: ElasticsearchClient, - templates: TemplateRef[] + templates: IndexTemplateEntry[] ): Promise => { const dataStreamPromises = templates.map((template) => { return getDataStreams(esClient, template); @@ -482,7 +482,7 @@ const queryDataStreamsFromTemplates = async ( const getDataStreams = async ( esClient: ElasticsearchClient, - template: TemplateRef + template: IndexTemplateEntry ): Promise => { const { templateName, indexTemplate } = template; const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 65d71ac5fdc179..1bbbb1bb9b6a24 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -10,10 +10,10 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } fro import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common'; import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; -import { ElasticsearchAssetType } from '../../../types'; import type { AssetReference, Installation, InstallType } from '../../../types'; import { installTemplates } from '../elasticsearch/template/install'; import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/'; +import { getAllTemplateRefs } from '../elasticsearch/template/install'; import { installILMPolicy } from '../elasticsearch/ilm/install'; import { installKibanaAssets, getKibanaAssets } from '../kibana/assets/install'; import { updateCurrentWriteIndices } from '../elasticsearch/template/template'; @@ -170,10 +170,7 @@ export async function _installPackage({ installedPkg.attributes.install_version ); } - const installedTemplateRefs = installedTemplates.map((template) => ({ - id: template.templateName, - type: ElasticsearchAssetType.indexTemplate, - })); + const installedTemplateRefs = getAllTemplateRefs(installedTemplates); // make sure the assets are installed (or didn't error) if (installKibanaAssetsError) throw installKibanaAssetsError; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 48d66f06e17b94..d70627ce63ef3a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -249,8 +249,7 @@ async function installPackageFromRegistry({ const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion); // try installing the package, if there was an error, call error handler and rethrow - // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status - // @ts-ignore + // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, esClient, @@ -326,8 +325,7 @@ async function installPackageByUpload({ version: packageInfo.version, packageInfo, }); - // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status - // @ts-ignore + // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed' return _installPackage({ savedObjectsClient, esClient, @@ -476,17 +474,17 @@ export const saveInstalledEsRefs = async ( return installedAssets; }; -export const removeAssetsFromInstalledEsByType = async ( +export const removeAssetTypesFromInstalledEs = async ( savedObjectsClient: SavedObjectsClientContract, pkgName: string, - assetType: AssetType + assetTypes: AssetType[] ) => { const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName }); const installedAssets = installedPkg?.attributes.installed_es; if (!installedAssets?.length) return; - const installedAssetsToSave = installedAssets?.filter(({ id, type }) => { - return type !== assetType; - }); + const installedAssetsToSave = installedAssets?.filter( + (asset) => !assetTypes.includes(asset.type) + ); return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { installed_es: installedAssetsToSave, diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index a48a389ae689e7..76a726e132c397 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -64,7 +64,7 @@ export { RegistrySearchResults, RegistrySearchResult, DefaultPackages, - TemplateRef, + IndexTemplateEntry, IndexTemplateMappings, Settings, SettingsSOAttributes, diff --git a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts index 2d749e7d96d1b0..7280ac5dcc7993 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts @@ -87,6 +87,40 @@ export default function (providerContext: FtrProviderContext) { ); expect(resMetricsTemplate.statusCode).equal(404); }); + it('should have uninstalled the component templates', async function () { + const resMappings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}-mappings`, + }, + { + ignore: [404], + } + ); + expect(resMappings.statusCode).equal(404); + + const resSettings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}-settings`, + }, + { + ignore: [404], + } + ); + expect(resSettings.statusCode).equal(404); + + const resUserSettings = await es.transport.request( + { + method: 'GET', + path: `/_component_template/${logsTemplateName}-user_settings`, + }, + { + ignore: [404], + } + ); + expect(resUserSettings.statusCode).equal(404); + }); it('should have uninstalled the pipelines', async function () { const res = await es.transport.request( { @@ -328,17 +362,22 @@ const expectAssetsInstalled = ({ }); expect(resPipeline2.statusCode).equal(200); }); - it('should have installed the template components', async function () { - const res = await es.transport.request({ + it('should have installed the component templates', async function () { + const resMappings = await es.transport.request({ method: 'GET', path: `/_component_template/${logsTemplateName}-mappings`, }); - expect(res.statusCode).equal(200); + expect(resMappings.statusCode).equal(200); const resSettings = await es.transport.request({ method: 'GET', path: `/_component_template/${logsTemplateName}-settings`, }); expect(resSettings.statusCode).equal(200); + const resUserSettings = await es.transport.request({ + method: 'GET', + path: `/_component_template/${logsTemplateName}-user_settings`, + }); + expect(resUserSettings.statusCode).equal(200); }); it('should have installed the transform components', async function () { const res = await es.transport.request({