Code & tests for correct component templates in SO on reinstall

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -432,7 +432,7 @@ export interface IndexTemplate {
   _meta: object;
 }
 
-export interface TemplateRef {
+export interface IndexTemplateEntry {
   templateName: string;
   indexTemplate: IndexTemplate;
 }

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts

@@ -11,15 +11,15 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/s
 import { ElasticsearchAssetType } from '../../../../types';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   RegistryElasticsearch,
   InstallablePackage,
 } from '../../../../types';
 import { loadFieldsFromYaml, processFields } from '../../fields/field';
 import type { Field } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
 import { getAsset, getPathParts } from '../../archive';
-import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../packages/install';
+import { removeAssetTypesFromInstalledEs, saveInstalledEsRefs } from '../../packages/install';
 
 import {
   generateMappings,
@@ -34,24 +34,23 @@ export const installTemplates = async (
   esClient: ElasticsearchClient,
   paths: string[],
   savedObjectsClient: SavedObjectsClientContract
-): Promise<TemplateRef[]> => {
+): Promise<IndexTemplateEntry[]> => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global index templates
   // Install component templates first, as they are used by the index templates
   await installPreBuiltComponentTemplates(paths, esClient);
   await installPreBuiltTemplates(paths, esClient);
 
   // remove package installation's references to index templates
-  await removeAssetsFromInstalledEsByType(
-    savedObjectsClient,
-    installablePackage.name,
-    ElasticsearchAssetType.indexTemplate
-  );
+  await removeAssetTypesFromInstalledEs(savedObjectsClient, installablePackage.name, [
+    ElasticsearchAssetType.indexTemplate,
+    ElasticsearchAssetType.componentTemplate,
+  ]);
   // build templates per data stream from yml files
   const dataStreams = installablePackage.data_streams;
   if (!dataStreams) return [];
 
-  const installTemplatePromises = dataStreams.reduce<Array<Promise<TemplateRef>>>(
+  const installTemplatePromises = dataStreams.reduce<Array<Promise<IndexTemplateEntry>>>(
     (acc, dataStream) => {
       acc.push(
         installTemplateForDataStream({
@@ -69,25 +68,14 @@ export const installTemplates = async (
   const installedTemplates = res.flat();
 
   // get template refs to save
-  const templateRefs = installedTemplates.flatMap((installedTemplate) => {
-    const indexTemplates = [
-      {
-        id: installedTemplate.templateName,
-        type: ElasticsearchAssetType.indexTemplate,
-      },
-    ];
-    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
-      (componentTemplateId) => ({
-        id: componentTemplateId,
-        type: ElasticsearchAssetType.componentTemplate,
-      })
-    );
-    return indexTemplates.concat(componentTemplates);
-  });
+  const installedIndexTemplateRefs = getAllTemplateRefs(installedTemplates);
 
   // add package installation's references to index templates
-  // await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, installedTemplateRefs);
-  await saveInstalledEsRefs(savedObjectsClient, installablePackage.name, templateRefs);
+  await saveInstalledEsRefs(
+    savedObjectsClient,
+    installablePackage.name,
+    installedIndexTemplateRefs
+  );
 
   return installedTemplates;
 };
@@ -170,7 +158,7 @@ export async function installTemplateForDataStream({
   pkg: InstallablePackage;
   esClient: ElasticsearchClient;
   dataStream: RegistryDataStream;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const fields = await loadFieldsFromYaml(pkg, dataStream.path);
   return installTemplate({
     esClient,
@@ -298,7 +286,7 @@ export async function installTemplate({
   dataStream: RegistryDataStream;
   packageVersion: string;
   packageName: string;
-}): Promise<TemplateRef> {
+}): Promise<IndexTemplateEntry> {
   const validFields = processFields(fields);
   const mappings = generateMappings(validFields);
   const templateName = generateTemplateName(dataStream);
@@ -377,3 +365,21 @@ export async function installTemplate({
     indexTemplate: template,
   };
 }
+
+export function getAllTemplateRefs(installedTemplates: IndexTemplateEntry[]) {
+  return installedTemplates.flatMap((installedTemplate) => {
+    const indexTemplates = [
+      {
+        id: installedTemplate.templateName,
+        type: ElasticsearchAssetType.indexTemplate,
+      },
+    ];
+    const componentTemplates = installedTemplate.indexTemplate.composed_of.map(
+      (componentTemplateId) => ({
+        id: componentTemplateId,
+        type: ElasticsearchAssetType.componentTemplate,
+      })
+    );
+    return indexTemplates.concat(componentTemplates);
+  });
+}

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -10,7 +10,7 @@ import type { ElasticsearchClient } from 'kibana/server';
 import type { Field, Fields } from '../../fields/field';
 import type {
   RegistryDataStream,
-  TemplateRef,
+  IndexTemplateEntry,
   IndexTemplate,
   IndexTemplateMappings,
 } from '../../../../types';
@@ -456,7 +456,7 @@ function getBaseTemplate(
 
 export const updateCurrentWriteIndices = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<void> => {
   if (!templates.length) return;
 
@@ -471,7 +471,7 @@ function isCurrentDataStream(item: CurrentDataStream[] | undefined): item is Cur
 
 const queryDataStreamsFromTemplates = async (
   esClient: ElasticsearchClient,
-  templates: TemplateRef[]
+  templates: IndexTemplateEntry[]
 ): Promise<CurrentDataStream[]> => {
   const dataStreamPromises = templates.map((template) => {
     return getDataStreams(esClient, template);
@@ -482,7 +482,7 @@ const queryDataStreamsFromTemplates = async (
 
 const getDataStreams = async (
   esClient: ElasticsearchClient,
-  template: TemplateRef
+  template: IndexTemplateEntry
 ): Promise<CurrentDataStream[] | undefined> => {
   const { templateName, indexTemplate } = template;
   const { body } = await esClient.indices.getDataStream({ name: `${templateName}-*` });

x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts

@@ -10,10 +10,10 @@ import type { ElasticsearchClient, SavedObject, SavedObjectsClientContract } fro
 import { MAX_TIME_COMPLETE_INSTALL, ASSETS_SAVED_OBJECT_TYPE } from '../../../../common';
 import type { InstallablePackage, InstallSource, PackageAssetReference } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
-import { ElasticsearchAssetType } from '../../../types';
 import type { AssetReference, Installation, InstallType } from '../../../types';
 import { installTemplates } from '../elasticsearch/template/install';
 import { installPipelines, deletePreviousPipelines } from '../elasticsearch/ingest_pipeline/';
+import { getAllTemplateRefs } from '../elasticsearch/template/install';
 import { installILMPolicy } from '../elasticsearch/ilm/install';
 import { installKibanaAssets, getKibanaAssets } from '../kibana/assets/install';
 import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
@@ -170,10 +170,7 @@ export async function _installPackage({
         installedPkg.attributes.install_version
       );
     }
-    const installedTemplateRefs = installedTemplates.map((template) => ({
-      id: template.templateName,
-      type: ElasticsearchAssetType.indexTemplate,
-    }));
+    const installedTemplateRefs = getAllTemplateRefs(installedTemplates);
 
     // make sure the assets are installed (or didn't error)
     if (installKibanaAssetsError) throw installKibanaAssetsError;

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -249,8 +249,7 @@ async function installPackageFromRegistry({
     const { paths, packageInfo } = await Registry.getRegistryPackage(pkgName, pkgVersion);
 
     // try installing the package, if there was an error, call error handler and rethrow
-    // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status
-    // @ts-ignore
+    // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
     return _installPackage({
       savedObjectsClient,
       esClient,
@@ -326,8 +325,7 @@ async function installPackageByUpload({
       version: packageInfo.version,
       packageInfo,
     });
-    // TODO: without the ts-ignore, TS complains about the type of the value of the returned InstallResult.status
-    // @ts-ignore
+    // @ts-expect-error status is string instead of InstallResult.status 'installed' | 'already_installed'
     return _installPackage({
       savedObjectsClient,
       esClient,
@@ -476,17 +474,17 @@ export const saveInstalledEsRefs = async (
   return installedAssets;
 };
 
-export const removeAssetsFromInstalledEsByType = async (
+export const removeAssetTypesFromInstalledEs = async (
   savedObjectsClient: SavedObjectsClientContract,
   pkgName: string,
-  assetType: AssetType
+  assetTypes: AssetType[]
 ) => {
   const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
   const installedAssets = installedPkg?.attributes.installed_es;
   if (!installedAssets?.length) return;
-  const installedAssetsToSave = installedAssets?.filter(({ id, type }) => {
-    return type !== assetType;
-  });
+  const installedAssetsToSave = installedAssets?.filter(
+    (asset) => !assetTypes.includes(asset.type)
+  );
 
   return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
     installed_es: installedAssetsToSave,

x-pack/plugins/fleet/server/types/index.tsx

@@ -64,7 +64,7 @@ export {
   RegistrySearchResults,
   RegistrySearchResult,
   DefaultPackages,
-  TemplateRef,
+  IndexTemplateEntry,
   IndexTemplateMappings,
   Settings,
   SettingsSOAttributes,

x-pack/test/fleet_api_integration/apis/epm/install_remove_assets.ts

@@ -87,6 +87,40 @@ export default function (providerContext: FtrProviderContext) {
         );
         expect(resMetricsTemplate.statusCode).equal(404);
       });
+      it('should have uninstalled the component templates', async function () {
+        const resMappings = await es.transport.request(
+          {
+            method: 'GET',
+            path: `/_component_template/${logsTemplateName}-mappings`,
+          },
+          {
+            ignore: [404],
+          }
+        );
+        expect(resMappings.statusCode).equal(404);
+
+        const resSettings = await es.transport.request(
+          {
+            method: 'GET',
+            path: `/_component_template/${logsTemplateName}-settings`,
+          },
+          {
+            ignore: [404],
+          }
+        );
+        expect(resSettings.statusCode).equal(404);
+
+        const resUserSettings = await es.transport.request(
+          {
+            method: 'GET',
+            path: `/_component_template/${logsTemplateName}-user_settings`,
+          },
+          {
+            ignore: [404],
+          }
+        );
+        expect(resUserSettings.statusCode).equal(404);
+      });
       it('should have uninstalled the pipelines', async function () {
         const res = await es.transport.request(
           {
@@ -328,17 +362,22 @@ const expectAssetsInstalled = ({
     });
     expect(resPipeline2.statusCode).equal(200);
   });
-  it('should have installed the template components', async function () {
-    const res = await es.transport.request({
+  it('should have installed the component templates', async function () {
+    const resMappings = await es.transport.request({
       method: 'GET',
       path: `/_component_template/${logsTemplateName}-mappings`,
     });
-    expect(res.statusCode).equal(200);
+    expect(resMappings.statusCode).equal(200);
     const resSettings = await es.transport.request({
       method: 'GET',
       path: `/_component_template/${logsTemplateName}-settings`,
     });
     expect(resSettings.statusCode).equal(200);
+    const resUserSettings = await es.transport.request({
+      method: 'GET',
+      path: `/_component_template/${logsTemplateName}-user_settings`,
+    });
+    expect(resUserSettings.statusCode).equal(200);
   });
   it('should have installed the transform components', async function () {
     const res = await es.transport.request({