[Ingest Manager] check for packages stuck installing and reinstall (#…

…75226)

* check for packages stuck installing and reinstall

* update mock endpoint package

* diferentiate between reinstall and reupdate type of install, remove isUpdate, add integration test

* create new EpmPackageInstallStatus type instead of using InstallStatus

* fix merge conflict

* change EpmPackageInstallStatus to a union type

* change time to install to 1 minute

* used saved object find

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ingest_manager/common/constants/epm.ts

@@ -7,3 +7,4 @@
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
+export const MAX_TIME_COMPLETE_INSTALL = 60000;

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -19,6 +19,8 @@ export enum InstallStatus {
   uninstalling = 'uninstalling',
 }
 
+export type EpmPackageInstallStatus = 'installed' | 'installing';
+
 export type DetailViewPanelName = 'overview' | 'usages' | 'settings';
 export type ServiceName = 'kibana' | 'elasticsearch';
 export type AssetType = KibanaAssetType | ElasticsearchAssetType | AgentAssetType;
@@ -234,6 +236,9 @@ export interface Installation extends SavedObjectAttributes {
   es_index_patterns: Record<string, string>;
   name: string;
   version: string;
+  install_status: EpmPackageInstallStatus;
+  install_version: string;
+  install_started_at: string;
 }
 
 export type Installable<T> = Installed<T> | NotInstalled<T>;

x-pack/plugins/ingest_manager/server/constants/index.ts

@@ -14,6 +14,7 @@ export {
   AGENT_POLICY_ROLLOUT_RATE_LIMIT_INTERVAL_MS,
   AGENT_UPDATE_ACTIONS_INTERVAL_MS,
   INDEX_PATTERN_PLACEHOLDER_SUFFIX,
+  MAX_TIME_COMPLETE_INSTALL,
   // Routes
   LIMITED_CONCURRENCY_ROUTE_TAG,
   PLUGIN_ID,

x-pack/plugins/ingest_manager/server/saved_objects/index.ts

@@ -285,6 +285,9 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
             type: { type: 'keyword' },
           },
         },
+        install_started_at: { type: 'date' },
+        install_version: { type: 'keyword' },
+        install_status: { type: 'keyword' },
       },
     },
   },

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts

@@ -22,7 +22,6 @@ import { removeAssetsFromInstalledEsByType, saveInstalledEsRefs } from '../../pa
 
 export const installTemplates = async (
   registryPackage: RegistryPackage,
-  isUpdate: boolean,
   callCluster: CallESAsCurrentUser,
   paths: string[],
   savedObjectsClient: SavedObjectsClientContract

x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts

@@ -48,7 +48,6 @@ export async function installKibanaAssets(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgName: string;
   kibanaAssets: ArchiveAsset[];
-  isUpdate: boolean;
 }): Promise<SavedObject[]> {
   const { savedObjectsClient, kibanaAssets } = options;
 

x-pack/plugins/ingest_manager/server/services/epm/packages/get.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { SavedObjectsClientContract } from 'src/core/server';
+import { SavedObjectsClientContract, SavedObjectsFindOptions } from 'src/core/server';
 import { isPackageLimited } from '../../../../common';
 import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
 import { Installation, InstallationStatus, PackageInfo, KibanaAssetType } from '../../../types';
@@ -72,8 +72,12 @@ export async function getLimitedPackages(options: {
   return installedPackagesInfo.filter(isPackageLimited).map((pkgInfo) => pkgInfo.name);
 }
 
-export async function getPackageSavedObjects(savedObjectsClient: SavedObjectsClientContract) {
+export async function getPackageSavedObjects(
+  savedObjectsClient: SavedObjectsClientContract,
+  options?: Omit<SavedObjectsFindOptions, 'type'>
+) {
   return savedObjectsClient.find<Installation>({
+    ...(options || {}),
     type: PACKAGES_SAVED_OBJECT_TYPE,
   });
 }

x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts

@@ -6,7 +6,7 @@
 
 import { SavedObjectsClientContract } from 'src/core/server';
 import semver from 'semver';
-import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
+import { PACKAGES_SAVED_OBJECT_TYPE, MAX_TIME_COMPLETE_INSTALL } from '../../../constants';
 import {
   AssetReference,
   Installation,
@@ -33,6 +33,7 @@ import {
 import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
 import { deleteKibanaSavedObjectsAssets } from './remove';
 import { PackageOutdatedError } from '../../../errors';
+import { getPackageSavedObjects } from './get';
 
 export async function installLatestPackage(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -107,22 +108,24 @@ export async function installPackage({
   // TODO: calls to getInstallationObject, Registry.fetchInfo, and Registry.fetchFindLatestPackge
   // and be replaced by getPackageInfo after adjusting for it to not group/use archive assets
   const latestPackage = await Registry.fetchFindLatestPackage(pkgName);
-  if (semver.lt(pkgVersion, latestPackage.version) && !force)
-    throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
+  // get the currently installed package
+  const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
+  const reinstall = pkgVersion === installedPkg?.attributes.version;
+  const reupdate = pkgVersion === installedPkg?.attributes.install_version;
 
+  // let the user install if using the force flag or this is a reinstall or reupdate due to intallation interruption
+  if (semver.lt(pkgVersion, latestPackage.version) && !force && !reinstall && !reupdate) {
+    throw new PackageOutdatedError(`${pkgkey} is out-of-date and cannot be installed or updated`);
+  }
   const paths = await Registry.getArchiveInfo(pkgName, pkgVersion);
   const registryPackageInfo = await Registry.fetchInfo(pkgName, pkgVersion);
 
-  // get the currently installed package
-  const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
-  const isUpdate = installedPkg && installedPkg.attributes.version < pkgVersion ? true : false;
-
-  const reinstall = pkgVersion === installedPkg?.attributes.version;
   const removable = !isRequiredPackage(pkgName);
   const { internal = false } = registryPackageInfo;
   const toSaveESIndexPatterns = generateESIndexPatterns(registryPackageInfo.datasets);
 
-  // add the package installation to the saved object
+  // add the package installation to the saved object.
+  // if some installation already exists, just update install info
   if (!installedPkg) {
     await createInstallation({
       savedObjectsClient,
@@ -134,6 +137,12 @@ export async function installPackage({
       installed_es: [],
       toSaveESIndexPatterns,
     });
+  } else {
+    await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
+      install_version: pkgVersion,
+      install_status: 'installing',
+      install_started_at: new Date().toISOString(),
+    });
   }
   const installIndexPatternPromise = installIndexPatterns(savedObjectsClient, pkgName, pkgVersion);
   const kibanaAssets = await getKibanaAssets(paths);
@@ -152,7 +161,6 @@ export async function installPackage({
     savedObjectsClient,
     pkgName,
     kibanaAssets,
-    isUpdate,
   });
 
   // the rest of the installation must happen in sequential order
@@ -172,7 +180,6 @@ export async function installPackage({
   // install or update the templates referencing the newly installed pipelines
   const installedTemplates = await installTemplates(
     registryPackageInfo,
-    isUpdate,
     callCluster,
     paths,
     savedObjectsClient
@@ -197,9 +204,14 @@ export async function installPackage({
   }));
   await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
   // update to newly installed version when all assets are successfully installed
-  if (isUpdate) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
+  if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
+  await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
+    install_version: pkgVersion,
+    install_status: 'installed',
+  });
   return [...installedKibanaAssetsRefs, ...installedPipelines, ...installedTemplateRefs];
 }
+
 const updateVersion = async (
   savedObjectsClient: SavedObjectsClientContract,
   pkgName: string,
@@ -239,6 +251,9 @@ export async function createInstallation(options: {
       version: pkgVersion,
       internal,
       removable,
+      install_version: pkgVersion,
+      install_status: 'installing',
+      install_started_at: new Date().toISOString(),
     },
     { id: pkgName, overwrite: true }
   );
@@ -286,3 +301,28 @@ export const removeAssetsFromInstalledEsByType = async (
     installed_es: installedAssetsToSave,
   });
 };
+
+export async function ensurePackagesCompletedInstall(
+  savedObjectsClient: SavedObjectsClientContract,
+  callCluster: CallESAsCurrentUser
+) {
+  const installingPackages = await getPackageSavedObjects(savedObjectsClient, {
+    searchFields: ['install_status'],
+    search: 'installing',
+  });
+  const installingPromises = installingPackages.saved_objects.reduce<
+    Array<Promise<AssetReference[]>>
+  >((acc, pkg) => {
+    const startDate = pkg.attributes.install_started_at;
+    const nowDate = new Date().toISOString();
+    const elapsedTime = Date.parse(nowDate) - Date.parse(startDate);
+    const pkgkey = `${pkg.attributes.name}-${pkg.attributes.install_version}`;
+    // reinstall package
+    if (elapsedTime > MAX_TIME_COMPLETE_INSTALL) {
+      acc.push(installPackage({ savedObjectsClient, pkgkey, callCluster }));
+    }
+    return acc;
+  }, []);
+  await Promise.all(installingPromises);
+  return installingPackages;
+}

x-pack/plugins/ingest_manager/server/services/setup.ts

@@ -10,7 +10,10 @@ import { SavedObjectsClientContract } from 'src/core/server';
 import { CallESAsCurrentUser } from '../types';
 import { agentPolicyService } from './agent_policy';
 import { outputService } from './output';
-import { ensureInstalledDefaultPackages } from './epm/packages/install';
+import {
+  ensureInstalledDefaultPackages,
+  ensurePackagesCompletedInstall,
+} from './epm/packages/install';
 import { ensureDefaultIndices } from './epm/kibana/index_pattern/install';
 import {
   packageToPackagePolicy,
@@ -51,6 +54,7 @@ async function createSetupSideEffects(
     ensureInstalledDefaultPackages(soClient, callCluster),
     outputService.ensureDefaultOutput(soClient),
     agentPolicyService.ensureDefaultAgentPolicy(soClient),
+    ensurePackagesCompletedInstall(soClient, callCluster),
     ensureDefaultIndices(callCluster),
     settingsService.getSettings(soClient).catch((e: any) => {
       if (e.isBoom && e.output.statusCode === 404) {

x-pack/plugins/ingest_manager/server/types/index.tsx

@@ -37,6 +37,7 @@ export {
   EnrollmentAPIKey,
   EnrollmentAPIKeySOAttributes,
   Installation,
+  EpmPackageInstallStatus,
   InstallationStatus,
   PackageInfo,
   RegistryVarsEntry,

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1158,6 +1158,9 @@ export class EndpointDocGenerator {
           version: '0.5.0',
           internal: false,
           removable: false,
+          install_version: '0.5.0',
+          install_status: 'installed',
+          install_started_at: '2020-06-24T14:41:23.098Z',
         },
         references: [],
         updated_at: '2020-06-24T14:41:23.098Z',

x-pack/test/ingest_manager_api_integration/apis/epm/index.js

@@ -17,5 +17,6 @@ export default function loadTests({ loadTestFile }) {
     loadTestFile(require.resolve('./install_update'));
     loadTestFile(require.resolve('./update_assets'));
     loadTestFile(require.resolve('./data_stream'));
+    loadTestFile(require.resolve('./package_install_complete'));
   });
 }

x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts

@@ -170,6 +170,9 @@ export default function (providerContext: FtrProviderContext) {
           version: '0.1.0',
           internal: false,
           removable: true,
+          install_version: '0.1.0',
+          install_status: 'installed',
+          install_started_at: res.attributes.install_started_at,
         });
       });
     });

x-pack/test/ingest_manager_api_integration/apis/epm/install_update.ts

@@ -7,6 +7,10 @@
 import expect from '@kbn/expect';
 import { FtrProviderContext } from '../../../api_integration/ftr_provider_context';
 import { skipIfNoDockerRegistry } from '../../helpers';
+import {
+  PACKAGES_SAVED_OBJECT_TYPE,
+  MAX_TIME_COMPLETE_INSTALL,
+} from '../../../../plugins/ingest_manager/common';
 
 export default function (providerContext: FtrProviderContext) {
   const { getService } = providerContext;
@@ -62,6 +66,12 @@ export default function (providerContext: FtrProviderContext) {
         .send({ force: true })
         .expect(200);
     });
+    it('should return 200 if trying to reinstall an out-of-date package', async function () {
+      await supertest
+        .post(`/api/ingest_manager/epm/packages/multiple_versions-0.1.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .expect(200);
+    });
     it('should return 400 if trying to update to an out-of-date package', async function () {
       await supertest
         .post(`/api/ingest_manager/epm/packages/multiple_versions-0.2.0`)
@@ -75,6 +85,24 @@ export default function (providerContext: FtrProviderContext) {
         .send({ force: true })
         .expect(200);
     });
+    it('should return 200 if trying to reupdate an out-of-date package', async function () {
+      const previousInstallDate = new Date(Date.now() - MAX_TIME_COMPLETE_INSTALL).toISOString();
+      // mock package to be stuck installing an update
+      await kibanaServer.savedObjects.update({
+        id: 'multiple_versions',
+        type: PACKAGES_SAVED_OBJECT_TYPE,
+        attributes: {
+          install_status: 'installing',
+          install_started_at: previousInstallDate,
+          install_version: '0.2.0',
+          version: '0.1.0',
+        },
+      });
+      await supertest
+        .post(`/api/ingest_manager/epm/packages/multiple_versions-0.2.0`)
+        .set('kbn-xsrf', 'xxxx')
+        .expect(200);
+    });
     it('should return 200 if trying to update to the latest package', async function () {
       await supertest
         .post(`/api/ingest_manager/epm/packages/multiple_versions-0.3.0`)