diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 043b02ca93d7fd..8b472b8b4dfbbd 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -130,6 +130,7 @@ import { getPackageSpecTagId } from './services/epm/kibana/assets/tag_assets'; import { FleetMetricsTask } from './services/metrics/fleet_metrics_task'; import { fetchAgentMetrics } from './services/metrics/fetch_agent_metrics'; import { registerIntegrationFieldsExtractor } from './services/register_integration_fields_extractor'; +import { registerUpgradeManagedPackagePoliciesTask } from './services/setup/managed_package_policies'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -180,6 +181,7 @@ export interface FleetAppContext { auditLogger?: AuditLogger; uninstallTokenService: UninstallTokenServiceInterface; unenrollInactiveAgentsTask: UnenrollInactiveAgentsTask; + taskManagerStart?: TaskManagerStartContract; } export type FleetSetupContract = void; @@ -596,6 +598,8 @@ export class FleetPlugin registerRoutes(fleetAuthzRouter, config); this.telemetryEventsSender.setup(deps.telemetry); + // Register task + registerUpgradeManagedPackagePoliciesTask(deps.taskManager); this.bulkActionsResolver = new BulkActionsResolver(deps.taskManager, core); this.checkDeletedFilesTask = new CheckDeletedFilesTask({ core, @@ -653,6 +657,7 @@ export class FleetPlugin messageSigningService, uninstallTokenService, unenrollInactiveAgentsTask: this.unenrollInactiveAgentsTask!, + taskManagerStart: plugins.taskManager, }); licenseService.start(plugins.licensing.license$); this.telemetryEventsSender.start(plugins.telemetry, core).catch(() => {}); diff --git a/x-pack/plugins/fleet/server/services/app_context.ts b/x-pack/plugins/fleet/server/services/app_context.ts index f2965025d45d19..7dccb7ba1dfe0b 100644 --- a/x-pack/plugins/fleet/server/services/app_context.ts +++ b/x-pack/plugins/fleet/server/services/app_context.ts @@ -25,14 +25,12 @@ import type { EncryptedSavedObjectsPluginSetup, EncryptedSavedObjectsPluginStart, } from '@kbn/encrypted-saved-objects-plugin/server'; - import type { SecurityPluginStart, SecurityPluginSetup } from '@kbn/security-plugin/server'; - import type { CloudSetup } from '@kbn/cloud-plugin/server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common'; import type { SavedObjectTaggingStart } from '@kbn/saved-objects-tagging-plugin/server'; - import { SECURITY_EXTENSION_ID, SPACES_EXTENSION_ID } from '@kbn/core-saved-objects-server'; +import type { TaskManagerStartContract } from '@kbn/task-manager-plugin/server'; import type { FleetConfigType } from '../../common/types'; import { @@ -84,6 +82,7 @@ class AppContextService { private bulkActionsResolver: BulkActionsResolver | undefined; private messageSigningService: MessageSigningServiceInterface | undefined; private uninstallTokenService: UninstallTokenServiceInterface | undefined; + private taskManagerStart: TaskManagerStartContract | undefined; public start(appContext: FleetAppContext) { this.data = appContext.data; @@ -108,6 +107,7 @@ class AppContextService { this.bulkActionsResolver = appContext.bulkActionsResolver; this.messageSigningService = appContext.messageSigningService; this.uninstallTokenService = appContext.uninstallTokenService; + this.taskManagerStart = appContext.taskManagerStart; if (appContext.config$) { this.config$ = appContext.config$; @@ -282,6 +282,10 @@ class AppContextService { return this.kibanaInstanceId; } + public getTaskManagerStart() { + return this.taskManagerStart; + } + public addExternalCallback(type: ExternalCallback[0], callback: ExternalCallback[1]) { if (!this.externalCallbacks.has(type)) { this.externalCallbacks.set(type, new Set()); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/cache.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/cache.test.ts new file mode 100644 index 00000000000000..5588dbe35ef314 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/cache.test.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + getPackageAssetsMapCache, + getPackageInfoCache, + runWithCache, + setPackageAssetsMapCache, + setPackageInfoCache, +} from './cache'; + +const PKG_NAME = 'test'; +const PKG_VERSION = '1.0.0'; + +describe('EPM CacheSession', () => { + describe('outside of a cache session', () => { + it('should not cache package info', () => { + setPackageInfoCache(PKG_NAME, PKG_VERSION, { + name: 'test', + } as any); + const cache = getPackageInfoCache(PKG_NAME, PKG_VERSION); + expect(cache).toBeUndefined(); + }); + + it('should not cache assetsMap', () => { + setPackageAssetsMapCache(PKG_NAME, PKG_VERSION, new Map()); + const cache = getPackageAssetsMapCache(PKG_NAME, PKG_VERSION); + expect(cache).toBeUndefined(); + }); + }); + + describe('in of a cache session', () => { + it('should cache package info', async () => { + function setCache() { + setPackageInfoCache(PKG_NAME, PKG_VERSION, { + name: 'test', + } as any); + } + function getCache() { + const cache = getPackageInfoCache(PKG_NAME, PKG_VERSION); + expect(cache).toEqual({ name: 'test' }); + } + + await runWithCache(async () => { + setCache(); + getCache(); + }); + }); + + it('should cache assetsMap', async () => { + function setCache() { + const map = new Map(); + map.set('test.yaml', Buffer.from('name: test')); + setPackageAssetsMapCache(PKG_NAME, PKG_VERSION, map); + } + function getCache() { + const cache = getPackageAssetsMapCache(PKG_NAME, PKG_VERSION); + expect(cache).not.toBeUndefined(); + expect(cache?.get('test.yaml')?.toString()).toEqual('name: test'); + } + + await runWithCache(async () => { + setCache(); + getCache(); + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/cache.ts b/x-pack/plugins/fleet/server/services/epm/packages/cache.ts new file mode 100644 index 00000000000000..63eb825365fbe2 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/cache.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AsyncLocalStorage } from 'async_hooks'; + +import LRUCache from 'lru-cache'; + +import type { AssetsMap } from '../../../../common/types'; + +import type { PackageInfo } from '../../../../common'; + +const cacheStore = new AsyncLocalStorage(); + +const PACKAGE_INFO_CACHE_SIZE = 20; +const PACKAGE_ASSETS_MAP_CACHE_SIZE = 1; + +class CacheSession { + private _packageInfoCache?: LRUCache; + + private _packageAssetsMap?: LRUCache; + + getPackageInfoCache() { + if (!this._packageInfoCache) { + this._packageInfoCache = new LRUCache({ + max: PACKAGE_INFO_CACHE_SIZE, + }); + } + return this._packageInfoCache; + } + + getPackageAssetsMapCache() { + if (!this._packageAssetsMap) { + this._packageAssetsMap = new LRUCache({ + max: PACKAGE_ASSETS_MAP_CACHE_SIZE, + }); + } + return this._packageAssetsMap; + } +} + +export function getPackageInfoCache(pkgName: string, pkgVersion: string) { + return cacheStore.getStore()?.getPackageInfoCache()?.get(`${pkgName}:${pkgVersion}`); +} + +export function setPackageInfoCache(pkgName: string, pkgVersion: string, packageInfo: PackageInfo) { + return cacheStore.getStore()?.getPackageInfoCache()?.set(`${pkgName}:${pkgVersion}`, packageInfo); +} + +export function getPackageAssetsMapCache(pkgName: string, pkgVersion: string) { + return cacheStore.getStore()?.getPackageAssetsMapCache()?.get(`${pkgName}:${pkgVersion}`); +} + +export function setPackageAssetsMapCache( + pkgName: string, + pkgVersion: string, + assetsMap: AssetsMap +) { + return cacheStore + .getStore() + ?.getPackageAssetsMapCache() + ?.set(`${pkgName}:${pkgVersion}`, assetsMap); +} + +export async function runWithCache(cb: () => Promise): Promise { + const cache = new CacheSession(); + + return cacheStore.run(cache, cb); +} diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 0e5f5be89cab30..2af0f1e9f1d2b7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -60,6 +60,12 @@ import { auditLoggingService } from '../../audit_logging'; import { getFilteredSearchPackages } from '../filtered_packages'; import { createInstallableFrom } from '.'; +import { + getPackageAssetsMapCache, + setPackageAssetsMapCache, + getPackageInfoCache, + setPackageInfoCache, +} from './cache'; export { getFile } from '../registry'; @@ -415,6 +421,10 @@ export async function getPackageInfo({ ignoreUnverified?: boolean; prerelease?: boolean; }): Promise { + const cacheResult = getPackageInfoCache(pkgName, pkgVersion); + if (cacheResult) { + return cacheResult; + } const [savedObject, latestPackage] = await Promise.all([ getInstallationObject({ savedObjectsClient, pkgName }), Registry.fetchFindLatestPackageOrUndefined(pkgName, { prerelease }), @@ -468,7 +478,10 @@ export async function getPackageInfo({ }; const updated = { ...packageInfo, ...additions }; - return createInstallableFrom(updated, savedObject); + const installable = createInstallableFrom(updated, savedObject); + setPackageInfoCache(pkgName, pkgVersion, installable); + + return installable; } export const getPackageUsageStats = async ({ @@ -720,6 +733,10 @@ export async function getPackageAssetsMap({ logger: Logger; ignoreUnverified?: boolean; }) { + const cache = getPackageAssetsMapCache(packageInfo.name, packageInfo.version); + if (cache) { + return cache; + } const installedPackageWithAssets = await getInstalledPackageWithAssets({ savedObjectsClient, pkgName: packageInfo.name, @@ -736,6 +753,7 @@ export async function getPackageAssetsMap({ } else { assetsMap = installedPackageWithAssets.assetsMap; } + setPackageAssetsMapCache(packageInfo.name, packageInfo.version, assetsMap); return assetsMap; } diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index 0d76fc307c74c1..5c19345a58f798 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -20,55 +20,16 @@ import { outputService } from '../output'; import { appContextService } from '../app_context'; export const mapPackagePolicySavedObjectToPackagePolicy = ({ - /* eslint-disable @typescript-eslint/naming-convention */ id, version, - attributes: { - name, - description, - namespace, - enabled, - is_managed, - policy_id, - policy_ids, - output_id, - // `package` is a reserved keyword - package: packageInfo, - inputs, - vars, - elasticsearch, - agents, - revision, - secret_references, - updated_at, - updated_by, - created_at, - created_by, - /* eslint-enable @typescript-eslint/naming-convention */ - }, + attributes, + namespaces, }: SavedObject): PackagePolicy => { return { id, - name, - description, - namespace, - enabled, - is_managed, - policy_id, - policy_ids, - output_id, - package: packageInfo, - inputs, - vars, - elasticsearch, version, - agents, - revision, - secret_references, - updated_at, - updated_by, - created_at, - created_by, + spaceIds: namespaces, + ...attributes, }; }; diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index f23ed5836afeb6..c1970548e98faa 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -118,7 +118,6 @@ import { mapPackagePolicySavedObjectToPackagePolicy, preflightCheckPackagePolicy, } from './package_policies'; -import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { PackagePolicyClient, PackagePolicyClientFetchAllItemsOptions, @@ -1651,13 +1650,6 @@ class PackagePolicyClientImpl implements PackagePolicyClient { await this.update(soClient, esClient, id, updatePackagePolicy, updateOptions); - // Persist any experimental feature opt-ins that come through the upgrade process to the Installation SO - await updateDatastreamExperimentalFeatures( - soClient, - packagePolicy.package!.name, - experimentalDataStreamFeatures - ); - result.push({ id, name: packagePolicy.name, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.ts b/x-pack/plugins/fleet/server/services/preconfiguration.ts index 853961f2fd77a2..18726cdab44522 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.ts @@ -42,7 +42,7 @@ import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy'; import { type InputsOverride, packagePolicyService } from './package_policy'; import { preconfigurePackageInputs } from './package_policy'; import { appContextService } from './app_context'; -import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; +import type { UpgradeManagedPackagePoliciesResult } from './setup/managed_package_policies'; import { isDefaultAgentlessPolicyEnabled } from './utils/agentless'; interface PreconfigurationResult { diff --git a/x-pack/plugins/fleet/server/services/setup.test.ts b/x-pack/plugins/fleet/server/services/setup.test.ts index 5e88fac35e1405..8add6942e9da76 100644 --- a/x-pack/plugins/fleet/server/services/setup.test.ts +++ b/x-pack/plugins/fleet/server/services/setup.test.ts @@ -8,13 +8,14 @@ import type { SavedObjectsClientContract } from '@kbn/core/server'; import type { ElasticsearchClientMock } from '@kbn/core/server/mocks'; +import { MessageSigningError } from '../../common/errors'; import { createAppContextStartContractMock, xpackMocks } from '../mocks'; import { ensurePreconfiguredPackagesAndPolicies } from '.'; import { appContextService } from './app_context'; import { getInstallations } from './epm/packages'; -import { upgradeManagedPackagePolicies } from './managed_package_policies'; +import { setupUpgradeManagedPackagePolicies } from './setup/managed_package_policies'; import { setupFleet } from './setup'; jest.mock('./preconfiguration'); @@ -25,7 +26,7 @@ jest.mock('./settings'); jest.mock('./output'); jest.mock('./download_source'); jest.mock('./epm/packages'); -jest.mock('./managed_package_policies'); +jest.mock('./setup/managed_package_policies'); jest.mock('./setup/upgrade_package_install_version'); jest.mock('./epm/elasticsearch/template/install', () => { return { @@ -64,7 +65,7 @@ describe('setupFleet', () => { nonFatalErrors: [], }); - (upgradeManagedPackagePolicies as jest.Mock).mockResolvedValue([]); + (setupUpgradeManagedPackagePolicies as jest.Mock).mockResolvedValue([]); soClient.get.mockResolvedValue({ attributes: {} } as any); soClient.find.mockResolvedValue({ saved_objects: [] } as any); @@ -80,7 +81,7 @@ describe('setupFleet', () => { describe('should reject with any error thrown underneath', () => { it('SO client throws plain Error', async () => { - mockedMethodThrowsError(upgradeManagedPackagePolicies as jest.Mock); + mockedMethodThrowsError(setupUpgradeManagedPackagePolicies as jest.Mock); const setupPromise = setupFleet(soClient, esClient); await expect(setupPromise).rejects.toThrow('SO method mocked to throw'); @@ -88,7 +89,7 @@ describe('setupFleet', () => { }); it('SO client throws other error', async () => { - mockedMethodThrowsCustom(upgradeManagedPackagePolicies as jest.Mock); + mockedMethodThrowsCustom(setupUpgradeManagedPackagePolicies as jest.Mock); const setupPromise = setupFleet(soClient, esClient); await expect(setupPromise).rejects.toThrow('method mocked to throw'); @@ -97,13 +98,6 @@ describe('setupFleet', () => { }); it('should not return non fatal errors when upgrade result has no errors', async () => { - (upgradeManagedPackagePolicies as jest.Mock).mockResolvedValue([ - { - errors: [], - packagePolicyId: '1', - }, - ]); - const result = await setupFleet(soClient, esClient); expect(result).toEqual({ @@ -112,13 +106,11 @@ describe('setupFleet', () => { }); }); - it('should return non fatal errors when upgrade result has errors', async () => { - (upgradeManagedPackagePolicies as jest.Mock).mockResolvedValue([ - { - errors: [{ key: 'key', message: 'message' }], - packagePolicyId: '1', - }, - ]); + it('should return non fatal errors when generateKeyPair result has errors', async () => { + const messageSigninError = new MessageSigningError('test'); + jest + .mocked(appContextService.getMessageSigningService()!.generateKeyPair) + .mockRejectedValue(messageSigninError); const result = await setupFleet(soClient, esClient); @@ -126,13 +118,7 @@ describe('setupFleet', () => { isInitialized: true, nonFatalErrors: [ { - errors: [ - { - key: 'key', - message: 'message', - }, - ], - packagePolicyId: '1', + error: messageSigninError, }, ], }); diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index e59eb229ad8e58..c0b86d63947690 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -40,8 +40,8 @@ import { ensureFleetFinalPipelineIsInstalled } from './epm/elasticsearch/ingest_ import { ensureDefaultComponentTemplates } from './epm/elasticsearch/template/install'; import { getInstallations, reinstallPackageForInstallation } from './epm/packages'; import { isPackageInstalled } from './epm/packages/install'; -import type { UpgradeManagedPackagePoliciesResult } from './managed_package_policies'; -import { upgradeManagedPackagePolicies } from './managed_package_policies'; +import type { UpgradeManagedPackagePoliciesResult } from './setup/managed_package_policies'; +import { setupUpgradeManagedPackagePolicies } from './setup/managed_package_policies'; import { upgradePackageInstallVersion } from './setup/upgrade_package_install_version'; import { upgradeAgentPolicySchemaVersion } from './setup/upgrade_agent_policy_schema_version'; import { migrateSettingsToFleetServerHost } from './fleet_server_host'; @@ -250,9 +250,7 @@ async function createSetupSideEffects( stepSpan?.end(); stepSpan = apm.startSpan('Upgrade managed package policies', 'preconfiguration'); - const packagePolicyUpgradeErrors = ( - await upgradeManagedPackagePolicies(soClient, esClient) - ).filter((result) => (result.errors ?? []).length > 0); + await setupUpgradeManagedPackagePolicies(soClient, esClient); stepSpan?.end(); logger.debug('Upgrade Fleet package install versions'); @@ -294,7 +292,6 @@ async function createSetupSideEffects( const nonFatalErrors = [ ...preconfiguredPackagesNonFatalErrors, - ...packagePolicyUpgradeErrors, ...(messageSigningServiceNonFatalError ? [messageSigningServiceNonFatalError] : []), ]; diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts b/x-pack/plugins/fleet/server/services/setup/managed_package_policies.test.ts similarity index 64% rename from x-pack/plugins/fleet/server/services/managed_package_policies.test.ts rename to x-pack/plugins/fleet/server/services/setup/managed_package_policies.test.ts index 0882fff0ec34f0..166ad8c4e8fa3c 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts +++ b/x-pack/plugins/fleet/server/services/setup/managed_package_policies.test.ts @@ -7,15 +7,16 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { packagePolicyService } from '../package_policy'; +import { getInstallation } from '../epm/packages'; + import { upgradeManagedPackagePolicies } from './managed_package_policies'; -import { packagePolicyService } from './package_policy'; -import { getInstallations } from './epm/packages'; -jest.mock('./package_policy'); -jest.mock('./epm/packages'); -jest.mock('./app_context', () => { +jest.mock('../package_policy'); +jest.mock('../epm/packages'); +jest.mock('../app_context', () => { return { - ...jest.requireActual('./app_context'), + ...jest.requireActual('../app_context'), appContextService: { getLogger: jest.fn(() => { return { error: jest.fn(), debug: jest.fn() }; @@ -23,22 +24,21 @@ jest.mock('./app_context', () => { }, }; }); -jest.mock('./audit_logging'); +jest.mock('../audit_logging'); describe('upgradeManagedPackagePolicies', () => { afterEach(() => { jest.clearAllMocks(); + jest.mocked(packagePolicyService.fetchAllItems).mockReset(); }); - it('should not upgrade policies for non-managed package', async () => { + it('should not upgrade policies for installed package', async () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const soClient = savedObjectsClientMock.create(); - (getInstallations as jest.Mock).mockResolvedValueOnce({ - saved_objects: [], - }); + (getInstallation as jest.Mock).mockResolvedValueOnce(undefined); - await upgradeManagedPackagePolicies(soClient, esClient); + await upgradeManagedPackagePolicies(soClient, esClient, 'testpkg'); expect(packagePolicyService.upgrade).not.toBeCalled(); }); @@ -62,9 +62,11 @@ describe('upgradeManagedPackagePolicies', () => { }, }; - (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({ - items: [packagePolicy], - }); + (packagePolicyService.fetchAllItems as jest.Mock).mockResolvedValueOnce( + (async function* () { + yield [packagePolicy]; + })() + ); (packagePolicyService.getUpgradeDryRunDiff as jest.Mock).mockResolvedValueOnce({ name: 'non-managed-package-policy', @@ -72,19 +74,13 @@ describe('upgradeManagedPackagePolicies', () => { hasErrors: false, }); - (getInstallations as jest.Mock).mockResolvedValueOnce({ - saved_objects: [ - { - attributes: { - id: 'test-installation', - version: '1.0.0', - keep_policies_up_to_date: true, - }, - }, - ], + (getInstallation as jest.Mock).mockResolvedValueOnce({ + id: 'test-installation', + version: '1.0.0', + keep_policies_up_to_date: true, }); - const results = await upgradeManagedPackagePolicies(soClient, esClient); + const results = await upgradeManagedPackagePolicies(soClient, esClient, 'pkgname'); expect(results).toEqual([ { packagePolicyId: 'managed-package-id', diff: [{ id: 'foo' }, { id: 'bar' }], errors: [] }, ]); @@ -103,39 +99,35 @@ describe('upgradeManagedPackagePolicies', () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const soClient = savedObjectsClientMock.create(); - (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({ - items: [ - { - id: 'managed-package-id', - inputs: {}, - version: '', - revision: 1, - updated_at: '', - updated_by: '', - created_at: '', - created_by: '', - package: { - name: 'managed-package', - title: 'Managed Package', - version: '1.0.1', + (packagePolicyService.fetchAllItems as jest.Mock).mockResolvedValueOnce( + (async function* () { + yield [ + { + id: 'managed-package-id', + inputs: {}, + version: '', + revision: 1, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + package: { + name: 'managed-package', + title: 'Managed Package', + version: '1.0.1', + }, }, - }, - ], - }); + ]; + })() + ); - (getInstallations as jest.Mock).mockResolvedValueOnce({ - saved_objects: [ - { - attributes: { - id: 'test-installation', - version: '1.0.0', - keep_policies_up_to_date: true, - }, - }, - ], + (getInstallation as jest.Mock).mockResolvedValueOnce({ + id: 'test-installation', + version: '1.0.0', + keep_policies_up_to_date: true, }); - await upgradeManagedPackagePolicies(soClient, esClient); + await upgradeManagedPackagePolicies(soClient, esClient, 'pkgname'); expect(packagePolicyService.getUpgradeDryRunDiff).not.toHaveBeenCalled(); expect(packagePolicyService.upgrade).not.toHaveBeenCalled(); @@ -146,25 +138,27 @@ describe('upgradeManagedPackagePolicies', () => { const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const soClient = savedObjectsClientMock.create(); - (packagePolicyService.list as jest.Mock).mockResolvedValueOnce({ - items: [ - { - id: 'conflicting-package-policy', - inputs: {}, - version: '', - revision: 1, - updated_at: '', - updated_by: '', - created_at: '', - created_by: '', - package: { - name: 'conflicting-package', - title: 'Conflicting Package', - version: '0.0.1', + (packagePolicyService.fetchAllItems as jest.Mock).mockResolvedValueOnce( + (async function* () { + yield [ + { + id: 'conflicting-package-policy', + inputs: {}, + version: '', + revision: 1, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + package: { + name: 'conflicting-package', + title: 'Conflicting Package', + version: '0.0.1', + }, }, - }, - ], - }); + ]; + })() + ); (packagePolicyService.getUpgradeDryRunDiff as jest.Mock).mockResolvedValueOnce({ name: 'conflicting-package-policy', @@ -175,19 +169,13 @@ describe('upgradeManagedPackagePolicies', () => { hasErrors: true, }); - (getInstallations as jest.Mock).mockResolvedValueOnce({ - saved_objects: [ - { - attributes: { - id: 'test-installation', - version: '1.0.0', - keep_policies_up_to_date: true, - }, - }, - ], + (getInstallation as jest.Mock).mockResolvedValueOnce({ + id: 'test-installation', + version: '1.0.0', + keep_policies_up_to_date: true, }); - const result = await upgradeManagedPackagePolicies(soClient, esClient); + const result = await upgradeManagedPackagePolicies(soClient, esClient, 'pkgname'); expect(result).toEqual([ { diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.ts b/x-pack/plugins/fleet/server/services/setup/managed_package_policies.ts similarity index 50% rename from x-pack/plugins/fleet/server/services/managed_package_policies.ts rename to x-pack/plugins/fleet/server/services/setup/managed_package_policies.ts index 053d46d1a087d5..edb701971cd65f 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.ts +++ b/x-pack/plugins/fleet/server/services/setup/managed_package_policies.ts @@ -7,16 +7,22 @@ import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; import semverLt from 'semver/functions/lt'; +import type { + ConcreteTaskInstance, + TaskManagerSetupContract, + TaskManagerStartContract, +} from '@kbn/task-manager-plugin/server'; -import type { UpgradePackagePolicyDryRunResponseItem } from '../../common/types'; +import type { UpgradePackagePolicyDryRunResponseItem } from '../../../common/types'; -import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../constants'; -import type { Installation, PackagePolicy } from '../types'; +import type { Installation, PackagePolicy } from '../../types'; -import { appContextService } from './app_context'; -import { getInstallations } from './epm/packages'; -import { packagePolicyService } from './package_policy'; +import { appContextService } from '../app_context'; +import { getInstallation, getInstallations } from '../epm/packages'; +import { packagePolicyService } from '../package_policy'; +import { runWithCache } from '../epm/packages/cache'; export interface UpgradeManagedPackagePoliciesResult { packagePolicyId: string; @@ -24,30 +30,130 @@ export interface UpgradeManagedPackagePoliciesResult { errors: any; } +const TASK_TYPE = 'fleet:setup:upgrade_managed_package_policies'; + +export function registerUpgradeManagedPackagePoliciesTask( + taskManagerSetup: TaskManagerSetupContract +) { + taskManagerSetup.registerTaskDefinitions({ + [TASK_TYPE]: { + title: 'Fleet Setup Upgrade managed package policies', + timeout: '1h', + maxAttempts: 1, + createTaskRunner: ({ taskInstance }: { taskInstance: ConcreteTaskInstance }) => { + const pkgName = taskInstance.params.packageName; + return { + async run() { + const esClient = appContextService.getInternalUserESClient(); + const soClient = appContextService.getInternalUserSOClientWithoutSpaceExtension(); + + await runWithCache(() => upgradeManagedPackagePolicies(soClient, esClient, pkgName)); + }, + async cancel() {}, + }; + }, + }, + }); +} + +async function runUpgradeManagedPackagePoliciesTask( + taskManagerStart: TaskManagerStartContract, + pkgName: string +) { + await taskManagerStart.ensureScheduled({ + id: `${TASK_TYPE}:${pkgName}`, + scope: ['fleet'], + params: { packageName: pkgName }, + taskType: TASK_TYPE, + runAt: new Date(Date.now() + 3 * 1000), + state: {}, + }); +} + /** - * Upgrade any package policies for packages installed through setup that are denoted as `AUTO_UPGRADE` packages - * or have the `keep_policies_up_to_date` flag set to `true` + * + * @param soClient + * @param esClient + * @returns */ -export const upgradeManagedPackagePolicies = async ( +export const setupUpgradeManagedPackagePolicies = async ( soClient: SavedObjectsClientContract, esClient: ElasticsearchClient -): Promise => { +) => { appContextService .getLogger() - .debug('Running required package policies upgrades for managed policies'); - const results: UpgradeManagedPackagePoliciesResult[] = []; + .debug('Scheduling required package policies upgrades for managed policies'); const installedPackages = await getInstallations(soClient, { filter: `${PACKAGES_SAVED_OBJECT_TYPE}.attributes.install_status:installed AND ${PACKAGES_SAVED_OBJECT_TYPE}.attributes.keep_policies_up_to_date:true`, }); - for (const { attributes: installedPackage } of installedPackages.saved_objects) { - const packagePolicies = await getPackagePoliciesNotMatchingVersion( + const packagePoliciesFinder = await getPackagePoliciesNotMatchingVersion( soClient, installedPackage.name, installedPackage.version ); + let shouldRegisterTask = false; + for await (const packagePolicies of packagePoliciesFinder) { + for (const packagePolicy of packagePolicies) { + if (isPolicyVersionLtInstalledVersion(packagePolicy, installedPackage)) { + shouldRegisterTask = true; + break; + } + } + if (shouldRegisterTask) { + break; + } + } + if (shouldRegisterTask) { + appContextService + .getLogger() + .debug( + `Scheduled package policies upgrades for package: ${installedPackage.name}@${installedPackage.version}` + ); + await runUpgradeManagedPackagePoliciesTask( + appContextService.getTaskManagerStart()!, + installedPackage.name + ); + } + } +}; + +/** + * Upgrade any package policies for packages installed through setup that are denoted as `AUTO_UPGRADE` packages + * or have the `keep_policies_up_to_date` flag set to `true` + */ +export const upgradeManagedPackagePolicies = async ( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + pkgName: string +): Promise => { + appContextService + .getLogger() + .debug('Running required package policies upgrades for managed policies'); + const results: UpgradeManagedPackagePoliciesResult[] = []; + + const installedPackage = await getInstallation({ + pkgName, + savedObjectsClient: soClient, + logger: appContextService.getLogger(), + }); + if (!installedPackage) { + appContextService + .getLogger() + .debug('Aborting upgrading managed package policies: package is not installed'); + + return []; + } + + const packagePoliciesFinder = await getPackagePoliciesNotMatchingVersion( + soClient, + installedPackage.name, + installedPackage.version + ); + + for await (const packagePolicies of packagePoliciesFinder) { for (const packagePolicy of packagePolicies) { if (isPolicyVersionLtInstalledVersion(packagePolicy, installedPackage)) { await upgradePackagePolicy(soClient, esClient, packagePolicy, installedPackage, results); @@ -61,14 +167,11 @@ async function getPackagePoliciesNotMatchingVersion( soClient: SavedObjectsClientContract, pkgName: string, pkgVersion: string -): Promise { - return ( - await packagePolicyService.list(soClient, { - page: 1, - perPage: 1000, - kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName} AND NOT ${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.version:${pkgVersion}`, - }) - ).items; +) { + return packagePolicyService.fetchAllItems(soClient, { + perPage: 50, + kuery: `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name:${pkgName} AND NOT ${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.version:${pkgVersion}`, + }); } function isPolicyVersionLtInstalledVersion( diff --git a/x-pack/test/fleet_api_integration/apis/fleet_setup.ts b/x-pack/test/fleet_api_integration/apis/fleet_setup.ts index 3f76f4594592fb..ae196f13d9dc35 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_setup.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_setup.ts @@ -6,8 +6,15 @@ */ import expect from '@kbn/expect'; +import { v4 as uuidV4 } from 'uuid'; +import { INGEST_SAVED_OBJECT_INDEX } from '@kbn/core-saved-objects-server'; +import { LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common/constants'; +import pRetry from 'p-retry'; +import { SearchTotalHits } from '@elastic/elasticsearch/lib/api/types'; + import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; import { skipIfNoDockerRegistry } from '../helpers'; +import { SpaceTestApiClient } from './space_awareness/api_helper'; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; @@ -48,23 +55,6 @@ export default function (providerContext: FtrProviderContext) { } }); - it('should not create a fleet_enroll role if one does not already exist', async () => { - const { body: apiResponse } = await supertest - .post(`/api/fleet/setup`) - .set('kbn-xsrf', 'xxxx') - .expect(200); - - expect(apiResponse.isInitialized).to.be(true); - - try { - await es.security.getUser({ - username: 'fleet_enroll', - }); - } catch (e) { - expect(e.meta?.statusCode).to.eql(404); - } - }); - it('should install default packages', async () => { await supertest.post(`/api/fleet/setup`).set('kbn-xsrf', 'xxxx').expect(200); @@ -78,5 +68,92 @@ export default function (providerContext: FtrProviderContext) { expect(installedPackages).to.eql(['endpoint']); }); + + describe('upgrade managed package policies', () => { + const apiClient = new SpaceTestApiClient(supertest); + before(async () => { + const pkgRes = await apiClient.getPackage({ + pkgName: 'synthetics', + }); + await apiClient.installPackage({ + pkgName: 'synthetics', + pkgVersion: pkgRes.item.version, + force: true, + }); + await apiClient.updatePackage({ + pkgName: 'synthetics', + pkgVersion: pkgRes.item.version, + data: { + keepPoliciesUpToDate: true, + }, + }); + + const agentPolicyRes = await apiClient.createAgentPolicy(); + + await es.bulk({ + index: INGEST_SAVED_OBJECT_INDEX, + refresh: 'wait_for', + operations: [...new Array(10).keys()].flatMap((_, index) => [ + { + create: { + _id: `${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}:${uuidV4()}`, + }, + }, + { + type: LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE, + [LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE]: { + name: `test-${index}`, + policy_ids: [agentPolicyRes.item.id], + inputs: [], + package: { + name: 'synthetics', + version: '1.2.1', + }, + }, + }, + ]), + }); + + await apiClient.getPackage({ + pkgName: 'synthetics', + }); + }); + it('should upgrade managed package policies', async () => { + await apiClient.setup(); + + await pRetry( + async () => { + const res = await es.search({ + index: INGEST_SAVED_OBJECT_INDEX, + track_total_hits: true, + query: { + bool: { + must: { + term: { + [`${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.version`]: '1.2.1', + }, + }, + filter: { + term: { + [`${LEGACY_PACKAGE_POLICY_SAVED_OBJECT_TYPE}.package.name`]: 'synthetics', + }, + }, + }, + }, + }); + if ((res.hits.total as SearchTotalHits).value > 0) { + throw new Error( + `Managed package policies not upgraded ${ + (res.hits.total as SearchTotalHits).value + }.` + ); + } + }, + { + maxRetryTime: 20 * 1000, + } + ); + }); + }); }); } diff --git a/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts b/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts index 0695dd8868d4a4..9b6a76c3ff6bc9 100644 --- a/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts +++ b/x-pack/test/fleet_api_integration/apis/space_awareness/api_helper.ts @@ -31,6 +31,8 @@ import { PostNewAgentActionResponse, UpdateAgentPolicyResponse, UpdateAgentPolicyRequest, + UpdatePackageResponse, + UpdatePackageRequest, } from '@kbn/fleet-plugin/common/types'; import { GetUninstallTokenResponse, @@ -331,11 +333,31 @@ export class SpaceTestApiClient { } // Package install async getPackage( - { pkgName, pkgVersion }: { pkgName: string; pkgVersion: string }, + { pkgName, pkgVersion }: { pkgName: string; pkgVersion?: string }, spaceId?: string ): Promise { const { body: res } = await this.supertest - .get(`${this.getBaseUrl(spaceId)}/api/fleet/epm/packages/${pkgName}/${pkgVersion}`) + .get( + pkgVersion + ? `${this.getBaseUrl(spaceId)}/api/fleet/epm/packages/${pkgName}/${pkgVersion}` + : `${this.getBaseUrl(spaceId)}/api/fleet/epm/packages/${pkgName}` + ) + .expect(200); + + return res; + } + async updatePackage( + { + pkgName, + pkgVersion, + data, + }: { pkgName: string; pkgVersion: string; data: UpdatePackageRequest['body'] }, + spaceId?: string + ): Promise { + const { body: res } = await this.supertest + .put(`${this.getBaseUrl(spaceId)}/api/fleet/epm/packages/${pkgName}/${pkgVersion}`) + .set('kbn-xsrf', 'xxxx') + .send({ ...data }) .expect(200); return res; diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts index 810db7295a79fb..a488cd1a471718 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/check_registered_task_types.ts @@ -140,6 +140,7 @@ export default function ({ getService }: FtrProviderContext) { 'fleet:check-deleted-files-task', 'fleet:reassign_action:retry', 'fleet:request_diagnostics:retry', + 'fleet:setup:upgrade_managed_package_policies', 'fleet:unenroll-inactive-agents-task', 'fleet:unenroll_action:retry', 'fleet:update_agent_tags:retry',