From ab401c98572780168ac589e1685b256004abe92e Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 9 Feb 2021 23:29:52 -0800 Subject: [PATCH 01/32] skip flaky suite (#90136) --- x-pack/test/api_integration/apis/security_solution/users.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/api_integration/apis/security_solution/users.ts b/x-pack/test/api_integration/apis/security_solution/users.ts index 45e06ab72adbb9..b888be2bf6276f 100644 --- a/x-pack/test/api_integration/apis/security_solution/users.ts +++ b/x-pack/test/api_integration/apis/security_solution/users.ts @@ -23,6 +23,7 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); // Failing: See https://github.com/elastic/kibana/issues/90135 + // Failing: See https://github.com/elastic/kibana/issues/90136 describe.skip('Users', () => { describe('With auditbeat', () => { before(() => esArchiver.load('auditbeat/default')); From af277b83962d4bef12094bbbc17bad805a5c02a1 Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Wed, 10 Feb 2021 09:16:19 +0100 Subject: [PATCH 02/32] Add deprecation warning to all Beats CM pages. (#90741) --- .../beats_management/public/application.tsx | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/x-pack/plugins/beats_management/public/application.tsx b/x-pack/plugins/beats_management/public/application.tsx index 6e81809b9c4930..5a9b0a768856ed 100644 --- a/x-pack/plugins/beats_management/public/application.tsx +++ b/x-pack/plugins/beats_management/public/application.tsx @@ -7,11 +7,13 @@ import * as euiVars from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; import ReactDOM from 'react-dom'; import { Router } from 'react-router-dom'; import { ThemeProvider } from 'styled-components'; import { Provider as UnstatedProvider, Subscribe } from 'unstated'; +import { EuiCallOut, EuiLink, EuiSpacer } from '@elastic/eui'; import { Background } from './components/layouts/background'; import { BreadcrumbProvider } from './components/navigation/breadcrumb'; import { Breadcrumb } from './components/navigation/breadcrumb/breadcrumb'; @@ -37,6 +39,38 @@ export const renderApp = ({ element, history }: ManagementAppMountParams, libs: defaultMessage: 'Management', })} /> + +

+ + + + ), + }} + /> +

+ + )} From 240da2bf2a999501eddc378466e67ebb8e26df76 Mon Sep 17 00:00:00 2001 From: Brandon Kobel Date: Wed, 10 Feb 2021 00:38:37 -0800 Subject: [PATCH 03/32] Actually deleting x-pack/tsconfig.refs.json (#90898) --- x-pack/tsconfig.refs.json | 60 --------------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 x-pack/tsconfig.refs.json diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json deleted file mode 100644 index a36f4e205ab7da..00000000000000 --- a/x-pack/tsconfig.refs.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "include": [], - "references": [ - { "path": "./plugins/actions/tsconfig.json" }, - { "path": "./plugins/alerts/tsconfig.json" }, - { "path": "./plugins/beats_management/tsconfig.json" }, - { "path": "./plugins/canvas/tsconfig.json" }, - { "path": "./plugins/cloud/tsconfig.json" }, - { "path": "./plugins/code/tsconfig.json" }, - { "path": "./plugins/console_extensions/tsconfig.json" }, - { "path": "./plugins/dashboard_enhanced/tsconfig.json" }, - { "path": "./plugins/data_enhanced/tsconfig.json" }, - { "path": "./plugins/dashboard_mode/tsconfig.json" }, - { "path": "./plugins/discover_enhanced/tsconfig.json" }, - { "path": "./plugins/embeddable_enhanced/tsconfig.json" }, - { "path": "./plugins/encrypted_saved_objects/tsconfig.json" }, - { "path": "./plugins/enterprise_search/tsconfig.json" }, - { "path": "./plugins/event_log/tsconfig.json" }, - { "path": "./plugins/features/tsconfig.json" }, - { "path": "./plugins/file_upload/tsconfig.json" }, - { "path": "./plugins/fleet/tsconfig.json" }, - { "path": "./plugins/global_search_bar/tsconfig.json" }, - { "path": "./plugins/global_search_providers/tsconfig.json" }, - { "path": "./plugins/global_search/tsconfig.json" }, - { "path": "./plugins/graph/tsconfig.json" }, - { "path": "./plugins/grokdebugger/tsconfig.json" }, - { "path": "./plugins/infra/tsconfig.json" }, - { "path": "./plugins/ingest_pipelines/tsconfig.json" }, - { "path": "./plugins/lens/tsconfig.json" }, - { "path": "./plugins/license_management/tsconfig.json" }, - { "path": "./plugins/licensing/tsconfig.json" }, - { "path": "./plugins/maps_legacy_licensing/tsconfig.json" }, - { "path": "./plugins/maps/tsconfig.json" }, - { "path": "./plugins/ml/tsconfig.json" }, - { "path": "./plugins/observability/tsconfig.json" }, - { "path": "./plugins/painless_lab/tsconfig.json" }, - { "path": "./plugins/reporting/tsconfig.json" }, - { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, - { "path": "./plugins/searchprofiler/tsconfig.json" }, - { "path": "./plugins/security/tsconfig.json" }, - { "path": "./plugins/snapshot_restore/tsconfig.json" }, - { "path": "./plugins/spaces/tsconfig.json" }, - { "path": "./plugins/stack_alerts/tsconfig.json" }, - { "path": "./plugins/task_manager/tsconfig.json" }, - { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, - { "path": "./plugins/transform/tsconfig.json" }, - { "path": "./plugins/translations/tsconfig.json" }, - { "path": "./plugins/triggers_actions_ui/tsconfig.json" }, - { "path": "./plugins/ui_actions_enhanced/tsconfig.json" }, - { "path": "./plugins/upgrade_assistant/tsconfig.json" }, - { "path": "./plugins/runtime_fields/tsconfig.json" }, - { "path": "./plugins/index_management/tsconfig.json" }, - { "path": "./plugins/watcher/tsconfig.json" }, - { "path": "./plugins/rollup/tsconfig.json"}, - { "path": "./plugins/remote_clusters/tsconfig.json"}, - { "path": "./plugins/cross_cluster_replication/tsconfig.json"}, - { "path": "./plugins/index_lifecycle_management/tsconfig.json"}, - { "path": "./plugins/uptime/tsconfig.json" } - ] -} From 634c0b34242adf87c78b72ed73051b0fd1c41014 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 10 Feb 2021 09:57:09 +0100 Subject: [PATCH 04/32] [Fleet] Use staging registry for snapshot builds (#90327) The staging registry is used in Kibana builds which are not built of the master branch or release version. This means, any build ending with `-SNAPSHOT` not the master branch will use the staging registry. Closes https://github.com/elastic/kibana/issues/90131 Co-authored-by: Jen Huang --- .../fleet/server/services/epm/registry/registry_url.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/epm/registry/registry_url.ts b/x-pack/plugins/fleet/server/services/epm/registry/registry_url.ts index 1394d2738482d0..8c637006fb0cd4 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/registry_url.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/registry_url.ts @@ -12,7 +12,7 @@ import { appContextService, licenseService } from '../../'; // chose to comment them out vs @ts-ignore or @ts-expect-error on each line const PRODUCTION_REGISTRY_URL_CDN = 'https://epr.elastic.co'; -// const STAGING_REGISTRY_URL_CDN = 'https://epr-staging.elastic.co'; +const STAGING_REGISTRY_URL_CDN = 'https://epr-staging.elastic.co'; const SNAPSHOT_REGISTRY_URL_CDN = 'https://epr-snapshot.elastic.co'; // const PRODUCTION_REGISTRY_URL_NO_CDN = 'https://epr.ea-web.elastic.dev'; @@ -23,6 +23,8 @@ const getDefaultRegistryUrl = (): string => { const branch = appContextService.getKibanaBranch(); if (branch === 'master') { return SNAPSHOT_REGISTRY_URL_CDN; + } else if (appContextService.getKibanaVersion().includes('-SNAPSHOT')) { + return STAGING_REGISTRY_URL_CDN; } else { return PRODUCTION_REGISTRY_URL_CDN; } From 03a53b9f39a54a6accb8f37eb9fe84c882681511 Mon Sep 17 00:00:00 2001 From: Aleh Zasypkin Date: Wed, 10 Feb 2021 11:27:31 +0100 Subject: [PATCH 05/32] Do not generate an ephemeral encryption key in production. (#81511) --- .../server/action_type_registry.test.ts | 4 +- .../actions/server/actions_client.test.ts | 8 +- .../server/builtin_action_types/index.test.ts | 4 +- .../server/create_execute_function.test.ts | 14 +- .../actions/server/create_execute_function.ts | 8 +- .../server/lib/action_executor.test.ts | 8 +- .../actions/server/lib/action_executor.ts | 10 +- .../server/lib/task_runner_factory.test.ts | 8 +- x-pack/plugins/actions/server/plugin.test.ts | 44 +-- x-pack/plugins/actions/server/plugin.ts | 27 +- x-pack/plugins/alerts/server/plugin.test.ts | 18 +- x-pack/plugins/alerts/server/plugin.ts | 15 +- .../alerts/server/routes/health.test.ts | 26 +- x-pack/plugins/alerts/server/routes/health.ts | 2 +- .../server/config.test.ts | 68 ++-- .../encrypted_saved_objects/server/config.ts | 24 +- .../encrypted_saved_objects_service.test.ts | 334 ++++++++++++++++++ .../crypto/encrypted_saved_objects_service.ts | 53 ++- .../encrypted_saved_objects/server/mocks.ts | 6 +- .../server/plugin.test.ts | 28 +- .../encrypted_saved_objects/server/plugin.ts | 37 +- .../server/routes/index.mock.ts | 4 +- x-pack/plugins/fleet/kibana.json | 5 +- x-pack/plugins/fleet/server/plugin.ts | 8 +- .../fleet/server/routes/setup/handlers.ts | 4 +- .../elasticsearch/verify_alerting_security.ts | 2 +- .../privileges/read_privileges_route.test.ts | 2 +- .../privileges/read_privileges_route.ts | 4 +- .../security_solution/server/plugin.ts | 2 +- .../security_solution/server/routes/index.ts | 4 +- 30 files changed, 543 insertions(+), 238 deletions(-) diff --git a/x-pack/plugins/actions/server/action_type_registry.test.ts b/x-pack/plugins/actions/server/action_type_registry.test.ts index 813e47c2e99574..c8972d8113f162 100644 --- a/x-pack/plugins/actions/server/action_type_registry.test.ts +++ b/x-pack/plugins/actions/server/action_type_registry.test.ts @@ -26,9 +26,7 @@ beforeEach(() => { actionTypeRegistryParams = { licensing: licensingMock.createSetup(), taskManager: mockTaskManager, - taskRunnerFactory: new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ), + taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })), actionsConfigUtils: mockedActionsConfig, licenseState: mockedLicenseState, preconfiguredActions: [ diff --git a/x-pack/plugins/actions/server/actions_client.test.ts b/x-pack/plugins/actions/server/actions_client.test.ts index 1bea3e1fc356d3..3bd8bb5f1ba52e 100644 --- a/x-pack/plugins/actions/server/actions_client.test.ts +++ b/x-pack/plugins/actions/server/actions_client.test.ts @@ -59,9 +59,7 @@ beforeEach(() => { actionTypeRegistryParams = { licensing: licensingMock.createSetup(), taskManager: mockTaskManager, - taskRunnerFactory: new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ), + taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })), actionsConfigUtils: actionsConfigMock.create(), licenseState: mockedLicenseState, preconfiguredActions: [], @@ -411,9 +409,7 @@ describe('create()', () => { const localActionTypeRegistryParams = { licensing: licensingMock.createSetup(), taskManager: mockTaskManager, - taskRunnerFactory: new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ), + taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })), actionsConfigUtils: localConfigUtils, licenseState: licenseStateMock.create(), preconfiguredActions: [], diff --git a/x-pack/plugins/actions/server/builtin_action_types/index.test.ts b/x-pack/plugins/actions/server/builtin_action_types/index.test.ts index bad709247d0804..10955af2f3b13d 100644 --- a/x-pack/plugins/actions/server/builtin_action_types/index.test.ts +++ b/x-pack/plugins/actions/server/builtin_action_types/index.test.ts @@ -33,9 +33,7 @@ export function createActionTypeRegistry(): { const actionTypeRegistry = new ActionTypeRegistry({ taskManager: taskManagerMock.createSetup(), licensing: licensingMock.createSetup(), - taskRunnerFactory: new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ), + taskRunnerFactory: new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })), actionsConfigUtils: actionsConfigMock.create(), licenseState: licenseStateMock.create(), preconfiguredActions: [], diff --git a/x-pack/plugins/actions/server/create_execute_function.test.ts b/x-pack/plugins/actions/server/create_execute_function.test.ts index aaf11669c1d03c..d4100537fa6b8e 100644 --- a/x-pack/plugins/actions/server/create_execute_function.test.ts +++ b/x-pack/plugins/actions/server/create_execute_function.test.ts @@ -28,7 +28,7 @@ describe('execute()', () => { const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, actionTypeRegistry, - isESOUsingEphemeralEncryptionKey: false, + isESOCanEncrypt: true, preconfiguredActions: [], }); savedObjectsClient.get.mockResolvedValueOnce({ @@ -87,7 +87,7 @@ describe('execute()', () => { const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, actionTypeRegistry: actionTypeRegistryMock.create(), - isESOUsingEphemeralEncryptionKey: false, + isESOCanEncrypt: true, preconfiguredActions: [ { id: '123', @@ -158,10 +158,10 @@ describe('execute()', () => { ); }); - test('throws when passing isESOUsingEphemeralEncryptionKey with true as a value', async () => { + test('throws when passing isESOCanEncrypt with false as a value', async () => { const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - isESOUsingEphemeralEncryptionKey: true, + isESOCanEncrypt: false, actionTypeRegistry: actionTypeRegistryMock.create(), preconfiguredActions: [], }); @@ -173,7 +173,7 @@ describe('execute()', () => { apiKey: null, }) ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Unable to execute action because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` + `"Unable to execute action because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` ); }); @@ -181,7 +181,7 @@ describe('execute()', () => { const mockedActionTypeRegistry = actionTypeRegistryMock.create(); const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - isESOUsingEphemeralEncryptionKey: false, + isESOCanEncrypt: true, actionTypeRegistry: mockedActionTypeRegistry, preconfiguredActions: [], }); @@ -211,7 +211,7 @@ describe('execute()', () => { const mockedActionTypeRegistry = actionTypeRegistryMock.create(); const executeFn = createExecutionEnqueuerFunction({ taskManager: mockTaskManager, - isESOUsingEphemeralEncryptionKey: false, + isESOCanEncrypt: true, actionTypeRegistry: mockedActionTypeRegistry, preconfiguredActions: [ { diff --git a/x-pack/plugins/actions/server/create_execute_function.ts b/x-pack/plugins/actions/server/create_execute_function.ts index 0d75c0b410e449..025b4d31077985 100644 --- a/x-pack/plugins/actions/server/create_execute_function.ts +++ b/x-pack/plugins/actions/server/create_execute_function.ts @@ -14,7 +14,7 @@ import { isSavedObjectExecutionSource } from './lib'; interface CreateExecuteFunctionOptions { taskManager: TaskManagerStartContract; - isESOUsingEphemeralEncryptionKey: boolean; + isESOCanEncrypt: boolean; actionTypeRegistry: ActionTypeRegistryContract; preconfiguredActions: PreConfiguredAction[]; } @@ -33,16 +33,16 @@ export type ExecutionEnqueuer = ( export function createExecutionEnqueuerFunction({ taskManager, actionTypeRegistry, - isESOUsingEphemeralEncryptionKey, + isESOCanEncrypt, preconfiguredActions, }: CreateExecuteFunctionOptions) { return async function execute( unsecuredSavedObjectsClient: SavedObjectsClientContract, { id, params, spaceId, source, apiKey }: ExecuteOptions ) { - if (isESOUsingEphemeralEncryptionKey === true) { + if (!isESOCanEncrypt) { throw new Error( - `Unable to execute action because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` + `Unable to execute action because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` ); } diff --git a/x-pack/plugins/actions/server/lib/action_executor.test.ts b/x-pack/plugins/actions/server/lib/action_executor.test.ts index e9b72f9bf0e4ed..8ec94c4d4a5521 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.test.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.test.ts @@ -17,7 +17,7 @@ import { ActionType } from '../types'; import { actionsMock, actionsClientMock } from '../mocks'; import { pick } from 'lodash'; -const actionExecutor = new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }); +const actionExecutor = new ActionExecutor({ isESOCanEncrypt: true }); const services = actionsMock.createServices(); const actionsClient = actionsClientMock.create(); @@ -310,8 +310,8 @@ test('should not throws an error if actionType is preconfigured', async () => { }); }); -test('throws an error when passing isESOUsingEphemeralEncryptionKey with value of true', async () => { - const customActionExecutor = new ActionExecutor({ isESOUsingEphemeralEncryptionKey: true }); +test('throws an error when passing isESOCanEncrypt with value of false', async () => { + const customActionExecutor = new ActionExecutor({ isESOCanEncrypt: false }); customActionExecutor.initialize({ logger: loggingSystemMock.create().get(), spaces: spacesMock, @@ -325,7 +325,7 @@ test('throws an error when passing isESOUsingEphemeralEncryptionKey with value o await expect( customActionExecutor.execute(executeParams) ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Unable to execute action because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` + `"Unable to execute action because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` ); }); diff --git a/x-pack/plugins/actions/server/lib/action_executor.ts b/x-pack/plugins/actions/server/lib/action_executor.ts index 7a54f88e2f27c5..6deaa4d587904d 100644 --- a/x-pack/plugins/actions/server/lib/action_executor.ts +++ b/x-pack/plugins/actions/server/lib/action_executor.ts @@ -48,10 +48,10 @@ export type ActionExecutorContract = PublicMethodsOf; export class ActionExecutor { private isInitialized = false; private actionExecutorContext?: ActionExecutorContext; - private readonly isESOUsingEphemeralEncryptionKey: boolean; + private readonly isESOCanEncrypt: boolean; - constructor({ isESOUsingEphemeralEncryptionKey }: { isESOUsingEphemeralEncryptionKey: boolean }) { - this.isESOUsingEphemeralEncryptionKey = isESOUsingEphemeralEncryptionKey; + constructor({ isESOCanEncrypt }: { isESOCanEncrypt: boolean }) { + this.isESOCanEncrypt = isESOCanEncrypt; } public initialize(actionExecutorContext: ActionExecutorContext) { @@ -72,9 +72,9 @@ export class ActionExecutor { throw new Error('ActionExecutor not initialized'); } - if (this.isESOUsingEphemeralEncryptionKey === true) { + if (!this.isESOCanEncrypt) { throw new Error( - `Unable to execute action because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` + `Unable to execute action because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` ); } diff --git a/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts b/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts index e42fc363f328b2..9e101f2ee76b0c 100644 --- a/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts +++ b/x-pack/plugins/actions/server/lib/task_runner_factory.test.ts @@ -84,18 +84,14 @@ beforeEach(() => { }); test(`throws an error if factory isn't initialized`, () => { - const factory = new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ); + const factory = new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })); expect(() => factory.create({ taskInstance: mockedTaskInstance }) ).toThrowErrorMatchingInlineSnapshot(`"TaskRunnerFactory not initialized"`); }); test(`throws an error if factory is already initialized`, () => { - const factory = new TaskRunnerFactory( - new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false }) - ); + const factory = new TaskRunnerFactory(new ActionExecutor({ isESOCanEncrypt: true })); factory.initialize(taskRunnerFactoryInitializerParams); expect(() => factory.initialize(taskRunnerFactoryInitializerParams) diff --git a/x-pack/plugins/actions/server/plugin.test.ts b/x-pack/plugins/actions/server/plugin.test.ts index 187cba9d3240c3..0e916220ca9468 100644 --- a/x-pack/plugins/actions/server/plugin.test.ts +++ b/x-pack/plugins/actions/server/plugin.test.ts @@ -51,25 +51,21 @@ describe('Actions Plugin', () => { }; }); - it('should log warning when Encrypted Saved Objects plugin is using an ephemeral encryption key', async () => { - // coreMock.createSetup doesn't support Plugin generics - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await plugin.setup(coreSetup as any, pluginsSetup); - expect(pluginsSetup.encryptedSavedObjects.usingEphemeralEncryptionKey).toEqual(true); + it('should log warning when Encrypted Saved Objects plugin is missing encryption key', async () => { + await plugin.setup(coreSetup, pluginsSetup); + expect(pluginsSetup.encryptedSavedObjects.canEncrypt).toEqual(false); expect(context.logger.get().warn).toHaveBeenCalledWith( - 'APIs are disabled because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' + 'APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' ); }); describe('routeHandlerContext.getActionsClient()', () => { - it('should not throw error when ESO plugin not using a generated key', async () => { - // coreMock.createSetup doesn't support Plugin generics - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await plugin.setup(coreSetup as any, { + it('should not throw error when ESO plugin has encryption key', async () => { + await plugin.setup(coreSetup, { ...pluginsSetup, encryptedSavedObjects: { ...pluginsSetup.encryptedSavedObjects, - usingEphemeralEncryptionKey: false, + canEncrypt: true, }, }); @@ -99,10 +95,8 @@ describe('Actions Plugin', () => { actionsContextHandler!.getActionsClient(); }); - it('should throw error when ESO plugin using a generated key', async () => { - // coreMock.createSetup doesn't support Plugin generics - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await plugin.setup(coreSetup as any, pluginsSetup); + it('should throw error when ESO plugin is missing encryption key', async () => { + await plugin.setup(coreSetup, pluginsSetup); expect(coreSetup.http.registerRouteHandlerContext).toHaveBeenCalledTimes(1); const handler = coreSetup.http.registerRouteHandlerContext.mock.calls[0] as [ @@ -123,7 +117,7 @@ describe('Actions Plugin', () => { httpServerMock.createResponseFactory() )) as unknown) as ActionsApiRequestHandlerContext; expect(() => actionsContextHandler!.getActionsClient()).toThrowErrorMatchingInlineSnapshot( - `"Unable to create actions client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` + `"Unable to create actions client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` ); }); }); @@ -234,14 +228,12 @@ describe('Actions Plugin', () => { expect(pluginStart.isActionExecutable('preconfiguredServerLog', '.server-log')).toBe(true); }); - it('should not throw error when ESO plugin not using a generated key', async () => { - // coreMock.createSetup doesn't support Plugin generics - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await plugin.setup(coreSetup as any, { + it('should not throw error when ESO plugin has encryption key', async () => { + await plugin.setup(coreSetup, { ...pluginsSetup, encryptedSavedObjects: { ...pluginsSetup.encryptedSavedObjects, - usingEphemeralEncryptionKey: false, + canEncrypt: true, }, }); const pluginStart = await plugin.start(coreStart, pluginsStart); @@ -249,17 +241,15 @@ describe('Actions Plugin', () => { await pluginStart.getActionsClientWithRequest(httpServerMock.createKibanaRequest()); }); - it('should throw error when ESO plugin using generated key', async () => { - // coreMock.createSetup doesn't support Plugin generics - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await plugin.setup(coreSetup as any, pluginsSetup); + it('should throw error when ESO plugin is missing encryption key', async () => { + await plugin.setup(coreSetup, pluginsSetup); const pluginStart = await plugin.start(coreStart, pluginsStart); - expect(pluginsSetup.encryptedSavedObjects.usingEphemeralEncryptionKey).toEqual(true); + expect(pluginsSetup.encryptedSavedObjects.canEncrypt).toEqual(false); await expect( pluginStart.getActionsClientWithRequest(httpServerMock.createKibanaRequest()) ).rejects.toThrowErrorMatchingInlineSnapshot( - `"Unable to create actions client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` + `"Unable to create actions client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` ); }); }); diff --git a/x-pack/plugins/actions/server/plugin.ts b/x-pack/plugins/actions/server/plugin.ts index 8fbacc71d30cb3..c4159c80e806f2 100644 --- a/x-pack/plugins/actions/server/plugin.ts +++ b/x-pack/plugins/actions/server/plugin.ts @@ -144,7 +144,7 @@ export class ActionsPlugin implements Plugin ) => { - if (isESOUsingEphemeralEncryptionKey === true) { + if (isESOCanEncrypt !== true) { throw new Error( - `Unable to create actions client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` + `Unable to create actions client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` ); } @@ -314,7 +313,7 @@ export class ActionsPlugin implements Plugin => { const { actionTypeRegistry, - isESOUsingEphemeralEncryptionKey, + isESOCanEncrypt, preconfiguredActions, actionExecutor, instantiateAuthorization, @@ -448,9 +447,9 @@ export class ActionsPlugin implements Plugin { - if (isESOUsingEphemeralEncryptionKey === true) { + if (isESOCanEncrypt !== true) { throw new Error( - `Unable to create actions client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` + `Unable to create actions client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` ); } return new ActionsClient({ @@ -468,7 +467,7 @@ export class ActionsPlugin implements Plugin { let coreSetup: ReturnType; let pluginsSetup: jest.Mocked; - it('should log warning when Encrypted Saved Objects plugin is using an ephemeral encryption key', async () => { + it('should log warning when Encrypted Saved Objects plugin is missing encryption key', async () => { const context = coreMock.createPluginInitializerContext({ healthCheck: { interval: '5m', @@ -40,7 +40,7 @@ describe('Alerting Plugin', () => { const encryptedSavedObjectsSetup = encryptedSavedObjectsMock.createSetup(); const setupMocks = coreMock.createSetup(); - // need await to test number of calls of setupMocks.status.set, becuase it is under async function which awaiting core.getStartServices() + // need await to test number of calls of setupMocks.status.set, because it is under async function which awaiting core.getStartServices() await plugin.setup(setupMocks, { licensing: licensingMock.createSetup(), encryptedSavedObjects: encryptedSavedObjectsSetup, @@ -51,9 +51,9 @@ describe('Alerting Plugin', () => { }); expect(setupMocks.status.set).toHaveBeenCalledTimes(1); - expect(encryptedSavedObjectsSetup.usingEphemeralEncryptionKey).toEqual(true); + expect(encryptedSavedObjectsSetup.canEncrypt).toEqual(false); expect(context.logger.get().warn).toHaveBeenCalledWith( - 'APIs are disabled because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' + 'APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' ); }); @@ -110,7 +110,7 @@ describe('Alerting Plugin', () => { describe('start()', () => { describe('getAlertsClientWithRequest()', () => { - it('throws error when encryptedSavedObjects plugin has usingEphemeralEncryptionKey set to true', async () => { + it('throws error when encryptedSavedObjects plugin is missing encryption key', async () => { const context = coreMock.createPluginInitializerContext({ healthCheck: { interval: '5m', @@ -141,15 +141,15 @@ describe('Alerting Plugin', () => { taskManager: taskManagerMock.createStart(), }); - expect(encryptedSavedObjectsSetup.usingEphemeralEncryptionKey).toEqual(true); + expect(encryptedSavedObjectsSetup.canEncrypt).toEqual(false); expect(() => startContract.getAlertsClientWithRequest({} as KibanaRequest) ).toThrowErrorMatchingInlineSnapshot( - `"Unable to create alerts client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` + `"Unable to create alerts client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command."` ); }); - it(`doesn't throw error when encryptedSavedObjects plugin has usingEphemeralEncryptionKey set to false`, async () => { + it(`doesn't throw error when encryptedSavedObjects plugin has encryption key`, async () => { const context = coreMock.createPluginInitializerContext({ healthCheck: { interval: '5m', @@ -163,7 +163,7 @@ describe('Alerting Plugin', () => { const encryptedSavedObjectsSetup = { ...encryptedSavedObjectsMock.createSetup(), - usingEphemeralEncryptionKey: false, + canEncrypt: true, }; plugin.setup(coreMock.createSetup(), { licensing: licensingMock.createSetup(), diff --git a/x-pack/plugins/alerts/server/plugin.ts b/x-pack/plugins/alerts/server/plugin.ts index aaec0bb8a080d5..8dba4453d56827 100644 --- a/x-pack/plugins/alerts/server/plugin.ts +++ b/x-pack/plugins/alerts/server/plugin.ts @@ -153,7 +153,7 @@ export class AlertingPlugin { private alertTypeRegistry?: AlertTypeRegistry; private readonly taskRunnerFactory: TaskRunnerFactory; private licenseState: ILicenseState | null = null; - private isESOUsingEphemeralEncryptionKey?: boolean; + private isESOCanEncrypt?: boolean; private security?: SecurityPluginSetup; private readonly alertsClientFactory: AlertsClientFactory; private readonly telemetryLogger: Logger; @@ -189,12 +189,11 @@ export class AlertingPlugin { }; }); - this.isESOUsingEphemeralEncryptionKey = - plugins.encryptedSavedObjects.usingEphemeralEncryptionKey; + this.isESOCanEncrypt = plugins.encryptedSavedObjects.canEncrypt; - if (this.isESOUsingEphemeralEncryptionKey) { + if (!this.isESOCanEncrypt) { this.logger.warn( - 'APIs are disabled because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' + 'APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' ); } @@ -311,7 +310,7 @@ export class AlertingPlugin { public start(core: CoreStart, plugins: AlertingPluginsStart): PluginStartContract { const { - isESOUsingEphemeralEncryptionKey, + isESOCanEncrypt, logger, taskRunnerFactory, alertTypeRegistry, @@ -353,9 +352,9 @@ export class AlertingPlugin { }); const getAlertsClientWithRequest = (request: KibanaRequest) => { - if (isESOUsingEphemeralEncryptionKey === true) { + if (isESOCanEncrypt !== true) { throw new Error( - `Unable to create alerts client because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` + `Unable to create alerts client because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.` ); } return alertsClientFactory!.create(request, core.savedObjects); diff --git a/x-pack/plugins/alerts/server/routes/health.test.ts b/x-pack/plugins/alerts/server/routes/health.test.ts index 38bae896e40ba9..22df0e6a000463 100644 --- a/x-pack/plugins/alerts/server/routes/health.test.ts +++ b/x-pack/plugins/alerts/server/routes/health.test.ts @@ -47,8 +47,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [config] = router.get.mock.calls[0]; @@ -60,8 +59,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -85,12 +83,11 @@ describe('healthRoute', () => { `); }); - it('evaluates whether Encrypted Saved Objects is using an ephemeral encryption key', async () => { + it('evaluates whether Encrypted Saved Objects is missing encryption key', async () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = true; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: false }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -129,8 +126,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -169,8 +165,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -209,8 +204,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -249,8 +243,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; @@ -291,8 +284,7 @@ describe('healthRoute', () => { const router = httpServiceMock.createRouter(); const licenseState = licenseStateMock.create(); - const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup(); - encryptedSavedObjects.usingEphemeralEncryptionKey = false; + const encryptedSavedObjects = encryptedSavedObjectsMock.createSetup({ canEncrypt: true }); healthRoute(router, licenseState, encryptedSavedObjects); const [, handler] = router.get.mock.calls[0]; diff --git a/x-pack/plugins/alerts/server/routes/health.ts b/x-pack/plugins/alerts/server/routes/health.ts index 24b3642ca20857..9e1f01041e0912 100644 --- a/x-pack/plugins/alerts/server/routes/health.ts +++ b/x-pack/plugins/alerts/server/routes/health.ts @@ -55,7 +55,7 @@ export function healthRoute( const frameworkHealth: AlertingFrameworkHealth = { isSufficientlySecure: !isSecurityEnabled || (isSecurityEnabled && isTLSEnabled), - hasPermanentEncryptionKey: !encryptedSavedObjects.usingEphemeralEncryptionKey, + hasPermanentEncryptionKey: encryptedSavedObjects.canEncrypt, alertingFrameworkHeath, }; diff --git a/x-pack/plugins/encrypted_saved_objects/server/config.test.ts b/x-pack/plugins/encrypted_saved_objects/server/config.test.ts index 3633dae824a2b9..1cc5f7974cb136 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/config.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/config.test.ts @@ -5,10 +5,7 @@ * 2.0. */ -jest.mock('crypto', () => ({ randomBytes: jest.fn() })); - -import { loggingSystemMock } from 'src/core/server/mocks'; -import { createConfig, ConfigSchema } from './config'; +import { ConfigSchema } from './config'; describe('config schema', () => { it('generates proper defaults', () => { @@ -32,6 +29,17 @@ describe('config schema', () => { } `); + expect(ConfigSchema.validate({ encryptionKey: 'z'.repeat(32) }, { dist: true })) + .toMatchInlineSnapshot(` + Object { + "enabled": true, + "encryptionKey": "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", + "keyRotation": Object { + "decryptionOnlyKeys": Array [], + }, + } + `); + expect(ConfigSchema.validate({}, { dist: true })).toMatchInlineSnapshot(` Object { "enabled": true, @@ -79,6 +87,18 @@ describe('config schema', () => { ); }); + it('should not allow `null` value for the encryption key', () => { + expect(() => ConfigSchema.validate({ encryptionKey: null })).toThrowErrorMatchingInlineSnapshot( + `"[encryptionKey]: expected value of type [string] but got [null]"` + ); + + expect(() => + ConfigSchema.validate({ encryptionKey: null }, { dist: true }) + ).toThrowErrorMatchingInlineSnapshot( + `"[encryptionKey]: expected value of type [string] but got [null]"` + ); + }); + it('should throw error if any of the xpack.encryptedSavedObjects.keyRotation.decryptionOnlyKeys is less than 32 characters', () => { expect(() => ConfigSchema.validate({ @@ -121,43 +141,3 @@ describe('config schema', () => { ); }); }); - -describe('createConfig()', () => { - it('should log a warning, set xpack.encryptedSavedObjects.encryptionKey and usingEphemeralEncryptionKey=true when encryptionKey is not set', () => { - const mockRandomBytes = jest.requireMock('crypto').randomBytes; - mockRandomBytes.mockReturnValue('ab'.repeat(16)); - - const logger = loggingSystemMock.create().get(); - const config = createConfig(ConfigSchema.validate({}, { dist: true }), logger); - expect(config).toEqual({ - enabled: true, - encryptionKey: 'ab'.repeat(16), - keyRotation: { decryptionOnlyKeys: [] }, - usingEphemeralEncryptionKey: true, - }); - - expect(loggingSystemMock.collect(logger).warn).toMatchInlineSnapshot(` - Array [ - Array [ - "Generating a random key for xpack.encryptedSavedObjects.encryptionKey. To decrypt encrypted saved objects attributes after restart, please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.", - ], - ] - `); - }); - - it('should not log a warning and set usingEphemeralEncryptionKey=false when encryptionKey is set', async () => { - const logger = loggingSystemMock.create().get(); - const config = createConfig( - ConfigSchema.validate({ encryptionKey: 'supersecret'.repeat(3) }, { dist: true }), - logger - ); - expect(config).toEqual({ - enabled: true, - encryptionKey: 'supersecret'.repeat(3), - keyRotation: { decryptionOnlyKeys: [] }, - usingEphemeralEncryptionKey: false, - }); - - expect(loggingSystemMock.collect(logger).warn).toEqual([]); - }); -}); diff --git a/x-pack/plugins/encrypted_saved_objects/server/config.ts b/x-pack/plugins/encrypted_saved_objects/server/config.ts index 40db0187162d0d..2bcf0e9b695111 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/config.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/config.ts @@ -5,11 +5,9 @@ * 2.0. */ -import crypto from 'crypto'; import { schema, TypeOf } from '@kbn/config-schema'; -import { Logger } from 'src/core/server'; -export type ConfigType = ReturnType; +export type ConfigType = TypeOf; export const ConfigSchema = schema.object( { @@ -33,23 +31,3 @@ export const ConfigSchema = schema.object( }, } ); - -export function createConfig(config: TypeOf, logger: Logger) { - let encryptionKey = config.encryptionKey; - const usingEphemeralEncryptionKey = encryptionKey === undefined; - if (encryptionKey === undefined) { - logger.warn( - 'Generating a random key for xpack.encryptedSavedObjects.encryptionKey. ' + - 'To decrypt encrypted saved objects attributes after restart, ' + - 'please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' - ); - - encryptionKey = crypto.randomBytes(16).toString('hex'); - } - - return { - ...config, - encryptionKey, - usingEphemeralEncryptionKey, - }; -} diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts index 1760a858067865..f70810943d179f 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.test.ts @@ -226,6 +226,72 @@ describe('#stripOrDecryptAttributes', () => { ); }); }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be encrypted', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + await expect( + service.stripOrDecryptAttributes({ id: 'known-id', type: 'known-type-1' }, attributes) + ).resolves.toEqual({ attributes: { attrOne: 'one', attrTwo: 'two', attrThree: 'three' } }); + }); + + it('does not fail if there are attributes are supposed to be encrypted, but should be stripped', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + await expect( + service.stripOrDecryptAttributes({ id: 'known-id', type: 'known-type-1' }, attributes) + ).resolves.toEqual({ attributes: { attrTwo: 'two' } }); + }); + + it('fails if needs to decrypt any attribute', async () => { + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set([ + 'attrOne', + { key: 'attrThree', dangerouslyExposeValue: true }, + ]), + }); + + const mockUser = mockAuthenticatedUser(); + const { attributes, error } = await service.stripOrDecryptAttributes( + { type: 'known-type-1', id: 'object-id' }, + { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }, + undefined, + { user: mockUser } + ); + + expect(attributes).toEqual({ attrTwo: 'two' }); + + const encryptionError = error as EncryptionError; + expect(encryptionError.attributeName).toBe('attrThree'); + expect(encryptionError.message).toBe('Unable to decrypt attribute "attrThree"'); + expect(encryptionError.cause).toEqual( + new Error('Decryption is disabled because of missing decryption keys.') + ); + + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrThree', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); }); describe('#encryptAttributes', () => { @@ -465,6 +531,58 @@ describe('#encryptAttributes', () => { mockUser ); }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be encrypted', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + await expect( + service.encryptAttributes({ type: 'known-type-1', id: 'object-id' }, attributes) + ).resolves.toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.encryptAttributesSuccess).not.toHaveBeenCalled(); + }); + + it('fails if needs to encrypt any attribute', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + const mockUser = mockAuthenticatedUser(); + await expect( + service.encryptAttributes({ type: 'known-type-1', id: 'object-id' }, attributes, { + user: mockUser, + }) + ).rejects.toThrowError(EncryptionError); + + expect(attributes).toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.encryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.encryptAttributeFailure).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.encryptAttributeFailure).toHaveBeenCalledWith( + 'attrOne', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); }); describe('#decryptAttributes', () => { @@ -1099,6 +1217,88 @@ describe('#decryptAttributes', () => { expect(decryptionOnlyCryptoTwo.decrypt).not.toHaveBeenCalled(); }); }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be decrypted', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service = new EncryptedSavedObjectsService({ + decryptionOnlyCryptos: [], + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + await expect( + service.decryptAttributes({ type: 'known-type-1', id: 'object-id' }, attributes) + ).resolves.toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + }); + + it('does not fail if can decrypt attributes with decryption only keys', async () => { + const decryptionOnlyCryptoOne = createNodeCryptMock('old-key-one'); + decryptionOnlyCryptoOne.decrypt.mockImplementation( + async (encryptedOutput: string | Buffer, aad?: string) => `${encryptedOutput}||${aad}` + ); + + service = new EncryptedSavedObjectsService({ + decryptionOnlyCryptos: [decryptionOnlyCryptoOne], + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree', 'attrFour']), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three', attrFour: null }; + await expect( + service.decryptAttributes({ type: 'known-type-1', id: 'object-id' }, attributes) + ).resolves.toEqual({ + attrOne: 'one||["known-type-1","object-id",{"attrTwo":"two"}]', + attrTwo: 'two', + attrThree: 'three||["known-type-1","object-id",{"attrTwo":"two"}]', + attrFour: null, + }); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledWith( + ['attrOne', 'attrThree'], + { type: 'known-type-1', id: 'object-id' }, + undefined + ); + }); + + it('fails if needs to decrypt any attribute', async () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrOne']) }); + + const mockUser = mockAuthenticatedUser(); + await expect( + service.decryptAttributes({ type: 'known-type-1', id: 'object-id' }, attributes, { + user: mockUser, + }) + ).rejects.toThrowError(EncryptionError); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrOne', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); }); describe('#encryptAttributesSync', () => { @@ -1283,6 +1483,58 @@ describe('#encryptAttributesSync', () => { attrThree: 'three', }); }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be encrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.encryptAttributesSync({ type: 'known-type-1', id: 'object-id' }, attributes) + ).toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.encryptAttributesSuccess).not.toHaveBeenCalled(); + }); + + it('fails if needs to encrypt any attribute', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree']), + }); + + const mockUser = mockAuthenticatedUser(); + expect(() => + service.encryptAttributesSync({ type: 'known-type-1', id: 'object-id' }, attributes, { + user: mockUser, + }) + ).toThrowError(EncryptionError); + + expect(attributes).toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.encryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.encryptAttributeFailure).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.encryptAttributeFailure).toHaveBeenCalledWith( + 'attrOne', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); }); describe('#decryptAttributesSync', () => { @@ -1784,4 +2036,86 @@ describe('#decryptAttributesSync', () => { expect(decryptionOnlyCryptoTwo.decryptSync).not.toHaveBeenCalled(); }); }); + + describe('without encryption key', () => { + beforeEach(() => { + service = new EncryptedSavedObjectsService({ + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + }); + + it('does not fail if none of attributes are supposed to be decrypted', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service = new EncryptedSavedObjectsService({ + decryptionOnlyCryptos: [], + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrFour']) }); + + expect( + service.decryptAttributesSync({ type: 'known-type-1', id: 'object-id' }, attributes) + ).toEqual({ + attrOne: 'one', + attrTwo: 'two', + attrThree: 'three', + }); + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + }); + + it('does not fail if can decrypt attributes with decryption only keys', () => { + const decryptionOnlyCryptoOne = createNodeCryptMock('old-key-one'); + decryptionOnlyCryptoOne.decryptSync.mockImplementation( + (encryptedOutput: string | Buffer, aad?: string) => `${encryptedOutput}||${aad}` + ); + + service = new EncryptedSavedObjectsService({ + decryptionOnlyCryptos: [decryptionOnlyCryptoOne], + logger: loggingSystemMock.create().get(), + audit: mockAuditLogger, + }); + service.registerType({ + type: 'known-type-1', + attributesToEncrypt: new Set(['attrOne', 'attrThree', 'attrFour']), + }); + + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three', attrFour: null }; + expect( + service.decryptAttributesSync({ type: 'known-type-1', id: 'object-id' }, attributes) + ).toEqual({ + attrOne: 'one||["known-type-1","object-id",{"attrTwo":"two"}]', + attrTwo: 'two', + attrThree: 'three||["known-type-1","object-id",{"attrTwo":"two"}]', + attrFour: null, + }); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledTimes(1); + expect(mockAuditLogger.decryptAttributesSuccess).toHaveBeenCalledWith( + ['attrOne', 'attrThree'], + { type: 'known-type-1', id: 'object-id' }, + undefined + ); + }); + + it('fails if needs to decrypt any attribute', () => { + const attributes = { attrOne: 'one', attrTwo: 'two', attrThree: 'three' }; + + service.registerType({ type: 'known-type-1', attributesToEncrypt: new Set(['attrOne']) }); + + const mockUser = mockAuthenticatedUser(); + expect(() => + service.decryptAttributesSync({ type: 'known-type-1', id: 'object-id' }, attributes, { + user: mockUser, + }) + ).toThrowError(EncryptionError); + + expect(mockAuditLogger.decryptAttributesSuccess).not.toHaveBeenCalled(); + expect(mockAuditLogger.decryptAttributeFailure).toHaveBeenCalledWith( + 'attrOne', + { type: 'known-type-1', id: 'object-id' }, + mockUser + ); + }); + }); }); diff --git a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts index 91a3cfc921624d..23aef07ff8781f 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/crypto/encrypted_saved_objects_service.ts @@ -77,7 +77,7 @@ interface EncryptedSavedObjectsServiceOptions { /** * NodeCrypto instance used for both encryption and decryption. */ - primaryCrypto: Crypto; + primaryCrypto?: Crypto; /** * NodeCrypto instances used ONLY for decryption (i.e. rotated encryption keys). @@ -293,12 +293,17 @@ export class EncryptedSavedObjectsService { let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - try { - iteratorResult = iterator.next( - await this.options.primaryCrypto.encrypt(attributeValue, encryptionAAD) - ); - } catch (err) { - iterator.throw!(err); + // We check this inside of the iterator to throw only if we do need to encrypt anything. + if (this.options.primaryCrypto) { + try { + iteratorResult = iterator.next( + await this.options.primaryCrypto.encrypt(attributeValue, encryptionAAD) + ); + } catch (err) { + iterator.throw!(err); + } + } else { + iterator.throw!(new Error('Encryption is disabled because of missing encryption key.')); } } @@ -324,12 +329,17 @@ export class EncryptedSavedObjectsService { let iteratorResult = iterator.next(); while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - try { - iteratorResult = iterator.next( - this.options.primaryCrypto.encryptSync(attributeValue, encryptionAAD) - ); - } catch (err) { - iterator.throw!(err); + // We check this inside of the iterator to throw only if we do need to encrypt anything. + if (this.options.primaryCrypto) { + try { + iteratorResult = iterator.next( + this.options.primaryCrypto.encryptSync(attributeValue, encryptionAAD) + ); + } catch (err) { + iterator.throw!(err); + } + } else { + iterator.throw!(new Error('Encryption is disabled because of missing encryption key.')); } } @@ -358,7 +368,11 @@ export class EncryptedSavedObjectsService { while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - let decryptionError; + // We check this inside of the iterator to throw only if we do need to decrypt anything. + let decryptionError = + decrypters.length === 0 + ? new Error('Decryption is disabled because of missing decryption keys.') + : undefined; for (const decrypter of decrypters) { try { iteratorResult = iterator.next(await decrypter.decrypt(attributeValue, encryptionAAD)); @@ -402,7 +416,11 @@ export class EncryptedSavedObjectsService { while (!iteratorResult.done) { const [attributeValue, encryptionAAD] = iteratorResult.value; - let decryptionError; + // We check this inside of the iterator to throw only if we do need to decrypt anything. + let decryptionError = + decrypters.length === 0 + ? new Error('Decryption is disabled because of missing decryption keys.') + : undefined; for (const decrypter of decrypters) { try { iteratorResult = iterator.next(decrypter.decryptSync(attributeValue, encryptionAAD)); @@ -541,6 +559,9 @@ export class EncryptedSavedObjectsService { return this.options.decryptionOnlyCryptos; } - return [this.options.primaryCrypto, ...(this.options.decryptionOnlyCryptos ?? [])]; + return [ + ...(this.options.primaryCrypto ? [this.options.primaryCrypto] : []), + ...(this.options.decryptionOnlyCryptos ?? []), + ]; } } diff --git a/x-pack/plugins/encrypted_saved_objects/server/mocks.ts b/x-pack/plugins/encrypted_saved_objects/server/mocks.ts index 6c8196b2ae03c9..edb55513aabf5d 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/mocks.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/mocks.ts @@ -8,11 +8,13 @@ import { EncryptedSavedObjectsPluginSetup, EncryptedSavedObjectsPluginStart } from './plugin'; import { EncryptedSavedObjectsClient, EncryptedSavedObjectsClientOptions } from './saved_objects'; -function createEncryptedSavedObjectsSetupMock() { +function createEncryptedSavedObjectsSetupMock( + { canEncrypt }: { canEncrypt: boolean } = { canEncrypt: false } +) { return { registerType: jest.fn(), __legacyCompat: { registerLegacyAPI: jest.fn() }, - usingEphemeralEncryptionKey: true, + canEncrypt, createMigration: jest.fn(), } as jest.Mocked; } diff --git a/x-pack/plugins/encrypted_saved_objects/server/plugin.test.ts b/x-pack/plugins/encrypted_saved_objects/server/plugin.test.ts index 823a6b0afa9dc8..e71332b1c5aa7b 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/plugin.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/plugin.test.ts @@ -19,12 +19,28 @@ describe('EncryptedSavedObjects Plugin', () => { ); expect(plugin.setup(coreMock.createSetup(), { security: securityMock.createSetup() })) .toMatchInlineSnapshot(` - Object { - "createMigration": [Function], - "registerType": [Function], - "usingEphemeralEncryptionKey": true, - } - `); + Object { + "canEncrypt": false, + "createMigration": [Function], + "registerType": [Function], + } + `); + }); + + it('exposes proper contract when encryption key is set', () => { + const plugin = new EncryptedSavedObjectsPlugin( + coreMock.createPluginInitializerContext( + ConfigSchema.validate({ encryptionKey: 'z'.repeat(32) }, { dist: true }) + ) + ); + expect(plugin.setup(coreMock.createSetup(), { security: securityMock.createSetup() })) + .toMatchInlineSnapshot(` + Object { + "canEncrypt": true, + "createMigration": [Function], + "registerType": [Function], + } + `); }); }); diff --git a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts index e846b133c26e0b..c99d6bd32287d6 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/plugin.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/plugin.ts @@ -6,10 +6,9 @@ */ import nodeCrypto from '@elastic/node-crypto'; -import { Logger, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/server'; -import { TypeOf } from '@kbn/config-schema'; -import { SecurityPluginSetup } from '../../security/server'; -import { createConfig, ConfigSchema } from './config'; +import type { Logger, PluginInitializerContext, CoreSetup, Plugin } from 'src/core/server'; +import type { SecurityPluginSetup } from '../../security/server'; +import type { ConfigType } from './config'; import { EncryptedSavedObjectsService, EncryptedSavedObjectTypeRegistration, @@ -26,8 +25,11 @@ export interface PluginsSetup { } export interface EncryptedSavedObjectsPluginSetup { + /** + * Indicates if Saved Object encryption is possible. Requires an encryption key to be explicitly set via `xpack.encryptedSavedObjects.encryptionKey`. + */ + canEncrypt: boolean; registerType: (typeRegistration: EncryptedSavedObjectTypeRegistration) => void; - usingEphemeralEncryptionKey: boolean; createMigration: CreateEncryptedSavedObjectsMigrationFn; } @@ -50,19 +52,24 @@ export class EncryptedSavedObjectsPlugin } public setup(core: CoreSetup, deps: PluginsSetup): EncryptedSavedObjectsPluginSetup { - const config = createConfig( - this.initializerContext.config.get>(), - this.initializerContext.logger.get('config') - ); - const auditLogger = new EncryptedSavedObjectsAuditLogger( - deps.security?.audit.getLogger('encryptedSavedObjects') - ); + const config = this.initializerContext.config.get(); + const canEncrypt = config.encryptionKey !== undefined; + if (!canEncrypt) { + this.logger.warn( + 'Saved objects encryption key is not set. This will severely limit Kibana functionality. ' + + 'Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' + ); + } - const primaryCrypto = nodeCrypto({ encryptionKey: config.encryptionKey }); + const primaryCrypto = config.encryptionKey + ? nodeCrypto({ encryptionKey: config.encryptionKey }) + : undefined; const decryptionOnlyCryptos = config.keyRotation.decryptionOnlyKeys.map((decryptionKey) => nodeCrypto({ encryptionKey: decryptionKey }) ); - + const auditLogger = new EncryptedSavedObjectsAuditLogger( + deps.security?.audit.getLogger('encryptedSavedObjects') + ); const service = Object.freeze( new EncryptedSavedObjectsService({ primaryCrypto, @@ -94,9 +101,9 @@ export class EncryptedSavedObjectsPlugin }); return { + canEncrypt, registerType: (typeRegistration: EncryptedSavedObjectTypeRegistration) => service.registerType(typeRegistration), - usingEphemeralEncryptionKey: config.usingEphemeralEncryptionKey, createMigration: getCreateMigration( service, (typeRegistration: EncryptedSavedObjectTypeRegistration) => { diff --git a/x-pack/plugins/encrypted_saved_objects/server/routes/index.mock.ts b/x-pack/plugins/encrypted_saved_objects/server/routes/index.mock.ts index c2dbc4c163b445..32ac1617f4a7eb 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/routes/index.mock.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/routes/index.mock.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ConfigSchema, createConfig } from '../config'; +import { ConfigSchema, ConfigType } from '../config'; import { httpServiceMock, loggingSystemMock } from '../../../../../src/core/server/mocks'; import { encryptionKeyRotationServiceMock } from '../crypto/index.mock'; @@ -14,7 +14,7 @@ export const routeDefinitionParamsMock = { create: (config: Record = {}) => ({ router: httpServiceMock.createRouter(), logger: loggingSystemMock.create().get(), - config: createConfig(ConfigSchema.validate(config), loggingSystemMock.create().get()), + config: ConfigSchema.validate(config) as ConfigType, encryptionKeyRotationService: encryptionKeyRotationServiceMock.create(), }), }; diff --git a/x-pack/plugins/fleet/kibana.json b/x-pack/plugins/fleet/kibana.json index aa0761c8a39bd4..4a4019e3e9e47f 100644 --- a/x-pack/plugins/fleet/kibana.json +++ b/x-pack/plugins/fleet/kibana.json @@ -4,14 +4,13 @@ "server": true, "ui": true, "configPath": ["xpack", "fleet"], - "requiredPlugins": ["licensing", "data"], + "requiredPlugins": ["licensing", "data", "encryptedSavedObjects"], "optionalPlugins": [ "security", "features", "cloud", "usageCollection", - "home", - "encryptedSavedObjects" + "home" ], "extraPublicDirs": ["common"], "requiredBundles": ["kibanaReact", "esUiShared", "home", "infra", "kibanaUtils"] diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 7378d45e1bb3aa..d89db7f1ac3415 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -95,7 +95,7 @@ export interface FleetSetupDeps { } export interface FleetStartDeps { - encryptedSavedObjects?: EncryptedSavedObjectsPluginStart; + encryptedSavedObjects: EncryptedSavedObjectsPluginStart; security?: SecurityPluginStart; } @@ -255,11 +255,11 @@ export class FleetPlugin // Conditional config routes if (config.agents.enabled) { - const isESOUsingEphemeralEncryptionKey = !deps.encryptedSavedObjects; - if (isESOUsingEphemeralEncryptionKey) { + const isESOCanEncrypt = deps.encryptedSavedObjects.canEncrypt; + if (!isESOCanEncrypt) { if (this.logger) { this.logger.warn( - 'Fleet APIs are disabled because the Encrypted Saved Objects plugin uses an ephemeral encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' + 'Fleet APIs are disabled because the Encrypted Saved Objects plugin is missing encryption key. Please set xpack.encryptedSavedObjects.encryptionKey in the kibana.yml or use the bin/kibana-encryption-keys command.' ); } } else { diff --git a/x-pack/plugins/fleet/server/routes/setup/handlers.ts b/x-pack/plugins/fleet/server/routes/setup/handlers.ts index 1e74469107db45..0c6ba6d14b1be3 100644 --- a/x-pack/plugins/fleet/server/routes/setup/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/setup/handlers.ts @@ -24,7 +24,7 @@ export const getFleetStatusHandler: RequestHandler = async (context, request, re const isProductionMode = appContextService.getIsProductionMode(); const isCloud = appContextService.getCloud()?.isCloudEnabled ?? false; const isTLSCheckDisabled = appContextService.getConfig()?.agents?.tlsCheckDisabled ?? false; - const isUsingEphemeralEncryptionKey = !appContextService.getEncryptedSavedObjectsSetup(); + const canEncrypt = appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt === true; const missingRequirements: GetFleetStatusResponse['missing_requirements'] = []; if (!isAdminUserSetup) { @@ -37,7 +37,7 @@ export const getFleetStatusHandler: RequestHandler = async (context, request, re missingRequirements.push('tls_required'); } - if (isUsingEphemeralEncryptionKey) { + if (!canEncrypt) { missingRequirements.push('encrypted_saved_object_encryption_key_required'); } diff --git a/x-pack/plugins/monitoring/server/lib/elasticsearch/verify_alerting_security.ts b/x-pack/plugins/monitoring/server/lib/elasticsearch/verify_alerting_security.ts index c81b9632f0cd74..facb6e29236e37 100644 --- a/x-pack/plugins/monitoring/server/lib/elasticsearch/verify_alerting_security.ts +++ b/x-pack/plugins/monitoring/server/lib/elasticsearch/verify_alerting_security.ts @@ -44,7 +44,7 @@ export class AlertingSecurity { return { isSufficientlySecure: !isSecurityEnabled || (isSecurityEnabled && isTLSEnabled), - hasPermanentEncryptionKey: Boolean(encryptedSavedObjects), + hasPermanentEncryptionKey: encryptedSavedObjects?.canEncrypt === true, }; }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.test.ts index 3ee3c6884a3ec1..2efb65c4a49a24 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.test.ts @@ -18,7 +18,7 @@ describe('read_privileges route', () => { ({ clients, context } = requestContextMock.createTools()); clients.clusterClient.callAsCurrentUser.mockResolvedValue(getMockPrivilegesResult()); - readPrivilegesRoute(server.router, false); + readPrivilegesRoute(server.router, true); }); describe('normal status codes', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.ts index a934f0a0ce1349..f006d9250d3698 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/privileges/read_privileges_route.ts @@ -14,7 +14,7 @@ import { readPrivileges } from '../../privileges/read_privileges'; export const readPrivilegesRoute = ( router: SecuritySolutionPluginRouter, - usingEphemeralEncryptionKey: boolean + hasEncryptionKey: boolean ) => { router.get( { @@ -39,7 +39,7 @@ export const readPrivilegesRoute = ( const clusterPrivileges = await readPrivileges(clusterClient.callAsCurrentUser, index); const privileges = merge(clusterPrivileges, { is_authenticated: request.auth.isAuthenticated ?? false, - has_encryption_key: !usingEphemeralEncryptionKey, + has_encryption_key: hasEncryptionKey, }); return response.ok({ body: privileges }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 8c35fd2ce8f8ba..a34193937c788a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -183,7 +183,7 @@ export class Plugin implements IPlugin { @@ -102,5 +102,5 @@ export const initRoutes = ( readTagsRoute(router); // Privileges API to get the generic user privileges - readPrivilegesRoute(router, usingEphemeralEncryptionKey); + readPrivilegesRoute(router, hasEncryptionKey); }; From 4ee960380152b825862fa621e8b30474818cd9bd Mon Sep 17 00:00:00 2001 From: Sonja Krause-Harder Date: Wed, 10 Feb 2021 11:33:17 +0100 Subject: [PATCH 06/32] Use new shortcut links to Fleet discuss forums. (#90786) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fleet/public/applications/fleet/components/alpha_flyout.tsx | 2 +- .../plugins/fleet/public/applications/fleet/layouts/default.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/alpha_flyout.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/alpha_flyout.tsx index 82b2d200052256..c91d80124dd35a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/alpha_flyout.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/alpha_flyout.tsx @@ -61,7 +61,7 @@ export const AlphaFlyout: React.FunctionComponent = ({ onClose }) => { ), forumLink: ( - + = ({ Date: Wed, 10 Feb 2021 13:50:41 +0100 Subject: [PATCH 07/32] [Lens] (Accessibility) Fix focus on drag and drop actions (#90561) --- .../config_panel/config_panel.tsx | 55 +++------------ .../draggable_dimension_button.tsx | 15 +++- .../config_panel/layer_panel.test.tsx | 20 ++++-- .../editor_frame/config_panel/layer_panel.tsx | 32 +++++++-- .../config_panel/use_focus_update.tsx | 69 +++++++++++++++++++ .../functional/apps/lens/drag_and_drop.ts | 9 +++ .../test/functional/page_objects/lens_page.ts | 26 +++++++ 7 files changed, 168 insertions(+), 58 deletions(-) create mode 100644 x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/use_focus_update.tsx diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx index e5b07aacee16ef..393c7363dc03f9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/config_panel.tsx @@ -7,7 +7,7 @@ import './config_panel.scss'; -import React, { useMemo, memo, useEffect, useState, useCallback } from 'react'; +import React, { useMemo, memo } from 'react'; import { EuiFlexItem, EuiToolTip, EuiButton, EuiForm } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { Visualization } from '../../../types'; @@ -16,6 +16,7 @@ import { trackUiEvent } from '../../../lens_ui_telemetry'; import { generateId } from '../../../id_generator'; import { removeLayer, appendLayer } from './layer_actions'; import { ConfigPanelWrapperProps } from './types'; +import { useFocusUpdate } from './use_focus_update'; export const ConfigPanelWrapper = memo(function ConfigPanelWrapper(props: ConfigPanelWrapperProps) { const activeVisualization = props.visualizationMap[props.activeVisualizationId || '']; @@ -26,50 +27,6 @@ export const ConfigPanelWrapper = memo(function ConfigPanelWrapper(props: Config ) : null; }); -function useFocusUpdate(layerIds: string[]) { - const [nextFocusedLayerId, setNextFocusedLayerId] = useState(null); - const [layerRefs, setLayersRefs] = useState>({}); - - useEffect(() => { - const focusable = nextFocusedLayerId && layerRefs[nextFocusedLayerId]; - if (focusable) { - focusable.focus(); - setNextFocusedLayerId(null); - } - }, [layerIds, layerRefs, nextFocusedLayerId]); - - const setLayerRef = useCallback((layerId, el) => { - if (el) { - setLayersRefs((refs) => ({ - ...refs, - [layerId]: el, - })); - } - }, []); - - const removeLayerRef = useCallback( - (layerId) => { - if (layerIds.length <= 1) { - return setNextFocusedLayerId(layerId); - } - - const removedLayerIndex = layerIds.findIndex((l) => l === layerId); - const nextFocusedLayerIdId = - removedLayerIndex === 0 ? layerIds[1] : layerIds[removedLayerIndex - 1]; - - setLayersRefs((refs) => { - const newLayerRefs = { ...refs }; - delete newLayerRefs[layerId]; - return newLayerRefs; - }); - return setNextFocusedLayerId(nextFocusedLayerIdId); - }, - [layerIds] - ); - - return { setNextFocusedLayerId, removeLayerRef, setLayerRef }; -} - export function LayerPanels( props: ConfigPanelWrapperProps & { activeDatasourceId: string; @@ -85,7 +42,11 @@ export function LayerPanels( } = props; const layerIds = activeVisualization.getLayerIds(visualizationState); - const { setNextFocusedLayerId, removeLayerRef, setLayerRef } = useFocusUpdate(layerIds); + const { + setNextFocusedId: setNextFocusedLayerId, + removeRef: removeLayerRef, + registerNewRef: registerNewLayerRef, + } = useFocusUpdate(layerIds); const setVisualizationState = useMemo( () => (newState: unknown) => { @@ -145,7 +106,7 @@ export function LayerPanels( void; }) { const dropType = layerDatasource.getDropTypes({ ...layerDatasourceDropProps, @@ -94,8 +96,17 @@ export function DraggableDimensionButton({ [group.accessors] ); + const registerNewButtonRefMemoized = useCallback((el) => registerNewButtonRef(columnId, el), [ + registerNewButtonRef, + columnId, + ]); + return ( -
+
{ dispatch: jest.fn(), core: coreMock.createStart(), layerIndex: 0, - setLayerRef: jest.fn(), + registerNewLayerRef: jest.fn(), }; } @@ -620,17 +620,26 @@ describe('LayerPanel', () => { ); - - component.find(DragDrop).at(1).prop('onDrop')!(draggingOperation, 'reorder'); + act(() => { + component.find(DragDrop).at(1).prop('onDrop')!(draggingOperation, 'reorder'); + }); expect(mockDatasource.onDrop).toHaveBeenCalledWith( expect.objectContaining({ dropType: 'reorder', droppedItem: draggingOperation, }) ); + const secondButton = component + .find(DragDrop) + .at(1) + .find('[data-test-subj="lnsDragDrop-keyboardHandler"]') + .instance(); + const focusedEl = document.activeElement; + expect(focusedEl).toEqual(secondButton); }); it('should copy when dropping on empty slot in the same group', () => { + (generateId as jest.Mock).mockReturnValue(`newid`); mockVisualization.getConfiguration.mockReturnValue({ groups: [ { @@ -657,9 +666,12 @@ describe('LayerPanel', () => { ); - component.find(DragDrop).at(2).prop('onDrop')!(draggingOperation, 'duplicate_in_group'); + act(() => { + component.find(DragDrop).at(2).prop('onDrop')!(draggingOperation, 'duplicate_in_group'); + }); expect(mockDatasource.onDrop).toHaveBeenCalledWith( expect.objectContaining({ + columnId: 'newid', dropType: 'duplicate_in_group', droppedItem: draggingOperation, }) diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx index 80e9ed05b982d0..5ba73e98b42c12 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/config_panel/layer_panel.tsx @@ -26,6 +26,7 @@ import { RemoveLayerButton } from './remove_layer_button'; import { EmptyDimensionButton } from './empty_dimension_button'; import { DimensionButton } from './dimension_button'; import { DraggableDimensionButton } from './draggable_dimension_button'; +import { useFocusUpdate } from './use_focus_update'; const initialActiveDimensionState = { isNew: false, @@ -45,7 +46,7 @@ export function LayerPanel( newVisualizationState: unknown ) => void; onRemoveLayer: () => void; - setLayerRef: (layerId: string, instance: HTMLDivElement | null) => void; + registerNewLayerRef: (layerId: string, instance: HTMLDivElement | null) => void; } ) { const dragDropContext = useContext(DragContext); @@ -58,7 +59,7 @@ export function LayerPanel( layerId, isOnlyLayer, onRemoveLayer, - setLayerRef, + registerNewLayerRef, layerIndex, activeVisualization, updateVisualization, @@ -70,7 +71,10 @@ export function LayerPanel( setActiveDimension(initialActiveDimensionState); }, [activeVisualization.id]); - const setLayerRefMemoized = useCallback((el) => setLayerRef(layerId, el), [layerId, setLayerRef]); + const registerLayerRef = useCallback((el) => registerNewLayerRef(layerId, el), [ + layerId, + registerNewLayerRef, + ]); const layerVisualizationConfigProps = { layerId, @@ -114,6 +118,16 @@ export function LayerPanel( const { setDimension, removeDimension } = activeVisualization; const layerDatasourceOnDrop = layerDatasource.onDrop; + const allAccessors = groups.flatMap((group) => + group.accessors.map((accessor) => accessor.columnId) + ); + + const { + setNextFocusedId: setNextFocusedButtonId, + removeRef: removeButtonRef, + registerNewRef: registerNewButtonRef, + } = useFocusUpdate(allAccessors); + const onDrop = useMemo(() => { return ( droppedItem: DragDropIdentifier, @@ -127,7 +141,12 @@ export function LayerPanel( columnId, groupId, layerId: targetLayerId, - } = (targetItem as unknown) as DraggedOperation; // TODO: correct misleading name + } = (targetItem as unknown) as DraggedOperation; + if (dropType === 'reorder' || dropType === 'field_replace' || dropType === 'field_add') { + setNextFocusedButtonId(droppedItem.id); + } else { + setNextFocusedButtonId(columnId); + } const filterOperations = groups.find(({ groupId: gId }) => gId === targetItem.groupId)?.filterOperations || @@ -171,11 +190,12 @@ export function LayerPanel( setDimension, removeDimension, layerDatasourceDropProps, + setNextFocusedButtonId, ]); return ( -
+
@@ -264,6 +284,7 @@ export function LayerPanel( return ( { + const focusableSelector = 'button, [href], input, select, textarea, [tabindex]'; + if (!el) { + return null; + } + if (el.matches(focusableSelector)) { + return el; + } + const firstFocusable = el.querySelector(focusableSelector); + if (!firstFocusable) { + return null; + } + return (firstFocusable as unknown) as { focus: () => void }; +}; + +type RefsById = Record; + +export function useFocusUpdate(ids: string[]) { + const [nextFocusedId, setNextFocusedId] = useState(null); + const [refsById, setRefsById] = useState({}); + + useEffect(() => { + const element = nextFocusedId && refsById[nextFocusedId]; + if (element) { + const focusable = getFirstFocusable(element); + focusable?.focus(); + setNextFocusedId(null); + } + }, [ids, refsById, nextFocusedId]); + + const registerNewRef = useCallback((id, el) => { + if (el) { + setRefsById((r) => ({ + ...r, + [id]: el, + })); + } + }, []); + + const removeRef = useCallback( + (id) => { + if (ids.length <= 1) { + return setNextFocusedId(id); + } + + const removedIndex = ids.findIndex((l) => l === id); + + setRefsById((refs) => { + const newRefsById = { ...refs }; + delete newRefsById[id]; + return newRefsById; + }); + const next = removedIndex === 0 ? ids[1] : ids[removedIndex - 1]; + return setNextFocusedId(next); + }, + [ids] + ); + + return { setNextFocusedId, removeRef, registerNewRef }; +} diff --git a/x-pack/test/functional/apps/lens/drag_and_drop.ts b/x-pack/test/functional/apps/lens/drag_and_drop.ts index 5b3a984f005192..a272b67de1b0ad 100644 --- a/x-pack/test/functional/apps/lens/drag_and_drop.ts +++ b/x-pack/test/functional/apps/lens/drag_and_drop.ts @@ -143,6 +143,7 @@ export default function ({ getPageObjects }: FtrProviderContext) { expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_xDimensionPanel')).to.eql( '@timestamp' ); + await PageObjects.lens.assertFocusedField('@timestamp'); }); it('should drop a field to empty dimension', async () => { await PageObjects.lens.dragFieldWithKeyboard('bytes', 4); @@ -154,12 +155,15 @@ export default function ({ getPageObjects }: FtrProviderContext) { expect( await PageObjects.lens.getDimensionTriggersTexts('lnsXY_splitDimensionPanel') ).to.eql(['Top values of @message.raw']); + await PageObjects.lens.assertFocusedField('@message.raw'); }); it('should drop a field to an existing dimension replacing the old one', async () => { await PageObjects.lens.dragFieldWithKeyboard('clientip', 1, true); expect( await PageObjects.lens.getDimensionTriggersTexts('lnsXY_splitDimensionPanel') ).to.eql(['Top values of clientip']); + + await PageObjects.lens.assertFocusedField('clientip'); }); it('should duplicate an element in a group', async () => { await PageObjects.lens.dimensionKeyboardDragDrop('lnsXY_yDimensionPanel', 0, 1); @@ -168,6 +172,8 @@ export default function ({ getPageObjects }: FtrProviderContext) { 'Average of bytes', 'Count of records [1]', ]); + + await PageObjects.lens.assertFocusedDimension('Count of records [1]'); }); it('should move dimension to compatible dimension', async () => { @@ -186,6 +192,7 @@ export default function ({ getPageObjects }: FtrProviderContext) { expect( await PageObjects.lens.getDimensionTriggersTexts('lnsXY_splitDimensionPanel') ).to.eql([]); + await PageObjects.lens.assertFocusedDimension('@timestamp'); }); it('should move dimension to incompatible dimension', async () => { await PageObjects.lens.dimensionKeyboardDragDrop('lnsXY_yDimensionPanel', 1, 2); @@ -198,6 +205,7 @@ export default function ({ getPageObjects }: FtrProviderContext) { 'Count of records', 'Unique count of @timestamp', ]); + await PageObjects.lens.assertFocusedDimension('Unique count of @timestamp'); }); it('should reorder elements with keyboard', async () => { await PageObjects.lens.dimensionKeyboardReorder('lnsXY_yDimensionPanel', 0, 1); @@ -205,6 +213,7 @@ export default function ({ getPageObjects }: FtrProviderContext) { 'Unique count of @timestamp', 'Count of records', ]); + await PageObjects.lens.assertFocusedDimension('Count of records'); }); }); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index aae161ef9fcf19..add6979c2dde1f 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -707,5 +707,31 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await this.saveAndReturn(); } }, + + /** + * Asserts that the focused element is a field with a specified text + * + * @param name - the element visible text + */ + async assertFocusedField(name: string) { + const input = await find.activeElement(); + const fieldAncestor = await input.findByXpath('./../../..'); + const focusedElementText = await fieldAncestor.getVisibleText(); + const dataTestSubj = await fieldAncestor.getAttribute('data-test-subj'); + expect(focusedElementText).to.eql(name); + expect(dataTestSubj).to.eql('lnsFieldListPanelField'); + }, + + /** + * Asserts that the focused element is a dimension with with a specified text + * + * @param name - the element visible text + */ + async assertFocusedDimension(name: string) { + const input = await find.activeElement(); + const fieldAncestor = await input.findByXpath('./../../..'); + const focusedElementText = await fieldAncestor.getVisibleText(); + expect(focusedElementText).to.eql(name); + }, }); } From ce441bdc3258f1f35da3c787236d14b4d133e54c Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Wed, 10 Feb 2021 13:54:52 +0100 Subject: [PATCH 08/32] RFC Improve saved object migrations algorithm (#84333) * Instead of cloning, reindex legacy index * Reindex for every v2 migration * Use _reindex?require_alias=true and a write block toggle to prevent lost deletes * Use a ..._reindex_in_progress alias so that waiting for and preventing other reindex operations is idempotent The first version of the reindex block had only the instance which was able to mark the migration as complete set and remove the write block. This means other instances couldn't know if any reindex operaitons were in progress if the migration was already marked as complete. It also meant that a failure in this critical step could result in a permanent write block. * Revert "Use a ..._reindex_in_progress alias so that waiting for and preventing other reindex operations is idempotent" This reverts commit 8baf9b13dbbe50c1dec0d4844f170304ecc0b883. * Revert "Use _reindex?require_alias=true and a write block toggle to prevent lost deletes" This reverts commit d7237ca42c4167b6931fe8c544ac7c40e27afc6c. * Use reindex + clone as a way to prevent lost deletes * Fix numbering and ignore index_not_found_exceptionfor temporary index * Apply suggestions from code review Co-authored-by: Josh Dover Co-authored-by: Josh Dover Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- rfcs/text/0013_saved_object_migrations.md | 48 ++++++++++++----------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/rfcs/text/0013_saved_object_migrations.md b/rfcs/text/0013_saved_object_migrations.md index 6f5ab280a46122..88879e5e706eb4 100644 --- a/rfcs/text/0013_saved_object_migrations.md +++ b/rfcs/text/0013_saved_object_migrations.md @@ -248,45 +248,49 @@ Note: 6. Use the reindexed legacy `.kibana_pre6.5.0_001` as the source for the rest of the migration algorithm. 3. If `.kibana` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed. 1. Because the same version can have plugins enabled at any point in time, - perform the mappings update in step (7) and migrate outdated documents - with step (8). - 2. Skip to step (10) to start serving traffic. + migrate outdated documents with step (9) and perform the mappings update in step (10). + 2. Skip to step (12) to start serving traffic. 4. Fail the migration if: 1. `.kibana` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001` 2. (Only in 8.x) The source index contains documents that belong to an unknown Saved Object type (from a disabled plugin). Log an error explaining that the plugin that created these documents needs to be enabled again or that these objects should be deleted. See section (4.2.1.4). -5. Mark the source index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `//_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds. -6. Clone the source index into a new target index which has writes enabled. All nodes on the same version will use the same fixed index name e.g. `.kibana_7.10.0_001`. The `001` postfix isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`. - 1. `POST /.kibana_n/_clone/.kibana_7.10.0_001?wait_for_active_shards=all {"settings": {"index.blocks.write": false}}`. Ignore errors if the clone already exists. - 2. Wait for the cloning to complete `GET /_cluster/health/.kibana_7.10.0_001?wait_for_status=green&timeout=60s` If cloning doesn’t complete within the 60s timeout, log a warning for visibility and poll again. -7. Update the mappings of the target index +5. Set a write block on the source index. This prevents any further writes from outdated nodes. +6. Create a new temporary index `.kibana_7.10.0_reindex_temp` with `dynamic: false` on the top-level mappings so that any kind of document can be written to the index. This allows us to write untransformed documents to the index which might have fields which have been removed from the latest mappings defined by the plugin. Define minimal mappings for the `migrationVersion` and `type` fields so that we're still able to search for outdated documents that need to be transformed. + 1. Ignore errors if the target index already exists. +7. Reindex the source index into the new temporary index. + 1. Use `op_type=create` `conflicts=proceed` and `wait_for_completion=false` so that multiple instances can perform the reindex in parallel but only one write per document will succeed. + 2. Wait for the reindex task to complete. If reindexing doesn’t complete within the 60s timeout, log a warning for visibility and poll again. +8. Clone the temporary index into the target index `.kibana_7.10.0_001`. Since any further writes will only happen against the cloned target index this prevents a lost delete from occuring where one instance finishes the migration and deletes a document and another instance's reindex operation re-creates the deleted document. + 1. Set a write block on the temporary index + 2. Clone the temporary index into the target index while specifying that the target index should have writes enabled. + 3. If the clone operation fails because the target index already exist, ignore the error and wait for the target index to become green before proceeding. + 4. (The `001` postfix in the target index name isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`.) +9. Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. + 1. Ignore any version conflict errors. + 2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration. +10. Update the mappings of the target index 1. Retrieve the existing mappings including the `migrationMappingPropertyHashes` metadata. - 2. Update the mappings with `PUT /.kibana_7.10.0_001/_mapping`. The API deeply merges any updates so this won't remove the mappings of any plugins that were enabled in a previous version but are now disabled. + 2. Update the mappings with `PUT /.kibana_7.10.0_001/_mapping`. The API deeply merges any updates so this won't remove the mappings of any plugins that are disabled on this instance but have been enabled on another instance that also migrated this index. 3. Ensure that fields are correctly indexed using the target index's latest mappings `POST /.kibana_7.10.0_001/_update_by_query?conflicts=proceed`. In the future we could optimize this query by only targeting documents: 1. That belong to a known saved object type. - 2. Which don't have outdated migrationVersion numbers since these will be transformed anyway. - 3. That belong to a type whose mappings were changed by comparing the `migrationMappingPropertyHashes`. (Metadata, unlike the mappings isn't commutative, so there is a small chance that the metadata hashes do not accurately reflect the latest mappings, however, this will just result in an less efficient query). -8. Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. - 1. Ignore any version conflict errors. - 2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration. -9. Mark the migration as complete. This is done as a single atomic +11. Mark the migration as complete. This is done as a single atomic operation (requires https://github.com/elastic/elasticsearch/pull/58100) - to guarantees when multiple versions of Kibana are performing the + to guarantee that when multiple versions of Kibana are performing the migration in parallel, only one version will win. E.g. if 7.11 and 7.12 are started in parallel and migrate from a 7.9 index, either 7.11 or 7.12 should succeed and accept writes, but not both. - 3. Checks that `.kibana` alias is still pointing to the source index - 4. Points the `.kibana_7.10.0` and `.kibana` aliases to the target index. - 5. If this fails with a "required alias [.kibana] does not exist" error fetch `.kibana` again: + 1. Check that `.kibana` alias is still pointing to the source index + 2. Point the `.kibana_7.10.0` and `.kibana` aliases to the target index. + 3. Remove the temporary index `.kibana_7.10.0_reindex_temp` + 4. If this fails with a "required alias [.kibana] does not exist" error or "index_not_found_exception" for the temporary index, fetch `.kibana` again: 1. If `.kibana` is _not_ pointing to our target index fail the migration. - 2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (10). -10. Start serving traffic. All saved object reads/writes happen through the + 2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (12). +12. Start serving traffic. All saved object reads/writes happen through the version-specific alias `.kibana_7.10.0`. Together with the limitations, this algorithm ensures that migrations are idempotent. If two nodes are started simultaneously, both of them will start transforming documents in that version's target index, but because migrations are idempotent, it doesn’t matter which node’s writes win. - #### Known weaknesses: (Also present in our existing migration algorithm since v7.4) When the task manager index gets reindexed a reindex script is applied. From 3e91bc728d7cf13163a4528b530246f1a34bd7a6 Mon Sep 17 00:00:00 2001 From: ymao1 Date: Wed, 10 Feb 2021 08:06:09 -0500 Subject: [PATCH 09/32] [Alerting] License Errors on Alert List View (#89920) * Adding tooltips to alert list and modal for license upgrade * Fixing typings * Custom License Error status. Moving modal to alerts list page * Adding unit test * Cleanup * Unit tests * Removing tooltip from alert name * License * PR fixes * Updating modal wording * Updating license state error message * i18n fix * Fixing functional test Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../alerts/server/lib/license_state.test.ts | 2 +- .../alerts/server/lib/license_state.ts | 8 +- .../translations/translations/ja-JP.json | 1 - .../translations/translations/zh-CN.json | 1 - .../components/alerts_list.test.tsx | 289 +++++++++++------- .../alerts_list/components/alerts_list.tsx | 132 +++++--- .../components/manage_license_modal.tsx | 62 ++++ .../sections/alerts_list/translations.ts | 7 + .../tests/alerts/gold_noop_alert_type.ts | 2 +- 9 files changed, 355 insertions(+), 149 deletions(-) create mode 100644 x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/manage_license_modal.tsx diff --git a/x-pack/plugins/alerts/server/lib/license_state.test.ts b/x-pack/plugins/alerts/server/lib/license_state.test.ts index 07074b91875474..a1c326656f735a 100644 --- a/x-pack/plugins/alerts/server/lib/license_state.test.ts +++ b/x-pack/plugins/alerts/server/lib/license_state.test.ts @@ -248,7 +248,7 @@ describe('ensureLicenseForAlertType()', () => { expect(() => licenseState.ensureLicenseForAlertType(alertType) ).toThrowErrorMatchingInlineSnapshot( - `"Alert test is disabled because it requires a Gold license. Contact your administrator to upgrade your license."` + `"Alert test is disabled because it requires a Gold license. Go to License Management to view upgrade options."` ); }); diff --git a/x-pack/plugins/alerts/server/lib/license_state.ts b/x-pack/plugins/alerts/server/lib/license_state.ts index f95c6cb42a17b3..238b2e97c4cdf9 100644 --- a/x-pack/plugins/alerts/server/lib/license_state.ts +++ b/x-pack/plugins/alerts/server/lib/license_state.ts @@ -9,6 +9,7 @@ import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { assertNever } from '@kbn/std'; +import { capitalize } from 'lodash'; import { Observable, Subscription } from 'rxjs'; import { LicensingPluginStart } from '../../../licensing/server'; import { ILicense, LicenseType } from '../../../licensing/common/types'; @@ -190,8 +191,11 @@ export class LicenseState { throw new AlertTypeDisabledError( i18n.translate('xpack.alerts.serverSideErrors.invalidLicenseErrorMessage', { defaultMessage: - 'Alert {alertTypeId} is disabled because it requires a Gold license. Contact your administrator to upgrade your license.', - values: { alertTypeId: alertType.id }, + 'Alert {alertTypeId} is disabled because it requires a {licenseType} license. Go to License Management to view upgrade options.', + values: { + alertTypeId: alertType.id, + licenseType: capitalize(alertType.minimumLicenseRequired), + }, }), 'license_invalid' ); diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index 018d2d572eea06..5e0bf7501eb118 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -4830,7 +4830,6 @@ "xpack.alerts.server.healthStatus.degraded": "アラートフレームワークは劣化しました", "xpack.alerts.server.healthStatus.unavailable": "アラートフレームワークを使用できません", "xpack.alerts.serverSideErrors.expirerdLicenseErrorMessage": "{licenseType} ライセンスの期限が切れたのでアラートタイプ {alertTypeId} は無効です。", - "xpack.alerts.serverSideErrors.invalidLicenseErrorMessage": "アラート {alertTypeId} は無効です。Gold ライセンスが必要です。ライセンスをアップグレードするには、管理者に問い合わせてください。", "xpack.alerts.serverSideErrors.unavailableLicenseErrorMessage": "現時点でライセンス情報を入手できないため、アラートタイプ {alertTypeId} は無効です。", "xpack.alerts.serverSideErrors.unavailableLicenseInformationErrorMessage": "アラートを利用できません。現在ライセンス情報が利用できません。", "xpack.apm.a.thresholdMet": "しきい値一致", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index 5a9695b8ddc3de..d0dbd750853a25 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -4836,7 +4836,6 @@ "xpack.alerts.server.healthStatus.degraded": "告警框架已降级", "xpack.alerts.server.healthStatus.unavailable": "告警框架不可用", "xpack.alerts.serverSideErrors.expirerdLicenseErrorMessage": "告警类型 {alertTypeId} 已禁用，因为您的{licenseType}许可证已过期。", - "xpack.alerts.serverSideErrors.invalidLicenseErrorMessage": "告警 {alertTypeId} 已禁用，因为它需要黄金级许可证。请联系管理员升级您的许可证。", "xpack.alerts.serverSideErrors.unavailableLicenseErrorMessage": "告警类型 {alertTypeId} 已禁用，因为许可证信息当前不可用。", "xpack.alerts.serverSideErrors.unavailableLicenseInformationErrorMessage": "告警不可用 - 许可信息当前不可用。", "xpack.apm.a.thresholdMet": "已达到阈值", diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx index fb34c95f93de2b..fc41022dfb7b01 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.test.tsx @@ -127,11 +127,16 @@ describe('alerts_list component empty', () => { wrapper.find('button[data-test-subj="createFirstAlertButton"]').simulate('click'); - // When the AlertAdd component is rendered, it waits for the healthcheck to resolve - await new Promise((resolve) => { - setTimeout(resolve, 1000); + await act(async () => { + // When the AlertAdd component is rendered, it waits for the healthcheck to resolve + await new Promise((resolve) => { + setTimeout(resolve, 1000); + }); + + await nextTick(); + wrapper.update(); }); - wrapper.update(); + expect(wrapper.find('AlertAdd').exists()).toEqual(true); }); }); @@ -139,104 +144,131 @@ describe('alerts_list component empty', () => { describe('alerts_list component with items', () => { let wrapper: ReactWrapper; + const mockedAlertsData = [ + { + id: '1', + name: 'test alert', + tags: ['tag1'], + enabled: true, + alertTypeId: 'test_alert_type', + schedule: { interval: '5d' }, + actions: [], + params: { name: 'test alert type name' }, + scheduledTaskId: null, + createdBy: null, + updatedBy: null, + apiKeyOwner: null, + throttle: '1m', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'active', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + error: null, + }, + }, + { + id: '2', + name: 'test alert ok', + tags: ['tag1'], + enabled: true, + alertTypeId: 'test_alert_type', + schedule: { interval: '5d' }, + actions: [], + params: { name: 'test alert type name' }, + scheduledTaskId: null, + createdBy: null, + updatedBy: null, + apiKeyOwner: null, + throttle: '1m', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'ok', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + error: null, + }, + }, + { + id: '3', + name: 'test alert pending', + tags: ['tag1'], + enabled: true, + alertTypeId: 'test_alert_type', + schedule: { interval: '5d' }, + actions: [], + params: { name: 'test alert type name' }, + scheduledTaskId: null, + createdBy: null, + updatedBy: null, + apiKeyOwner: null, + throttle: '1m', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'pending', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + error: null, + }, + }, + { + id: '4', + name: 'test alert error', + tags: ['tag1'], + enabled: true, + alertTypeId: 'test_alert_type', + schedule: { interval: '5d' }, + actions: [{ id: 'test', group: 'alert', params: { message: 'test' } }], + params: { name: 'test alert type name' }, + scheduledTaskId: null, + createdBy: null, + updatedBy: null, + apiKeyOwner: null, + throttle: '1m', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'error', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + error: { + reason: AlertExecutionStatusErrorReasons.Unknown, + message: 'test', + }, + }, + }, + { + id: '5', + name: 'test alert license error', + tags: ['tag1'], + enabled: true, + alertTypeId: 'test_alert_type', + schedule: { interval: '5d' }, + actions: [{ id: 'test', group: 'alert', params: { message: 'test' } }], + params: { name: 'test alert type name' }, + scheduledTaskId: null, + createdBy: null, + updatedBy: null, + apiKeyOwner: null, + throttle: '1m', + muteAll: false, + mutedInstanceIds: [], + executionStatus: { + status: 'error', + lastExecutionDate: new Date('2020-08-20T19:23:38Z'), + error: { + reason: AlertExecutionStatusErrorReasons.License, + message: 'test', + }, + }, + }, + ]; + async function setup() { loadAlerts.mockResolvedValue({ page: 1, perPage: 10000, total: 4, - data: [ - { - id: '1', - name: 'test alert', - tags: ['tag1'], - enabled: true, - alertTypeId: 'test_alert_type', - schedule: { interval: '5d' }, - actions: [], - params: { name: 'test alert type name' }, - scheduledTaskId: null, - createdBy: null, - updatedBy: null, - apiKeyOwner: null, - throttle: '1m', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'active', - lastExecutionDate: new Date('2020-08-20T19:23:38Z'), - error: null, - }, - }, - { - id: '2', - name: 'test alert ok', - tags: ['tag1'], - enabled: true, - alertTypeId: 'test_alert_type', - schedule: { interval: '5d' }, - actions: [], - params: { name: 'test alert type name' }, - scheduledTaskId: null, - createdBy: null, - updatedBy: null, - apiKeyOwner: null, - throttle: '1m', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'ok', - lastExecutionDate: new Date('2020-08-20T19:23:38Z'), - error: null, - }, - }, - { - id: '3', - name: 'test alert pending', - tags: ['tag1'], - enabled: true, - alertTypeId: 'test_alert_type', - schedule: { interval: '5d' }, - actions: [], - params: { name: 'test alert type name' }, - scheduledTaskId: null, - createdBy: null, - updatedBy: null, - apiKeyOwner: null, - throttle: '1m', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'pending', - lastExecutionDate: new Date('2020-08-20T19:23:38Z'), - error: null, - }, - }, - { - id: '4', - name: 'test alert error', - tags: ['tag1'], - enabled: true, - alertTypeId: 'test_alert_type', - schedule: { interval: '5d' }, - actions: [{ id: 'test', group: 'alert', params: { message: 'test' } }], - params: { name: 'test alert type name' }, - scheduledTaskId: null, - createdBy: null, - updatedBy: null, - apiKeyOwner: null, - throttle: '1m', - muteAll: false, - mutedInstanceIds: [], - executionStatus: { - status: 'error', - lastExecutionDate: new Date('2020-08-20T19:23:38Z'), - error: { - reason: AlertExecutionStatusErrorReasons.Unknown, - message: 'test', - }, - }, - }, - ], + data: mockedAlertsData, }); loadActionTypes.mockResolvedValue([ { @@ -271,21 +303,66 @@ describe('alerts_list component with items', () => { it('renders table of alerts', async () => { await setup(); expect(wrapper.find('EuiBasicTable')).toHaveLength(1); - expect(wrapper.find('EuiTableRow')).toHaveLength(4); - expect(wrapper.find('[data-test-subj="alertsTableCell-status"]').length).toBeGreaterThan(0); - expect(wrapper.find('[data-test-subj="alertStatus-active"]').length).toBeGreaterThan(0); - expect(wrapper.find('[data-test-subj="alertStatus-error"]').length).toBeGreaterThan(0); - expect(wrapper.find('[data-test-subj="alertStatus-ok"]').length).toBeGreaterThan(0); - expect(wrapper.find('[data-test-subj="alertStatus-pending"]').length).toBeGreaterThan(0); - expect(wrapper.find('[data-test-subj="alertStatus-unknown"]').length).toBe(0); + expect(wrapper.find('EuiTableRow')).toHaveLength(mockedAlertsData.length); + expect(wrapper.find('EuiTableRowCell[data-test-subj="alertsTableCell-status"]').length).toEqual( + mockedAlertsData.length + ); + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-active"]').length).toEqual(1); + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-ok"]').length).toEqual(1); + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-pending"]').length).toEqual(1); + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-unknown"]').length).toEqual(0); + + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-error"]').length).toEqual(2); + expect(wrapper.find('[data-test-subj="alertStatus-error-tooltip"]').length).toEqual(2); + expect( + wrapper.find('EuiButtonEmpty[data-test-subj="alertStatus-error-license-fix"]').length + ).toEqual(1); + expect(wrapper.find('[data-test-subj="refreshAlertsButton"]').exists()).toBeTruthy(); + + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-error"]').first().text()).toEqual( + 'Error' + ); + expect(wrapper.find('EuiHealth[data-test-subj="alertStatus-error"]').last().text()).toEqual( + 'License Error' + ); }); it('loads alerts when refresh button is clicked', async () => { await setup(); wrapper.find('[data-test-subj="refreshAlertsButton"]').first().simulate('click'); + + await act(async () => { + await nextTick(); + wrapper.update(); + }); + expect(loadAlerts).toHaveBeenCalled(); }); + + it('renders license errors and manage license modal on click', async () => { + global.open = jest.fn(); + await setup(); + expect(wrapper.find('ManageLicenseModal').exists()).toBeFalsy(); + expect( + wrapper.find('EuiButtonEmpty[data-test-subj="alertStatus-error-license-fix"]').length + ).toEqual(1); + wrapper + .find('EuiButtonEmpty[data-test-subj="alertStatus-error-license-fix"]') + .simulate('click'); + + await act(async () => { + await nextTick(); + wrapper.update(); + }); + + expect(wrapper.find('ManageLicenseModal').exists()).toBeTruthy(); + expect(wrapper.find('EuiButton[data-test-subj="confirmModalConfirmButton"]').text()).toEqual( + 'Manage license' + ); + wrapper.find('EuiButton[data-test-subj="confirmModalConfirmButton"]').simulate('click'); + expect(global.open).toHaveBeenCalled(); + }); }); describe('alerts_list component empty with show only capability', () => { @@ -308,7 +385,9 @@ describe('alerts_list component empty with show only capability', () => { name: 'Test2', }, ]); - loadAlertTypes.mockResolvedValue([{ id: 'test_alert_type', name: 'some alert type' }]); + loadAlertTypes.mockResolvedValue([ + { id: 'test_alert_type', name: 'some alert type', authorizedConsumers: {} }, + ]); loadAllActions.mockResolvedValue([]); // eslint-disable-next-line react-hooks/rules-of-hooks useKibanaMock().services.alertTypeRegistry = alertTypeRegistry; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx index 76680a60a24e1b..11761cec7cdbb9 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx @@ -53,14 +53,15 @@ import { AlertExecutionStatus, AlertExecutionStatusValues, ALERTS_FEATURE_ID, + AlertExecutionStatusErrorReasons, } from '../../../../../../alerts/common'; import { hasAllPrivilege } from '../../../lib/capabilities'; -import { alertsStatusesTranslationsMapping } from '../translations'; +import { alertsStatusesTranslationsMapping, ALERT_STATUS_LICENSE_ERROR } from '../translations'; import { useKibana } from '../../../../common/lib/kibana'; -import { checkAlertTypeEnabled } from '../../../lib/check_alert_type_enabled'; import { DEFAULT_HIDDEN_ACTION_TYPES } from '../../../../common/constants'; import './alerts_list.scss'; import { CenterJustifiedSpinner } from '../../../components/center_justified_spinner'; +import { ManageLicenseModal } from './manage_license_modal'; const ENTER_KEY = 13; @@ -97,7 +98,11 @@ export const AlertsList: React.FunctionComponent = () => { const [actionTypesFilter, setActionTypesFilter] = useState([]); const [alertStatusesFilter, setAlertStatusesFilter] = useState([]); const [alertFlyoutVisible, setAlertFlyoutVisibility] = useState(false); - const [dissmissAlertErrors, setDissmissAlertErrors] = useState(false); + const [dismissAlertErrors, setDismissAlertErrors] = useState(false); + const [manageLicenseModalOpts, setManageLicenseModalOpts] = useState<{ + licenseType: string; + alertTypeId: string; + } | null>(null); const [alertsStatusesTotal, setAlertsStatusesTotal] = useState>( AlertExecutionStatusValues.reduce( (prev: Record, status: string) => @@ -238,25 +243,64 @@ export const AlertsList: React.FunctionComponent = () => { } } + const renderAlertExecutionStatus = ( + executionStatus: AlertExecutionStatus, + item: AlertTableItem + ) => { + const healthColor = getHealthColor(executionStatus.status); + const tooltipMessage = + executionStatus.status === 'error' ? `Error: ${executionStatus?.error?.message}` : null; + const isLicenseError = + executionStatus.error?.reason === AlertExecutionStatusErrorReasons.License; + const statusMessage = isLicenseError + ? ALERT_STATUS_LICENSE_ERROR + : alertsStatusesTranslationsMapping[executionStatus.status]; + + const health = ( + + {statusMessage} + + ); + + const healthWithTooltip = tooltipMessage ? ( + + {health} + + ) : ( + health + ); + + return ( + + {healthWithTooltip} + {isLicenseError && ( + + + setManageLicenseModalOpts({ + licenseType: alertTypesState.data.get(item.alertTypeId)?.minimumLicenseRequired!, + alertTypeId: item.alertTypeId, + }) + } + > + + + + )} + + ); + }; + const alertsTableColumns = [ - { - field: 'executionStatus', - name: i18n.translate( - 'xpack.triggersActionsUI.sections.alertsList.alertsListTable.columns.statusTitle', - { defaultMessage: 'Status' } - ), - sortable: false, - truncateText: false, - 'data-test-subj': 'alertsTableCell-status', - render: (executionStatus: AlertExecutionStatus) => { - const healthColor = getHealthColor(executionStatus.status); - return ( - - {alertsStatusesTranslationsMapping[executionStatus.status]} - - ); - }, - }, { field: 'name', name: i18n.translate( @@ -265,12 +309,10 @@ export const AlertsList: React.FunctionComponent = () => { ), sortable: false, truncateText: true, + width: '35%', 'data-test-subj': 'alertsTableCell-name', render: (name: string, alert: AlertTableItem) => { - const checkEnabledResult = checkAlertTypeEnabled( - alertTypesState.data.get(alert.alertTypeId) - ); - const link = ( + return ( { @@ -280,17 +322,20 @@ export const AlertsList: React.FunctionComponent = () => { {name} ); - return checkEnabledResult.isEnabled ? ( - link - ) : ( - - {link} - - ); + }, + }, + { + field: 'executionStatus', + name: i18n.translate( + 'xpack.triggersActionsUI.sections.alertsList.alertsListTable.columns.statusTitle', + { defaultMessage: 'Status' } + ), + sortable: false, + truncateText: false, + width: '150px', + 'data-test-subj': 'alertsTableCell-status', + render: (executionStatus: AlertExecutionStatus, item: AlertTableItem) => { + return renderAlertExecutionStatus(executionStatus, item); }, }, { @@ -492,7 +537,7 @@ export const AlertsList: React.FunctionComponent = () => { - {!dissmissAlertErrors && alertsStatusesTotal.error > 0 ? ( + {!dismissAlertErrors && alertsStatusesTotal.error > 0 ? ( { defaultMessage="View" /> - setDissmissAlertErrors(true)}> + setDismissAlertErrors(true)}> { setPage(changedPage); }} /> + {manageLicenseModalOpts && ( + { + window.open(`${http.basePath.get()}/app/management/stack/license_management`, '_blank'); + setManageLicenseModalOpts(null); + }} + onCancel={() => setManageLicenseModalOpts(null)} + /> + )} ); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/manage_license_modal.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/manage_license_modal.tsx new file mode 100644 index 00000000000000..f13e5fd96d2ad8 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/manage_license_modal.tsx @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiOverlayMask, EuiConfirmModal } from '@elastic/eui'; +import { capitalize } from 'lodash'; + +interface Props { + licenseType: string; + alertTypeId: string; + onConfirm: () => void; + onCancel: () => void; +} + +export const ManageLicenseModal: React.FC = ({ + licenseType, + alertTypeId, + onConfirm, + onCancel, +}) => { + const licenseRequired = capitalize(licenseType); + return ( + + +

+ +

+ + + ); +}; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/translations.ts b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/translations.ts index 0b8bba9ffe95a5..1a2c576b1fa28c 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/translations.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/translations.ts @@ -28,6 +28,13 @@ export const ALERT_STATUS_ERROR = i18n.translate( } ); +export const ALERT_STATUS_LICENSE_ERROR = i18n.translate( + 'xpack.triggersActionsUI.sections.alertsList.alertStatusLicenseError', + { + defaultMessage: 'License Error', + } +); + export const ALERT_STATUS_PENDING = i18n.translate( 'xpack.triggersActionsUI.sections.alertsList.alertStatusPending', { diff --git a/x-pack/test/alerting_api_integration/basic/tests/alerts/gold_noop_alert_type.ts b/x-pack/test/alerting_api_integration/basic/tests/alerts/gold_noop_alert_type.ts index 488b39eabb6377..211d1acb2a0054 100644 --- a/x-pack/test/alerting_api_integration/basic/tests/alerts/gold_noop_alert_type.ts +++ b/x-pack/test/alerting_api_integration/basic/tests/alerts/gold_noop_alert_type.ts @@ -22,7 +22,7 @@ export default function emailTest({ getService }: FtrProviderContext) { statusCode: 403, error: 'Forbidden', message: - 'Alert test.gold.noop is disabled because it requires a Gold license. Contact your administrator to upgrade your license.', + 'Alert test.gold.noop is disabled because it requires a Gold license. Go to License Management to view upgrade options.', }); }); }); From e17878ef330b152dac9fb0933b13be668c5f1867 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Wed, 10 Feb 2021 14:25:53 +0100 Subject: [PATCH 10/32] [ILM] Revisit searchable snapshot field after new redesign (#90793) * moved searchable snapshot field out of cold phase accordian * refactor styling to padding top and bottom to get advanced settings drop down to sit flush with side of panel * Error clearing fix and cosmetic changes - the error state of the form would not clear correctly if the erroring field was unmounted. The logic for clearing form errors was also incorrectly using "keys" instead of "values". - updated the width of wait for snapshot policy field to be the same as other fields * fix hook dependency causing clearError to be called * slight improvement to component integration test * re-add singleSelection to snapshot policiy field config * refactored Phase component API and fixed typo in comment Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../edit_policy/edit_policy.helpers.tsx | 4 +- .../edit_policy/edit_policy.test.ts | 37 ++++++++++++++++--- .../phases/cold_phase/cold_phase.tsx | 10 ++--- .../components/phases/phase/phase.scss | 3 +- .../components/phases/phase/phase.tsx | 16 +++++++- .../searchable_snapshot_field.tsx | 2 + .../shared_fields/snapshot_policies_field.tsx | 1 + .../form/components/enhanced_use_field.tsx | 9 +++++ .../edit_policy/form/form_errors_context.tsx | 15 +++++--- 9 files changed, 76 insertions(+), 21 deletions(-) diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx index 38049dd7c6cfac..7e1b7c5267a8bf 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx @@ -237,7 +237,9 @@ export const setup = async (arg?: { appServicesContext: Partial { - await toggleSearchableSnapshot(true); + if (!exists(`searchableSnapshotField-${phase}.searchableSnapshotCombobox`)) { + await toggleSearchableSnapshot(true); + } act(() => { find(`searchableSnapshotField-${phase}.searchableSnapshotCombobox`).simulate('change', [ { label: value }, diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts index 282daf780b86ce..6f325084938e8d 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts @@ -916,7 +916,7 @@ describe('', () => { await actions.warm.enable(true); await actions.warm.toggleForceMerge(true); await actions.warm.setForcemergeSegmentsCount('-22'); - await runTimers(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(true); expect(actions.hot.hasErrorIndicator()).toBe(true); expect(actions.warm.hasErrorIndicator()).toBe(true); @@ -925,7 +925,7 @@ describe('', () => { // 3. Cold phase validation issue await actions.cold.enable(true); await actions.cold.setReplicas('-33'); - await runTimers(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(true); expect(actions.hot.hasErrorIndicator()).toBe(true); expect(actions.warm.hasErrorIndicator()).toBe(true); @@ -933,7 +933,7 @@ describe('', () => { // 4. Fix validation issue in hot await actions.hot.setForcemergeSegmentsCount('1'); - await runTimers(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(true); expect(actions.hot.hasErrorIndicator()).toBe(false); expect(actions.warm.hasErrorIndicator()).toBe(true); @@ -941,7 +941,7 @@ describe('', () => { // 5. Fix validation issue in warm await actions.warm.setForcemergeSegmentsCount('1'); - await runTimers(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(true); expect(actions.hot.hasErrorIndicator()).toBe(false); expect(actions.warm.hasErrorIndicator()).toBe(false); @@ -949,7 +949,7 @@ describe('', () => { // 6. Fix validation issue in cold await actions.cold.setReplicas('1'); - await runTimers(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(false); expect(actions.hot.hasErrorIndicator()).toBe(false); expect(actions.warm.hasErrorIndicator()).toBe(false); @@ -966,11 +966,36 @@ describe('', () => { await actions.saveAsNewPolicy(true); await actions.setPolicyName(''); - await runTimers(); + runTimers(); + + expect(actions.hasGlobalErrorCallout()).toBe(true); + expect(actions.hot.hasErrorIndicator()).toBe(false); + expect(actions.warm.hasErrorIndicator()).toBe(false); + expect(actions.cold.hasErrorIndicator()).toBe(false); + }); + + test('clears all error indicators if last erroring field is unmounted', async () => { + const { actions } = testBed; + + await actions.cold.enable(true); + // introduce validation error + await actions.cold.setSearchableSnapshot(''); + runTimers(); + + await actions.savePolicy(); + runTimers(); expect(actions.hasGlobalErrorCallout()).toBe(true); expect(actions.hot.hasErrorIndicator()).toBe(false); expect(actions.warm.hasErrorIndicator()).toBe(false); + expect(actions.cold.hasErrorIndicator()).toBe(true); + + // unmount the field + await actions.cold.toggleSearchableSnapshot(false); + + expect(actions.hasGlobalErrorCallout()).toBe(false); + expect(actions.hot.hasErrorIndicator()).toBe(false); + expect(actions.warm.hasErrorIndicator()).toBe(false); expect(actions.cold.hasErrorIndicator()).toBe(false); }); }); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx index 1e1e97789e105d..27aacef1a368bd 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase/cold_phase.tsx @@ -51,10 +51,8 @@ export const ColdPhase: FunctionComponent = () => { const showReplicasField = get(formData, formFieldPaths.searchableSnapshot) == null; return ( - - - - {showReplicasField && } + }> + {showReplicasField && } {/* Freeze section */} {!isUsingSearchableSnapshotInHotPhase && ( @@ -90,10 +88,10 @@ export const ColdPhase: FunctionComponent = () => { {/* Data tier allocation section */} - + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.scss b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.scss index 15f2dc508a365f..75d25c0bffa501 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.scss +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.scss @@ -9,7 +9,8 @@ } .ilmSettingsButton { color: $euiColorPrimary; - padding: $euiSizeS; + padding-top: $euiSizeS; + padding-bottom: $euiSizeS; } .euiCommentTimeline { padding-top: $euiSize; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.tsx index 0ac6f6922ec1e1..3a057f6204e24f 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/phase/phase.tsx @@ -37,9 +37,13 @@ import './phase.scss'; interface Props { phase: PhasesExceptDelete; + /** + * Settings that should always be visible on the phase when it is enabled. + */ + topLevelSettings?: React.ReactNode; } -export const Phase: FunctionComponent = ({ children, phase }) => { +export const Phase: FunctionComponent = ({ children, topLevelSettings, phase }) => { const enabledPath = `_meta.${phase}.enabled`; const [formData] = useFormData({ watch: [enabledPath], @@ -102,7 +106,15 @@ export const Phase: FunctionComponent = ({ children, phase }) => { {enabled && ( <> - + {!!topLevelSettings ? ( + <> + + {topLevelSettings} + + ) : ( + + )} + = ({ phase }) =>
config={{ + label: i18nTexts.editPolicy.searchableSnapshotsFieldLabel, defaultValue: cloud?.isCloudEnabled ? CLOUD_DEFAULT_REPO : undefined, validations: [ { @@ -209,6 +210,7 @@ export const SearchableSnapshotField: FunctionComponent = ({ phase }) => value: singleSelectionArray, } as any } + label={field.label} fullWidth={false} euiFieldProps={{ 'data-test-subj': 'searchableSnapshotCombobox', diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared_fields/snapshot_policies_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared_fields/snapshot_policies_field.tsx index f9c973d14b3e22..21dd083ccf7c55 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared_fields/snapshot_policies_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared_fields/snapshot_policies_field.tsx @@ -194,6 +194,7 @@ export const SnapshotPoliciesField: React.FunctionComponent = () => { } euiFieldProps={{ 'data-test-subj': 'snapshotPolicyCombobox', + fullWidth: false, options: policies, singleSelection: { asPlainText: true }, isLoading, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/components/enhanced_use_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/components/enhanced_use_field.tsx index 85e854fb5f0047..7210dc6b7ce2b7 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/components/enhanced_use_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/components/enhanced_use_field.tsx @@ -70,5 +70,14 @@ export const EnhancedUseField = ( }; }, []); + // Make sure to clear error message if the field is unmounted. + useEffect(() => { + return () => { + if (isMounted.current === false) { + clearError(phase, path); + } + }; + }, [phase, path, clearError]); + return ; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/form_errors_context.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/form_errors_context.tsx index b0903dbbc1b1a0..9877a2ea9449cf 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/form_errors_context.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form/form_errors_context.tsx @@ -54,6 +54,8 @@ export const FormErrorsProvider: FunctionComponent = ({ children }) => { const [errors, setErrors] = useState(createEmptyErrors); const form = useFormContext(); + const { getErrors: getFormErrors } = form; + const addError: ContextValue['addError'] = useCallback( (phase, fieldPath, errorMessages) => { setErrors((previousErrors) => ({ @@ -70,20 +72,23 @@ export const FormErrorsProvider: FunctionComponent = ({ children }) => { const clearError: ContextValue['clearError'] = useCallback( (phase, fieldPath) => { - if (form.getErrors().length) { + if (getFormErrors().length) { setErrors((previousErrors) => { const { [phase]: { [fieldPath]: fieldErrorToOmit, ...restOfPhaseErrors }, + hasErrors, ...otherPhases } = previousErrors; - const hasErrors = + const nextHasErrors = Object.keys(restOfPhaseErrors).length === 0 && - Object.keys(otherPhases).some((phaseErrors) => !!Object.keys(phaseErrors).length); + Object.values(otherPhases).some((phaseErrors) => { + return !!Object.keys(phaseErrors).length; + }); return { ...previousErrors, - hasErrors, + hasErrors: nextHasErrors, [phase]: restOfPhaseErrors, }; }); @@ -91,7 +96,7 @@ export const FormErrorsProvider: FunctionComponent = ({ children }) => { setErrors(createEmptyErrors); } }, - [form, setErrors] + [getFormErrors, setErrors] ); return ( From 7e80bb32744ce3fa5481a82bad92da0e20b8f6a6 Mon Sep 17 00:00:00 2001 From: Liza Katz Date: Wed, 10 Feb 2021 15:26:25 +0200 Subject: [PATCH 11/32] [Search Sessions] added an info flyout to session management (#90559) * added an info flyout to session management * better filter serialization * Fix jest tests * jest * display the originalState as a json object * code review * Text improvements Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../components/actions/extend_button.tsx | 2 +- .../components/actions/get_action.tsx | 11 +- .../components/actions/inspect_button.scss | 6 + .../components/actions/inspect_button.tsx | 134 ++++++++++++++++++ .../sessions_mgmt/components/actions/types.ts | 1 + .../sessions_mgmt/components/status.test.tsx | 2 + .../search/sessions_mgmt/lib/api.test.ts | 11 +- .../public/search/sessions_mgmt/lib/api.ts | 3 + .../sessions_mgmt/lib/get_columns.test.tsx | 2 + .../public/search/sessions_mgmt/types.ts | 2 + 10 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.scss create mode 100644 x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.tsx diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/extend_button.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/extend_button.tsx index 06459db154f4ae..381c44b1bf7bef 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/extend_button.tsx +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/extend_button.tsx @@ -38,7 +38,7 @@ const ExtendConfirm = ({ defaultMessage: 'Extend search session expiration', }); const confirm = i18n.translate('xpack.data.mgmt.searchSessions.extendModal.extendButton', { - defaultMessage: 'Extend', + defaultMessage: 'Extend expiration', }); const extend = i18n.translate('xpack.data.mgmt.searchSessions.extendModal.dontExtendButton', { defaultMessage: 'Cancel', diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/get_action.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/get_action.tsx index edc5037f1dbec9..1a2b2cfb4ececd 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/get_action.tsx +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/get_action.tsx @@ -12,15 +12,24 @@ import { SearchSessionsMgmtAPI } from '../../lib/api'; import { UISession } from '../../types'; import { DeleteButton } from './delete_button'; import { ExtendButton } from './extend_button'; +import { InspectButton } from './inspect_button'; import { ACTION, OnActionComplete } from './types'; export const getAction = ( api: SearchSessionsMgmtAPI, actionType: string, - { id, name, expires }: UISession, + uiSession: UISession, onActionComplete: OnActionComplete ): IClickActionDescriptor | null => { + const { id, name, expires } = uiSession; switch (actionType) { + case ACTION.INSPECT: + return { + iconType: 'document', + textColor: 'default', + label: , + }; + case ACTION.DELETE: return { iconType: 'crossInACircleFilled', diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.scss b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.scss new file mode 100644 index 00000000000000..a43bb65927ed47 --- /dev/null +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.scss @@ -0,0 +1,6 @@ +.searchSessionsFlyout .euiFlyoutBody__overflowContent { + height: 100%; + > div { + height: 100%; + } +} \ No newline at end of file diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.tsx new file mode 100644 index 00000000000000..86dca64909b55c --- /dev/null +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/inspect_button.tsx @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlyout, + EuiFlyoutBody, + EuiFlyoutHeader, + EuiPortal, + EuiSpacer, + EuiText, + EuiTitle, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { Component, Fragment } from 'react'; +import { UISession } from '../../types'; +import { TableText } from '..'; +import { CodeEditor } from '../../../../../../../../src/plugins/kibana_react/public'; +import './inspect_button.scss'; + +interface Props { + searchSession: UISession; +} + +interface State { + isFlyoutVisible: boolean; +} + +export class InspectButton extends Component { + constructor(props: Props) { + super(props); + + this.state = { + isFlyoutVisible: false, + }; + + this.closeFlyout = this.closeFlyout.bind(this); + this.showFlyout = this.showFlyout.bind(this); + } + + public renderInfo() { + return ( + + {}} + options={{ + readOnly: true, + lineNumbers: 'off', + fontSize: 12, + minimap: { + enabled: false, + }, + scrollBeyondLastLine: false, + wordWrap: 'on', + wrappingIndent: 'indent', + automaticLayout: true, + }} + /> + + ); + } + + public render() { + let flyout; + + if (this.state.isFlyoutVisible) { + flyout = ( + + + + +

+ +

+ + + + + +

+ +

+ + + {this.renderInfo()} + + + + + ); + } + + return ( + + + + + {flyout} + + ); + } + + private closeFlyout = () => { + this.setState({ + isFlyoutVisible: false, + }); + }; + + private showFlyout = () => { + this.setState({ isFlyoutVisible: true }); + }; +} diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/types.ts b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/types.ts index 5f82f16adcbb6f..c94b6aa8495c7f 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/types.ts +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/actions/types.ts @@ -8,6 +8,7 @@ export type OnActionComplete = () => void; export enum ACTION { + INSPECT = 'inspect', EXTEND = 'extend', DELETE = 'delete', } diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/status.test.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/status.test.tsx index 3d92f349fd2d6f..f1d4f2ab379a01 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/status.test.tsx +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/components/status.test.tsx @@ -31,6 +31,8 @@ describe('Background Search Session management status labels', () => { status: SearchSessionStatus.IN_PROGRESS, created: '2020-12-02T00:19:32Z', expires: '2020-12-07T00:19:32Z', + initialState: {}, + restoreState: {}, }; }); diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.test.ts b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.test.ts index 0fa13ac145223c..10b2ac3ec1d4c1 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.test.ts +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.test.ts @@ -46,7 +46,13 @@ describe('Search Sessions Management API', () => { saved_objects: [ { id: 'hello-pizza-123', - attributes: { name: 'Veggie', appId: 'pizza', status: 'complete' }, + attributes: { + name: 'Veggie', + appId: 'pizza', + status: 'complete', + initialState: {}, + restoreState: {}, + }, }, ], } as SavedObjectsFindResponse; @@ -61,6 +67,7 @@ describe('Search Sessions Management API', () => { Array [ Object { "actions": Array [ + "inspect", "extend", "delete", ], @@ -68,8 +75,10 @@ describe('Search Sessions Management API', () => { "created": undefined, "expires": undefined, "id": "hello-pizza-123", + "initialState": Object {}, "name": "Veggie", "reloadUrl": "hello-cool-undefined-url", + "restoreState": Object {}, "restoreUrl": "hello-cool-undefined-url", "status": "complete", }, diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.ts b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.ts index 42e9384cce2d86..39da58cb769182 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.ts +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/api.ts @@ -21,6 +21,7 @@ type UrlGeneratorsStart = SharePluginStart['urlGenerators']; function getActions(status: SearchSessionStatus) { const actions: ACTION[] = []; + actions.push(ACTION.INSPECT); if (status === SearchSessionStatus.IN_PROGRESS || status === SearchSessionStatus.COMPLETE) { actions.push(ACTION.EXTEND); actions.push(ACTION.DELETE); @@ -78,6 +79,8 @@ const mapToUISession = (urls: UrlGeneratorsStart, config: SessionsConfigSchema) actions, restoreUrl, reloadUrl, + initialState, + restoreState, }; }; diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/get_columns.test.tsx b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/get_columns.test.tsx index 2aab35e34a2d0b..fc0a8849006d3c 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/get_columns.test.tsx +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/lib/get_columns.test.tsx @@ -66,6 +66,8 @@ describe('Search Sessions Management table column factory', () => { status: SearchSessionStatus.IN_PROGRESS, created: '2020-12-02T00:19:32Z', expires: '2020-12-07T00:19:32Z', + initialState: {}, + restoreState: {}, }; }); diff --git a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/types.ts b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/types.ts index d9aea4ddae93e9..e7b48f319a8a8b 100644 --- a/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/types.ts +++ b/x-pack/plugins/data_enhanced/public/search/sessions_mgmt/types.ts @@ -32,4 +32,6 @@ export interface UISession { actions?: ACTION[]; reloadUrl: string; restoreUrl: string; + initialState: Record; + restoreState: Record; } From f95bfe83b706cea92b0862ad16d2b3d054433dae Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Wed, 10 Feb 2021 08:28:22 -0500 Subject: [PATCH 12/32] [Fleet] Use Fleet Server indices in the search bar (#90835) --- .../applications/fleet/components/search_bar.tsx | 14 ++++++++------ .../public/applications/fleet/constants/index.ts | 3 +++ .../components/search_and_filter_bar.tsx | 12 ++++++++++-- .../agents/enrollment_token_list_page/index.tsx | 15 +++++++++++++-- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx index a402fd995a42e3..9897d898814509 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/components/search_bar.tsx @@ -14,13 +14,14 @@ import { import { useStartServices } from '../hooks'; import { INDEX_NAME, AGENT_SAVED_OBJECT_TYPE } from '../constants'; -const HIDDEN_FIELDS = [`${AGENT_SAVED_OBJECT_TYPE}.actions`]; +const HIDDEN_FIELDS = [`${AGENT_SAVED_OBJECT_TYPE}.actions`, '_id', '_index']; interface Props { value: string; - fieldPrefix: string; + fieldPrefix?: string; onChange: (newValue: string, submit?: boolean) => void; placeholder?: string; + indexPattern?: string; } export const SearchBar: React.FunctionComponent = ({ @@ -28,6 +29,7 @@ export const SearchBar: React.FunctionComponent = ({ fieldPrefix, onChange, placeholder, + indexPattern = INDEX_NAME, }) => { const { data } = useStartServices(); const [indexPatternFields, setIndexPatternFields] = useState(); @@ -49,10 +51,10 @@ export const SearchBar: React.FunctionComponent = ({ const fetchFields = async () => { try { const _fields: IFieldType[] = await data.indexPatterns.getFieldsForWildcard({ - pattern: INDEX_NAME, + pattern: indexPattern, }); const fields = (_fields || []).filter((field) => { - if (fieldPrefix && field.name.startsWith(fieldPrefix)) { + if (!fieldPrefix || field.name.startsWith(fieldPrefix)) { for (const hiddenField of HIDDEN_FIELDS) { if (field.name.startsWith(hiddenField)) { return false; @@ -67,7 +69,7 @@ export const SearchBar: React.FunctionComponent = ({ } }; fetchFields(); - }, [data.indexPatterns, fieldPrefix]); + }, [data.indexPatterns, fieldPrefix, indexPattern]); return ( = ({ indexPatternFields ? [ { - title: INDEX_NAME, + title: indexPattern, fields: indexPatternFields, }, ] diff --git a/x-pack/plugins/fleet/public/applications/fleet/constants/index.ts b/x-pack/plugins/fleet/public/applications/fleet/constants/index.ts index 249087eda5cb12..6686aa21a9f2e7 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/constants/index.ts +++ b/x-pack/plugins/fleet/public/applications/fleet/constants/index.ts @@ -15,6 +15,9 @@ export { AGENT_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE, + // Fleet Server index + AGENTS_INDEX, + ENROLLMENT_API_KEYS_INDEX, } from '../../../../common'; export * from './page_paths'; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx index e6f681e4b39ea9..af990a36a74155 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_list_page/components/search_and_filter_bar.tsx @@ -18,7 +18,8 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { AgentPolicy } from '../../../../types'; import { SearchBar } from '../../../../components'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../../../constants'; +import { AGENTS_INDEX, AGENT_SAVED_OBJECT_TYPE } from '../../../../constants'; +import { useConfig } from '../../../../hooks'; const statusFilters = [ { @@ -76,6 +77,7 @@ export const SearchAndFilterBar: React.FunctionComponent<{ showUpgradeable, onShowUpgradeableChange, }) => { + const config = useConfig(); // Policies state for filtering const [isAgentPoliciesFilterOpen, setIsAgentPoliciesFilterOpen] = useState(false); @@ -109,7 +111,13 @@ export const SearchAndFilterBar: React.FunctionComponent<{ onSubmitSearch(newSearch); } }} - fieldPrefix={AGENT_SAVED_OBJECT_TYPE} + {...(config.agents.fleetServerEnabled + ? { + indexPattern: AGENTS_INDEX, + } + : { + fieldPrefix: AGENT_SAVED_OBJECT_TYPE, + })} /> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/enrollment_token_list_page/index.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/enrollment_token_list_page/index.tsx index 1871e0c1f537b0..bab3763ea4f6af 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/enrollment_token_list_page/index.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/enrollment_token_list_page/index.tsx @@ -20,7 +20,10 @@ import { HorizontalAlignment, } from '@elastic/eui'; import { FormattedMessage, FormattedDate } from '@kbn/i18n/react'; -import { ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE } from '../../../constants'; +import { + ENROLLMENT_API_KEYS_INDEX, + ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, +} from '../../../constants'; import { useBreadcrumbs, usePagination, @@ -29,6 +32,7 @@ import { sendGetOneEnrollmentAPIKey, useStartServices, sendDeleteOneEnrollmentAPIKey, + useConfig, } from '../../../hooks'; import { EnrollmentAPIKey } from '../../../types'; import { SearchBar } from '../../../components/search_bar'; @@ -154,6 +158,7 @@ const DeleteButton: React.FunctionComponent<{ apiKey: EnrollmentAPIKey; refresh: export const EnrollmentTokenListPage: React.FunctionComponent<{}> = () => { useBreadcrumbs('fleet_enrollment_tokens'); + const config = useConfig(); const [flyoutOpen, setFlyoutOpen] = useState(false); const [search, setSearch] = useState(''); const { pagination, setPagination, pageSizeOptions } = usePagination(); @@ -281,7 +286,13 @@ export const EnrollmentTokenListPage: React.FunctionComponent<{}> = () => { }); setSearch(newSearch); }} - fieldPrefix={ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE} + {...(config.agents.fleetServerEnabled + ? { + indexPattern: ENROLLMENT_API_KEYS_INDEX, + } + : { + fieldPrefix: ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + })} /> From 061cb50c712131077d4063a52f9eec0af7ccfa06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Wed, 10 Feb 2021 14:13:27 +0000 Subject: [PATCH 13/32] [Telemetry] Add stakeholders to schema changes (#90143) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 87dc99fa337492..4b0479eedea988 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -223,9 +223,8 @@ /x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-core /.telemetryrc.json @elastic/kibana-core /x-pack/.telemetryrc.json @elastic/kibana-core -src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-core -src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-core -x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-core +src/plugins/telemetry/schema/ @elastic/kibana-core @elastic/kibana-telemetry @elastic/infra-telemetry +x-pack/plugins/telemetry_collection_xpack/schema/ @elastic/kibana-core @elastic/kibana-telemetry @elastic/infra-telemetry # Kibana Localization /src/dev/i18n/ @elastic/kibana-localization @elastic/kibana-core From 3497c04d286a53716b2a02bd8f850530b4db8d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20Haro?= Date: Wed, 10 Feb 2021 14:18:11 +0000 Subject: [PATCH 14/32] Exclude telemetry schemas from the distributables (#90819) --- src/dev/build/tasks/copy_source_task.ts | 1 + x-pack/tasks/build.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/dev/build/tasks/copy_source_task.ts b/src/dev/build/tasks/copy_source_task.ts index 5571e0df2a9625..06ec40e8fcfa09 100644 --- a/src/dev/build/tasks/copy_source_task.ts +++ b/src/dev/build/tasks/copy_source_task.ts @@ -29,6 +29,7 @@ export const CopySource: Task = { '!src/cli/dev.js', '!src/functional_test_runner/**', '!src/dev/**', + '!src/plugins/telemetry/schema/**', // Skip telemetry schemas // this is the dev-only entry '!src/setup_node_env/index.js', '!**/public/**/*.{js,ts,tsx,json}', diff --git a/x-pack/tasks/build.ts b/x-pack/tasks/build.ts index 2cad1de9095211..4b6bc29284748a 100644 --- a/x-pack/tasks/build.ts +++ b/x-pack/tasks/build.ts @@ -77,6 +77,7 @@ async function copySourceAndBabelify() { '**/public/**/*.{js,ts,tsx,json}', '**/{__tests__,__mocks__,__snapshots__}/**', 'plugins/canvas/shareable_runtime/test/**', + 'plugins/telemetry_collection_xpack/schema/**', // Skip telemetry schemas ], allowEmpty: true, } From a87535624db850e8f9c4aafbc0796ab66d187d3b Mon Sep 17 00:00:00 2001 From: Marta Bondyra Date: Wed, 10 Feb 2021 15:25:47 +0100 Subject: [PATCH 15/32] [Lens] (performance) replace operationsSupportMatrix by getOperationsByField for drag and drop (#90744) --- .../lens/public/drag_drop/drag_drop.tsx | 99 +++++++++---------- .../dimension_panel/droppable.ts | 40 +++----- .../operations/operations.test.ts | 45 ++++++++- .../operations/operations.ts | 19 ++-- 4 files changed, 115 insertions(+), 88 deletions(-) diff --git a/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx b/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx index 898071e85ea79a..07c1368e534566 100644 --- a/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx +++ b/x-pack/plugins/lens/public/drag_drop/drag_drop.tsx @@ -119,8 +119,14 @@ interface DragInnerProps extends BaseProps { /** * The props for a non-draggable instance of that component. */ -interface DropInnerProps extends BaseProps, DragContextState { - isDragging: boolean; +interface DropInnerProps extends BaseProps { + dragging: DragContextState['dragging']; + setKeyboardMode: DragContextState['setKeyboardMode']; + setDragging: DragContextState['setDragging']; + setActiveDropTarget: DragContextState['setActiveDropTarget']; + setA11yMessage: DragContextState['setA11yMessage']; + registerDropTarget: DragContextState['registerDropTarget']; + isActiveDropTarget: boolean; isNotDroppable: boolean; } @@ -141,27 +147,42 @@ export const DragDrop = (props: BaseProps) => { const { value, draggable, dropType, reorderableGroup } = props; const isDragging = !!(draggable && value.id === dragging?.id); - const dragProps = { - ...props, - isDragging, - keyboardMode: isDragging ? keyboardMode : false, // optimization to not rerender all dragging components - activeDropTarget: isDragging ? activeDropTarget : undefined, // optimization to not rerender all dragging components - setKeyboardMode, - setDragging, - setActiveDropTarget, - setA11yMessage, - }; + if (draggable && !dropType) { + const dragProps = { + ...props, + isDragging, + keyboardMode: isDragging ? keyboardMode : false, // optimization to not rerender all dragging components + activeDropTarget: isDragging ? activeDropTarget : undefined, // optimization to not rerender all dragging components + setKeyboardMode, + setDragging, + setActiveDropTarget, + setA11yMessage, + }; + if (reorderableGroup && reorderableGroup.length > 1) { + return ( + + ); + } else { + return ; + } + } + const isActiveDropTarget = Boolean( + activeDropTarget?.activeDropTarget && activeDropTarget.activeDropTarget.id === value.id + ); const dropProps = { ...props, setKeyboardMode, - keyboardMode, dragging, setDragging, - activeDropTarget, + isActiveDropTarget, setActiveDropTarget, registerDropTarget, - isDragging, setA11yMessage, isNotDroppable: // If the configuration has provided a droppable flag, but this particular item is not @@ -169,21 +190,6 @@ export const DragDrop = (props: BaseProps) => { // draggable and drop targets !!(!dropType && dragging && value.id !== dragging.id), }; - - if (draggable && !dropType) { - if (reorderableGroup && reorderableGroup.length > 1) { - return ( - - ); - } else { - return ; - } - } if ( reorderableGroup && reorderableGroup.length > 1 && @@ -340,19 +346,16 @@ const DropInner = memo(function DropInner(props: DropInnerProps) { children, draggable, dragging, - isDragging, isNotDroppable, - dragType = 'copy', dropType, - keyboardMode, - activeDropTarget, - registerDropTarget, - setActiveDropTarget, + order, getAdditionalClassesOnEnter, getAdditionalClassesOnDroppable, + isActiveDropTarget, + registerDropTarget, + setActiveDropTarget, setKeyboardMode, setDragging, - order, setA11yMessage, } = props; @@ -365,11 +368,6 @@ const DropInner = memo(function DropInner(props: DropInnerProps) { } }, [order, value, registerDropTarget, dropType]); - const activeDropTargetMatches = - activeDropTarget?.activeDropTarget && activeDropTarget.activeDropTarget.id === value.id; - - const isMoveDragging = isDragging && dragType === 'move'; - const classesOnEnter = getAdditionalClassesOnEnter?.(dropType); const classesOnDroppable = getAdditionalClassesOnDroppable?.(dropType); @@ -377,15 +375,12 @@ const DropInner = memo(function DropInner(props: DropInnerProps) { 'lnsDragDrop', { 'lnsDragDrop-isDraggable': draggable, - 'lnsDragDrop-isDragging': isDragging, - 'lnsDragDrop-isHidden': isMoveDragging && !keyboardMode, 'lnsDragDrop-isDroppable': !draggable, 'lnsDragDrop-isDropTarget': dropType && dropType !== 'reorder', - 'lnsDragDrop-isActiveDropTarget': - dropType && activeDropTargetMatches && dropType !== 'reorder', - 'lnsDragDrop-isNotDroppable': !isMoveDragging && isNotDroppable, + 'lnsDragDrop-isActiveDropTarget': dropType && isActiveDropTarget && dropType !== 'reorder', + 'lnsDragDrop-isNotDroppable': isNotDroppable, }, - classesOnEnter && { [classesOnEnter]: activeDropTargetMatches }, + classesOnEnter && { [classesOnEnter]: isActiveDropTarget }, classesOnDroppable && { [classesOnDroppable]: dropType } ); @@ -396,7 +391,7 @@ const DropInner = memo(function DropInner(props: DropInnerProps) { e.preventDefault(); // An optimization to prevent a bunch of React churn. - if (!activeDropTargetMatches && dragging && onDrop) { + if (!isActiveDropTarget && dragging && onDrop) { setActiveDropTarget({ ...value, dropType, onDrop }); setA11yMessage(announce.selectedTarget(dragging.humanData, value.humanData, dropType)); } @@ -602,7 +597,7 @@ const ReorderableDrop = memo(function ReorderableDrop( dragging, setDragging, setKeyboardMode, - activeDropTarget, + isActiveDropTarget, setActiveDropTarget, reorderableGroup, setA11yMessage, @@ -610,8 +605,6 @@ const ReorderableDrop = memo(function ReorderableDrop( } = props; const currentIndex = reorderableGroup.findIndex((i) => i.id === value.id); - const activeDropTargetMatches = - activeDropTarget?.activeDropTarget && activeDropTarget.activeDropTarget.id === value.id; const { reorderState: { isReorderOn, reorderedItems, draggingHeight, direction }, @@ -646,7 +639,7 @@ const ReorderableDrop = memo(function ReorderableDrop( e.preventDefault(); // An optimization to prevent a bunch of React churn. - if (!activeDropTargetMatches && dropType && onDrop) { + if (!isActiveDropTarget && dropType && onDrop) { setActiveDropTarget({ ...value, dropType, onDrop }); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable.ts index cbd599743f8132..69c7e8c3c2ae61 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable.ts @@ -12,12 +12,11 @@ import { DraggedOperation, } from '../../types'; import { IndexPatternColumn } from '../indexpattern'; -import { insertOrReplaceColumn, deleteColumn } from '../operations'; +import { insertOrReplaceColumn, deleteColumn, getOperationTypesForField } from '../operations'; import { mergeLayer } from '../state_helpers'; import { hasField, isDraggedField } from '../utils'; import { IndexPatternPrivateState, IndexPatternField, DraggedField } from '../types'; import { trackUiEvent } from '../../lens_ui_telemetry'; -import { getOperationSupportMatrix } from './operation_support'; type DropHandlerProps = DatasourceDimensionDropHandlerProps & { droppedItem: T; @@ -34,7 +33,8 @@ export function getDropTypes( const layerIndexPatternId = props.state.layers[props.layerId].indexPatternId; function hasOperationForField(field: IndexPatternField) { - return !!getOperationSupportMatrix(props).operationByField[field.name]; + const operationsForNewField = getOperationTypesForField(field, props.filterOperations); + return !!operationsForNewField.length; } const currentColumn = props.state.layers[props.layerId].columns[props.columnId]; @@ -171,10 +171,9 @@ function onMoveDropToNonCompatibleGroup(props: DropHandlerProps) { const { columnId, setState, state, layerId, droppedItem } = props; - const operationSupportMatrix = getOperationSupportMatrix(props); - function hasOperationForField(field: IndexPatternField) { - return !!operationSupportMatrix.operationByField[field.name]; - } + const operationsForNewField = getOperationTypesForField( + droppedItem.field, + props.filterOperations + ); - if (!isDraggedField(droppedItem) || !hasOperationForField(droppedItem.field)) { + if (!isDraggedField(droppedItem) || !operationsForNewField.length) { // TODO: What do we do if we couldn't find a column? return false; } - // dragged field, not operation - - const operationsForNewField = operationSupportMatrix.operationByField[droppedItem.field.name]; - - if (!operationsForNewField || operationsForNewField.size === 0) { - return false; - } - const layer = state.layers[layerId]; const selectedColumn: IndexPatternColumn | null = layer.columns[columnId] || null; @@ -308,18 +299,13 @@ function onFieldDrop(props: DropHandlerProps) { // Detects if we can change the field only, otherwise change field + operation const fieldIsCompatibleWithCurrent = - selectedColumn && - operationSupportMatrix.operationByField[droppedItem.field.name]?.has( - selectedColumn.operationType - ); + selectedColumn && operationsForNewField.includes(selectedColumn.operationType); const newLayer = insertOrReplaceColumn({ layer, columnId, indexPattern: currentIndexPattern, - op: fieldIsCompatibleWithCurrent - ? selectedColumn.operationType - : operationsForNewField.values().next().value, + op: fieldIsCompatibleWithCurrent ? selectedColumn.operationType : operationsForNewField[0], field: droppedItem.field, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 4249f8397716a1..360e1697ae58dc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -56,7 +56,22 @@ describe('getOperationTypesForField', () => { aggregatable: true, searchable: true, }) - ).toEqual(expect.arrayContaining(['terms'])); + ).toEqual(['terms', 'cardinality', 'last_value']); + }); + + it('should return only bucketed operations on strings when passed proper filterOperations function', () => { + expect( + getOperationTypesForField( + { + type: 'string', + name: 'a', + displayName: 'aLabel', + aggregatable: true, + searchable: true, + }, + (op) => op.isBucketed + ) + ).toEqual(['terms']); }); it('should return operations on numbers', () => { @@ -68,7 +83,33 @@ describe('getOperationTypesForField', () => { aggregatable: true, searchable: true, }) - ).toEqual(expect.arrayContaining(['avg', 'sum', 'min', 'max'])); + ).toEqual([ + 'range', + 'terms', + 'avg', + 'sum', + 'min', + 'max', + 'cardinality', + 'median', + 'percentile', + 'last_value', + ]); + }); + + it('should return only metric operations on numbers when passed proper filterOperations function', () => { + expect( + getOperationTypesForField( + { + type: 'number', + name: 'a', + displayName: 'aLabel', + aggregatable: true, + searchable: true, + }, + (op) => !op.isBucketed + ) + ).toEqual(['avg', 'sum', 'min', 'max', 'cardinality', 'median', 'percentile', 'last_value']); }); it('should return operations on dates', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts index 2869b14208e1ae..63671fe35e99e1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts @@ -58,13 +58,20 @@ export function getSortScoreByPriority( * Returns all `OperationType`s that can build a column using `buildColumn` based on the * passed in field. */ -export function getOperationTypesForField(field: IndexPatternField): OperationType[] { +export function getOperationTypesForField( + field: IndexPatternField, + filterOperations?: (operation: OperationMetadata) => boolean +): OperationType[] { return operationDefinitions - .filter( - (operationDefinition) => - operationDefinition.input === 'field' && - operationDefinition.getPossibleOperationForField(field) - ) + .filter((operationDefinition) => { + if (operationDefinition.input !== 'field') { + return false; + } + const possibleOperation = operationDefinition.getPossibleOperationForField(field); + return filterOperations + ? possibleOperation && filterOperations(possibleOperation) + : possibleOperation; + }) .sort(getSortScoreByPriority) .map(({ type }) => type); } From 40f44a91c8902bcbd460ca6b5dbee740b3754f52 Mon Sep 17 00:00:00 2001 From: Matthias Wilhelm Date: Wed, 10 Feb 2021 16:08:20 +0100 Subject: [PATCH 16/32] [Discover] Add "Hide chart" / "Show chart" persistence (#88603) --- .../public/application/angular/discover.js | 22 +++++---- .../application/angular/discover_legacy.html | 1 - .../application/angular/discover_state.ts | 4 ++ .../application/components/discover.test.tsx | 5 +- .../application/components/discover.tsx | 32 +++++++++---- .../components/discover_topnav.test.tsx | 11 +++-- .../components/discover_topnav.tsx | 18 ++++--- .../public/application/components/types.ts | 24 ++++++---- .../helpers/persist_saved_search.ts | 3 ++ .../public/saved_searches/_saved_search.ts | 2 + .../discover/public/saved_searches/types.ts | 1 + .../discover/server/saved_objects/search.ts | 1 + .../apps/discover/_discover_histogram.ts | 48 +++++++++++++++++-- 13 files changed, 128 insertions(+), 44 deletions(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index af63485507d05c..3733e866989588 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -252,6 +252,12 @@ function discoverController($route, $scope, Promise) { (prop) => !_.isEqual(newStatePartial[prop], oldStatePartial[prop]) ); + if (oldStatePartial.hideChart && !newStatePartial.hideChart) { + // in case the histogram is hidden, no data is requested + // so when changing this state data needs to be fetched + changes.push(true); + } + if (changes.length) { refetch$.next(); } @@ -313,6 +319,8 @@ function discoverController($route, $scope, Promise) { setAppState, data, stateContainer, + searchSessionManager, + refetch$, }; const inspectorAdapters = ($scope.opts.inspectorAdapters = { @@ -412,6 +420,9 @@ function discoverController($route, $scope, Promise) { if (savedSearch.grid) { defaultState.grid = savedSearch.grid; } + if (savedSearch.hideChart) { + defaultState.hideChart = savedSearch.hideChart; + } return defaultState; } @@ -562,13 +573,6 @@ function discoverController($route, $scope, Promise) { }); }; - $scope.handleRefresh = function (_payload, isUpdate) { - if (isUpdate === false) { - searchSessionManager.removeSearchSessionIdFromURL({ replace: false }); - refetch$.next(); - } - }; - function getDimensions(aggs, timeRange) { const [metric, agg] = aggs; agg.params.timeRange = timeRange; @@ -601,7 +605,7 @@ function discoverController($route, $scope, Promise) { function onResults(resp) { inspectorRequest.stats(getResponseInspectorStats(resp, $scope.searchSource)).ok({ json: resp }); - if (getTimeField()) { + if (getTimeField() && !$scope.state.hideChart) { const tabifiedData = tabifyAggResponse($scope.opts.chartAggConfigs, resp); $scope.searchSource.rawResponse = resp; $scope.histogramData = discoverResponseHandler( @@ -704,7 +708,7 @@ function discoverController($route, $scope, Promise) { async function setupVisualization() { // If no timefield has been specified we don't create a histogram of messages - if (!getTimeField()) return; + if (!getTimeField() || $scope.state.hideChart) return; const { interval: histogramInterval } = $scope.state; const visStateAggs = [ diff --git a/src/plugins/discover/public/application/angular/discover_legacy.html b/src/plugins/discover/public/application/angular/discover_legacy.html index dc18b7929318bd..501496494106af 100644 --- a/src/plugins/discover/public/application/angular/discover_legacy.html +++ b/src/plugins/discover/public/application/angular/discover_legacy.html @@ -17,7 +17,6 @@ state="state" time-range="timeRange" top-nav-menu="topNavMenu" - update-query="handleRefresh" use-new-fields-api="useNewFieldsApi" unmapped-fields-config="unmappedFieldsConfig" > diff --git a/src/plugins/discover/public/application/angular/discover_state.ts b/src/plugins/discover/public/application/angular/discover_state.ts index 5e93966d78d9fa..93fc49b65cbc92 100644 --- a/src/plugins/discover/public/application/angular/discover_state.ts +++ b/src/plugins/discover/public/application/angular/discover_state.ts @@ -44,6 +44,10 @@ export interface AppState { * Data Grid related state */ grid?: DiscoverGridSettings; + /** + * Hide chart + */ + hideChart?: boolean; /** * id of the used index pattern */ diff --git a/src/plugins/discover/public/application/components/discover.test.tsx b/src/plugins/discover/public/application/components/discover.test.tsx index f0f11558abd65c..00554196e11fd2 100644 --- a/src/plugins/discover/public/application/components/discover.test.tsx +++ b/src/plugins/discover/public/application/components/discover.test.tsx @@ -25,6 +25,8 @@ import { indexPatternWithTimefieldMock } from '../../__mocks__/index_pattern_wit import { calcFieldCounts } from '../helpers/calc_field_counts'; import { DiscoverProps } from './types'; import { RequestAdapter } from '../../../../inspector/common'; +import { Subject } from 'rxjs'; +import { DiscoverSearchSessionManager } from '../angular/discover_search_session'; const mockNavigation = navigationPluginMock.createStartContract(); @@ -73,8 +75,10 @@ function getProps(indexPattern: IndexPattern): DiscoverProps { indexPatternList: (indexPattern as unknown) as Array>, inspectorAdapters: { requests: {} as RequestAdapter }, navigateTo: jest.fn(), + refetch$: {} as Subject, sampleSize: 10, savedSearch: savedSearchMock, + searchSessionManager: {} as DiscoverSearchSessionManager, setHeaderActionMenu: jest.fn(), timefield: indexPattern.timeFieldName || '', setAppState: jest.fn(), @@ -86,7 +90,6 @@ function getProps(indexPattern: IndexPattern): DiscoverProps { rows: esHits, searchSource: searchSourceMock, state: { columns: [] }, - updateQuery: jest.fn(), }; } diff --git a/src/plugins/discover/public/application/components/discover.tsx b/src/plugins/discover/public/application/components/discover.tsx index 99baa30e18c7a7..71650a4a38472e 100644 --- a/src/plugins/discover/public/application/components/discover.tsx +++ b/src/plugins/discover/public/application/components/discover.tsx @@ -6,7 +6,7 @@ * Side Public License, v 1. */ import './discover.scss'; -import React, { useCallback, useMemo, useRef, useState } from 'react'; +import React, { useState, useRef, useMemo, useCallback } from 'react'; import { EuiButtonEmpty, EuiButtonIcon, @@ -66,7 +66,6 @@ export function Discover({ searchSource, state, timeRange, - updateQuery, unmappedFieldsConfig, }: DiscoverProps) { const [expandedDoc, setExpandedDoc] = useState(undefined); @@ -76,8 +75,11 @@ export function Discover({ // collapse icon isn't displayed in mobile view, use it to detect which view is displayed return collapseIcon && !collapseIcon.current; }; - - const [toggleOn, toggleChart] = useState(true); + const toggleHideChart = useCallback(() => { + const newState = { ...state, hideChart: !state.hideChart }; + opts.stateContainer.setAppState(newState); + }, [state, opts]); + const hideChart = useMemo(() => state.hideChart, [state]); const { savedSearch, indexPatternList, config, services, data, setAppState } = opts; const { trackUiMetric, capabilities, indexPatterns } = services; const [isSidebarClosed, setIsSidebarClosed] = useState(false); @@ -89,6 +91,15 @@ export function Discover({ const contentCentered = resultState === 'uninitialized'; const isLegacy = services.uiSettings.get('doc_table:legacy'); const useNewFieldsApi = !services.uiSettings.get(SEARCH_FIELDS_FROM_SOURCE); + const updateQuery = useCallback( + (_payload, isUpdate?: boolean) => { + if (isUpdate === false) { + opts.searchSessionManager.removeSearchSessionIdFromURL({ replace: false }); + opts.refetch$.next(); + } + }, + [opts] + ); const { onAddColumn, onRemoveColumn, onMoveColumn, onSetColumns } = useMemo( () => @@ -192,7 +203,8 @@ export function Discover({ indexPattern={indexPattern} opts={opts} onOpenInspector={onOpenInspector} - state={state} + query={state.query} + savedQuery={state.savedQuery} updateQuery={updateQuery} /> @@ -277,7 +289,7 @@ export function Discover({ onResetQuery={resetQuery} /> - {toggleOn && ( + {!hideChart && ( { - toggleChart(!toggleOn); + toggleHideChart(); }} data-test-subj="discoverChartToggle" > - {toggleOn + {!hideChart ? i18n.translate('discover.hideChart', { defaultMessage: 'Hide chart', }) @@ -312,7 +324,7 @@ export function Discover({ {isLegacy && } - {toggleOn && opts.timefield && ( + {!hideChart && opts.timefield && (
>, inspectorAdapters: { requests: {} as RequestAdapter }, navigateTo: jest.fn(), + refetch$: {} as Subject, sampleSize: 10, savedSearch: savedSearchMock, + searchSessionManager: {} as DiscoverSearchSessionManager, services, setAppState: jest.fn(), setHeaderActionMenu: jest.fn(), stateContainer: {} as GetStateReturn, timefield: indexPattern.timeFieldName || '', }, - state, + query: {} as Query, + savedQuery: '', updateQuery: jest.fn(), onOpenInspector: jest.fn(), }; diff --git a/src/plugins/discover/public/application/components/discover_topnav.tsx b/src/plugins/discover/public/application/components/discover_topnav.tsx index 69a1433b6505c0..fd2aba22aa41db 100644 --- a/src/plugins/discover/public/application/components/discover_topnav.tsx +++ b/src/plugins/discover/public/application/components/discover_topnav.tsx @@ -8,17 +8,21 @@ import React, { useMemo } from 'react'; import { DiscoverProps } from './types'; import { getTopNavLinks } from './top_nav/get_top_nav_links'; +import { Query, TimeRange } from '../../../../data/common/query'; -export type DiscoverTopNavProps = Pick< - DiscoverProps, - 'indexPattern' | 'updateQuery' | 'state' | 'opts' -> & { onOpenInspector: () => void }; +export type DiscoverTopNavProps = Pick & { + onOpenInspector: () => void; + query?: Query; + savedQuery?: string; + updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; +}; export const DiscoverTopNav = ({ indexPattern, opts, onOpenInspector, - state, + query, + savedQuery, updateQuery, }: DiscoverTopNavProps) => { const showDatePicker = useMemo(() => indexPattern.isTimeBased(), [indexPattern]); @@ -58,9 +62,9 @@ export const DiscoverTopNav = ({ indexPatterns={[indexPattern]} onQuerySubmit={updateQuery} onSavedQueryIdChange={updateSavedQueryId} - query={state.query} + query={query} setMenuMountPoint={opts.setHeaderActionMenu} - savedQueryId={state.savedQuery} + savedQueryId={savedQuery} screenTitle={opts.savedSearch.title} showDatePicker={showDatePicker} showSaveQuery={!!opts.services.capabilities.discover.saveQuery} diff --git a/src/plugins/discover/public/application/components/types.ts b/src/plugins/discover/public/application/components/types.ts index ee06bcab6528b2..e276795f9ed7fd 100644 --- a/src/plugins/discover/public/application/components/types.ts +++ b/src/plugins/discover/public/application/components/types.ts @@ -7,6 +7,7 @@ */ import { IUiSettingsClient, MountPoint, SavedObject } from 'kibana/public'; +import { Subject } from 'rxjs'; import { Chart } from '../angular/helpers/point_series'; import { IndexPattern } from '../../../../data/common/index_patterns/index_patterns'; import { ElasticSearchHit } from '../doc_views/doc_views_types'; @@ -17,13 +18,12 @@ import { FilterManager, IndexPatternAttributes, ISearchSource, - Query, - TimeRange, } from '../../../../data/public'; import { SavedSearch } from '../../saved_searches'; import { AppState, GetStateReturn } from '../angular/discover_state'; import { RequestAdapter } from '../../../../inspector/common'; import { DiscoverServices } from '../../build_services'; +import { DiscoverSearchSessionManager } from '../angular/discover_search_session'; export interface DiscoverProps { /** @@ -97,10 +97,18 @@ export interface DiscoverProps { * List of available index patterns */ indexPatternList: Array>; + /** + * Refetch observable + */ + refetch$: Subject; /** * Kibana core services used by discover */ services: DiscoverServices; + /** + * Helps with state management of search session + */ + searchSessionManager: DiscoverSearchSessionManager; /** * The number of documents that can be displayed in the table/grid */ @@ -113,10 +121,6 @@ export interface DiscoverProps { * Function to set the header menu */ setHeaderActionMenu: (menuMount: MountPoint | undefined) => void; - /** - * Functions for retrieving/mutating state - */ - stateContainer: GetStateReturn; /** * Timefield of the currently used index pattern */ @@ -125,6 +129,10 @@ export interface DiscoverProps { * Function to set the current state */ setAppState: (state: Partial) => void; + /** + * State container providing globalState, appState and functions + */ + stateContainer: GetStateReturn; }; /** * Function to reset the current query @@ -150,10 +158,6 @@ export interface DiscoverProps { * Currently selected time range */ timeRange?: { from: string; to: string }; - /** - * Function to update the actual query - */ - updateQuery: (payload: { dateRange: TimeRange; query?: Query }, isUpdate?: boolean) => void; /** * An object containing properties for proper handling of unmapped fields in the UI */ diff --git a/src/plugins/discover/public/application/helpers/persist_saved_search.ts b/src/plugins/discover/public/application/helpers/persist_saved_search.ts index 1bebf60c0a8051..06e90c93bc77c9 100644 --- a/src/plugins/discover/public/application/helpers/persist_saved_search.ts +++ b/src/plugins/discover/public/application/helpers/persist_saved_search.ts @@ -48,6 +48,9 @@ export async function persistSavedSearch( if (state.grid) { savedSearch.grid = state.grid; } + if (state.hideChart) { + savedSearch.hideChart = state.hideChart; + } try { const id = await savedSearch.save(saveOptions); diff --git a/src/plugins/discover/public/saved_searches/_saved_search.ts b/src/plugins/discover/public/saved_searches/_saved_search.ts index d5bd3ea4011bb4..a7b6ef49cacd21 100644 --- a/src/plugins/discover/public/saved_searches/_saved_search.ts +++ b/src/plugins/discover/public/saved_searches/_saved_search.ts @@ -14,6 +14,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { public static mapping = { title: 'text', description: 'text', + hideChart: 'boolean', hits: 'integer', columns: 'keyword', grid: 'object', @@ -35,6 +36,7 @@ export function createSavedSearchClass(savedObjects: SavedObjectsStart) { mapping: { title: 'text', description: 'text', + hideChart: 'boolean', hits: 'integer', columns: 'keyword', grid: 'object', diff --git a/src/plugins/discover/public/saved_searches/types.ts b/src/plugins/discover/public/saved_searches/types.ts index 24fbbcb61cb489..4646744ee0ef3c 100644 --- a/src/plugins/discover/public/saved_searches/types.ts +++ b/src/plugins/discover/public/saved_searches/types.ts @@ -24,6 +24,7 @@ export interface SavedSearch { lastSavedTitle?: string; copyOnSave?: boolean; pre712?: boolean; + hideChart?: boolean; } export interface SavedSearchLoader { get: (id: string) => Promise; diff --git a/src/plugins/discover/server/saved_objects/search.ts b/src/plugins/discover/server/saved_objects/search.ts index 43f107399ac365..de3a2197fe0acc 100644 --- a/src/plugins/discover/server/saved_objects/search.ts +++ b/src/plugins/discover/server/saved_objects/search.ts @@ -34,6 +34,7 @@ export const searchSavedObjectType: SavedObjectsType = { properties: { columns: { type: 'keyword', index: false, doc_values: false }, description: { type: 'text' }, + hideChart: { type: 'boolean', index: false, doc_values: false }, hits: { type: 'integer', index: false, doc_values: false }, kibanaSavedObjectMeta: { properties: { diff --git a/test/functional/apps/discover/_discover_histogram.ts b/test/functional/apps/discover/_discover_histogram.ts index 56dc784eac70be..9a6692dc793d67 100644 --- a/test/functional/apps/discover/_discover_histogram.ts +++ b/test/functional/apps/discover/_discover_histogram.ts @@ -19,6 +19,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { defaultIndex: 'long-window-logstash-*', 'dateFormat:tz': 'Europe/Berlin', }; + const testSubjects = getService('testSubjects'); + const browser = getService('browser'); describe('discover histogram', function describeIndexTests() { before(async () => { @@ -35,11 +37,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await security.testUser.restoreDefaults(); }); - async function prepareTest(fromTime: string, toTime: string, interval: string) { + async function prepareTest(fromTime: string, toTime: string, interval?: string) { await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); await PageObjects.discover.waitUntilSearchingHasFinished(); - await PageObjects.discover.setChartInterval(interval); - await PageObjects.header.waitUntilLoadingHasFinished(); + if (interval) { + await PageObjects.discover.setChartInterval(interval); + await PageObjects.header.waitUntilLoadingHasFinished(); + } } it('should visualize monthly data with different day intervals', async () => { @@ -65,5 +69,43 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const chartIntervalIconTip = await PageObjects.discover.getChartIntervalWarningIcon(); expect(chartIntervalIconTip).to.be(true); }); + it('should allow hide/show histogram, persisted in url state', async () => { + const fromTime = 'Jan 01, 2010 @ 00:00:00.000'; + const toTime = 'Mar 21, 2019 @ 00:00:00.000'; + await prepareTest(fromTime, toTime); + let canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(true); + await testSubjects.click('discoverChartToggle'); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(false); + // histogram is hidden, when reloading the page it should remain hidden + await browser.refresh(); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(false); + await testSubjects.click('discoverChartToggle'); + await PageObjects.header.waitUntilLoadingHasFinished(); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(true); + }); + it('should allow hiding the histogram, persisted in saved search', async () => { + const fromTime = 'Jan 01, 2010 @ 00:00:00.000'; + const toTime = 'Mar 21, 2019 @ 00:00:00.000'; + const savedSearch = 'persisted hidden histogram'; + await prepareTest(fromTime, toTime); + await testSubjects.click('discoverChartToggle'); + let canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(false); + await PageObjects.discover.saveSearch(savedSearch); + await PageObjects.header.waitUntilLoadingHasFinished(); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(false); + await testSubjects.click('discoverChartToggle'); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(true); + await PageObjects.discover.clickResetSavedSearchButton(); + await PageObjects.header.waitUntilLoadingHasFinished(); + canvasExists = await elasticChart.canvasExists(); + expect(canvasExists).to.be(false); + }); }); } From 4aabf358b02d9daaf8cc40eb139745f238a34e07 Mon Sep 17 00:00:00 2001 From: Scotty Bollinger Date: Wed, 10 Feb 2021 09:08:59 -0600 Subject: [PATCH 17/32] [Workplace Search] Fix error message in Schema (#90869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In SchemaErrors, we have an edge case where a user navigates to a non-existent error message state. When migrating I took a stab at passing the custom message to the `flashAPIErrors` helper. It turns out that since we don’t have to parse an error object, we can just set the error message directly --- .../components/schema/schema_logic.test.ts | 12 +++++++----- .../components/schema/schema_logic.ts | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.test.ts index af650d95efaf6b..28850531ebb94f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.test.ts @@ -35,7 +35,12 @@ import { SchemaLogic, dataTypeOptions } from './schema_logic'; describe('SchemaLogic', () => { const { http } = mockHttpValues; - const { clearFlashMessages, flashAPIErrors, setSuccessMessage } = mockFlashMessageHelpers; + const { + clearFlashMessages, + flashAPIErrors, + setSuccessMessage, + setErrorMessage, + } = mockFlashMessageHelpers; const { mount } = new LogicMounter(SchemaLogic); const defaultValues = { @@ -298,10 +303,7 @@ describe('SchemaLogic', () => { ); await nextTick(); - expect(flashAPIErrors).toHaveBeenCalledWith({ - error: 'this is an error', - message: SCHEMA_FIELD_ERRORS_ERROR_MESSAGE, - }); + expect(setErrorMessage).toHaveBeenCalledWith(SCHEMA_FIELD_ERRORS_ERROR_MESSAGE); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.ts b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.ts index 9906efe707d850..10b7f85a631bc1 100644 --- a/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/workplace_search/views/content_sources/components/schema/schema_logic.ts @@ -13,6 +13,7 @@ import { ADD, UPDATE } from '../../../../../shared/constants/operations'; import { flashAPIErrors, setSuccessMessage, + setErrorMessage, clearFlashMessages, } from '../../../../../shared/flash_messages'; import { HttpLogic } from '../../../../../shared/http'; @@ -295,7 +296,7 @@ export const SchemaLogic = kea>({ fieldCoercionErrors: response.fieldCoercionErrors, }); } catch (e) { - flashAPIErrors({ ...e, message: SCHEMA_FIELD_ERRORS_ERROR_MESSAGE }); + setErrorMessage(SCHEMA_FIELD_ERRORS_ERROR_MESSAGE); } }, addNewField: ({ fieldName, newFieldType }) => { From fa18be9beb49ac8152cc39a0705946f0efe33bd2 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Wed, 10 Feb 2021 17:19:34 +0200 Subject: [PATCH 18/32] [Telemetry] use fresh keys (#90446) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/encryption/encrypt.test.ts | 14 +++++++------- .../server/encryption/encrypt.ts | 2 +- .../server/encryption/telemetry_jwks.ts | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.test.ts b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.test.ts index 9d4443d6e70350..c1a1a32e3c7f07 100644 --- a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.test.ts +++ b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.test.ts @@ -14,13 +14,13 @@ describe('getKID', () => { it(`returns 'kibana_dev' kid for development`, async () => { const useProdKey = false; const kid = getKID(useProdKey); - expect(kid).toBe('kibana_dev'); + expect(kid).toBe('kibana_dev1'); }); - it(`returns 'kibana_prod' kid for development`, async () => { + it(`returns 'kibana_1' kid for production`, async () => { const useProdKey = true; const kid = getKID(useProdKey); - expect(kid).toBe('kibana'); + expect(kid).toBe('kibana1'); }); }); @@ -35,15 +35,15 @@ describe('encryptTelemetry', () => { expect(createRequestEncryptor).toBeCalledWith(telemetryJWKS); }); - it('uses kibana kid on { useProdKey: true }', async () => { + it('uses kibana1 kid on { useProdKey: true }', async () => { const payload = { some: 'value' }; await encryptTelemetry(payload, { useProdKey: true }); - expect(mockEncrypt).toBeCalledWith('kibana', payload); + expect(mockEncrypt).toBeCalledWith('kibana1', payload); }); - it('uses kibana_dev kid on { useProdKey: false }', async () => { + it('uses kibana_dev1 kid on { useProdKey: false }', async () => { const payload = { some: 'value' }; await encryptTelemetry(payload, { useProdKey: false }); - expect(mockEncrypt).toBeCalledWith('kibana_dev', payload); + expect(mockEncrypt).toBeCalledWith('kibana_dev1', payload); }); }); diff --git a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts index 331aedce8cbc6d..a2c24627f6fd78 100644 --- a/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts +++ b/src/plugins/telemetry_collection_manager/server/encryption/encrypt.ts @@ -10,7 +10,7 @@ import { createRequestEncryptor } from '@elastic/request-crypto'; import { telemetryJWKS } from './telemetry_jwks'; export function getKID(useProdKey = false): string { - return useProdKey ? 'kibana' : 'kibana_dev'; + return useProdKey ? 'kibana1' : 'kibana_dev1'; } export async function encryptTelemetry( diff --git a/src/plugins/telemetry_collection_manager/server/encryption/telemetry_jwks.ts b/src/plugins/telemetry_collection_manager/server/encryption/telemetry_jwks.ts index c3ac71969e4493..bf4c2a952c431c 100644 --- a/src/plugins/telemetry_collection_manager/server/encryption/telemetry_jwks.ts +++ b/src/plugins/telemetry_collection_manager/server/encryption/telemetry_jwks.ts @@ -12,21 +12,21 @@ export const telemetryJWKS: PublicJWKS = { keys: [ { kty: 'RSA', - kid: 'kibana', + kid: 'kibana1', use: 'enc', alg: 'RSA-OAEP', e: 'AQAB', n: - 'xYYa5XzvENaAzElCxQurloQM2KEQ058YSjZqmOwa-IN-EZMSUaYPY3qfYCG78ioRaKTHq4mgnkyrDKgjY_1pWKytiRD61FG2ZUeOCwzydnqO8Qpz2vFnibEHkZBRsKkLHgm90RgGpcXfz8vwxkz_nu59aWy5Qr7Ct99H0pEV1HoiCvy5Yw3QfWSAeV-3DWmq_0kX49tqk5yZE-vKnUhNMgqM22lMFTE5-vlaeHgv4ZcvCQx_HrOeea8LyZa5YOdqN-9st0g0G-aWp3CNI2-KJlMUTBAfIAtjwmJ-8QlgeIB1aA7OI2Ceh3kd4dNLesGdLvZ0y4f8IMOsO1dsRWSEsQ', + 'gjwVNVkOqbTZ6QrxdeYbKDnBzhCZGXM97Iq0dXJlpa-7UegcBemI1ALZkbX6AaDrCmqzetsnxJbVz2gr-uzkc97zzjTvPAn-jM-9cfjfsb-nd70qLY7ru3qdyOLb5-ho8cjmjnAp7VaEPuiNOjZ6V6tXq8Cn5LHH8G6K8oZLU1N4RWqkcAvEIlfaLusfMnl15fe7aZkYaKfVFjD-pti_2JGRV9XZ0knRI2oIRMaroBYpfYJxbpR0NLhR7ND6U5WlvxfnaVvRK4c_plVLOtcROqZVn98Z8yZ6GU14vCcvkIBox2D_xd1gSkpMammTQ3tVAEAvoq_wEn_qEbls1Uucgw', }, { kty: 'RSA', - kid: 'kibana_dev', + kid: 'kibana_dev1', use: 'enc', alg: 'RSA-OAEP', e: 'AQAB', n: - 'juVHivsYFznjrDC449oL3xKVTvux_7dEgBGOgJdfzA2R2GspEAOzupT-VkBnqrJnRP_lznM8bQIvvst1f_DNQ1me_Lr9u9cwL5Vq6SWlmw_u9ur_-ewkShU4tBoJDArksOS-ciTaUJoMaxanb7jWexp0pCDlrLrQyAOCnKQL701mD1gdT4rIw7F-jkb5fLUNUVzOGaGyVy6DHAHZx7Tnyw8rswhyRVvuS73imbRp9XcdOFhBDOeSbrSuZGqrVCjoIlWw-UsiW2ueRd8brBoOIHSmTOMIrIMjpPmzMFRKyCvvhnbjrw8j3fQtFII8urhXCVAw8aIHZhiBc5t9ZuwbJw', + 'rEi54h-9hCbqy9Mj_tJmx-dJdtrMmMzkhX5Wd63Pp3dABHpnLJSy--y8QoEa9K9ACaRfExSxgYQ-3K17Yy-UYj3ChAl3hrqZcP2AT3O18Lr2BN7EBjy88lTM0oeck9KLL_iGf8wz8_jeqQFIo3AWrBBuR3VFE0_k-_N1KCenSVm_fE3Nk_ZXm1ByFbgxWUFrYgLfEQn2v0FQYVpfTlbV_awtqoZLYGtuHmaLZhErzJFh6W8zrx8oSpGn8VlVLjF-AR3ugfw2F_HM8ZR8zY1dHVxvoLGz13F5aY8DHn0_ao9t0Yz2Y_SUNviyxMx0eIEJeo2njM2vMzYQNaT1Ghgc-w', }, ], }; From a60e4752609ccff554c3e76ca1a853afe83390f5 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Wed, 10 Feb 2021 15:30:43 +0000 Subject: [PATCH 19/32] [Security Solution] Split test cases into blocks (#89978) * split test cases into blocks * run tests individually * fix up timeline creation * fix up timeline creation * add assertions for timeline creation * create timeline by api * create timeline by api * fix lint error * add selector to timeline screen * fix type error Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../integration/timelines/creation.spec.ts | 97 ++++++++--------- .../integration/timelines/notes_tab.spec.ts | 60 ++++++++++ .../timelines/open_timeline.spec.ts | 103 ++++++++++++++++++ .../integration/timelines/query_tab.spec.ts | 77 +++++++++++++ .../cypress/screens/timeline.ts | 4 + .../cypress/tasks/api_calls/notes.ts | 23 ++++ .../cypress/tasks/timeline.ts | 27 ++++- 7 files changed, 336 insertions(+), 55 deletions(-) create mode 100644 x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts create mode 100644 x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts create mode 100644 x-pack/plugins/security_solution/cypress/integration/timelines/query_tab.spec.ts create mode 100644 x-pack/plugins/security_solution/cypress/tasks/api_calls/notes.ts diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/creation.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/creation.spec.ts index 3c1a49bbf17925..b08bae26bf7edf 100644 --- a/x-pack/plugins/security_solution/cypress/integration/timelines/creation.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/timelines/creation.spec.ts @@ -10,25 +10,12 @@ import { timeline } from '../../objects/timeline'; import { FAVORITE_TIMELINE, LOCKED_ICON, - UNLOCKED_ICON, - NOTES_TAB_BUTTON, NOTES_TEXT, - // NOTES_COUNT, - NOTES_TEXT_AREA, PIN_EVENT, - TIMELINE_DESCRIPTION, TIMELINE_FILTER, - // TIMELINE_FILTER, - TIMELINE_QUERY, - TIMELINE_TITLE, - OPEN_TIMELINE_MODAL, + TIMELINE_PANEL, } from '../../screens/timeline'; -import { - TIMELINES_DESCRIPTION, - TIMELINES_PINNED_EVENT_COUNT, - TIMELINES_NOTES_COUNT, - TIMELINES_FAVORITE, -} from '../../screens/timelines'; + import { cleanKibana } from '../../tasks/common'; import { loginAndWaitForPage } from '../../tasks/login'; @@ -39,70 +26,72 @@ import { addNotesToTimeline, closeTimeline, createNewTimeline, + goToQueryTab, markAsFavorite, - openTimelineFromSettings, pinFirstEvent, populateTimeline, waitForTimelineChanges, } from '../../tasks/timeline'; -import { openTimeline } from '../../tasks/timelines'; import { OVERVIEW_URL } from '../../urls/navigation'; -// Skipped at the moment as there looks to be in-deterministic bugs with the open timeline dialog. -describe.skip('Timelines', () => { - beforeEach(() => { +describe('Timelines', (): void => { + before(() => { cleanKibana(); + loginAndWaitForPage(OVERVIEW_URL); }); - it('Creates a timeline', () => { - cy.intercept('PATCH', '/api/timeline').as('timeline'); + describe('Toggle create timeline from plus icon', () => { + after(() => { + closeTimeline(); + }); - loginAndWaitForPage(OVERVIEW_URL); - openTimelineUsingToggle(); - addNameAndDescriptionToTimeline(timeline); + it('toggle create timeline ', () => { + createNewTimeline(); + cy.get(TIMELINE_PANEL).should('be.visible'); + }); + }); - cy.wait('@timeline').then(({ response }) => { - const timelineId = response!.body.data.persistTimeline.timeline.savedObjectId; + describe('Creates a timeline by clicking untitled timeline from bottom bar', () => { + after(() => { + closeTimeline(); + }); + before(() => { + openTimelineUsingToggle(); + addNameAndDescriptionToTimeline(timeline); populateTimeline(); + }); + + beforeEach(() => { + goToQueryTab(); + }); + + it('can be added filter', () => { addFilter(timeline.filter); - pinFirstEvent(); + cy.get(TIMELINE_FILTER(timeline.filter)).should('exist'); + }); + it('pins an event', () => { + pinFirstEvent(); cy.get(PIN_EVENT) .should('have.attr', 'aria-label') .and('match', /Unpin the event in row 2/); + }); + + it('has a lock icon', () => { cy.get(LOCKED_ICON).should('be.visible'); + }); + it('can be added notes', () => { addNotesToTimeline(timeline.notes); + cy.get(NOTES_TEXT).should('have.text', timeline.notes); + }); + + it('can be marked as favorite', () => { markAsFavorite(); waitForTimelineChanges(); - createNewTimeline(); - closeTimeline(); - openTimelineFromSettings(); - - cy.get(OPEN_TIMELINE_MODAL).should('be.visible'); - cy.contains(timeline.title).should('exist'); - cy.get(TIMELINES_DESCRIPTION).first().should('have.text', timeline.description); - cy.get(TIMELINES_PINNED_EVENT_COUNT).first().should('have.text', '1'); - cy.get(TIMELINES_NOTES_COUNT).first().should('have.text', '1'); - cy.get(TIMELINES_FAVORITE).first().should('exist'); - - openTimeline(timelineId); - - cy.get(FAVORITE_TIMELINE).should('exist'); - cy.get(TIMELINE_TITLE).should('have.text', timeline.title); - cy.get(TIMELINE_DESCRIPTION).should('have.text', timeline.description); // This is the flake part where it sometimes does not show/load the timelines correctly - cy.get(TIMELINE_QUERY).should('have.text', `${timeline.query} `); - cy.get(TIMELINE_FILTER(timeline.filter)).should('exist'); - cy.get(PIN_EVENT) - .should('have.attr', 'aria-label') - .and('match', /Unpin the event in row 2/); - cy.get(UNLOCKED_ICON).should('be.visible'); - cy.get(NOTES_TAB_BUTTON).click(); - cy.get(NOTES_TEXT_AREA).should('exist'); - - cy.get(NOTES_TEXT).should('have.text', timeline.notes); + cy.get(FAVORITE_TIMELINE).should('have.text', 'Remove from favorites'); }); }); }); diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts new file mode 100644 index 00000000000000..6653290fc2ebba --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/integration/timelines/notes_tab.spec.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timeline } from '../../objects/timeline'; + +import { NOTES_TEXT, NOTES_TEXT_AREA } from '../../screens/timeline'; +import { createTimeline } from '../../tasks/api_calls/timelines'; + +import { cleanKibana } from '../../tasks/common'; + +import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; +import { + addNotesToTimeline, + closeTimeline, + goToNotesTab, + openTimelineById, + waitForEventsPanelToBeLoaded, +} from '../../tasks/timeline'; +import { waitForTimelinesPanelToBeLoaded } from '../../tasks/timelines'; + +import { TIMELINES_URL } from '../../urls/navigation'; + +describe('Timeline notes tab', () => { + let timelineId: string | null = null; + before(() => { + cleanKibana(); + loginAndWaitForPageWithoutDateRange(TIMELINES_URL); + waitForTimelinesPanelToBeLoaded(); + + createTimeline(timeline) + .then((response) => { + timelineId = response.body.data.persistTimeline.timeline.savedObjectId; + }) + .then(() => { + waitForTimelinesPanelToBeLoaded(); + openTimelineById(timelineId!) + .click({ force: true }) + .then(() => { + waitForEventsPanelToBeLoaded(); + goToNotesTab(); + addNotesToTimeline(timeline.notes); + }); + }); + }); + after(() => { + closeTimeline(); + }); + + it('should contain notes', () => { + cy.get(NOTES_TEXT).should('have.text', timeline.notes); + }); + + it('should render mockdown', () => { + cy.get(NOTES_TEXT_AREA).should('exist'); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts new file mode 100644 index 00000000000000..5d5d125082b8b0 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/integration/timelines/open_timeline.spec.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timeline } from '../../objects/timeline'; + +import { TIMELINE_DESCRIPTION, TIMELINE_TITLE, OPEN_TIMELINE_MODAL } from '../../screens/timeline'; +import { + TIMELINES_DESCRIPTION, + TIMELINES_PINNED_EVENT_COUNT, + TIMELINES_NOTES_COUNT, + TIMELINES_FAVORITE, +} from '../../screens/timelines'; +import { addNoteToTimeline } from '../../tasks/api_calls/notes'; + +import { createTimeline } from '../../tasks/api_calls/timelines'; + +import { cleanKibana } from '../../tasks/common'; + +import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; +import { + closeOpenTimelineModal, + markAsFavorite, + openTimelineById, + openTimelineFromSettings, + pinFirstEvent, + waitForEventsPanelToBeLoaded, +} from '../../tasks/timeline'; +import { waitForTimelinesPanelToBeLoaded } from '../../tasks/timelines'; + +import { TIMELINES_URL } from '../../urls/navigation'; + +describe('Open timeline', () => { + let timelineId: string | null = null; + before(() => { + cleanKibana(); + loginAndWaitForPageWithoutDateRange(TIMELINES_URL); + waitForTimelinesPanelToBeLoaded(); + + createTimeline(timeline) + .then((response) => { + timelineId = response.body.data.persistTimeline.timeline.savedObjectId; + }) + .then(() => { + const note = timeline.notes; + addNoteToTimeline(note, timelineId!).should((response) => { + expect(response.status).to.equal(200); + waitForTimelinesPanelToBeLoaded(); + openTimelineById(timelineId!) + .click({ force: true }) + .then(() => { + waitForEventsPanelToBeLoaded(); + pinFirstEvent(); + markAsFavorite(); + }); + }); + }); + }); + describe('Open timeline modal', () => { + before(() => { + openTimelineFromSettings(); + }); + + after(() => { + closeOpenTimelineModal(); + }); + + it('should open a modal', () => { + cy.get(OPEN_TIMELINE_MODAL).should('be.visible'); + }); + + it('should display timeline info - title', () => { + cy.contains(timeline.title).should('exist'); + }); + + it('should display timeline info - description', () => { + cy.get(TIMELINES_DESCRIPTION).first().should('have.text', timeline.description); + }); + + it('should display timeline info - pinned event count', () => { + cy.get(TIMELINES_PINNED_EVENT_COUNT).first().should('have.text', '1'); + }); + + it('should display timeline info - notes count', () => { + cy.get(TIMELINES_NOTES_COUNT).first().should('have.text', '1'); + }); + + it('should display timeline info - favorite timeline', () => { + cy.get(TIMELINES_FAVORITE).first().should('exist'); + }); + + it('should display timeline content - title', () => { + cy.get(TIMELINE_TITLE).should('have.text', timeline.title); + }); + + it('should display timeline content - description', () => { + cy.get(TIMELINE_DESCRIPTION).should('have.text', timeline.description); // This is the flake part where it sometimes does not show/load the timelines correctly + }); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/integration/timelines/query_tab.spec.ts b/x-pack/plugins/security_solution/cypress/integration/timelines/query_tab.spec.ts new file mode 100644 index 00000000000000..56cb5d870d795a --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/integration/timelines/query_tab.spec.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { timeline } from '../../objects/timeline'; + +import { UNLOCKED_ICON, PIN_EVENT, TIMELINE_FILTER, TIMELINE_QUERY } from '../../screens/timeline'; +import { addNoteToTimeline } from '../../tasks/api_calls/notes'; +import { createTimeline } from '../../tasks/api_calls/timelines'; + +import { cleanKibana } from '../../tasks/common'; + +import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login'; +import { + addFilter, + closeTimeline, + openTimelineById, + pinFirstEvent, + waitForEventsPanelToBeLoaded, +} from '../../tasks/timeline'; +import { waitForTimelinesPanelToBeLoaded } from '../../tasks/timelines'; + +import { TIMELINES_URL } from '../../urls/navigation'; + +describe('Timeline query tab', () => { + let timelineId: string | null = null; + before(() => { + cleanKibana(); + loginAndWaitForPageWithoutDateRange(TIMELINES_URL); + waitForTimelinesPanelToBeLoaded(); + + createTimeline(timeline) + .then((response) => { + timelineId = response.body.data.persistTimeline.timeline.savedObjectId; + }) + .then(() => { + const note = timeline.notes; + addNoteToTimeline(note, timelineId!).should((response) => { + expect(response.status).to.equal(200); + waitForTimelinesPanelToBeLoaded(); + openTimelineById(timelineId!) + .click({ force: true }) + .then(() => { + waitForEventsPanelToBeLoaded(); + pinFirstEvent(); + addFilter(timeline.filter); + }); + }); + }); + }); + + describe('Query tab', () => { + after(() => { + closeTimeline(); + }); + it('should contain the right query', () => { + cy.get(TIMELINE_QUERY).should('have.text', `${timeline.query}`); + }); + + it('should display timeline filter', () => { + cy.get(TIMELINE_FILTER(timeline.filter)).should('exist'); + }); + + it('should display pinned events', () => { + cy.get(PIN_EVENT) + .should('have.attr', 'aria-label') + .and('match', /Unpin the event in row 2/); + }); + + it('should have an unlock icon', () => { + cy.get(UNLOCKED_ICON).should('be.visible'); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/cypress/screens/timeline.ts b/x-pack/plugins/security_solution/cypress/screens/timeline.ts index 78bc091e8db7c2..92f96a591ab5d0 100644 --- a/x-pack/plugins/security_solution/cypress/screens/timeline.ts +++ b/x-pack/plugins/security_solution/cypress/screens/timeline.ts @@ -73,6 +73,8 @@ export const OPEN_TIMELINE_ICON = '[data-test-subj="open-timeline-button"]'; export const OPEN_TIMELINE_MODAL = '[data-test-subj="open-timeline-modal"]'; +export const CLOSE_OPEN_TIMELINE_MODAL_BTN = `${OPEN_TIMELINE_MODAL} > button`; + export const OPEN_TIMELINE_TEMPLATE_ICON = '[data-test-subj="open-timeline-modal-body-filter-template"]'; @@ -148,6 +150,8 @@ export const TIMELINE_FLYOUT_BODY = '[data-test-subj="query-tab-flyout-body"]'; export const TIMELINE_INSPECT_BUTTON = `${TIMELINE_FLYOUT} [data-test-subj="inspect-icon-button"]`; +export const TIMELINE_PANEL = `[data-test-subj="timeline-flyout-header-panel"]`; + export const TIMELINE_QUERY = '[data-test-subj="timelineQueryInput"]'; export const TIMELINE_SETTINGS_ICON = '[data-test-subj="settings-plus-in-circle"]'; diff --git a/x-pack/plugins/security_solution/cypress/tasks/api_calls/notes.ts b/x-pack/plugins/security_solution/cypress/tasks/api_calls/notes.ts new file mode 100644 index 00000000000000..0fc1a863956054 --- /dev/null +++ b/x-pack/plugins/security_solution/cypress/tasks/api_calls/notes.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const addNoteToTimeline = (note: string, timelineId: string) => + cy.request({ + method: 'POST', + url: '/api/solutions/security/graphql', + body: { + operationName: 'PersistTimelineNoteMutation', + variables: { + noteId: null, + version: null, + note: { note, timelineId }, + }, + query: + 'mutation PersistTimelineNoteMutation($noteId: ID, $version: String, $note: NoteInput!) {\n persistNote(noteId: $noteId, version: $version, note: $note) {\n code\n message\n note {\n eventId\n note\n timelineId\n timelineVersion\n noteId\n created\n createdBy\n updated\n updatedBy\n version\n __typename\n }\n __typename\n }\n}\n', + }, + headers: { 'kbn-xsrf': 'cypress-creds' }, + }); diff --git a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts index edaa5be487a0eb..ca4c869e0f2d38 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/timeline.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/timeline.ts @@ -50,6 +50,7 @@ import { TIMELINE_EDIT_MODAL_OPEN_BUTTON, TIMELINE_EDIT_MODAL_SAVE_BUTTON, QUERY_TAB_BUTTON, + CLOSE_OPEN_TIMELINE_MODAL_BTN, } from '../screens/timeline'; import { TIMELINES_TABLE } from '../screens/timelines'; @@ -83,8 +84,20 @@ export const addNameAndDescriptionToTimeline = (timeline: Timeline) => { cy.get(TIMELINE_TITLE_INPUT).should('not.exist'); }; +export const goToNotesTab = () => { + return cy.get(NOTES_TAB_BUTTON).click({ force: true }); +}; + +export const getNotePreviewByNoteId = (noteId: string) => { + return cy.get(`[data-test-subj="note-preview-${noteId}"]`); +}; + +export const goToQueryTab = () => { + cy.get(QUERY_TAB_BUTTON).click({ force: true }); +}; + export const addNotesToTimeline = (notes: string) => { - cy.get(NOTES_TAB_BUTTON).click(); + goToNotesTab(); cy.get(NOTES_TEXT_AREA).type(notes); cy.get(ADD_NOTE_BUTTON).click(); cy.get(QUERY_TAB_BUTTON).click(); @@ -123,6 +136,10 @@ export const checkIdToggleField = () => { }); }; +export const closeOpenTimelineModal = () => { + cy.get(CLOSE_OPEN_TIMELINE_MODAL_BTN).click({ force: true }); +}; + export const closeTimeline = () => { cy.get(CLOSE_TIMELINE_BTN).filter(':visible').click({ force: true }); }; @@ -170,6 +187,10 @@ export const openTimelineTemplateFromSettings = (id: string) => { cy.get(TIMELINE_TITLE_BY_ID(id)).click({ force: true }); }; +export const openTimelineById = (timelineId: string) => { + return cy.get(TIMELINE_TITLE_BY_ID(timelineId)).click({ force: true }); +}; + export const pinFirstEvent = () => { cy.get(PIN_EVENT).first().click({ force: true }); }; @@ -223,3 +244,7 @@ export const waitForTimelineChanges = () => { export const waitForTimelinesPanelToBeLoaded = () => { cy.get(TIMELINES_TABLE).should('exist'); }; + +export const waitForEventsPanelToBeLoaded = () => { + cy.get(QUERY_TAB_BUTTON).find('.euiBadge').should('exist'); +}; From bcdc022efedcf949abfcef58f5705caba3c28bac Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 10 Feb 2021 08:40:13 -0700 Subject: [PATCH 20/32] [Maps] Add redux devtools back to Maps (#90863) --- .../plugins/maps/public/reducers/non_serializable_instances.js | 1 - x-pack/plugins/maps/public/reducers/store.js | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/maps/public/reducers/non_serializable_instances.js b/x-pack/plugins/maps/public/reducers/non_serializable_instances.js index 4cc4e91a308a52..402d7727cd6fe7 100644 --- a/x-pack/plugins/maps/public/reducers/non_serializable_instances.js +++ b/x-pack/plugins/maps/public/reducers/non_serializable_instances.js @@ -77,7 +77,6 @@ export const getEventHandlers = ({ nonSerializableInstances }) => { }; export function getChartsPaletteServiceGetColor({ nonSerializableInstances }) { - console.log('getChartsPaletteServiceGetColor', nonSerializableInstances); return nonSerializableInstances.chartsPaletteServiceGetColor; } diff --git a/x-pack/plugins/maps/public/reducers/store.js b/x-pack/plugins/maps/public/reducers/store.js index 4e355add59fee0..76199de5b24c92 100644 --- a/x-pack/plugins/maps/public/reducers/store.js +++ b/x-pack/plugins/maps/public/reducers/store.js @@ -36,5 +36,6 @@ export function createMapStore() { }; const storeConfig = {}; - return createStore(rootReducer, storeConfig, compose(...enhancers)); + const composeEnhancers = window.__REDUX_DEVTOOLS_EXTENSION_COMPOSE__ || compose; + return createStore(rootReducer, storeConfig, composeEnhancers(...enhancers)); } From e94a164b7eb11939eb31d1a53e29dfcec93b09c0 Mon Sep 17 00:00:00 2001 From: Bohdan Tsymbala Date: Wed, 10 Feb 2021 16:47:56 +0100 Subject: [PATCH 21/32] Initial version of adding artifacts per policy to the manifest. (#89130) * Initial version of adding artifacts per policy to the manifest. * Minor renaming to convey the purpose of the variable. * Added ability to override list item mock data. * Changed function signature to be more reusable. * Implementationg of support of artifacts per policy in the manifest data structure. * Added saved objects migrations. * Renamed the endpoint to reflect that it's artifacts endpoint. * Fixed tests. * Fixed the manifest data. * Fixed linting errors (result of merge). * Updated ES mappings for manifest in all test setups. * Updated hash in the mappings. * Fixed the typo that lead to failing test. * Fixed the problem with manifest not being dispatched to policies if there are same artifact names but different content. Artifact name in the ManifestSchema is not unique id, hence added decoded_sha256 to the comparison. Added test case to cover this. * Fixed the problem with the task flow when failure to dispatch to policies will result in commited manifest and no redispatch on next task run. Changed tests to reflect new flow (actually restored previous flow). * Forgot to commit changes in mock. * Made other tests more readable using same varialbe naming pattern. Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../es_archiver/apm_8.0.0/mappings.json | 21 +- .../exception_list_item_schema.mock.ts | 5 +- .../endpoint/ingest_integration.test.ts | 304 +++-- .../server/endpoint/ingest_integration.ts | 36 +- .../server/endpoint/lib/artifacts/lists.ts | 18 +- .../endpoint/lib/artifacts/manifest.test.ts | 828 ++++++++++--- .../server/endpoint/lib/artifacts/manifest.ts | 169 +-- .../endpoint/lib/artifacts/migrations.test.ts | 57 + .../endpoint/lib/artifacts/migrations.ts | 35 + .../server/endpoint/lib/artifacts/mocks.ts | 146 ++- .../lib/artifacts/saved_object_mappings.ts | 17 +- .../endpoint/lib/artifacts/task.test.ts | 277 ++++- .../server/endpoint/lib/artifacts/task.ts | 42 +- ...list.test.ts => download_artifact.test.ts} | 4 +- ...exception_list.ts => download_artifact.ts} | 6 +- .../server/endpoint/routes/artifacts/index.ts | 2 +- .../schemas/artifacts/saved_objects.mock.ts | 12 +- .../schemas/artifacts/saved_objects.ts | 10 +- .../artifacts/artifact_client.mock.ts | 19 - .../artifacts/artifact_client.test.ts | 7 +- .../manifest_manager/manifest_manager.mock.ts | 118 +- .../manifest_manager/manifest_manager.test.ts | 1078 ++++++++++++----- .../manifest_manager/manifest_manager.ts | 173 +-- .../security_solution/server/plugin.ts | 10 +- .../es_archiver/apm_8.0.0/mappings.json | 21 +- .../es_archiver/key_rotation/mappings.json | 19 +- .../es_archives/actions/mappings.json | 21 +- .../es_archives/alerts_legacy/mappings.json | 21 +- .../es_archives/canvas/filter/mappings.json | 21 +- .../es_archives/canvas/reports/mappings.json | 21 +- .../es_archives/cases/mappings.json | 21 +- .../data/search_sessions/mappings.json | 19 +- .../endpoint/artifacts/api_feature/data.json | 12 +- .../telemetry/agent_only/mappings.json | 21 +- .../mappings.json | 21 +- .../cloned_endpoint_installed/mappings.json | 21 +- .../cloned_endpoint_uninstalled/mappings.json | 21 +- .../endpoint_malware_disabled/mappings.json | 21 +- .../endpoint_malware_enabled/mappings.json | 21 +- .../endpoint_uninstalled/mappings.json | 21 +- .../event_log_multiple_indicies/mappings.json | 4 +- .../es_archives/lists/mappings.json | 24 +- .../canvas_disallowed_url/mappings.json | 20 +- .../reporting/ecommerce_kibana/mappings.json | 21 +- .../ecommerce_kibana_spaces/mappings.json | 21 +- .../task_manager_removed_types/mappings.json | 4 +- .../visualize/default/mappings.json | 21 +- 47 files changed, 2717 insertions(+), 1115 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.test.ts create mode 100644 x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.ts rename x-pack/plugins/security_solution/server/endpoint/routes/artifacts/{download_exception_list.test.ts => download_artifact.test.ts} (98%) rename x-pack/plugins/security_solution/server/endpoint/routes/artifacts/{download_exception_list.ts => download_artifact.ts} (94%) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts diff --git a/x-pack/plugins/apm/ftr_e2e/cypress/fixtures/es_archiver/apm_8.0.0/mappings.json b/x-pack/plugins/apm/ftr_e2e/cypress/fixtures/es_archiver/apm_8.0.0/mappings.json index 13bfec74269b7e..9f84f4885c4ce8 100644 --- a/x-pack/plugins/apm/ftr_e2e/cypress/fixtures/es_archiver/apm_8.0.0/mappings.json +++ b/x-pack/plugins/apm/ftr_e2e/cypress/fixtures/es_archiver/apm_8.0.0/mappings.json @@ -31,7 +31,7 @@ "core-usage-stats": "3d1b76c39bfb2cc8296b024d73854724", "dashboard": "40554caf09725935e2c02e02563a2d07", "endpoint:user-artifact": "4a11183eee21e6fbad864f7a30b39ad0", - "endpoint:user-artifact-manifest": "4b9c0e7cfaf86d82a7ee9ed68065e50d", + "endpoint:user-artifact-manifest": "a0d7b04ad405eed54d76e279c3727862", "enterprise_search_telemetry": "3d1b76c39bfb2cc8296b024d73854724", "epm-packages": "2b83397e3eaaaa8ef15e38813f3721c3", "exception-list": "67f055ab8c10abd7b2ebfd969b836788", @@ -818,16 +818,25 @@ "index": false, "type": "date" }, - "ids": { - "index": false, - "type": "keyword" - }, "schemaVersion": { "type": "keyword" }, "semanticVersion": { "index": false, "type": "keyword" + }, + "artifacts": { + "type": "nested", + "properties": { + "policyId": { + "type": "keyword", + "index": false + }, + "artifactId": { + "type": "keyword", + "index": false + } + } } } }, @@ -22352,4 +22361,4 @@ } } } -} \ No newline at end of file +} diff --git a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts index e2c5ea504bd478..ab2aac39c19d25 100644 --- a/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/exception_list_item_schema.mock.ts @@ -24,7 +24,9 @@ import { import { ExceptionListItemSchema } from './exception_list_item_schema'; -export const getExceptionListItemSchemaMock = (): ExceptionListItemSchema => ({ +export const getExceptionListItemSchemaMock = ( + overrides?: Partial +): ExceptionListItemSchema => ({ _version: undefined, comments: COMMENTS, created_at: DATE_NOW, @@ -43,6 +45,7 @@ export const getExceptionListItemSchemaMock = (): ExceptionListItemSchema => ({ type: ITEM_TYPE, updated_at: DATE_NOW, updated_by: USER, + ...(overrides || {}), }); export const getExceptionListItemSchemaXMock = (count = 1): ExceptionListItemSchema[] => { diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts index f4cdb28a01b8c7..733e25947347a6 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts @@ -11,10 +11,7 @@ import { policyFactory, policyFactoryWithoutPaidFeatures, } from '../../common/endpoint/models/policy_config'; -import { - getManifestManagerMock, - ManifestManagerMockType, -} from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { buildManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackagePolicyCreateCallback, getPackagePolicyUpdateCallback, @@ -32,6 +29,12 @@ import { ProtectionModes } from '../../common/endpoint/types'; import type { SecuritySolutionRequestHandlerContext } from '../types'; import { getExceptionListClientMock } from '../../../lists/server/services/exception_lists/exception_list_client.mock'; import { ExceptionListClient } from '../../../lists/server'; +import { InternalArtifactCompleteSchema } from './schemas/artifacts'; +import { ManifestManager } from './services/artifacts/manifest_manager'; +import { getMockArtifacts, toArtifactRecords } from './lib/artifacts/mocks'; +import { Manifest } from './lib/artifacts'; +import { NewPackagePolicy } from '../../../fleet/common/types/models'; +import { ManifestSchema } from '../../common/endpoint/schema/manifest'; describe('ingest_integration tests ', () => { let endpointAppContextMock: EndpointAppContextServiceStartContract; @@ -53,21 +56,25 @@ describe('ingest_integration tests ', () => { licenseService = new LicenseService(); licenseService.start(licenseEmitter); }); + afterEach(() => { licenseService.stop(); licenseEmitter.complete(); }); - describe('ingest_integration sanity checks', () => { - beforeEach(() => { - licenseEmitter.next(Platinum); // set license level to platinum + describe('package policy init callback (atifacts manifest initialisation tests)', () => { + const createNewEndpointPolicyInput = (manifest: ManifestSchema) => ({ + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { value: policyFactory() }, + artifact_manifest: { value: manifest }, + }, }); - test('policy is updated with initial manifest', async () => { - const logger = loggingSystemMock.create().get('ingest_integration.test'); - const manifestManager = getManifestManagerMock({ - mockType: ManifestManagerMockType.InitialSystemState, - }); + const invokeCallback = async (manifestManager: ManifestManager): Promise => { + const logger = loggingSystemMock.create().get('ingest_integration.test'); const callback = getPackagePolicyCreateCallback( logger, manifestManager, @@ -78,175 +85,153 @@ describe('ingest_integration tests ', () => { licenseService, exceptionListClient ); - const policyConfig = createNewPackagePolicyMock(); // policy config without manifest - const newPolicyConfig = await callback(policyConfig, ctx, req); // policy config WITH manifest - expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); - expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyFactory()); - expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({ - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - decoded_size: 14, - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - encoded_size: 22, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, - 'endpoint-trustlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - 'endpoint-trustlist-macos-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - 'endpoint-trustlist-windows-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - }, - manifest_version: '1.0.0', - schema_version: 'v1', - }); + return callback(createNewPackagePolicyMock(), ctx, req); + }; + + const TEST_POLICY_ID_1 = 'c6d16e42-c32d-4dce-8a88-113cfe276ad1'; + const TEST_POLICY_ID_2 = '93c46720-c217-11ea-9906-b5b8a21b268e'; + const ARTIFACT_NAME_EXCEPTIONS_MACOS = 'endpoint-exceptionlist-macos-v1'; + const ARTIFACT_NAME_TRUSTED_APPS_MACOS = 'endpoint-trustlist-macos-v1'; + const ARTIFACT_NAME_TRUSTED_APPS_WINDOWS = 'endpoint-trustlist-windows-v1'; + let ARTIFACT_EXCEPTIONS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_EXCEPTIONS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema; + + beforeAll(async () => { + const artifacts = await getMockArtifacts({ compress: true }); + ARTIFACT_EXCEPTIONS_MACOS = artifacts[0]; + ARTIFACT_EXCEPTIONS_WINDOWS = artifacts[1]; + ARTIFACT_TRUSTED_APPS_MACOS = artifacts[2]; + ARTIFACT_TRUSTED_APPS_WINDOWS = artifacts[3]; }); - test('policy is returned even if error is encountered during artifact creation', async () => { - const logger = loggingSystemMock.create().get('ingest_integration.test'); - const manifestManager = getManifestManagerMock(); - manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]); - const lastComputed = await manifestManager.getLastComputedManifest(); + beforeEach(() => { + licenseEmitter.next(Platinum); // set license level to platinum + }); - const callback = getPackagePolicyCreateCallback( - logger, - manifestManager, - endpointAppContextMock.appClientFactory, - maxTimelineImportExportSize, - endpointAppContextMock.security, - endpointAppContextMock.alerts, - licenseService, - exceptionListClient - ); - const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig, ctx, req); + test('default manifest is taken when there is none and there are errors building new one', async () => { + const manifestManager = buildManifestManagerMock(); + manifestManager.getLastComputedManifest = jest.fn().mockResolvedValue(null); + manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error()); - expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); - expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyFactory()); - expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - lastComputed!.toEndpointFormat() + expect((await invokeCallback(manifestManager)).inputs[0]).toStrictEqual( + createNewEndpointPolicyInput({ + artifacts: {}, + manifest_version: '1.0.0', + schema_version: 'v1', + }) ); + + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(); + expect(manifestManager.pushArtifacts).not.toHaveBeenCalled(); + expect(manifestManager.commit).not.toHaveBeenCalled(); }); - test('initial policy creation succeeds if manifest retrieval fails', async () => { - const logger = loggingSystemMock.create().get('ingest_integration.test'); - const manifestManager = getManifestManagerMock({ - mockType: ManifestManagerMockType.InitialSystemState, - }); - const lastComputed = await manifestManager.getLastComputedManifest(); - expect(lastComputed).toEqual(null); + test('default manifest is taken when there is none and there are errors pushing artifacts', async () => { + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); - manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd')); - const callback = getPackagePolicyCreateCallback( - logger, - manifestManager, - endpointAppContextMock.appClientFactory, - maxTimelineImportExportSize, - endpointAppContextMock.security, - endpointAppContextMock.alerts, - licenseService, - exceptionListClient + const manifestManager = buildManifestManagerMock(); + manifestManager.getLastComputedManifest = jest.fn().mockResolvedValue(null); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error()]); + + expect((await invokeCallback(manifestManager)).inputs[0]).toStrictEqual( + createNewEndpointPolicyInput({ + artifacts: {}, + manifest_version: '1.0.0', + schema_version: 'v1', + }) ); - const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig, ctx, req); - expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); - expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyFactory()); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_EXCEPTIONS_MACOS]); + expect(manifestManager.commit).not.toHaveBeenCalled(); }); - test('subsequent policy creations succeed', async () => { - const logger = loggingSystemMock.create().get('ingest_integration.test'); - const manifestManager = getManifestManagerMock(); - const lastComputed = await manifestManager.getLastComputedManifest(); + test('default manifest is taken when there is none and there are errors commiting manifest', async () => { + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); - manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs - const callback = getPackagePolicyCreateCallback( - logger, - manifestManager, - endpointAppContextMock.appClientFactory, - maxTimelineImportExportSize, - endpointAppContextMock.security, - endpointAppContextMock.alerts, - licenseService, - exceptionListClient - ); - const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig, ctx, req); + const manifestManager = buildManifestManagerMock(); + manifestManager.getLastComputedManifest = jest.fn().mockResolvedValue(null); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockRejectedValue(new Error()); - expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); - expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyFactory()); - expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - lastComputed!.toEndpointFormat() + expect((await invokeCallback(manifestManager)).inputs[0]).toStrictEqual( + createNewEndpointPolicyInput({ + artifacts: {}, + manifest_version: '1.0.0', + schema_version: 'v1', + }) ); + + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_EXCEPTIONS_MACOS]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); }); - test('policy creation succeeds even if endpoint exception list creation fails', async () => { - const mockError = new Error('error creating endpoint list'); - const logger = loggingSystemMock.create().get('ingest_integration.test'); - const manifestManager = getManifestManagerMock(); - const lastComputed = await manifestManager.getLastComputedManifest(); - exceptionListClient.createEndpointList = jest.fn().mockRejectedValue(mockError); - const callback = getPackagePolicyCreateCallback( - logger, - manifestManager, - endpointAppContextMock.appClientFactory, - maxTimelineImportExportSize, - endpointAppContextMock.security, - endpointAppContextMock.alerts, - licenseService, - exceptionListClient + test('manifest is created successfuly when there is none', async () => { + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + const manifestManager = buildManifestManagerMock(); + manifestManager.getLastComputedManifest = jest.fn().mockResolvedValue(null); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockResolvedValue(null); + + expect((await invokeCallback(manifestManager)).inputs[0]).toStrictEqual( + createNewEndpointPolicyInput({ + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_MACOS]: ARTIFACT_EXCEPTIONS_MACOS, + [ARTIFACT_NAME_TRUSTED_APPS_MACOS]: ARTIFACT_TRUSTED_APPS_MACOS, + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }) ); - const policyConfig = createNewPackagePolicyMock(); - const newPolicyConfig = await callback(policyConfig, ctx, req); - expect(exceptionListClient.createEndpointList).toHaveBeenCalled(); - expect(newPolicyConfig.inputs[0]!.type).toEqual('endpoint'); - expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyFactory()); - expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual( - lastComputed!.toEndpointFormat() + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ + ARTIFACT_EXCEPTIONS_MACOS, + ARTIFACT_TRUSTED_APPS_MACOS, + ]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); + }); + + test('policy is updated with only default entries from manifest', async () => { + const manifest = new Manifest({ soVersion: '1.0.1', semanticVersion: '1.0.1' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS); + + const manifestManager = buildManifestManagerMock(); + manifestManager.getLastComputedManifest = jest.fn().mockResolvedValue(manifest); + + expect((await invokeCallback(manifestManager)).inputs[0]).toStrictEqual( + createNewEndpointPolicyInput({ + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_MACOS]: ARTIFACT_EXCEPTIONS_MACOS, + [ARTIFACT_NAME_TRUSTED_APPS_WINDOWS]: ARTIFACT_TRUSTED_APPS_WINDOWS, + }), + manifest_version: '1.0.1', + schema_version: 'v1', + }) ); + + expect(manifestManager.buildNewManifest).not.toHaveBeenCalled(); + expect(manifestManager.pushArtifacts).not.toHaveBeenCalled(); + expect(manifestManager.commit).not.toHaveBeenCalled(); }); }); - describe('when the license is below platinum', () => { + + describe('package policy update callback (when the license is below platinum)', () => { beforeEach(() => { licenseEmitter.next(Gold); // set license level to gold }); @@ -271,7 +256,8 @@ describe('ingest_integration tests ', () => { expect(updatedPolicyConfig.inputs[0]!.config!.policy.value).toEqual(mockPolicy); }); }); - describe('when the license is at least platinum', () => { + + describe('package policy update callback (when the license is at least platinum)', () => { beforeEach(() => { licenseEmitter.next(Platinum); // set license level to platinum }); diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index 4dab1c305d17a1..080a8474da54e2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -39,37 +39,18 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr if (manifest == null) { // New computed manifest based on current state of exception list const newManifest = await manifestManager.buildNewManifest(); - const diffs = newManifest.diff(Manifest.getDefault()); - - // Compress new artifacts - const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); - for (const artifactId of adds) { - const compressError = await newManifest.compressArtifact(artifactId); - if (compressError) { - throw compressError; - } - } // Persist new artifacts - const artifacts = adds - .map((artifactId) => newManifest.getArtifact(artifactId)) - .filter((artifact): artifact is InternalArtifactCompleteSchema => artifact !== undefined); - if (artifacts.length !== adds.length) { - throw new Error('Invalid artifact encountered.'); - } - const persistErrors = await manifestManager.pushArtifacts(artifacts); + const persistErrors = await manifestManager.pushArtifacts( + newManifest.getAllArtifacts() as InternalArtifactCompleteSchema[] + ); if (persistErrors.length) { reportErrors(logger, persistErrors); throw new Error('Unable to persist new artifacts.'); } // Commit the manifest state - if (diffs.length) { - const error = await manifestManager.commit(newManifest); - if (error) { - throw error; - } - } + await manifestManager.commit(newManifest); manifest = newManifest; } @@ -93,7 +74,7 @@ export const getPackagePolicyCreateCallback = ( licenseService: LicenseService, exceptionsClient: ExceptionListClient | undefined ): ExternalCallback[1] => { - const handlePackagePolicyCreate = async ( + return async ( newPackagePolicy: NewPackagePolicy, context: RequestHandlerContext, request: KibanaRequest @@ -143,7 +124,7 @@ export const getPackagePolicyCreateCallback = ( // Get most recent manifest const manifest = await getManifest(logger, manifestManager); - const serializedManifest = manifest.toEndpointFormat(); + const serializedManifest = manifest.toPackagePolicyManifest(); if (!manifestDispatchSchema.is(serializedManifest)) { // This should not happen. // But if it does, we log it and return it anyway. @@ -183,15 +164,13 @@ export const getPackagePolicyCreateCallback = ( return updatedPackagePolicy; }; - - return handlePackagePolicyCreate; }; export const getPackagePolicyUpdateCallback = ( logger: Logger, licenseService: LicenseService ): ExternalCallback[1] => { - const handlePackagePolicyUpdate = async ( + return async ( newPackagePolicy: NewPackagePolicy, context: RequestHandlerContext, request: KibanaRequest @@ -213,5 +192,4 @@ export const getPackagePolicyUpdateCallback = ( } return newPackagePolicy; }; - return handlePackagePolicyUpdate; }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index c408cb56a7fd75..6cc6a821eba334 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -11,7 +11,6 @@ import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { validate } from '../../../../common/validate'; import { Entry, EntryNested } from '../../../../../lists/common/schemas/types'; -import { FoundExceptionListItemSchema } from '../../../../../lists/common/schemas/response/found_exception_list_item_schema'; import { ExceptionListClient } from '../../../../../lists/server'; import { ENDPOINT_LIST_ID } from '../../../../common/shared_imports'; import { @@ -58,14 +57,14 @@ export async function maybeCompressArtifact( ): Promise { const compressedArtifact = { ...uncompressedArtifact }; if (internalArtifactCompleteSchema.is(uncompressedArtifact)) { - const compressedExceptionList = await compressExceptionList( + const compressedArtifactBody = await compressExceptionList( Buffer.from(uncompressedArtifact.body, 'base64') ); - compressedArtifact.body = compressedExceptionList.toString('base64'); - compressedArtifact.encodedSize = compressedExceptionList.byteLength; + compressedArtifact.body = compressedArtifactBody.toString('base64'); + compressedArtifact.encodedSize = compressedArtifactBody.byteLength; compressedArtifact.compressionAlgorithm = 'zlib'; compressedArtifact.encodedSha256 = createHash('sha256') - .update(compressedExceptionList) + .update(compressedArtifactBody) .digest('hex'); } return compressedArtifact; @@ -98,7 +97,7 @@ export async function getFullEndpointExceptionList( if (response?.data !== undefined) { exceptions.entries = exceptions.entries.concat( - translateToEndpointExceptions(response, schemaVersion) + translateToEndpointExceptions(response.data, schemaVersion) ); paging = (page - 1) * 100 + response.data.length < response.total; @@ -117,16 +116,17 @@ export async function getFullEndpointExceptionList( /** * Translates Exception list items to Exceptions the endpoint can understand - * @param exc + * @param exceptions + * @param schemaVersion */ export function translateToEndpointExceptions( - exc: FoundExceptionListItemSchema, + exceptions: ExceptionListItemSchema[], schemaVersion: string ): TranslatedExceptionListItem[] { const entrySet = new Set(); const entriesFiltered: TranslatedExceptionListItem[] = []; if (schemaVersion === 'v1') { - exc.data.forEach((entry) => { + exceptions.forEach((entry) => { const translatedItem = translateItem(schemaVersion, entry); const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex'); if (!entrySet.has(entryHash)) { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 047b79ea01efcb..beaf0c06299fac 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -8,31 +8,48 @@ import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common'; import { InternalArtifactCompleteSchema } from '../../schemas'; import { getArtifactId } from './common'; -import { Manifest } from './manifest'; -import { - getMockArtifacts, - getMockManifest, - getMockManifestWithDiffs, - getEmptyMockManifest, -} from './mocks'; +import { isEmptyManifestDiff, Manifest } from './manifest'; +import { getMockArtifacts, toArtifactRecords } from './mocks'; describe('manifest', () => { - describe('Manifest object sanity checks', () => { - let artifacts: InternalArtifactCompleteSchema[] = []; - let manifest1: Manifest; - let manifest2: Manifest; - let emptyManifest: Manifest; + const TEST_POLICY_ID_1 = 'c6d16e42-c32d-4dce-8a88-113cfe276ad1'; + const TEST_POLICY_ID_2 = '93c46720-c217-11ea-9906-b5b8a21b268e'; + const ARTIFACT_ID_EXCEPTIONS_MACOS = + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_EXCEPTIONS_WINDOWS = + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_TRUSTED_APPS_MACOS = + 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_TRUSTED_APPS_WINDOWS = + 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; - beforeAll(async () => { - artifacts = await getMockArtifacts({ compress: true }); - manifest1 = await getMockManifest({ compress: true }); - manifest2 = await getMockManifestWithDiffs({ compress: true }); - emptyManifest = await getEmptyMockManifest({ compress: true }); - }); + let ARTIFACTS: InternalArtifactCompleteSchema[] = []; + let ARTIFACTS_COPY: InternalArtifactCompleteSchema[] = []; + let ARTIFACT_EXCEPTIONS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_EXCEPTIONS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_COPY_EXCEPTIONS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_COPY_EXCEPTIONS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_COPY_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_COPY_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema; + + beforeAll(async () => { + ARTIFACTS = await getMockArtifacts({ compress: true }); + ARTIFACTS_COPY = await getMockArtifacts({ compress: true }); + ARTIFACT_EXCEPTIONS_MACOS = ARTIFACTS[0]; + ARTIFACT_EXCEPTIONS_WINDOWS = ARTIFACTS[1]; + ARTIFACT_TRUSTED_APPS_MACOS = ARTIFACTS[2]; + ARTIFACT_TRUSTED_APPS_WINDOWS = ARTIFACTS[3]; + ARTIFACT_COPY_EXCEPTIONS_MACOS = ARTIFACTS_COPY[0]; + ARTIFACT_COPY_EXCEPTIONS_WINDOWS = ARTIFACTS_COPY[1]; + ARTIFACT_COPY_TRUSTED_APPS_MACOS = ARTIFACTS_COPY[2]; + ARTIFACT_COPY_TRUSTED_APPS_WINDOWS = ARTIFACTS_COPY[3]; + }); + describe('Manifest constructor', () => { test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(); - expect(manifest).toBeInstanceOf(Manifest); + expect(new Manifest()).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { @@ -43,177 +60,638 @@ describe('manifest', () => { }).toThrow(); }); - test('Empty manifest transforms correctly to expected endpoint format', async () => { - expect(emptyManifest.toEndpointFormat()).toStrictEqual({ - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, - }, + test('Can create manifest with valid constructor parameters', () => { + const manifest = new Manifest({ + schemaVersion: 'v1', + semanticVersion: '1.1.1', + soVersion: '2.2.2', + }); + + expect(manifest.getAllArtifacts()).toStrictEqual([]); + expect(manifest.getSchemaVersion()).toBe('v1'); + expect(manifest.getSemanticVersion()).toBe('1.1.1'); + expect(manifest.getSavedObjectVersion()).toBe('2.2.2'); + }); + }); + + describe('Manifest.getDefault()', () => { + test('Creates empty default manifest', () => { + const manifest = Manifest.getDefault(); + + expect(manifest.getAllArtifacts()).toStrictEqual([]); + expect(manifest.getSchemaVersion()).toBe('v1'); + expect(manifest.getSemanticVersion()).toBe('1.0.0'); + expect(manifest.getSavedObjectVersion()).toBe(undefined); + }); + }); + + describe('bumpSemanticVersion', () => { + test('Bumps the version properly', () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.1.1' }); + + manifest.bumpSemanticVersion(); + + expect(manifest.getSemanticVersion()).toBe('1.1.2'); + }); + }); + + describe('addEntry', () => { + test('Adds default artifact', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.getAllArtifacts()).toStrictEqual([ARTIFACT_EXCEPTIONS_MACOS]); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set() + ); + }); + + test('Adds policy specific artifact', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.getAllArtifacts()).toStrictEqual([ARTIFACT_EXCEPTIONS_MACOS]); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(false); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + }); + + test('Adds same artifact as default and policy specific', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.getAllArtifacts()).toStrictEqual([ARTIFACT_EXCEPTIONS_MACOS]); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + }); + + test('Adds multiple artifacts as default and policy specific', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS); + + expect(manifest.getAllArtifacts()).toStrictEqual(ARTIFACTS.slice(0, 4)); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBe(false); + expect(manifest.isDefaultArtifact(ARTIFACT_TRUSTED_APPS_MACOS)).toBe(true); + expect(manifest.isDefaultArtifact(ARTIFACT_TRUSTED_APPS_WINDOWS)).toBe(true); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_WINDOWS)).toStrictEqual( + new Set([TEST_POLICY_ID_2]) + ); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_TRUSTED_APPS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1, TEST_POLICY_ID_2]) + ); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_TRUSTED_APPS_WINDOWS)).toStrictEqual( + new Set([]) + ); + }); + + test('Adding same artifact as default multiple times has no effect', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.getAllArtifacts()).toStrictEqual(ARTIFACTS.slice(0, 1)); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([]) + ); + }); + + test('Adding same artifact as policy specific for same policy multiple times has no effect', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.getAllArtifacts()).toStrictEqual(ARTIFACTS.slice(0, 1)); + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(false); + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + }); + }); + + describe('getAllArtifacts', () => { + test('Returns empty list initially', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + expect(manifest.getAllArtifacts()).toStrictEqual([]); + }); + + test('Returns only unique artifacts', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + expect(manifest.getAllArtifacts()).toStrictEqual(ARTIFACTS.slice(0, 2)); + }); + }); + + describe('getArtifact', () => { + test('Returns undefined for non existing artifact id', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.getArtifact('non-existing-artifact-macos-v1')).toBeUndefined(); + }); + + test('Returns default artifact', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + expect(manifest.getArtifact(getArtifactId(ARTIFACT_EXCEPTIONS_MACOS))).toStrictEqual( + ARTIFACT_EXCEPTIONS_MACOS + ); + }); + + test('Returns policy specific artifact', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + + expect(manifest.getArtifact(getArtifactId(ARTIFACT_EXCEPTIONS_MACOS))).toStrictEqual( + ARTIFACT_EXCEPTIONS_MACOS + ); + }); + }); + + describe('containsArtifact', () => { + test('Returns false for artifact that is not in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.containsArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBe(false); + }); + + test('Returns true for default artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + expect(manifest.containsArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBe(true); + }); + + test('Returns true for policy specific artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + + expect(manifest.containsArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBe(true); + }); + + test('Returns true for different instances but same ids', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.containsArtifact(ARTIFACT_COPY_EXCEPTIONS_MACOS)).toBe(true); + }); + }); + + describe('isDefaultArtifact', () => { + test('Returns undefined for artifact that is not in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBeUndefined(); + }); + + test('Returns true for default artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + }); + + test('Returns false for policy specific artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(false); + }); + + test('Returns true for different instances but same ids', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.isDefaultArtifact(ARTIFACT_COPY_EXCEPTIONS_MACOS)).toBe(true); + }); + }); + + describe('getArtifactTargetPolicies', () => { + test('Returns undefined for artifact that is not in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_WINDOWS)).toBeUndefined(); + }); + + test('Returns empty set for default artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set() + ); + }); + + test('Returns policy set for policy specific artifact that is in the manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_2); + + expect(manifest.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1, TEST_POLICY_ID_2]) + ); + }); + + test('Returns policy set for different instances but same ids', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_2); + + expect(manifest.getArtifactTargetPolicies(ARTIFACT_COPY_EXCEPTIONS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1, TEST_POLICY_ID_2]) + ); + }); + }); + + describe('diff', () => { + test('Returns empty diff between empty manifests', async () => { + expect(Manifest.getDefault().diff(Manifest.getDefault())).toStrictEqual({ + additions: [], + removals: [], + transitions: [], + }); + }); + + test('Returns diff from empty manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + expect(manifest.diff(Manifest.getDefault())).toStrictEqual({ + additions: ARTIFACTS.slice(0, 3), + removals: [], + transitions: [], + }); + }); + + test('Returns empty diff for equal manifests', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS, TEST_POLICY_ID_2); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_WINDOWS); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_WINDOWS, TEST_POLICY_ID_2); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [], + transitions: [], + }); + }); + + test('Returns additions diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_WINDOWS); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [ARTIFACT_COPY_TRUSTED_APPS_MACOS, ARTIFACT_COPY_TRUSTED_APPS_WINDOWS], + removals: [], + transitions: [], + }); + }); + + test('Returns removals diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [ARTIFACT_COPY_TRUSTED_APPS_MACOS, ARTIFACT_COPY_TRUSTED_APPS_WINDOWS], + transitions: [], + }); + }); + + test('Returns transitions from one policy to another in diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + // policy transition + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [], + transitions: [ARTIFACT_COPY_TRUSTED_APPS_MACOS], + }); + }); + + test('Returns transitions from policy to default in diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + // transition to default + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [], + transitions: [ARTIFACT_COPY_TRUSTED_APPS_MACOS], + }); + }); + + test('Returns transitions from default to specific policy in diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + // transition to specific policy + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [], + transitions: [ARTIFACT_COPY_TRUSTED_APPS_MACOS], + }); + }); + + test('Returns complex transitions diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + // transition to default policy only + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + // transition to second policy + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + // transition to one policy only + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [], + removals: [], + transitions: ARTIFACTS_COPY.slice(0, 3), + }); + }); + + test('Returns complex diff properly', async () => { + const manifest1 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest1.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + manifest1.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS); + + const manifest2 = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.1' }); + + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_MACOS); + manifest2.addEntry(ARTIFACT_COPY_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest2.addEntry(ARTIFACT_COPY_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest2.diff(manifest1)).toStrictEqual({ + additions: [ARTIFACT_COPY_EXCEPTIONS_WINDOWS], + removals: [ARTIFACT_TRUSTED_APPS_WINDOWS], + transitions: [ARTIFACT_COPY_EXCEPTIONS_MACOS, ARTIFACT_COPY_TRUSTED_APPS_MACOS], + }); + }); + }); + + describe('toPackagePolicyManifest', () => { + test('Returns empty manifest', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + expect(manifest.toPackagePolicyManifest()).toStrictEqual({ + schema_version: 'v1', manifest_version: '1.0.0', + artifacts: {}, + }); + }); + + test('Returns default policy manifest when no policy id provided', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.toPackagePolicyManifest()).toStrictEqual({ schema_version: 'v1', + manifest_version: '1.0.0', + artifacts: toArtifactRecords({ + 'endpoint-exceptionlist-windows-v1': ARTIFACT_EXCEPTIONS_MACOS, + 'endpoint-exceptionlist-macos-v1': ARTIFACT_EXCEPTIONS_WINDOWS, + }), }); }); - test('Manifest transforms correctly to expected endpoint format', async () => { - expect(manifest1.toEndpointFormat()).toStrictEqual({ - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-trustlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - decoded_size: 432, - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - encoded_size: 147, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-trustlist-macos-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - decoded_size: 432, - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - encoded_size: 147, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-trustlist-windows-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - decoded_size: 432, - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - encoded_size: 147, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - }, + test('Returns default policy manifest when no policy specific artifacts present', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.toPackagePolicyManifest(TEST_POLICY_ID_2)).toStrictEqual({ + schema_version: 'v1', manifest_version: '1.0.0', + artifacts: toArtifactRecords({ + 'endpoint-exceptionlist-windows-v1': ARTIFACT_EXCEPTIONS_MACOS, + 'endpoint-exceptionlist-macos-v1': ARTIFACT_EXCEPTIONS_WINDOWS, + }), + }); + }); + + test('Returns policy specific manifest when policy specific artifacts present', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + + expect(manifest.toPackagePolicyManifest(TEST_POLICY_ID_2)).toStrictEqual({ schema_version: 'v1', + manifest_version: '1.0.0', + artifacts: toArtifactRecords({ + 'endpoint-exceptionlist-windows-v1': ARTIFACT_TRUSTED_APPS_MACOS, + 'endpoint-exceptionlist-macos-v1': ARTIFACT_EXCEPTIONS_WINDOWS, + }), }); }); + }); + + describe('toSavedObject', () => { + test('Returns empty saved object', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + expect(manifest.toSavedObject()).toStrictEqual({ + schemaVersion: 'v1', + semanticVersion: '1.0.0', + artifacts: [], + }); + }); + + test('Returns populated saved object', async () => { + const manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_WINDOWS, TEST_POLICY_ID_2); - test('Manifest transforms correctly to expected saved object format', async () => { - expect(manifest1.toSavedObject()).toStrictEqual({ + expect(manifest.toSavedObject()).toStrictEqual({ schemaVersion: 'v1', semanticVersion: '1.0.0', - ids: [ - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + artifacts: [ + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_WINDOWS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_WINDOWS, policyId: TEST_POLICY_ID_2 }, ], }); }); + }); - test('Manifest returns diffs since supplied manifest', async () => { - const diffs = manifest2.diff(manifest1); - expect(diffs).toEqual([ - { - id: - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - type: 'add', - }, - ]); - }); - - test('Manifest returns data for given artifact', async () => { - const artifact = artifacts[0]; - const returned = manifest1.getArtifact(getArtifactId(artifact)); - expect(returned).toEqual(artifact); - }); - - test('Manifest returns entries map', async () => { - const entries = manifest1.getEntries(); - const keys = Object.keys(entries); - expect(keys).toEqual([ - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - 'endpoint-trustlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - ]); - }); - - test('Manifest returns true if contains artifact', async () => { - const found = manifest1.contains( - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' - ); - expect(found).toEqual(true); - }); - - test('Manifest can be created from list of artifacts', async () => { - const oldManifest = new Manifest(); - const manifest = Manifest.fromArtifacts(artifacts, oldManifest); - expect( - manifest.contains( - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' - ) - ).toEqual(true); - expect( - manifest.contains( - 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3' - ) - ).toEqual(true); + describe('isEmptyManifestDiff', () => { + test('Returns true when no additions, removals or transitions', async () => { + expect(isEmptyManifestDiff({ additions: [], removals: [], transitions: [] })).toBe(true); + }); + + test('Returns false when there are additions', async () => { + const diff = { additions: [ARTIFACT_EXCEPTIONS_MACOS], removals: [], transitions: [] }; + + expect(isEmptyManifestDiff(diff)).toBe(false); + }); + + test('Returns false when there are removals', async () => { + const diff = { additions: [], removals: [ARTIFACT_EXCEPTIONS_MACOS], transitions: [] }; + + expect(isEmptyManifestDiff(diff)).toBe(false); + }); + + test('Returns false when there are transitions', async () => { + const diff = { additions: [], removals: [], transitions: [ARTIFACT_EXCEPTIONS_MACOS] }; + + expect(isEmptyManifestDiff(diff)).toBe(false); + }); + + test('Returns false when there are all typesof changes', async () => { + const diff = { + additions: [ARTIFACT_EXCEPTIONS_MACOS], + removals: [ARTIFACT_EXCEPTIONS_WINDOWS], + transitions: [ARTIFACT_TRUSTED_APPS_MACOS], + }; + + expect(isEmptyManifestDiff(diff)).toBe(false); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index e2065b6bbc3751..7e1accac37cf02 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -5,13 +5,13 @@ * 2.0. */ +import { flatMap, isEqual } from 'lodash'; import semver from 'semver'; -import { validate } from '../../../../common/validate'; +import { validate } from '../../../../common'; import { InternalArtifactSchema, InternalManifestSchema, - internalArtifactCompleteSchema, - InternalArtifactCompleteSchema, + InternalManifestEntrySchema, } from '../../schemas/artifacts'; import { ManifestSchemaVersion, @@ -20,21 +20,46 @@ import { } from '../../../../common/endpoint/schema/common'; import { manifestSchema, ManifestSchema } from '../../../../common/endpoint/schema/manifest'; import { ManifestEntry } from './manifest_entry'; -import { maybeCompressArtifact, isCompressed } from './lists'; import { getArtifactId } from './common'; import { ManifestVersion, manifestVersion } from '../../schemas/artifacts/manifest'; +function createInternalManifestEntries( + artifactIds: string[], + policyId?: string +): InternalManifestEntrySchema[] { + return artifactIds.map((artifactId) => ({ policyId, artifactId })); +} + export interface ManifestDiff { - type: string; - id: string; + additions: InternalArtifactSchema[]; + removals: InternalArtifactSchema[]; + transitions: InternalArtifactSchema[]; +} + +export function isEmptyManifestDiff(diff: ManifestDiff) { + return diff.additions.length === 0 && diff.removals.length === 0 && diff.transitions.length === 0; +} + +interface ManifestEntryDescriptor { + isDefaultEntry: boolean; + specificTargetPolicies: Set; + entry: ManifestEntry; +} + +function addValueToSet(set?: Set, value?: T) { + return new Set([...(set?.values() || []), ...(value !== undefined ? [value] : [])]); } export class Manifest { - private entries: Record; + private readonly allEntries: Map; + private readonly defaultEntries: Map; + private readonly policySpecificEntries: Map>; private version: ManifestVersion; constructor(version?: Partial) { - this.entries = {}; + this.allEntries = new Map(); + this.defaultEntries = new Map(); + this.policySpecificEntries = new Map(); const decodedVersion = { schemaVersion: version?.schemaVersion ?? 'v1', @@ -54,28 +79,6 @@ export class Manifest { return new Manifest({ schemaVersion, semanticVersion: '1.0.0' }); } - public static fromArtifacts( - artifacts: InternalArtifactCompleteSchema[], - oldManifest: Manifest, - schemaVersion?: ManifestSchemaVersion - ): Manifest { - const manifest = new Manifest({ - schemaVersion, - semanticVersion: oldManifest.getSemanticVersion(), - soVersion: oldManifest.getSavedObjectVersion(), - }); - artifacts.forEach((artifact) => { - const id = getArtifactId(artifact); - const existingArtifact = oldManifest.getArtifact(id); - if (existingArtifact) { - manifest.addEntry(existingArtifact); - } else { - manifest.addEntry(artifact); - } - }); - return manifest; - } - public bumpSemanticVersion() { const newSemanticVersion = semver.inc(this.getSemanticVersion(), 'patch'); if (!semanticVersion.is(newSemanticVersion)) { @@ -84,26 +87,6 @@ export class Manifest { this.version.semanticVersion = newSemanticVersion; } - public async compressArtifact(id: string): Promise { - try { - const artifact = this.getArtifact(id); - if (artifact == null) { - throw new Error(`Corrupted manifest detected. Artifact ${id} not in manifest.`); - } - - const compressedArtifact = await maybeCompressArtifact(artifact); - if (!isCompressed(compressedArtifact)) { - throw new Error(`Unable to compress artifact: ${id}`); - } else if (!internalArtifactCompleteSchema.is(compressedArtifact)) { - throw new Error(`Incomplete artifact detected: ${id}`); - } - this.addEntry(compressedArtifact); - } catch (err) { - return err; - } - return null; - } - public getSchemaVersion(): ManifestSchemaVersion { return this.version.schemaVersion; } @@ -116,53 +99,85 @@ export class Manifest { return this.version.semanticVersion; } - public addEntry(artifact: InternalArtifactSchema) { - const entry = new ManifestEntry(artifact); - this.entries[entry.getDocId()] = entry; + public addEntry(artifact: InternalArtifactSchema, policyId?: string) { + const existingDescriptor = this.allEntries.get(getArtifactId(artifact)); + const descriptor = { + isDefaultEntry: existingDescriptor?.isDefaultEntry || policyId === undefined, + specificTargetPolicies: addValueToSet(existingDescriptor?.specificTargetPolicies, policyId), + entry: existingDescriptor?.entry || new ManifestEntry(artifact), + }; + + this.allEntries.set(descriptor.entry.getDocId(), descriptor); + + if (policyId) { + const entries = this.policySpecificEntries.get(policyId) || new Map(); + entries.set(descriptor.entry.getDocId(), descriptor.entry); + + this.policySpecificEntries.set(policyId, entries); + } else { + this.defaultEntries.set(descriptor.entry.getDocId(), descriptor.entry); + } } - public contains(artifactId: string): boolean { - return artifactId in this.entries; + public getAllArtifacts(): InternalArtifactSchema[] { + return [...this.allEntries.values()].map((descriptor) => descriptor.entry.getArtifact()); } - public getEntries(): Record { - return this.entries; + public getArtifact(artifactId: string): InternalArtifactSchema | undefined { + return this.allEntries.get(artifactId)?.entry.getArtifact(); } - public getEntry(artifactId: string): ManifestEntry | undefined { - return this.entries[artifactId]; + public containsArtifact(artifact: InternalArtifactSchema): boolean { + return this.allEntries.has(getArtifactId(artifact)); } - public getArtifact(artifactId: string): InternalArtifactSchema | undefined { - return this.getEntry(artifactId)?.getArtifact(); + public isDefaultArtifact(artifact: InternalArtifactSchema): boolean | undefined { + return this.allEntries.get(getArtifactId(artifact))?.isDefaultEntry; } - public diff(manifest: Manifest): ManifestDiff[] { - const diffs: ManifestDiff[] = []; + public getArtifactTargetPolicies(artifact: InternalArtifactSchema): Set | undefined { + return this.allEntries.get(getArtifactId(artifact))?.specificTargetPolicies; + } + + public diff(manifest: Manifest): ManifestDiff { + const diff: ManifestDiff = { + additions: [], + removals: [], + transitions: [], + }; - for (const id in manifest.getEntries()) { - if (!this.contains(id)) { - diffs.push({ type: 'delete', id }); + for (const artifact of manifest.getAllArtifacts()) { + if (!this.containsArtifact(artifact)) { + diff.removals.push(artifact); + } else if ( + this.isDefaultArtifact(artifact) !== manifest.isDefaultArtifact(artifact) || + !isEqual( + this.getArtifactTargetPolicies(artifact), + manifest.getArtifactTargetPolicies(artifact) + ) + ) { + diff.transitions.push(artifact); } } - for (const id in this.entries) { - if (!manifest.contains(id)) { - diffs.push({ type: 'add', id }); + for (const artifact of this.getAllArtifacts()) { + if (!manifest.containsArtifact(artifact)) { + diff.additions.push(artifact); } } - return diffs; + return diff; } - public toEndpointFormat(): ManifestSchema { + public toPackagePolicyManifest(policyId?: string): ManifestSchema { + const entries = (!!policyId && this.policySpecificEntries.get(policyId)) || this.defaultEntries; const manifestObj: ManifestSchema = { manifest_version: this.getSemanticVersion(), schema_version: this.getSchemaVersion(), artifacts: {}, }; - for (const entry of Object.values(this.entries)) { + for (const entry of entries.values()) { manifestObj.artifacts[entry.getIdentifier()] = entry.getRecord(); } @@ -176,7 +191,15 @@ export class Manifest { public toSavedObject(): InternalManifestSchema { return { - ids: Object.keys(this.getEntries()), + artifacts: [ + ...createInternalManifestEntries([...this.defaultEntries.keys()]), + ...flatMap([...this.policySpecificEntries.keys()], (policyId) => + createInternalManifestEntries( + [...(this.policySpecificEntries.get(policyId)?.keys() || [])], + policyId + ) + ), + ], schemaVersion: this.getSchemaVersion(), semanticVersion: this.getSemanticVersion(), }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.test.ts new file mode 100644 index 00000000000000..814a9880014cd8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SavedObjectUnsanitizedDoc } from 'kibana/server'; +import { migrationMocks } from 'src/core/server/mocks'; +import { ManifestConstants } from './common'; +import { migrations, OldInternalManifestSchema } from './migrations'; + +describe('7.12.0 manifest migrations', () => { + const ARTIFACT_ID_0 = + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_1 = + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_2 = + 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_3 = + 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + + const migration = migrations['7.12.0']; + + test('Migrates ids property', () => { + const doc: SavedObjectUnsanitizedDoc = { + attributes: { + ids: [ARTIFACT_ID_0, ARTIFACT_ID_1, ARTIFACT_ID_2, ARTIFACT_ID_3], + schemaVersion: 'v1', + semanticVersion: '1.0.1', + }, + id: 'endpoint-manifest-v1', + migrationVersion: {}, + references: [], + type: ManifestConstants.SAVED_OBJECT_TYPE, + updated_at: '2020-06-09T20:18:20.349Z', + }; + + expect(migration(doc, migrationMocks.createContext())).toStrictEqual({ + attributes: { + artifacts: [ + { artifactId: ARTIFACT_ID_0, policyId: undefined }, + { artifactId: ARTIFACT_ID_1, policyId: undefined }, + { artifactId: ARTIFACT_ID_2, policyId: undefined }, + { artifactId: ARTIFACT_ID_3, policyId: undefined }, + ], + schemaVersion: 'v1', + semanticVersion: '1.0.1', + }, + id: 'endpoint-manifest-v1', + migrationVersion: {}, + references: [], + type: ManifestConstants.SAVED_OBJECT_TYPE, + updated_at: '2020-06-09T20:18:20.349Z', + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.ts new file mode 100644 index 00000000000000..e419c4297b23aa --- /dev/null +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/migrations.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SavedObjectMigrationMap, + SavedObjectSanitizedDoc, + SavedObjectUnsanitizedDoc, +} from 'kibana/server'; + +import { InternalManifestSchema } from '../../schemas/artifacts'; + +export type OldInternalManifestSchema = Omit & { + ids: string[]; +}; + +export const migrations: SavedObjectMigrationMap = { + '7.12.0': ( + doc: SavedObjectUnsanitizedDoc + ): SavedObjectSanitizedDoc => { + const { ids, ...rest } = doc.attributes; + + return { + ...doc, + references: doc.references || [], + attributes: { + ...rest, + artifacts: (ids || []).map((artifactId) => ({ artifactId, policyId: undefined })), + }, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts index 738a995f9fc621..1a582a51c52c1b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/mocks.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { PackagePolicy } from '../../../../../fleet/common'; +import { mapValues } from 'lodash'; +import { PackagePolicy, PackagePolicyConfigRecord } from '../../../../../fleet/common'; import { createPackagePolicyMock } from '../../../../../fleet/common/mocks'; import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; import { @@ -63,83 +64,94 @@ export const getMockManifest = async (opts?: { compress: boolean }) => { return manifest; }; -export const getMockManifestWithDiffs = async (opts?: { compress: boolean }) => { - const manifest = new Manifest(); - const artifacts = await getMockArtifactsWithDiff(opts); - artifacts.forEach((artifact) => manifest.addEntry(artifact)); - return manifest; -}; +const toArtifactRecord = (artifactName: string, artifact: InternalArtifactCompleteSchema) => ({ + compression_algorithm: artifact.compressionAlgorithm, + decoded_sha256: artifact.decodedSha256, + decoded_size: artifact.decodedSize, + encoded_sha256: artifact.encodedSha256, + encoded_size: artifact.encodedSize, + encryption_algorithm: artifact.encryptionAlgorithm, + relative_url: `/api/endpoint/artifacts/download/${artifactName}/${artifact.decodedSha256}`, +}); -export const getEmptyMockManifest = async (opts?: { compress: boolean }) => { - const manifest = new Manifest(); - const artifacts = await getEmptyMockArtifacts(opts); - artifacts.forEach((artifact) => manifest.addEntry(artifact)); - return manifest; +export const toArtifactRecords = (artifacts: Record) => + mapValues(artifacts, (artifact, key) => toArtifactRecord(key, artifact)); + +export const createPackagePolicyWithConfigMock = ( + options: Partial & { config?: PackagePolicyConfigRecord } +): PackagePolicy => { + const { config, ...packagePolicyOverrides } = options; + const packagePolicy = createPackagePolicyMock(); + packagePolicy.inputs[0].config = options.config; + return { ...packagePolicy, ...packagePolicyOverrides }; }; export const createPackagePolicyWithInitialManifestMock = (): PackagePolicy => { - const packagePolicy = createPackagePolicyMock(); - packagePolicy.inputs[0].config!.artifact_manifest = { - value: { - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', - encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', - decoded_size: 14, - encoded_size: 22, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + return createPackagePolicyWithConfigMock({ + config: { + artifact_manifest: { + value: { + artifacts: { + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda', + decoded_size: 14, + encoded_size: 22, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658', + }, + }, + manifest_version: '1.0.0', + schema_version: 'v1', }, }, - manifest_version: '1.0.0', - schema_version: 'v1', }, - }; - return packagePolicy; + }); }; export const createPackagePolicyWithManifestMock = (): PackagePolicy => { - const packagePolicy = createPackagePolicyMock(); - packagePolicy.inputs[0].config!.artifact_manifest = { - value: { - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + return createPackagePolicyWithConfigMock({ + config: { + artifact_manifest: { + value: { + artifacts: { + 'endpoint-exceptionlist-macos-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + }, + 'endpoint-exceptionlist-windows-v1': { + compression_algorithm: 'zlib', + encryption_algorithm: 'none', + decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', + decoded_size: 432, + encoded_size: 147, + relative_url: + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', + }, + }, + manifest_version: '1.0.1', + schema_version: 'v1', }, }, - manifest_version: '1.0.1', - schema_version: 'v1', }, - }; - - return packagePolicy; + }); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 8596e6b9917afc..2202336ef4519b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -8,6 +8,7 @@ import { SavedObjectsType } from '../../../../../../../src/core/server'; import { ArtifactConstants, ManifestConstants } from './common'; +import { migrations } from './migrations'; export const exceptionsArtifactSavedObjectType = ArtifactConstants.SAVED_OBJECT_TYPE; export const manifestSavedObjectType = ManifestConstants.SAVED_OBJECT_TYPE; @@ -63,9 +64,18 @@ export const manifestSavedObjectMappings: SavedObjectsType['mappings'] = { type: 'keyword', index: false, }, - ids: { - type: 'keyword', - index: false, + artifacts: { + type: 'nested', + properties: { + policyId: { + type: 'keyword', + index: false, + }, + artifactId: { + type: 'keyword', + index: false, + }, + }, }, }, }; @@ -82,4 +92,5 @@ export const manifestType: SavedObjectsType = { hidden: false, namespaceType: 'agnostic', mappings: manifestSavedObjectMappings, + migrations, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index 08f83595180135..9fac617f1f06d0 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -12,8 +12,27 @@ import { createMockEndpointAppContext } from '../../mocks'; import { ManifestTaskConstants, ManifestTask } from './task'; import { MockManifestTask } from './task.mock'; +import { ManifestManager } from '../../services/artifacts/manifest_manager'; +import { buildManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; +import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; +import { getMockArtifacts } from './mocks'; +import { Manifest } from './manifest'; describe('task', () => { + const MOCK_TASK_INSTANCE = { + id: `${ManifestTaskConstants.TYPE}:1.0.0`, + runAt: new Date(), + attempts: 0, + ownerId: '', + status: TaskStatus.Running, + startedAt: new Date(), + scheduledAt: new Date(), + retryAt: new Date(), + params: {}, + state: {}, + taskType: ManifestTaskConstants.TYPE, + }; + describe('Periodic task sanity checks', () => { test('can create task', () => { const manifestTask = new ManifestTask({ @@ -50,25 +69,255 @@ describe('task', () => { endpointAppContext: mockContext, taskManager: mockTaskManager, }); - const mockTaskInstance = { - id: ManifestTaskConstants.TYPE, - runAt: new Date(), - attempts: 0, - ownerId: '', - status: TaskStatus.Running, - startedAt: new Date(), - scheduledAt: new Date(), - retryAt: new Date(), - params: {}, - state: {}, - taskType: ManifestTaskConstants.TYPE, - }; const createTaskRunner = mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ManifestTaskConstants.TYPE] .createTaskRunner; - const taskRunner = createTaskRunner({ taskInstance: mockTaskInstance }); + const taskRunner = createTaskRunner({ taskInstance: MOCK_TASK_INSTANCE }); await taskRunner.run(); expect(mockManifestTask.runTask).toHaveBeenCalled(); }); }); + + describe('Artifacts generation flow tests', () => { + const runTask = async (manifestManager: ManifestManager) => { + const mockContext = createMockEndpointAppContext(); + const mockTaskManager = taskManagerMock.createSetup(); + + new ManifestTask({ + endpointAppContext: mockContext, + taskManager: mockTaskManager, + }); + + mockContext.service.getManifestManager = jest.fn().mockReturnValue(manifestManager); + + const createTaskRunner = + mockTaskManager.registerTaskDefinitions.mock.calls[0][0][ManifestTaskConstants.TYPE] + .createTaskRunner; + const taskRunner = createTaskRunner({ taskInstance: MOCK_TASK_INSTANCE }); + await taskRunner.run(); + }; + + const TEST_POLICY_ID_1 = 'c6d16e42-c32d-4dce-8a88-113cfe276ad1'; + const TEST_POLICY_ID_2 = '93c46720-c217-11ea-9906-b5b8a21b268e'; + const ARTIFACT_ID_1 = + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + let ARTIFACT_EXCEPTIONS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_EXCEPTIONS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema; + + beforeAll(async () => { + const artifacts = await getMockArtifacts({ compress: true }); + ARTIFACT_EXCEPTIONS_MACOS = artifacts[0]; + ARTIFACT_EXCEPTIONS_WINDOWS = artifacts[1]; + ARTIFACT_TRUSTED_APPS_MACOS = artifacts[2]; + }); + + test('Should not run the process when no current manifest manager', async () => { + const manifestManager = buildManifestManagerMock(); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(null); + + await runTask(manifestManager); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).not.toHaveBeenCalled(); + expect(manifestManager.pushArtifacts).not.toHaveBeenCalled(); + expect(manifestManager.commit).not.toHaveBeenCalled(); + expect(manifestManager.tryDispatch).not.toHaveBeenCalled(); + expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled(); + }); + + test('Should stop the process when no building new manifest throws error', async () => { + const manifestManager = buildManifestManagerMock(); + const lastManifest = Manifest.getDefault(); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error()); + + await runTask(manifestManager); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).not.toHaveBeenCalled(); + expect(manifestManager.commit).not.toHaveBeenCalled(); + expect(manifestManager.tryDispatch).not.toHaveBeenCalled(); + expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled(); + }); + + test('Should not bump version and commit manifest when no diff in the manifest', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.tryDispatch = jest.fn().mockResolvedValue([]); + manifestManager.deleteArtifacts = jest.fn().mockResolvedValue([]); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.0'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]); + expect(manifestManager.commit).not.toHaveBeenCalled(); + expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest); + expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]); + }); + + test('Should stop the process when there are errors pushing new artifacts', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error()]); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.0'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ + ARTIFACT_EXCEPTIONS_MACOS, + ARTIFACT_TRUSTED_APPS_MACOS, + ]); + expect(manifestManager.commit).not.toHaveBeenCalled(); + expect(manifestManager.tryDispatch).not.toHaveBeenCalled(); + expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled(); + }); + + test('Should stop the process when there are errors committing manifest', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockRejectedValue(new Error()); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.1'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ + ARTIFACT_EXCEPTIONS_MACOS, + ARTIFACT_TRUSTED_APPS_MACOS, + ]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); + expect(manifestManager.tryDispatch).not.toHaveBeenCalled(); + expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled(); + }); + + test('Should stop the process when there are errors dispatching manifest', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockResolvedValue(null); + manifestManager.tryDispatch = jest.fn().mockResolvedValue([new Error()]); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.1'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ + ARTIFACT_EXCEPTIONS_MACOS, + ARTIFACT_TRUSTED_APPS_MACOS, + ]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); + expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest); + expect(manifestManager.deleteArtifacts).not.toHaveBeenCalled(); + }); + + test('Should succeed the process and delete old artifacts', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockResolvedValue(null); + manifestManager.tryDispatch = jest.fn().mockResolvedValue([]); + manifestManager.deleteArtifacts = jest.fn().mockResolvedValue([]); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.1'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([ARTIFACT_TRUSTED_APPS_MACOS]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); + expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest); + expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([ARTIFACT_ID_1]); + }); + + test('Should succeed the process but not add or delete artifacts when there are only transitions', async () => { + const manifestManager = buildManifestManagerMock(); + + const lastManifest = Manifest.getDefault(); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + + const newManifest = Manifest.getDefault(); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + newManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + + manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); + manifestManager.buildNewManifest = jest.fn().mockResolvedValue(newManifest); + manifestManager.pushArtifacts = jest.fn().mockResolvedValue([]); + manifestManager.commit = jest.fn().mockResolvedValue(null); + manifestManager.tryDispatch = jest.fn().mockResolvedValue([]); + manifestManager.deleteArtifacts = jest.fn().mockResolvedValue([]); + + await runTask(manifestManager); + + expect(newManifest.getSemanticVersion()).toBe('1.0.1'); + + expect(manifestManager.getLastComputedManifest).toHaveBeenCalled(); + expect(manifestManager.buildNewManifest).toHaveBeenCalledWith(lastManifest); + expect(manifestManager.pushArtifacts).toHaveBeenCalledWith([]); + expect(manifestManager.commit).toHaveBeenCalledWith(newManifest); + expect(manifestManager.tryDispatch).toHaveBeenCalledWith(newManifest); + expect(manifestManager.deleteArtifacts).toHaveBeenCalledWith([]); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 4f0d8671fb177e..04dcb36bf4ea74 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -12,8 +12,9 @@ import { TaskManagerStartContract, } from '../../../../../task_manager/server'; import { EndpointAppContext } from '../../types'; -import { reportErrors } from './common'; +import { getArtifactId, reportErrors } from './common'; import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; +import { isEmptyManifestDiff } from './manifest'; export const ManifestTaskConstants = { TIMEOUT: '1m', @@ -114,39 +115,23 @@ export class ManifestTask { return; } - // New computed manifest based on current state of exception list + // New computed manifest based on current manifest const newManifest = await manifestManager.buildNewManifest(oldManifest); - const diffs = newManifest.diff(oldManifest); - - // Compress new artifacts - const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id); - for (const artifactId of adds) { - const compressError = await newManifest.compressArtifact(artifactId); - if (compressError) { - throw compressError; - } - } - // Persist new artifacts - const artifacts = adds - .map((artifactId) => newManifest.getArtifact(artifactId)) - .filter((artifact): artifact is InternalArtifactCompleteSchema => artifact !== undefined); - if (artifacts.length !== adds.length) { - throw new Error('Invalid artifact encountered.'); - } - const persistErrors = await manifestManager.pushArtifacts(artifacts); + const diff = newManifest.diff(oldManifest); + + const persistErrors = await manifestManager.pushArtifacts( + diff.additions as InternalArtifactCompleteSchema[] + ); if (persistErrors.length) { reportErrors(this.logger, persistErrors); throw new Error('Unable to persist new artifacts.'); } - // Commit latest manifest state, if different - if (diffs.length) { + if (!isEmptyManifestDiff(diff)) { + // Commit latest manifest state newManifest.bumpSemanticVersion(); - const error = await manifestManager.commit(newManifest); - if (error) { - throw error; - } + await manifestManager.commit(newManifest); } // Try dispatching to ingest-manager package policies @@ -157,8 +142,9 @@ export class ManifestTask { } // Try to clean up superceded artifacts - const deletes = diffs.filter((diff) => diff.type === 'delete').map((diff) => diff.id); - const deleteErrors = await manifestManager.deleteArtifacts(deletes); + const deleteErrors = await manifestManager.deleteArtifacts( + diff.removals.map((artifact) => getArtifactId(artifact)) + ); if (deleteErrors.length) { reportErrors(this.logger, deleteErrors); } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.test.ts similarity index 98% rename from x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.test.ts index a2aff41b68df70..32bd7379ade21f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.test.ts @@ -25,7 +25,7 @@ import { loggingSystemMock, } from 'src/core/server/mocks'; import { ArtifactConstants } from '../../lib/artifacts'; -import { registerDownloadExceptionListRoute } from './download_exception_list'; +import { registerDownloadArtifactRoute } from './download_artifact'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; @@ -121,7 +121,7 @@ describe('test alerts route', () => { ); endpointAppContextService.start(startContract); - registerDownloadExceptionListRoute( + registerDownloadArtifactRoute( routerMock, { logFactory: loggingSystemMock.create(), diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.ts similarity index 94% rename from x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts rename to x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.ts index 3dbaa137bb9281..020b70ca0553ca 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_artifact.ts @@ -29,9 +29,9 @@ import { EndpointAppContext } from '../../types'; const allowlistBaseRoute: string = '/api/endpoint/artifacts'; /** - * Registers the exception list route to enable sensors to download an allowlist artifact + * Registers the artifact download route to enable sensors to download an allowlist artifact */ -export function registerDownloadExceptionListRoute( +export function registerDownloadArtifactRoute( router: IRouter, endpointContext: EndpointAppContext, cache: LRU @@ -49,7 +49,7 @@ export function registerDownloadExceptionListRoute( }, async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; - const logger = endpointContext.logFactory.get('download_exception_list'); + const logger = endpointContext.logFactory.get('download_artifact'); // The ApiKey must be associated with an enrolled Fleet agent try { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts index a279a9546d3e1b..a651f93cab09dc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/index.ts @@ -5,4 +5,4 @@ * 2.0. */ -export * from './download_exception_list'; +export * from './download_artifact'; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 46d6e36d3debcd..dedbcc25e2373e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -46,14 +46,10 @@ export const getInternalArtifactMock = async ( export const getEmptyInternalArtifactMock = async ( os: string, schemaVersion: string, - opts?: { compress: boolean } + opts?: { compress: boolean }, + artifactName: string = ArtifactConstants.GLOBAL_ALLOWLIST_NAME ): Promise => { - const artifact = await buildArtifact( - { entries: [] }, - os, - schemaVersion, - ArtifactConstants.GLOBAL_ALLOWLIST_NAME - ); + const artifact = await buildArtifact({ entries: [] }, os, schemaVersion, artifactName); return opts?.compress ? compressArtifact(artifact) : artifact; }; @@ -74,7 +70,7 @@ export const getInternalArtifactMockWithDiffs = async ( }; export const getInternalManifestMock = (): InternalManifestSchema => ({ - ids: [], + artifacts: [], schemaVersion: 'v1', semanticVersion: '1.0.0', }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index 47941c0686572a..675ed41e394aa4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -58,9 +58,17 @@ export const internalArtifactCreateSchema = t.intersection([ ]); export type InternalArtifactCreateSchema = t.TypeOf; +export const internalManifestEntrySchema = t.exact( + t.type({ + policyId: t.union([identifier, t.undefined]), + artifactId: identifier, + }) +); +export type InternalManifestEntrySchema = t.TypeOf; + export const internalManifestSchema = t.exact( t.type({ - ids: t.array(identifier), + artifacts: t.array(internalManifestEntrySchema), schemaVersion: manifestSchemaVersion, semanticVersion, }) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts deleted file mode 100644 index c16b10b965cc0f..00000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.mock.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { SavedObjectsClientContract } from 'src/core/server'; -import { ArtifactClient } from './artifact_client'; - -export const getArtifactClientMock = ( - savedObjectsClient?: SavedObjectsClientContract -): ArtifactClient => { - if (savedObjectsClient !== undefined) { - return new ArtifactClient(savedObjectsClient); - } - return new ArtifactClient(savedObjectsClientMock.create()); -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index a5fcd24dbc753b..b3f098a9693363 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -8,7 +8,6 @@ import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ArtifactConstants, getArtifactId } from '../../lib/artifacts'; import { getInternalArtifactMock } from '../../schemas/artifacts/saved_objects.mock'; -import { getArtifactClientMock } from './artifact_client.mock'; import { ArtifactClient } from './artifact_client'; describe('artifact_client', () => { @@ -20,14 +19,14 @@ describe('artifact_client', () => { test('can get artifact', async () => { const savedObjectsClient = savedObjectsClientMock.create(); - const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifactClient = new ArtifactClient(savedObjectsClient); await artifactClient.getArtifact('abcd'); expect(savedObjectsClient.get).toHaveBeenCalled(); }); test('can create artifact', async () => { const savedObjectsClient = savedObjectsClientMock.create(); - const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifactClient = new ArtifactClient(savedObjectsClient); const artifact = await getInternalArtifactMock('linux', 'v1'); await artifactClient.createArtifact(artifact); expect(savedObjectsClient.create).toHaveBeenCalledWith( @@ -42,7 +41,7 @@ describe('artifact_client', () => { test('can delete artifact', async () => { const savedObjectsClient = savedObjectsClientMock.create(); - const artifactClient = getArtifactClientMock(savedObjectsClient); + const artifactClient = new ArtifactClient(savedObjectsClient); await artifactClient.deleteArtifact('abcd'); expect(savedObjectsClient.delete).toHaveBeenCalledWith( ArtifactConstants.SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 53d3bdfcb656c4..a8bbfca0d41e58 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -5,16 +5,14 @@ * 2.0. */ +import LRU from 'lru-cache'; import { savedObjectsClientMock, loggingSystemMock } from 'src/core/server/mocks'; import { Logger } from 'src/core/server'; import { PackagePolicyServiceInterface } from '../../../../../../fleet/server'; import { createPackagePolicyServiceMock } from '../../../../../../fleet/server/mocks'; import { ExceptionListClient } from '../../../../../../lists/server'; import { listMock } from '../../../../../../lists/server/mocks'; -import LRU from 'lru-cache'; -import { getArtifactClientMock } from '../artifact_client.mock'; -import { getManifestClientMock } from '../manifest_client.mock'; -import { ManifestManager } from './manifest_manager'; +import { ExceptionListItemSchema } from '../../../../../../lists/common/schemas/response'; import { createPackagePolicyWithManifestMock, createPackagePolicyWithInitialManifestMock, @@ -22,6 +20,32 @@ import { getMockArtifactsWithDiff, getEmptyMockArtifacts, } from '../../../lib/artifacts/mocks'; +import { ArtifactClient } from '../artifact_client'; +import { getManifestClientMock } from '../manifest_client.mock'; +import { ManifestManager, ManifestManagerContext } from './manifest_manager'; + +export const createExceptionListResponse = (data: ExceptionListItemSchema[], total?: number) => ({ + data, + page: 1, + per_page: 100, + total: total || data.length, +}); + +type FindExceptionListItemOptions = Parameters[0]; + +const FILTER_REGEXP = /^exception-list-agnostic\.attributes\.os_types:"(\w+)"$/; + +export const mockFindExceptionListItemResponses = ( + responses: Record> +) => { + return jest.fn().mockImplementation((options: FindExceptionListItemOptions) => { + const os = FILTER_REGEXP.test(options.filter || '') + ? options.filter!.match(FILTER_REGEXP)![1] + : ''; + + return createExceptionListResponse(responses[options.listId]?.[os] || []); + }); +}; export enum ManifestManagerMockType { InitialSystemState, @@ -29,28 +53,54 @@ export enum ManifestManagerMockType { NormalFlow, } -export const getManifestManagerMock = (opts?: { - mockType?: ManifestManagerMockType; - cache?: LRU; - exceptionListClient?: ExceptionListClient; - packagePolicyService?: jest.Mocked; - savedObjectsClient?: ReturnType; -}): ManifestManager => { - let cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); - if (opts?.cache != null) { - cache = opts.cache; - } +export interface ManifestManagerMockOptions { + cache: LRU; + exceptionListClient: ExceptionListClient; + packagePolicyService: jest.Mocked; + savedObjectsClient: ReturnType; +} - let exceptionListClient = listMock.getExceptionListClient(); - if (opts?.exceptionListClient != null) { - exceptionListClient = opts.exceptionListClient; - } +export const buildManifestManagerMockOptions = ( + opts: Partial +): ManifestManagerMockOptions => ({ + cache: new LRU({ max: 10, maxAge: 1000 * 60 * 60 }), + exceptionListClient: listMock.getExceptionListClient(), + packagePolicyService: createPackagePolicyServiceMock(), + savedObjectsClient: savedObjectsClientMock.create(), + ...opts, +}); - let packagePolicyService = createPackagePolicyServiceMock(); - if (opts?.packagePolicyService != null) { - packagePolicyService = opts.packagePolicyService; - } - packagePolicyService.list = jest.fn().mockResolvedValue({ +export const buildManifestManagerContextMock = ( + opts: Partial +): ManifestManagerContext => { + const fullOpts = buildManifestManagerMockOptions(opts); + + return { + ...fullOpts, + artifactClient: new ArtifactClient(fullOpts.savedObjectsClient), + logger: loggingSystemMock.create().get() as jest.Mocked, + }; +}; + +export const buildManifestManagerMock = (opts?: Partial) => { + const manifestManager = new ManifestManager(buildManifestManagerContextMock(opts || {})); + manifestManager.getLastComputedManifest = jest.fn(); + manifestManager.buildNewManifest = jest.fn(); + manifestManager.pushArtifacts = jest.fn(); + manifestManager.deleteArtifacts = jest.fn(); + manifestManager.commit = jest.fn(); + manifestManager.tryDispatch = jest.fn(); + + return manifestManager; +}; + +export const getManifestManagerMock = ( + opts?: Partial & { mockType?: ManifestManagerMockType } +): ManifestManager => { + const { mockType = ManifestManagerMockType.NormalFlow, ...restOptions } = opts || {}; + const context = buildManifestManagerContextMock(restOptions); + + context.packagePolicyService.list = jest.fn().mockResolvedValue({ total: 1, items: [ { version: 'policy-1-version', ...createPackagePolicyWithManifestMock() }, @@ -59,19 +109,13 @@ export const getManifestManagerMock = (opts?: { ], }); - let savedObjectsClient = savedObjectsClientMock.create(); - if (opts?.savedObjectsClient != null) { - savedObjectsClient = opts.savedObjectsClient; - } - class ManifestManagerMock extends ManifestManager { protected buildExceptionListArtifacts = jest.fn().mockImplementation(() => { - const mockType = opts?.mockType ?? ManifestManagerMockType.NormalFlow; switch (mockType) { case ManifestManagerMockType.InitialSystemState: return getEmptyMockArtifacts(); case ManifestManagerMockType.ListClientPromiseRejection: - exceptionListClient.findExceptionListItem = jest + context.exceptionListClient.findExceptionListItem = jest .fn() .mockRejectedValue(new Error('unexpected thing happened')); return super.buildExceptionListArtifacts('v1'); @@ -81,7 +125,6 @@ export const getManifestManagerMock = (opts?: { }); public getLastComputedManifest = jest.fn().mockImplementation(() => { - const mockType = opts?.mockType ?? ManifestManagerMockType.NormalFlow; switch (mockType) { case ManifestManagerMockType.InitialSystemState: return null; @@ -95,14 +138,5 @@ export const getManifestManagerMock = (opts?: { .mockReturnValue(getManifestClientMock(this.savedObjectsClient)); } - const manifestManager = new ManifestManagerMock({ - artifactClient: getArtifactClientMock(savedObjectsClient), - cache, - packagePolicyService, - exceptionListClient, - logger: loggingSystemMock.create().get() as jest.Mocked, - savedObjectsClient, - }); - - return manifestManager; + return new ManifestManagerMock(context); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index eedd3dad2cdb3f..52897f473189fb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -6,326 +6,854 @@ */ import { inflateSync } from 'zlib'; +import { SavedObjectsErrorHelpers } from 'src/core/server'; import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { createPackagePolicyServiceMock } from '../../../../../../fleet/server/mocks'; -import { ArtifactConstants, ManifestConstants, isCompleteArtifact } from '../../../lib/artifacts'; - -import { getManifestManagerMock, ManifestManagerMockType } from './manifest_manager.mock'; -import LRU from 'lru-cache'; - -describe('manifest_manager', () => { - describe('ManifestManager sanity checks', () => { - test('ManifestManager can retrieve and diff manifests', async () => { - const manifestManager = getManifestManagerMock(); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - expect(newManifest.diff(oldManifest!)).toEqual([ - { - id: - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - type: 'add', - }, - { - id: - 'endpoint-trustlist-macos-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - { - id: - 'endpoint-trustlist-windows-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - { - id: - 'endpoint-trustlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - ]); +import { ENDPOINT_LIST_ID } from '../../../../../../lists/common'; +import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../../lists/common/constants'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; +import { PackagePolicy } from '../../../../../../fleet/common/types/models'; +import { getEmptyInternalArtifactMock } from '../../../schemas/artifacts/saved_objects.mock'; +import { + InternalArtifactCompleteSchema, + InternalArtifactSchema, + InternalManifestSchema, +} from '../../../schemas/artifacts'; +import { + createPackagePolicyWithConfigMock, + getMockArtifacts, + toArtifactRecords, +} from '../../../lib/artifacts/mocks'; +import { + ArtifactConstants, + ManifestConstants, + getArtifactId, + isCompressed, + translateToEndpointExceptions, + Manifest, +} from '../../../lib/artifacts'; + +import { + buildManifestManagerContextMock, + mockFindExceptionListItemResponses, +} from './manifest_manager.mock'; + +import { ManifestManager } from './manifest_manager'; + +const uncompressData = async (data: Buffer) => JSON.parse(await inflateSync(data).toString()); + +const uncompressArtifact = async (artifact: InternalArtifactSchema) => + uncompressData(Buffer.from(artifact.body!, 'base64')); + +describe('ManifestManager', () => { + const TEST_POLICY_ID_1 = 'c6d16e42-c32d-4dce-8a88-113cfe276ad1'; + const TEST_POLICY_ID_2 = '93c46720-c217-11ea-9906-b5b8a21b268e'; + const ARTIFACT_ID_EXCEPTIONS_MACOS = + 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_EXCEPTIONS_WINDOWS = + 'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_TRUSTED_APPS_MACOS = + 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + const ARTIFACT_ID_TRUSTED_APPS_WINDOWS = + 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'; + + const ARTIFACT_NAME_EXCEPTIONS_MACOS = 'endpoint-exceptionlist-macos-v1'; + const ARTIFACT_NAME_EXCEPTIONS_WINDOWS = 'endpoint-exceptionlist-windows-v1'; + const ARTIFACT_NAME_TRUSTED_APPS_MACOS = 'endpoint-trustlist-macos-v1'; + const ARTIFACT_NAME_TRUSTED_APPS_WINDOWS = 'endpoint-trustlist-windows-v1'; + const ARTIFACT_NAME_TRUSTED_APPS_LINUX = 'endpoint-trustlist-linux-v1'; + + let ARTIFACTS: InternalArtifactCompleteSchema[] = []; + let ARTIFACTS_BY_ID: { [K: string]: InternalArtifactCompleteSchema } = {}; + let ARTIFACT_EXCEPTIONS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_EXCEPTIONS_WINDOWS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_MACOS: InternalArtifactCompleteSchema; + let ARTIFACT_TRUSTED_APPS_WINDOWS: InternalArtifactCompleteSchema; + + beforeAll(async () => { + ARTIFACTS = await getMockArtifacts({ compress: true }); + ARTIFACTS_BY_ID = { + [ARTIFACT_ID_EXCEPTIONS_MACOS]: ARTIFACTS[0], + [ARTIFACT_ID_EXCEPTIONS_WINDOWS]: ARTIFACTS[1], + [ARTIFACT_ID_TRUSTED_APPS_MACOS]: ARTIFACTS[2], + [ARTIFACT_ID_TRUSTED_APPS_WINDOWS]: ARTIFACTS[3], + }; + ARTIFACT_EXCEPTIONS_MACOS = ARTIFACTS[0]; + ARTIFACT_EXCEPTIONS_WINDOWS = ARTIFACTS[1]; + ARTIFACT_TRUSTED_APPS_MACOS = ARTIFACTS[2]; + ARTIFACT_TRUSTED_APPS_WINDOWS = ARTIFACTS[3]; + }); + + describe('getLastComputedManifest', () => { + test('Returns null when saved object not found', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestManager = new ManifestManager( + buildManifestManagerContextMock({ savedObjectsClient }) + ); + + savedObjectsClient.get = jest.fn().mockRejectedValue({ output: { statusCode: 404 } }); + + expect(await manifestManager.getLastComputedManifest()).toBe(null); }); - test('ManifestManager populates cache properly', async () => { - const cache = new LRU({ max: 10, maxAge: 1000 * 60 * 60 }); - const manifestManager = getManifestManagerMock({ cache }); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - const diffs = newManifest.diff(oldManifest!); - expect(diffs).toEqual([ - { - id: - 'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-trustlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - type: 'delete', - }, - { - id: - 'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - type: 'add', - }, - { - id: - 'endpoint-trustlist-macos-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - { - id: - 'endpoint-trustlist-windows-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - { - id: - 'endpoint-trustlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - type: 'add', - }, - ]); + test('Throws error when saved object client responds with 500', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestManager = new ManifestManager( + buildManifestManagerContextMock({ savedObjectsClient }) + ); + const error = { output: { statusCode: 500 } }; - const firstNewArtifactId = diffs.find((diff) => diff.type === 'add')!.id; + savedObjectsClient.get = jest.fn().mockRejectedValue(error); - // Compress all `add` artifacts - for (const artifactDiff of diffs) { - if (artifactDiff.type === 'add') { - await newManifest.compressArtifact(artifactDiff.id); - } - } + await expect(manifestManager.getLastComputedManifest()).rejects.toStrictEqual(error); + }); + + test('Throws error when no version on the manifest', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestManager = new ManifestManager( + buildManifestManagerContextMock({ savedObjectsClient }) + ); - const artifact = newManifest.getArtifact(firstNewArtifactId)!; + savedObjectsClient.get = jest.fn().mockResolvedValue({}); - if (isCompleteArtifact(artifact)) { - await manifestManager.pushArtifacts([artifact]); // caches the artifact - } else { - throw new Error('Artifact is missing a body.'); - } + await expect(manifestManager.getLastComputedManifest()).rejects.toStrictEqual( + new Error('No version returned for manifest.') + ); + }); + + test('Retrieves empty manifest successfully', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestManager = new ManifestManager( + buildManifestManagerContextMock({ savedObjectsClient }) + ); + + savedObjectsClient.get = jest.fn().mockResolvedValue({ + attributes: { + created: '20-01-2020 10:00:00.000Z', + schemaVersion: 'v2', + semanticVersion: '1.0.0', + artifacts: [], + }, + version: '2.0.0', + }); + + const manifest = await manifestManager.getLastComputedManifest(); + + expect(manifest?.getSchemaVersion()).toStrictEqual('v1'); + expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0'); + expect(manifest?.getSavedObjectVersion()).toStrictEqual('2.0.0'); + expect(manifest?.getAllArtifacts()).toStrictEqual([]); + }); + + test('Retrieves non empty manifest successfully', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + const manifestManager = new ManifestManager( + buildManifestManagerContextMock({ savedObjectsClient }) + ); - const entry = JSON.parse(inflateSync(cache.get(firstNewArtifactId)! as Buffer).toString()); - expect(entry).toEqual({ - entries: [ - { - type: 'simple', - entries: [ - { - entries: [ - { - field: 'some.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', - }, + savedObjectsClient.get = jest + .fn() + .mockImplementation(async (objectType: string, id: string) => { + if (objectType === ManifestConstants.SAVED_OBJECT_TYPE) { + return { + attributes: { + created: '20-01-2020 10:00:00.000Z', + schemaVersion: 'v2', + semanticVersion: '1.0.0', + artifacts: [ + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_WINDOWS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_WINDOWS, policyId: TEST_POLICY_ID_2 }, ], - field: 'some.parentField', - type: 'nested', - }, - { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', }, - ], - }, - ], - }); + version: '2.0.0', + }; + } else if (objectType === ArtifactConstants.SAVED_OBJECT_TYPE) { + return { attributes: ARTIFACTS_BY_ID[id], version: '2.1.1' }; + } else { + return null; + } + }); + + const manifest = await manifestManager.getLastComputedManifest(); + + expect(manifest?.getSchemaVersion()).toStrictEqual('v1'); + expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0'); + expect(manifest?.getSavedObjectVersion()).toStrictEqual('2.0.0'); + expect(manifest?.getAllArtifacts()).toStrictEqual(ARTIFACTS.slice(0, 4)); + expect(manifest?.isDefaultArtifact(ARTIFACT_EXCEPTIONS_MACOS)).toBe(true); + expect(manifest?.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_MACOS)).toStrictEqual( + new Set() + ); + expect(manifest?.isDefaultArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)).toBe(true); + expect(manifest?.getArtifactTargetPolicies(ARTIFACT_EXCEPTIONS_WINDOWS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + expect(manifest?.isDefaultArtifact(ARTIFACT_TRUSTED_APPS_MACOS)).toBe(false); + expect(manifest?.getArtifactTargetPolicies(ARTIFACT_TRUSTED_APPS_MACOS)).toStrictEqual( + new Set([TEST_POLICY_ID_1]) + ); + expect(manifest?.isDefaultArtifact(ARTIFACT_TRUSTED_APPS_WINDOWS)).toBe(false); + expect(manifest?.getArtifactTargetPolicies(ARTIFACT_TRUSTED_APPS_WINDOWS)).toStrictEqual( + new Set([TEST_POLICY_ID_1, TEST_POLICY_ID_2]) + ); }); + }); + + describe('buildNewManifest', () => { + const SUPPORTED_ARTIFACT_NAMES = [ + ARTIFACT_NAME_EXCEPTIONS_MACOS, + ARTIFACT_NAME_EXCEPTIONS_WINDOWS, + ARTIFACT_NAME_TRUSTED_APPS_MACOS, + ARTIFACT_NAME_TRUSTED_APPS_WINDOWS, + ARTIFACT_NAME_TRUSTED_APPS_LINUX, + ]; + + const getArtifactIds = (artifacts: InternalArtifactSchema[]) => + artifacts.map((artifact) => artifact.identifier); - test('ManifestManager cannot dispatch incomplete (uncompressed) artifact', async () => { - const packagePolicyService = createPackagePolicyServiceMock(); - const manifestManager = getManifestManagerMock({ packagePolicyService }); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - const dispatchErrors = await manifestManager.tryDispatch(newManifest); - expect(dispatchErrors.length).toEqual(1); - expect(dispatchErrors[0].message).toEqual('Invalid manifest'); + test('Fails when exception list list client fails', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + context.exceptionListClient.findExceptionListItem = jest.fn().mockRejectedValue(new Error()); + + await expect(manifestManager.buildNewManifest()).rejects.toThrow(); }); - test('ManifestManager can dispatch manifest', async () => { - const packagePolicyService = createPackagePolicyServiceMock(); - const manifestManager = getManifestManagerMock({ packagePolicyService }); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - const diffs = newManifest.diff(oldManifest!); + test('Builds fully new manifest if no baseline parameter passed and no exception list items', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); - for (const artifactDiff of diffs) { - if (artifactDiff.type === 'add') { - await newManifest.compressArtifact(artifactDiff.id); - } - } + context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({}); - newManifest.bumpSemanticVersion(); + const manifest = await manifestManager.buildNewManifest(); - const dispatchErrors = await manifestManager.tryDispatch(newManifest); + expect(manifest?.getSchemaVersion()).toStrictEqual('v1'); + expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0'); + expect(manifest?.getSavedObjectVersion()).toBeUndefined(); - expect(dispatchErrors).toEqual([]); + const artifacts = manifest.getAllArtifacts(); - // 2 policies updated... 1 is already up-to-date - expect(packagePolicyService.update.mock.calls.length).toEqual(2); + expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES); + expect(artifacts.every(isCompressed)).toBe(true); - expect( - packagePolicyService.update.mock.calls[0][3].inputs[0].config!.artifact_manifest.value - ).toEqual({ - manifest_version: '1.0.1', - schema_version: 'v1', - artifacts: { - 'endpoint-exceptionlist-macos-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - encoded_sha256: '57941169bb2c5416f9bd7224776c8462cb9a2be0fe8b87e6213e77a1d29be824', - decoded_size: 292, - encoded_size: 131, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8', - }, - 'endpoint-exceptionlist-windows-v1': { - compression_algorithm: 'zlib', - encryption_algorithm: 'none', - decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e', - decoded_size: 432, - encoded_size: 147, - relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3', - }, - 'endpoint-trustlist-linux-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - 'endpoint-trustlist-macos-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - 'endpoint-trustlist-windows-v1': { - compression_algorithm: 'zlib', - decoded_sha256: '1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - decoded_size: 287, - encoded_sha256: 'c3dec543df1177561ab2aa74a37997ea3c1d748d532a597884f5a5c16670d56c', - encoded_size: 133, - encryption_algorithm: 'none', - relative_url: - '/api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', - }, - }, - }); + for (const artifact of artifacts) { + expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] }); + } }); - test('ManifestManager fails to dispatch on conflict', async () => { - const packagePolicyService = createPackagePolicyServiceMock(); - const manifestManager = getManifestManagerMock({ packagePolicyService }); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - const diffs = newManifest.diff(oldManifest!); + test('Builds fully new manifest if no baseline parameter passed and present exception list items', async () => { + const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); + const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({ + [ENDPOINT_LIST_ID]: { macos: [exceptionListItem] }, + [ENDPOINT_TRUSTED_APPS_LIST_ID]: { linux: [trustedAppListItem] }, + }); + + const manifest = await manifestManager.buildNewManifest(); + + expect(manifest?.getSchemaVersion()).toStrictEqual('v1'); + expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0'); + expect(manifest?.getSavedObjectVersion()).toBeUndefined(); + + const artifacts = manifest.getAllArtifacts(); - for (const artifactDiff of diffs) { - if (artifactDiff.type === 'add') { - await newManifest.compressArtifact(artifactDiff.id); + expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES); + expect(artifacts.every(isCompressed)).toBe(true); + + for (const artifact of artifacts) { + if (artifact.identifier === ARTIFACT_NAME_EXCEPTIONS_MACOS) { + expect(await uncompressArtifact(artifact)).toStrictEqual({ + entries: translateToEndpointExceptions([exceptionListItem], 'v1'), + }); + } else if (artifact.identifier === 'endpoint-trustlist-linux-v1') { + expect(await uncompressArtifact(artifact)).toStrictEqual({ + entries: translateToEndpointExceptions([trustedAppListItem], 'v1'), + }); + } else { + expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] }); } } + }); - newManifest.bumpSemanticVersion(); + test('Reuses artifacts when baseline parameter passed and present exception list items', async () => { + const exceptionListItem = getExceptionListItemSchemaMock({ os_types: ['macos'] }); + const trustedAppListItem = getExceptionListItemSchemaMock({ os_types: ['linux'] }); + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); - packagePolicyService.update.mockRejectedValueOnce({ status: 409 }); - const dispatchErrors = await manifestManager.tryDispatch(newManifest); - expect(dispatchErrors).toEqual([{ status: 409 }]); - }); + context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({ + [ENDPOINT_LIST_ID]: { macos: [exceptionListItem] }, + }); - test('ManifestManager can commit manifest', async () => { - const savedObjectsClient: ReturnType< - typeof savedObjectsClientMock.create - > = savedObjectsClientMock.create(); - const manifestManager = getManifestManagerMock({ - savedObjectsClient, + const oldManifest = await manifestManager.buildNewManifest(); + + context.exceptionListClient.findExceptionListItem = mockFindExceptionListItemResponses({ + [ENDPOINT_LIST_ID]: { macos: [exceptionListItem] }, + [ENDPOINT_TRUSTED_APPS_LIST_ID]: { linux: [trustedAppListItem] }, }); - const oldManifest = await manifestManager.getLastComputedManifest(); - const newManifest = await manifestManager.buildNewManifest(oldManifest!); - const diffs = newManifest.diff(oldManifest!); - const firstOldArtifactId = diffs.find((diff) => diff.type === 'delete')!.id; - const FirstNewArtifactId = diffs.find((diff) => diff.type === 'add')!.id; + const manifest = await manifestManager.buildNewManifest(oldManifest); + + expect(manifest?.getSchemaVersion()).toStrictEqual('v1'); + expect(manifest?.getSemanticVersion()).toStrictEqual('1.0.0'); + expect(manifest?.getSavedObjectVersion()).toBeUndefined(); + + const artifacts = manifest.getAllArtifacts(); - // Compress all new artifacts - for (const artifactDiff of diffs) { - if (artifactDiff.type === 'add') { - await newManifest.compressArtifact(artifactDiff.id); + expect(getArtifactIds(artifacts)).toStrictEqual(SUPPORTED_ARTIFACT_NAMES); + expect(artifacts.every(isCompressed)).toBe(true); + + for (const artifact of artifacts) { + if (artifact.identifier === ARTIFACT_NAME_EXCEPTIONS_MACOS) { + expect(artifact).toStrictEqual(oldManifest.getAllArtifacts()[0]); + } else if (artifact.identifier === 'endpoint-trustlist-linux-v1') { + expect(await uncompressArtifact(artifact)).toStrictEqual({ + entries: translateToEndpointExceptions([trustedAppListItem], 'v1'), + }); + } else { + expect(await uncompressArtifact(artifact)).toStrictEqual({ entries: [] }); } } + }); + }); - const artifact = newManifest.getArtifact(FirstNewArtifactId)!; - if (isCompleteArtifact(artifact)) { - await manifestManager.pushArtifacts([artifact]); - } else { - throw new Error('Artifact is missing a body.'); - } + describe('deleteArtifacts', () => { + test('Successfully invokes saved objects client', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + context.savedObjectsClient.delete = jest.fn().mockResolvedValue({}); + + await expect( + manifestManager.deleteArtifacts([ + ARTIFACT_ID_EXCEPTIONS_MACOS, + ARTIFACT_ID_EXCEPTIONS_WINDOWS, + ]) + ).resolves.toStrictEqual([]); + + expect(context.savedObjectsClient.delete).toHaveBeenNthCalledWith( + 1, + ArtifactConstants.SAVED_OBJECT_TYPE, + ARTIFACT_ID_EXCEPTIONS_MACOS + ); + expect(context.savedObjectsClient.delete).toHaveBeenNthCalledWith( + 2, + ArtifactConstants.SAVED_OBJECT_TYPE, + ARTIFACT_ID_EXCEPTIONS_WINDOWS + ); + }); + + test('Returns errors for partial failures', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const error = new Error(); + + context.savedObjectsClient.delete = jest + .fn() + .mockImplementation(async (type: string, id: string) => { + if (id === ARTIFACT_ID_EXCEPTIONS_WINDOWS) { + throw error; + } else { + return {}; + } + }); - await manifestManager.commit(newManifest); - await manifestManager.deleteArtifacts([firstOldArtifactId]); + await expect( + manifestManager.deleteArtifacts([ + ARTIFACT_ID_EXCEPTIONS_MACOS, + ARTIFACT_ID_EXCEPTIONS_WINDOWS, + ]) + ).resolves.toStrictEqual([error]); - // created new artifact - expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( - ArtifactConstants.SAVED_OBJECT_TYPE + expect(context.savedObjectsClient.delete).toHaveBeenCalledTimes(2); + expect(context.savedObjectsClient.delete).toHaveBeenNthCalledWith( + 1, + ArtifactConstants.SAVED_OBJECT_TYPE, + ARTIFACT_ID_EXCEPTIONS_MACOS + ); + expect(context.savedObjectsClient.delete).toHaveBeenNthCalledWith( + 2, + ArtifactConstants.SAVED_OBJECT_TYPE, + ARTIFACT_ID_EXCEPTIONS_WINDOWS ); + }); + }); + + describe('pushArtifacts', () => { + test('Successfully invokes saved objects client and stores in the cache', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + context.savedObjectsClient.create = jest + .fn() + .mockImplementation((type: string, artifact: InternalArtifactCompleteSchema) => artifact); + + await expect( + manifestManager.pushArtifacts([ARTIFACT_EXCEPTIONS_MACOS, ARTIFACT_EXCEPTIONS_WINDOWS]) + ).resolves.toStrictEqual([]); + + expect(context.savedObjectsClient.create).toHaveBeenCalledTimes(2); + expect(context.savedObjectsClient.create).toHaveBeenNthCalledWith( + 1, + ArtifactConstants.SAVED_OBJECT_TYPE, + { ...ARTIFACT_EXCEPTIONS_MACOS, created: expect.anything() }, + { id: ARTIFACT_ID_EXCEPTIONS_MACOS } + ); + expect(context.savedObjectsClient.create).toHaveBeenNthCalledWith( + 2, + ArtifactConstants.SAVED_OBJECT_TYPE, + { ...ARTIFACT_EXCEPTIONS_WINDOWS, created: expect.anything() }, + { id: ARTIFACT_ID_EXCEPTIONS_WINDOWS } + ); + expect( + await uncompressData(context.cache.get(getArtifactId(ARTIFACT_EXCEPTIONS_MACOS))!) + ).toStrictEqual(await uncompressArtifact(ARTIFACT_EXCEPTIONS_MACOS)); + expect( + await uncompressData(context.cache.get(getArtifactId(ARTIFACT_EXCEPTIONS_WINDOWS))!) + ).toStrictEqual(await uncompressArtifact(ARTIFACT_EXCEPTIONS_WINDOWS)); + }); + + test('Returns errors for partial failures', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const error = new Error(); + const { body, ...incompleteArtifact } = ARTIFACT_TRUSTED_APPS_MACOS; + + context.savedObjectsClient.create = jest + .fn() + .mockImplementation(async (type: string, artifact: InternalArtifactCompleteSchema) => { + if (getArtifactId(artifact) === ARTIFACT_ID_EXCEPTIONS_WINDOWS) { + throw error; + } else { + return artifact; + } + }); + + await expect( + manifestManager.pushArtifacts([ + ARTIFACT_EXCEPTIONS_MACOS, + ARTIFACT_EXCEPTIONS_WINDOWS, + incompleteArtifact as InternalArtifactCompleteSchema, + ]) + ).resolves.toStrictEqual([ + error, + new Error(`Incomplete artifact: ${ARTIFACT_ID_TRUSTED_APPS_MACOS}`), + ]); - // committed new manifest - expect(savedObjectsClient.create.mock.calls[1][0]).toEqual( - ManifestConstants.SAVED_OBJECT_TYPE + expect(context.savedObjectsClient.create).toHaveBeenCalledTimes(2); + expect(context.savedObjectsClient.create).toHaveBeenNthCalledWith( + 1, + ArtifactConstants.SAVED_OBJECT_TYPE, + { ...ARTIFACT_EXCEPTIONS_MACOS, created: expect.anything() }, + { id: ARTIFACT_ID_EXCEPTIONS_MACOS } ); + expect( + await uncompressData(context.cache.get(getArtifactId(ARTIFACT_EXCEPTIONS_MACOS))!) + ).toStrictEqual(await uncompressArtifact(ARTIFACT_EXCEPTIONS_MACOS)); + expect(context.cache.get(getArtifactId(ARTIFACT_EXCEPTIONS_WINDOWS))).toBeUndefined(); + }); - // deleted old artifact - expect(savedObjectsClient.delete).toHaveBeenCalledWith( + test('Tolerates saved objects client conflict', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + context.savedObjectsClient.create = jest + .fn() + .mockRejectedValue( + SavedObjectsErrorHelpers.createConflictError( + ArtifactConstants.SAVED_OBJECT_TYPE, + ARTIFACT_ID_EXCEPTIONS_MACOS + ) + ); + + await expect( + manifestManager.pushArtifacts([ARTIFACT_EXCEPTIONS_MACOS]) + ).resolves.toStrictEqual([]); + + expect(context.savedObjectsClient.create).toHaveBeenCalledTimes(1); + expect(context.savedObjectsClient.create).toHaveBeenNthCalledWith( + 1, ArtifactConstants.SAVED_OBJECT_TYPE, - firstOldArtifactId + { ...ARTIFACT_EXCEPTIONS_MACOS, created: expect.anything() }, + { id: ARTIFACT_ID_EXCEPTIONS_MACOS } ); + expect(context.cache.get(getArtifactId(ARTIFACT_EXCEPTIONS_MACOS))).toBeUndefined(); }); + }); + + describe('commit', () => { + test('Creates new saved object if no saved object version', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const manifest = Manifest.getDefault(); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + context.savedObjectsClient.create = jest + .fn() + .mockImplementation((type: string, object: InternalManifestSchema) => object); - test('ManifestManager handles promise rejections when building artifacts', async () => { - // This test won't fail on an unhandled promise rejection, but it will cause - // an UnhandledPromiseRejectionWarning to be printed. - const manifestManager = getManifestManagerMock({ - mockType: ManifestManagerMockType.ListClientPromiseRejection, + await expect(manifestManager.commit(manifest)).resolves.toBeUndefined(); + + expect(context.savedObjectsClient.create).toHaveBeenCalledTimes(1); + expect(context.savedObjectsClient.create).toHaveBeenNthCalledWith( + 1, + ManifestConstants.SAVED_OBJECT_TYPE, + { + artifacts: [ + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: TEST_POLICY_ID_2 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_2 }, + ], + schemaVersion: 'v1', + semanticVersion: '1.0.0', + created: expect.anything(), + }, + { id: 'endpoint-manifest-v1' } + ); + }); + + test('Updates existing saved object if has saved object version', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const manifest = new Manifest({ soVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + context.savedObjectsClient.update = jest + .fn() + .mockImplementation((type: string, id: string, object: InternalManifestSchema) => object); + + await expect(manifestManager.commit(manifest)).resolves.toBeUndefined(); + + expect(context.savedObjectsClient.update).toHaveBeenCalledTimes(1); + expect(context.savedObjectsClient.update).toHaveBeenNthCalledWith( + 1, + ManifestConstants.SAVED_OBJECT_TYPE, + 'endpoint-manifest-v1', + { + artifacts: [ + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: undefined }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_1 }, + { artifactId: ARTIFACT_ID_EXCEPTIONS_WINDOWS, policyId: TEST_POLICY_ID_2 }, + { artifactId: ARTIFACT_ID_TRUSTED_APPS_MACOS, policyId: TEST_POLICY_ID_2 }, + ], + schemaVersion: 'v1', + semanticVersion: '1.0.0', + }, + { version: '1.0.0' } + ); + }); + + test('Throws error when saved objects client fails', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const manifest = new Manifest({ soVersion: '1.0.0' }); + const error = new Error(); + + context.savedObjectsClient.update = jest.fn().mockRejectedValue(error); + + await expect(manifestManager.commit(manifest)).rejects.toBe(error); + + expect(context.savedObjectsClient.update).toHaveBeenCalledTimes(1); + expect(context.savedObjectsClient.update).toHaveBeenNthCalledWith( + 1, + ManifestConstants.SAVED_OBJECT_TYPE, + 'endpoint-manifest-v1', + { + artifacts: [], + schemaVersion: 'v1', + semanticVersion: '1.0.0', + }, + { version: '1.0.0' } + ); + }); + }); + + describe('tryDispatch', () => { + const mockPolicyListResponse = (items: PackagePolicy[]) => + jest.fn().mockResolvedValue({ + items, + page: 1, + per_page: 100, + total: items.length, }); - await expect(manifestManager.buildNewManifest()).rejects.toThrow(); + + const toNewPackagePolicy = (packagePolicy: PackagePolicy) => { + const { id, revision, updated_at: updatedAt, updated_by: updatedBy, ...rest } = packagePolicy; + + return rest; + }; + + test('Should not dispatch if no policies', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const manifest = new Manifest({ soVersion: '1.0.0' }); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + context.packagePolicyService.list = mockPolicyListResponse([]); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + }); + + test('Should return errors if invalid config for package policy', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + const manifest = new Manifest({ soVersion: '1.0.0' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + context.packagePolicyService.list = mockPolicyListResponse([ + createPackagePolicyWithConfigMock({ id: TEST_POLICY_ID_1 }), + ]); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([ + new Error(`Package Policy ${TEST_POLICY_ID_1} has no config.`), + ]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + }); + + test('Should not dispatch if semantic version has not changed', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + const manifest = new Manifest({ soVersion: '1.0.0' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + context.packagePolicyService.list = mockPolicyListResponse([ + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_WINDOWS]: ARTIFACT_EXCEPTIONS_MACOS, + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + ]); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(0); + }); + + test('Should dispatch to only policies where list of artifacts changed', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + const manifest = new Manifest({ soVersion: '1.0.0', semanticVersion: '1.0.1' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + context.packagePolicyService.list = mockPolicyListResponse([ + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_MACOS]: ARTIFACT_EXCEPTIONS_MACOS, + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_2, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_WINDOWS]: ARTIFACT_EXCEPTIONS_WINDOWS, + [ARTIFACT_NAME_TRUSTED_APPS_MACOS]: ARTIFACT_TRUSTED_APPS_MACOS, + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + ]); + context.packagePolicyService.update = jest.fn().mockResolvedValue({}); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(1); + expect(context.packagePolicyService.update).toHaveBeenNthCalledWith( + 1, + expect.anything(), + undefined, + TEST_POLICY_ID_1, + toNewPackagePolicy( + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_WINDOWS]: ARTIFACT_EXCEPTIONS_WINDOWS, + }), + manifest_version: '1.0.1', + schema_version: 'v1', + }, + }, + }, + }) + ) + ); + }); + + test('Should dispatch to only policies where artifact content changed', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + + const manifest = new Manifest({ soVersion: '1.0.0', semanticVersion: '1.0.1' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_2); + + context.packagePolicyService.list = mockPolicyListResponse([ + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_MACOS]: await getEmptyInternalArtifactMock( + 'macos', + 'v1', + { + compress: true, + } + ), + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_2, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_WINDOWS]: ARTIFACT_EXCEPTIONS_WINDOWS, + [ARTIFACT_NAME_TRUSTED_APPS_MACOS]: ARTIFACT_TRUSTED_APPS_MACOS, + }), + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + ]); + context.packagePolicyService.update = jest.fn().mockResolvedValue({}); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(1); + expect(context.packagePolicyService.update).toHaveBeenNthCalledWith( + 1, + expect.anything(), + undefined, + TEST_POLICY_ID_1, + toNewPackagePolicy( + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: toArtifactRecords({ + [ARTIFACT_NAME_EXCEPTIONS_MACOS]: ARTIFACT_EXCEPTIONS_MACOS, + }), + manifest_version: '1.0.1', + schema_version: 'v1', + }, + }, + }, + }) + ) + ); + }); + + test('Should return partial errors', async () => { + const context = buildManifestManagerContextMock({}); + const manifestManager = new ManifestManager(context); + const error = new Error(); + + const manifest = new Manifest({ soVersion: '1.0.0', semanticVersion: '1.0.1' }); + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + + context.packagePolicyService.list = mockPolicyListResponse([ + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_1, + config: { + artifact_manifest: { + value: { + artifacts: {}, + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + createPackagePolicyWithConfigMock({ + id: TEST_POLICY_ID_2, + config: { + artifact_manifest: { + value: { + artifacts: {}, + manifest_version: '1.0.0', + schema_version: 'v1', + }, + }, + }, + }), + ]); + context.packagePolicyService.update = jest.fn().mockImplementation(async (...args) => { + if (args[2] === TEST_POLICY_ID_2) { + throw error; + } else { + return {}; + } + }); + + await expect(manifestManager.tryDispatch(manifest)).resolves.toStrictEqual([error]); + + expect(context.packagePolicyService.update).toHaveBeenCalledTimes(2); }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 8e4d2d9349bb2b..6b9cbb55415a01 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -6,20 +6,25 @@ */ import semver from 'semver'; -import { Logger, SavedObjectsClientContract } from 'src/core/server'; import LRU from 'lru-cache'; +import { isEqual } from 'lodash'; +import { Logger, SavedObjectsClientContract } from 'src/core/server'; import { PackagePolicyServiceInterface } from '../../../../../../fleet/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; -import { manifestDispatchSchema } from '../../../../../common/endpoint/schema/manifest'; +import { + manifestDispatchSchema, + ManifestSchema, +} from '../../../../../common/endpoint/schema/manifest'; import { ArtifactConstants, buildArtifact, getArtifactId, getFullEndpointExceptionList, + isCompressed, Manifest, - ManifestDiff, + maybeCompressArtifact, } from '../../../lib/artifacts'; import { InternalArtifactCompleteSchema, @@ -29,6 +34,7 @@ import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; import { ENDPOINT_LIST_ID } from '../../../../../../lists/common'; import { ENDPOINT_TRUSTED_APPS_LIST_ID } from '../../../../../../lists/common/constants'; +import { PackagePolicy } from '../../../../../../fleet/common/types/models'; export interface ManifestManagerContext { savedObjectsClient: SavedObjectsClientContract; @@ -39,14 +45,13 @@ export interface ManifestManagerContext { cache: LRU; } -export interface ManifestSnapshotOpts { - initialize?: boolean; -} +const getArtifactIds = (manifest: ManifestSchema) => + [...Object.keys(manifest.artifacts)].map( + (key) => `${key}-${manifest.artifacts[key].decoded_sha256}` + ); -export interface ManifestSnapshot { - manifest: Manifest; - diffs: ManifestDiff[]; -} +const manifestsEqual = (manifest1: ManifestSchema, manifest2: ManifestSchema) => + isEqual(new Set(getArtifactIds(manifest1)), new Set(getArtifactIds(manifest2))); export class ManifestManager { protected artifactClient: ArtifactClient; @@ -209,8 +214,7 @@ export class ManifestManager { */ public async getLastComputedManifest(): Promise { try { - const manifestClient = this.getManifestClient(); - const manifestSo = await manifestClient.getManifest(); + const manifestSo = await this.getManifestClient().getManifest(); if (manifestSo.version === undefined) { throw new Error('No version returned for manifest.'); @@ -222,14 +226,17 @@ export class ManifestManager { soVersion: manifestSo.version, }); - for (const id of manifestSo.attributes.ids) { - const artifactSo = await this.artifactClient.getArtifact(id); - manifest.addEntry(artifactSo.attributes); + for (const entry of manifestSo.attributes.artifacts) { + manifest.addEntry( + (await this.artifactClient.getArtifact(entry.artifactId)).attributes, + entry.policyId + ); } + return manifest; - } catch (err) { - if (err.output.statusCode !== 404) { - throw err; + } catch (error) { + if (!error.output || error.output.statusCode !== 404) { + throw error; } return null; } @@ -241,17 +248,36 @@ export class ManifestManager { * @param baselineManifest A baseline manifest to use for initializing pre-existing artifacts. * @returns {Promise} A new Manifest object reprenting the current exception list. */ - public async buildNewManifest(baselineManifest?: Manifest): Promise { + public async buildNewManifest( + baselineManifest: Manifest = Manifest.getDefault(this.schemaVersion) + ): Promise { // Build new exception list artifacts const artifacts = ( await Promise.all([this.buildExceptionListArtifacts(), this.buildTrustedAppsArtifacts()]) ).flat(); // Build new manifest - const manifest = Manifest.fromArtifacts( - artifacts, - baselineManifest ?? Manifest.getDefault(this.schemaVersion) - ); + const manifest = new Manifest({ + schemaVersion: this.schemaVersion, + semanticVersion: baselineManifest.getSemanticVersion(), + soVersion: baselineManifest.getSavedObjectVersion(), + }); + + for (const artifact of artifacts) { + let artifactToAdd = baselineManifest.getArtifact(getArtifactId(artifact)) || artifact; + + if (!isCompressed(artifactToAdd)) { + artifactToAdd = await maybeCompressArtifact(artifactToAdd); + + if (!isCompressed(artifactToAdd)) { + throw new Error(`Unable to compress artifact: ${getArtifactId(artifactToAdd)}`); + } else if (!internalArtifactCompleteSchema.is(artifactToAdd)) { + throw new Error(`Incomplete artifact detected: ${getArtifactId(artifactToAdd)}`); + } + } + + manifest.addEntry(artifactToAdd); + } return manifest; } @@ -264,35 +290,24 @@ export class ManifestManager { * @returns {Promise} Any errors encountered. */ public async tryDispatch(manifest: Manifest): Promise { - const serializedManifest = manifest.toEndpointFormat(); - if (!manifestDispatchSchema.is(serializedManifest)) { - return [new Error('Invalid manifest')]; - } - - let paging = true; - let page = 1; const errors: Error[] = []; - while (paging) { - const { items, total } = await this.packagePolicyService.list(this.savedObjectsClient, { - page, - perPage: 20, - kuery: 'ingest-package-policies.package.name:endpoint', - }); + await this.forEachPolicy(async (packagePolicy) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { id, revision, updated_at, updated_by, ...newPackagePolicy } = packagePolicy; + if (newPackagePolicy.inputs.length > 0 && newPackagePolicy.inputs[0].config !== undefined) { + const oldManifest = newPackagePolicy.inputs[0].config.artifact_manifest ?? { + value: {}, + }; - for (const packagePolicy of items) { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { id, revision, updated_at, updated_by, ...newPackagePolicy } = packagePolicy; - if (newPackagePolicy.inputs.length > 0 && newPackagePolicy.inputs[0].config !== undefined) { - const oldManifest = newPackagePolicy.inputs[0].config.artifact_manifest ?? { - value: {}, - }; - - const newManifestVersion = manifest.getSemanticVersion(); - if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { - newPackagePolicy.inputs[0].config.artifact_manifest = { - value: serializedManifest, - }; + const newManifestVersion = manifest.getSemanticVersion(); + if (semver.gt(newManifestVersion, oldManifest.value.manifest_version)) { + const serializedManifest = manifest.toPackagePolicyManifest(packagePolicy.id); + + if (!manifestDispatchSchema.is(serializedManifest)) { + errors.push(new Error(`Invalid manifest for policy ${packagePolicy.id}`)); + } else if (!manifestsEqual(serializedManifest, oldManifest.value)) { + newPackagePolicy.inputs[0].config.artifact_manifest = { value: serializedManifest }; try { await this.packagePolicyService.update( @@ -309,15 +324,17 @@ export class ManifestManager { errors.push(err); } } else { - this.logger.debug(`No change in package policy: ${id}`); + this.logger.debug( + `No change in manifest content for package policy: ${id}. Staying on old version` + ); } } else { - errors.push(new Error(`Package Policy ${id} has no config.`)); + this.logger.debug(`No change in manifest version for package policy: ${id}`); } + } else { + errors.push(new Error(`Package Policy ${id} has no config.`)); } - paging = (page - 1) * 20 + items.length < total; - page++; - } + }); return errors; } @@ -328,27 +345,41 @@ export class ManifestManager { * @param manifest The Manifest to commit. * @returns {Promise} An error, if encountered, or null. */ - public async commit(manifest: Manifest): Promise { - try { - const manifestClient = this.getManifestClient(); + public async commit(manifest: Manifest) { + const manifestClient = this.getManifestClient(); + + // Commit the new manifest + const manifestSo = manifest.toSavedObject(); + const version = manifest.getSavedObjectVersion(); + + if (version == null) { + await manifestClient.createManifest(manifestSo); + } else { + await manifestClient.updateManifest(manifestSo, { + version, + }); + } - // Commit the new manifest - const manifestSo = manifest.toSavedObject(); - const version = manifest.getSavedObjectVersion(); + this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`); + } - if (version == null) { - await manifestClient.createManifest(manifestSo); - } else { - await manifestClient.updateManifest(manifestSo, { - version, - }); + private async forEachPolicy(callback: (policy: PackagePolicy) => Promise) { + let paging = true; + let page = 1; + + while (paging) { + const { items, total } = await this.packagePolicyService.list(this.savedObjectsClient, { + page, + perPage: 20, + kuery: 'ingest-package-policies.package.name:endpoint', + }); + + for (const packagePolicy of items) { + await callback(packagePolicy); } - this.logger.info(`Committed manifest ${manifest.getSemanticVersion()}`); - } catch (err) { - return err; + paging = (page - 1) * 20 + items.length < total; + page++; } - - return null; } } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index a34193937c788a..6e03d81a7d3561 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -62,7 +62,7 @@ import { registerPolicyRoutes } from './endpoint/routes/policy'; import { ArtifactClient, ManifestManager } from './endpoint/services'; import { EndpointAppContextService } from './endpoint/endpoint_app_context_services'; import { EndpointAppContext } from './endpoint/types'; -import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts'; +import { registerDownloadArtifactRoute } from './endpoint/routes/artifacts'; import { initUsageCollectors } from './usage'; import type { SecuritySolutionRequestHandlerContext } from './types'; import { registerTrustedAppsRoutes } from './endpoint/routes/trusted_apps'; @@ -130,7 +130,7 @@ export class Plugin implements IPlugin; + private artifactsCache: LRU; constructor(context: PluginInitializerContext) { this.context = context; @@ -138,7 +138,7 @@ export class Plugin implements IPlugin({ max: 3, maxAge: 1000 * 60 * 5 }); + this.artifactsCache = new LRU({ max: 3, maxAge: 1000 * 60 * 5 }); this.telemetryEventsSender = new TelemetryEventsSender(this.logger); this.logger.debug('plugin initialized'); @@ -192,7 +192,7 @@ export class Plugin implements IPlugin Date: Wed, 10 Feb 2021 10:52:46 -0500 Subject: [PATCH 22/32] [ML] Data Frame Analytics creation wizard: adds support for extended hyper-parameters (#90843) * add support for new hyperparameters in the creation wizard * fix translation error --- .../ml/common/util/errors/process_errors.ts | 12 +- x-pack/plugins/ml/common/util/errors/types.ts | 2 + .../data_frame_analytics/common/analytics.ts | 6 + .../advanced_step/advanced_step_form.tsx | 16 +- .../advanced_step/hyper_parameters.tsx | 224 +++++++++++++++++- .../components/shared/fetch_explain_data.ts | 10 +- .../action_clone/clone_action_name.tsx | 48 ++++ .../hooks/use_create_analytics_form/state.ts | 28 ++- 8 files changed, 340 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ml/common/util/errors/process_errors.ts b/x-pack/plugins/ml/common/util/errors/process_errors.ts index dd97886f2ff3de..821ba670e2ddbf 100644 --- a/x-pack/plugins/ml/common/util/errors/process_errors.ts +++ b/x-pack/plugins/ml/common/util/errors/process_errors.ts @@ -59,11 +59,21 @@ export const extractErrorProperties = (error: ErrorType): MLErrorObject => { typeof error.body.attributes === 'object' && typeof error.body.attributes.body?.error?.reason === 'string' ) { - return { + const errObj: MLErrorObject = { message: error.body.attributes.body.error.reason, statusCode: error.body.statusCode, fullError: error.body.attributes.body, }; + if ( + typeof error.body.attributes.body.error.caused_by === 'object' && + (typeof error.body.attributes.body.error.caused_by?.reason === 'string' || + typeof error.body.attributes.body.error.caused_by?.caused_by?.reason === 'string') + ) { + errObj.causedBy = + error.body.attributes.body.error.caused_by?.caused_by?.reason || + error.body.attributes.body.error.caused_by?.reason; + } + return errObj; } else { return { message: error.body.message, diff --git a/x-pack/plugins/ml/common/util/errors/types.ts b/x-pack/plugins/ml/common/util/errors/types.ts index 23cd91a57c4f86..39e9ed4e2575f8 100644 --- a/x-pack/plugins/ml/common/util/errors/types.ts +++ b/x-pack/plugins/ml/common/util/errors/types.ts @@ -11,6 +11,7 @@ import Boom from '@hapi/boom'; export interface EsErrorRootCause { type: string; reason: string; + caused_by?: EsErrorRootCause; } export interface EsErrorBody { @@ -37,6 +38,7 @@ export interface ErrorMessage { } export interface MLErrorObject { + causedBy?: string; message: string; statusCode?: number; fullError?: EsErrorBody; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts index 929b055b5f7b52..4f1799ed26f872 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/analytics.ts @@ -33,18 +33,24 @@ export { getAnalysisType } from '../../../../common/util/analytics_utils'; export type IndexPattern = string; export enum ANALYSIS_ADVANCED_FIELDS { + ALPHA = 'alpha', ETA = 'eta', + ETA_GROWTH_RATE_PER_TREE = 'eta_growth_rate_per_tree', + DOWNSAMPLE_FACTOR = 'downsample_factor', FEATURE_BAG_FRACTION = 'feature_bag_fraction', FEATURE_INFLUENCE_THRESHOLD = 'feature_influence_threshold', GAMMA = 'gamma', LAMBDA = 'lambda', MAX_TREES = 'max_trees', + MAX_OPTIMIZATION_ROUNDS_PER_HYPERPARAMETER = 'max_optimization_rounds_per_hyperparameter', METHOD = 'method', N_NEIGHBORS = 'n_neighbors', NUM_TOP_CLASSES = 'num_top_classes', NUM_TOP_FEATURE_IMPORTANCE_VALUES = 'num_top_feature_importance_values', OUTLIER_FRACTION = 'outlier_fraction', RANDOMIZE_SEED = 'randomize_seed', + SOFT_TREE_DEPTH_LIMIT = 'soft_tree_depth_limit', + SOFT_TREE_DEPTH_TOLERANCE = 'soft_tree_depth_tolerance', } export enum OUTLIER_ANALYSIS_METHOD { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/advanced_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/advanced_step_form.tsx index 67ade39d6fa79d..8e25fc961c7c28 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/advanced_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/advanced_step_form.tsx @@ -138,14 +138,18 @@ export const AdvancedStepForm: FC = ({ const { setEstimatedModelMemoryLimit, setFormState } = actions; const { form, isJobCreated, estimatedModelMemoryLimit } = state; const { + alpha, computeFeatureInfluence, + downsampleFactor, eta, + etaGrowthRatePerTree, featureBagFraction, featureInfluenceThreshold, gamma, jobType, lambda, maxNumThreads, + maxOptimizationRoundsPerHyperparameter, maxTrees, method, modelMemoryLimit, @@ -157,6 +161,8 @@ export const AdvancedStepForm: FC = ({ outlierFraction, predictionFieldName, randomizeSeed, + softTreeDepthLimit, + softTreeDepthTolerance, useEstimatedMml, } = form; @@ -197,7 +203,7 @@ export const AdvancedStepForm: FC = ({ useEffect(() => { setFetchingAdvancedParamErrors(true); (async function () { - const { success, errorMessage, expectedMemory } = await fetchExplainData(form); + const { success, errorMessage, errorReason, expectedMemory } = await fetchExplainData(form); const paramErrors: AdvancedParamErrors = {}; if (success) { @@ -212,6 +218,8 @@ export const AdvancedStepForm: FC = ({ Object.values(ANALYSIS_ADVANCED_FIELDS).forEach((param) => { if (errorMessage.includes(`[${param}]`)) { paramErrors[param] = errorMessage; + } else if (errorReason?.includes(`[${param}]`)) { + paramErrors[param] = errorReason; } }); } @@ -219,12 +227,16 @@ export const AdvancedStepForm: FC = ({ setAdvancedParamErrors(paramErrors); })(); }, [ + alpha, + downsampleFactor, eta, + etaGrowthRatePerTree, featureBagFraction, featureInfluenceThreshold, gamma, lambda, maxNumThreads, + maxOptimizationRoundsPerHyperparameter, maxTrees, method, nNeighbors, @@ -232,6 +244,8 @@ export const AdvancedStepForm: FC = ({ numTopFeatureImportanceValues, outlierFraction, randomizeSeed, + softTreeDepthLimit, + softTreeDepthTolerance, ]); const outlierDetectionAdvancedConfig = ( diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/hyper_parameters.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/hyper_parameters.tsx index 0bd817f5e2757f..03dfc09d97b0e0 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/hyper_parameters.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/advanced_step/hyper_parameters.tsx @@ -21,7 +21,20 @@ interface Props extends CreateAnalyticsFormProps { export const HyperParameters: FC = ({ actions, state, advancedParamErrors }) => { const { setFormState } = actions; - const { eta, featureBagFraction, gamma, lambda, maxTrees, randomizeSeed } = state.form; + const { + alpha, + downsampleFactor, + eta, + etaGrowthRatePerTree, + featureBagFraction, + gamma, + lambda, + maxOptimizationRoundsPerHyperparameter, + maxTrees, + randomizeSeed, + softTreeDepthLimit, + softTreeDepthTolerance, + } = state.form; return ( @@ -203,6 +216,215 @@ export const HyperParameters: FC = ({ actions, state, advancedParamErrors /> + + + + setFormState({ alpha: e.target.value === '' ? undefined : +e.target.value }) + } + step={0.001} + min={0} + value={getNumberValue(alpha)} + /> + + + + + + setFormState({ + downsampleFactor: e.target.value === '' ? undefined : +e.target.value, + }) + } + step={0.001} + min={0} + max={1} + value={getNumberValue(downsampleFactor)} + /> + + + + + + setFormState({ + etaGrowthRatePerTree: e.target.value === '' ? undefined : +e.target.value, + }) + } + step={0.001} + min={0.5} + max={2} + value={getNumberValue(etaGrowthRatePerTree)} + /> + + + + + + setFormState({ + maxOptimizationRoundsPerHyperparameter: + e.target.value === '' ? undefined : +e.target.value, + }) + } + min={0} + max={20} + step={1} + value={getNumberValue(maxOptimizationRoundsPerHyperparameter)} + /> + + + + + + setFormState({ + softTreeDepthLimit: e.target.value === '' ? undefined : +e.target.value, + }) + } + step={0.001} + min={0} + value={getNumberValue(softTreeDepthLimit)} + /> + + + + + + setFormState({ + softTreeDepthTolerance: e.target.value === '' ? undefined : +e.target.value, + }) + } + step={0.001} + min={0.01} + value={getNumberValue(softTreeDepthTolerance)} + /> + + ); }; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/shared/fetch_explain_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/shared/fetch_explain_data.ts index 9b71b5d29c0f1e..ec567f1f96156c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/shared/fetch_explain_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/shared/fetch_explain_data.ts @@ -6,7 +6,7 @@ */ import { ml } from '../../../../../services/ml_api_service'; -import { extractErrorMessage } from '../../../../../../../common/util/errors'; +import { extractErrorProperties } from '../../../../../../../common/util/errors'; import { DfAnalyticsExplainResponse, FieldSelectionItem } from '../../../../common/analytics'; import { getJobConfigFromFormState, @@ -23,6 +23,7 @@ export interface FetchExplainDataReturnType { export const fetchExplainData = async (formState: State['form']) => { const jobConfig = getJobConfigFromFormState(formState); let errorMessage = ''; + let errorReason = ''; let success = true; let expectedMemory = ''; let fieldSelection: FieldSelectionItem[] = []; @@ -36,8 +37,12 @@ export const fetchExplainData = async (formState: State['form']) => { expectedMemory = resp.memory_estimation?.expected_memory_without_disk; fieldSelection = resp.field_selection || []; } catch (error) { + const errObj = extractErrorProperties(error); success = false; - errorMessage = extractErrorMessage(error); + errorMessage = errObj.message; + if (errObj.causedBy) { + errorReason = errObj.causedBy; + } } return { @@ -45,5 +50,6 @@ export const fetchExplainData = async (formState: State['form']) => { expectedMemory, fieldSelection, errorMessage, + errorReason, }; }; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx index ee66612de97acb..2ce6e7ac0e33df 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/action_clone/clone_action_name.tsx @@ -121,6 +121,30 @@ const getAnalyticsJobMeta = (config: CloneDataFrameAnalyticsConfig): AnalyticsJo optional: true, ignore: true, }, + alpha: { + optional: true, + formKey: 'alpha', + }, + downsample_factor: { + optional: true, + formKey: 'downsampleFactor', + }, + eta_growth_rate_per_tree: { + optional: true, + formKey: 'etaGrowthRatePerTree', + }, + max_optimization_rounds_per_hyperparameter: { + optional: true, + formKey: 'maxOptimizationRoundsPerHyperparameter', + }, + soft_tree_depth_limit: { + optional: true, + formKey: 'softTreeDepthLimit', + }, + soft_tree_depth_tolerance: { + optional: true, + formKey: 'softTreeDepthTolerance', + }, }, } : {}), @@ -215,6 +239,30 @@ const getAnalyticsJobMeta = (config: CloneDataFrameAnalyticsConfig): AnalyticsJo optional: true, ignore: true, }, + alpha: { + optional: true, + formKey: 'alpha', + }, + downsample_factor: { + optional: true, + formKey: 'downsampleFactor', + }, + eta_growth_rate_per_tree: { + optional: true, + formKey: 'etaGrowthRatePerTree', + }, + max_optimization_rounds_per_hyperparameter: { + optional: true, + formKey: 'maxOptimizationRoundsPerHyperparameter', + }, + soft_tree_depth_limit: { + optional: true, + formKey: 'softTreeDepthLimit', + }, + soft_tree_depth_tolerance: { + optional: true, + formKey: 'softTreeDepthTolerance', + }, }, } : {}), diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts index a70962c45ffcb7..131da93a2328a0 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/hooks/use_create_analytics_form/state.ts @@ -47,6 +47,7 @@ export interface State { advancedEditorRawString: string; disableSwitchToForm: boolean; form: { + alpha: undefined | number; computeFeatureInfluence: string; createIndexPattern: boolean; dependentVariable: DependentVariable; @@ -57,7 +58,9 @@ export interface State { destinationIndexNameValid: boolean; destinationIndexPatternTitleExists: boolean; earlyStoppingEnabled: undefined | boolean; + downsampleFactor: undefined | number; eta: undefined | number; + etaGrowthRatePerTree: undefined | number; featureBagFraction: undefined | number; featureInfluenceThreshold: undefined | number; gamma: undefined | number; @@ -73,6 +76,7 @@ export interface State { lambda: number | undefined; loadingFieldOptions: boolean; maxNumThreads: undefined | number; + maxOptimizationRoundsPerHyperparameter: undefined | number; maxTrees: undefined | number; method: undefined | string; modelMemoryLimit: string | undefined; @@ -88,6 +92,8 @@ export interface State { requiredFieldsError: string | undefined; randomizeSeed: undefined | number; resultsField: undefined | string; + softTreeDepthLimit: undefined | number; + softTreeDepthTolerance: undefined | number; sourceIndex: EsIndexName; sourceIndexNameEmpty: boolean; sourceIndexNameValid: boolean; @@ -117,6 +123,7 @@ export const getInitialState = (): State => ({ advancedEditorRawString: '', disableSwitchToForm: false, form: { + alpha: undefined, computeFeatureInfluence: 'true', createIndexPattern: true, dependentVariable: '', @@ -127,7 +134,9 @@ export const getInitialState = (): State => ({ destinationIndexNameValid: false, destinationIndexPatternTitleExists: false, earlyStoppingEnabled: undefined, + downsampleFactor: undefined, eta: undefined, + etaGrowthRatePerTree: undefined, featureBagFraction: undefined, featureInfluenceThreshold: undefined, gamma: undefined, @@ -143,6 +152,7 @@ export const getInitialState = (): State => ({ lambda: undefined, loadingFieldOptions: false, maxNumThreads: DEFAULT_MAX_NUM_THREADS, + maxOptimizationRoundsPerHyperparameter: undefined, maxTrees: undefined, method: undefined, modelMemoryLimit: undefined, @@ -158,6 +168,8 @@ export const getInitialState = (): State => ({ requiredFieldsError: undefined, randomizeSeed: undefined, resultsField: undefined, + softTreeDepthLimit: undefined, + softTreeDepthTolerance: undefined, sourceIndex: '', sourceIndexNameEmpty: true, sourceIndexNameValid: false, @@ -233,17 +245,31 @@ export const getJobConfigFromFormState = ( analysis = Object.assign( analysis, - formState.predictionFieldName && { prediction_field_name: formState.predictionFieldName }, + formState.alpha && { alpha: formState.alpha }, formState.eta && { eta: formState.eta }, + formState.etaGrowthRatePerTree && { + eta_growth_rate_per_tree: formState.etaGrowthRatePerTree, + }, + formState.downsampleFactor && { downsample_factor: formState.downsampleFactor }, formState.featureBagFraction && { feature_bag_fraction: formState.featureBagFraction, }, formState.gamma && { gamma: formState.gamma }, formState.lambda && { lambda: formState.lambda }, + formState.maxOptimizationRoundsPerHyperparameter && { + max_optimization_rounds_per_hyperparameter: + formState.maxOptimizationRoundsPerHyperparameter, + }, formState.maxTrees && { max_trees: formState.maxTrees }, formState.randomizeSeed && { randomize_seed: formState.randomizeSeed }, formState.earlyStoppingEnabled !== undefined && { early_stopping_enabled: formState.earlyStoppingEnabled, + }, + formState.predictionFieldName && { prediction_field_name: formState.predictionFieldName }, + formState.randomizeSeed && { randomize_seed: formState.randomizeSeed }, + formState.softTreeDepthLimit && { soft_tree_depth_limit: formState.softTreeDepthLimit }, + formState.softTreeDepthTolerance && { + soft_tree_depth_tolerance: formState.softTreeDepthTolerance, } ); From ec8305baa7c81587028caed5db7042e5a02c2e28 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 10 Feb 2021 10:01:49 -0600 Subject: [PATCH 23/32] [docs] Include setup instructions for environment variable KBN_PATH_CONF (#90794) --- docs/setup/install/deb.asciidoc | 1 + docs/setup/install/rpm.asciidoc | 1 + docs/setup/install/targz.asciidoc | 1 + docs/setup/install/windows.asciidoc | 1 + docs/setup/settings.asciidoc | 7 ++++++- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/setup/install/deb.asciidoc b/docs/setup/install/deb.asciidoc index 6012ae394c832a..1ec73e8c3c7f5a 100644 --- a/docs/setup/install/deb.asciidoc +++ b/docs/setup/install/deb.asciidoc @@ -156,6 +156,7 @@ locations for a Debian-based system: | config | Configuration files including `kibana.yml` | /etc/kibana + | <> d| | data diff --git a/docs/setup/install/rpm.asciidoc b/docs/setup/install/rpm.asciidoc index 216ec849147b41..a87d2f89b6ddb3 100644 --- a/docs/setup/install/rpm.asciidoc +++ b/docs/setup/install/rpm.asciidoc @@ -149,6 +149,7 @@ locations for an RPM-based system: | config | Configuration files including `kibana.yml` | /etc/kibana + | <> d| | data diff --git a/docs/setup/install/targz.asciidoc b/docs/setup/install/targz.asciidoc index bb51d98a4f922b..f0a90723a8ed1a 100644 --- a/docs/setup/install/targz.asciidoc +++ b/docs/setup/install/targz.asciidoc @@ -134,6 +134,7 @@ important data later on. | config | Configuration files including `kibana.yml` | $KIBANA_HOME\config + | <> d| | data diff --git a/docs/setup/install/windows.asciidoc b/docs/setup/install/windows.asciidoc index b4204cc623f0f3..4138fc1886a6cf 100644 --- a/docs/setup/install/windows.asciidoc +++ b/docs/setup/install/windows.asciidoc @@ -81,6 +81,7 @@ important data later on. | config | Configuration files including `kibana.yml` | $KIBANA_HOME\config + | <> d| | data diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc index 9b9c26fd0e1db3..b57152646dda16 100644 --- a/docs/setup/settings.asciidoc +++ b/docs/setup/settings.asciidoc @@ -5,7 +5,12 @@ The {kib} server reads properties from the `kibana.yml` file on startup. The location of this file differs depending on how you installed {kib}. For example, if you installed {kib} from an archive distribution (`.tar.gz` or `.zip`), by default it is in `$KIBANA_HOME/config`. By default, with package distributions -(Debian or RPM), it is in `/etc/kibana`. +(Debian or RPM), it is in `/etc/kibana`. The config directory can be changed via the +`KBN_PATH_CONF` environment variable: + +``` +KBN_PATH_CONF=/home/kibana/config ./bin/kibana +``` The default host and port settings configure {kib} to run on `localhost:5601`. To change this behavior and allow remote users to connect, you'll need to update your `kibana.yml` file. You can also enable SSL and set a variety of other options. Finally, environment variables can be injected into From 0fc9467e5101b408573840a50f14f9ec817e1d6f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 10 Feb 2021 17:08:38 +0100 Subject: [PATCH 24/32] fix popover state (#90942) --- .../indexpattern_datasource/dimension_panel/time_scaling.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx index 0dd54118d03051..a9362060b2dd04 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/time_scaling.tsx @@ -90,7 +90,7 @@ export function TimeScaling({ iconSide="right" data-test-subj="indexPattern-time-scaling-popover" onClick={() => { - setPopoverOpen(true); + setPopoverOpen(!popoverOpen); }} > {i18n.translate('xpack.lens.indexPattern.timeScale.advancedSettings', { From 874960e1c54dba901abc64ffd1eca8faf784afc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Wed, 10 Feb 2021 17:13:05 +0100 Subject: [PATCH 25/32] [Logs UI] Handle undefined composite key in category dataset response (#90472) This fixes the handling of Elasticsearch responses that don't contain a composite key when querying for the available category datasets. --- .../log_analysis/log_entry_categories_datasets_stats.ts | 2 +- .../latest_log_entry_categories_datasets_stats.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts index 850fab6937f4b2..7c92a81e1a8962 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts @@ -79,7 +79,7 @@ export async function getLatestLogEntriesCategoriesDatasetsStats( return { categorization_status: latestHitSource.categorization_status, categorized_doc_count: latestHitSource.categorized_doc_count, - dataset: bucket.key.dataset ?? '', + dataset: bucket.key?.dataset ?? '', dead_category_count: latestHitSource.dead_category_count, failed_category_count: latestHitSource.failed_category_count, frequent_category_count: latestHitSource.frequent_category_count, diff --git a/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts b/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts index c7b2590e4be98c..2cbb0ef60cd168 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts @@ -98,9 +98,12 @@ export const logEntryCategorizerStatsHitRT = rt.type({ export type LogEntryCategorizerStatsHit = rt.TypeOf; -const compositeDatasetKeyRT = rt.type({ - dataset: rt.union([rt.string, rt.null]), -}); +const compositeDatasetKeyRT = rt.union([ + rt.type({ + dataset: rt.union([rt.string, rt.null]), + }), + rt.undefined, +]); export type CompositeDatasetKey = rt.TypeOf; From f563a82903d96a74be5707c563fe105336571054 Mon Sep 17 00:00:00 2001 From: Kevin Qualters <56408403+kqualters-elastic@users.noreply.github.com> Date: Wed, 10 Feb 2021 11:15:21 -0500 Subject: [PATCH 26/32] [Security Solution] Use sourcerer selected indices in resolver (#90727) * Use sourcer indices * Add indices to panel requests * Use a separate indices selector for resolver events * Use valid timeline id in tests * Update TimelineId type usage, make selector test clearer * Update tests to use TimelineId type --- .../common/types/timeline/index.ts | 1 + .../events_viewer/events_viewer.test.tsx | 15 ++-- .../events_viewer/events_viewer.tsx | 2 +- .../components/events_viewer/index.test.tsx | 3 +- .../common/components/events_viewer/index.tsx | 3 +- .../public/resolver/store/data/reducer.ts | 2 + .../resolver/store/data/selectors.test.ts | 81 +++++++++++++++++++ .../public/resolver/store/data/selectors.ts | 27 ++++++- .../current_related_event_fetcher.ts | 2 +- .../store/middleware/node_data_fetcher.ts | 2 +- .../middleware/related_events_fetcher.ts | 2 +- .../public/resolver/store/selectors.ts | 7 +- .../public/resolver/types.ts | 2 + .../components/flyout/index.test.tsx | 3 +- .../timelines/components/flyout/index.tsx | 2 +- .../components/flyout/pane/index.test.tsx | 3 +- .../components/flyout/pane/index.tsx | 3 +- .../components/graph_overlay/index.test.tsx | 10 ++- .../components/graph_overlay/index.tsx | 30 ++++--- .../timeline/graph_tab_content/index.tsx | 3 +- .../components/timeline/index.test.tsx | 6 +- .../timelines/components/timeline/index.tsx | 4 +- .../timeline/tabs_content/index.tsx | 4 +- 23 files changed, 172 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/common/types/timeline/index.ts b/x-pack/plugins/security_solution/common/types/timeline/index.ts index 1ea9b5752e0ccf..26a30e7c8f239e 100644 --- a/x-pack/plugins/security_solution/common/types/timeline/index.ts +++ b/x-pack/plugins/security_solution/common/types/timeline/index.ts @@ -280,6 +280,7 @@ export enum TimelineId { active = 'timeline-1', casePage = 'timeline-case', test = 'test', // Reserved for testing purposes + test2 = 'test2', } export const TimelineIdLiteralRt = runtimeTypes.union([ diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx index 4364ca2d3465de..6dad6c439ce46c 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx @@ -119,7 +119,7 @@ describe('EventsViewer', () => { let testProps = { defaultModel: eventsDefaultModel, end: to, - id: 'test-stateful-events-viewer', + id: TimelineId.test, start: from, scopeId: SourcererScopeName.timeline, }; @@ -155,7 +155,7 @@ describe('EventsViewer', () => { indexName: 'auditbeat-7.10.1-2020.12.18-000001', }, tabType: 'query', - timelineId: 'test-stateful-events-viewer', + timelineId: TimelineId.test, }, type: 'x-pack/security_solution/local/timeline/TOGGLE_EXPANDED_EVENT', }); @@ -199,17 +199,22 @@ describe('EventsViewer', () => { defaultHeaders.forEach((header) => { test(`it renders the ${header.id} default EventsViewer column header`, () => { + testProps = { + ...testProps, + // Update with a new id, to force columns back to default. + id: TimelineId.test2, + }; const wrapper = mount( ); - defaultHeaders.forEach((h) => + defaultHeaders.forEach((h) => { expect(wrapper.find(`[data-test-subj="header-text-${header.id}"]`).first().exists()).toBe( true - ) - ); + ); + }); }); }); }); diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx index 77573dbab0a53b..254309aee906bc 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx @@ -117,7 +117,7 @@ interface Props { filters: Filter[]; headerFilterGroup?: React.ReactNode; height?: number; - id: string; + id: TimelineId; indexNames: string[]; indexPattern: IIndexPattern; isLive: boolean; diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.test.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.test.tsx index c2fbfdb666e04e..5004c23f9111c4 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.test.tsx @@ -16,6 +16,7 @@ import { useMountAppended } from '../../utils/use_mount_appended'; import { mockEventViewerResponse } from './mock'; import { StatefulEventsViewer } from '.'; import { eventsDefaultModel } from './default_model'; +import { TimelineId } from '../../../../common/types/timeline'; import { SourcererScopeName } from '../../store/sourcerer/model'; import { useTimelineEvents } from '../../../timelines/containers'; @@ -36,7 +37,7 @@ const testProps = { defaultModel: eventsDefaultModel, end: to, indexNames: [], - id: 'test-stateful-events-viewer', + id: TimelineId.test, scopeId: SourcererScopeName.default, start: from, }; diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx index 526bc312172b05..2b5420674b89c3 100644 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/events_viewer/index.tsx @@ -12,6 +12,7 @@ import styled from 'styled-components'; import { inputsModel, inputsSelectors, State } from '../../store'; import { inputsActions } from '../../store/actions'; +import { TimelineId } from '../../../../common/types/timeline'; import { timelineSelectors, timelineActions } from '../../../timelines/store/timeline'; import { SubsetTimelineModel, TimelineModel } from '../../../timelines/store/timeline/model'; import { Filter } from '../../../../../../../src/plugins/data/public'; @@ -34,7 +35,7 @@ const FullScreenContainer = styled.div<{ $isFullScreen: boolean }>` export interface OwnProps { defaultModel: SubsetTimelineModel; end: string; - id: string; + id: TimelineId; scopeId: SourcererScopeName; start: string; headerFilterGroup?: React.ReactNode; diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts index 8df0c92ca83e50..b5864a0a83cf24 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/reducer.ts @@ -19,6 +19,7 @@ const initialState: DataState = { data: null, }, resolverComponentInstanceID: undefined, + indices: [], }; /* eslint-disable complexity */ export const dataReducer: Reducer = (state = initialState, action) => { @@ -35,6 +36,7 @@ export const dataReducer: Reducer = (state = initialS }, resolverComponentInstanceID: action.payload.resolverComponentInstanceID, locationSearch: action.payload.locationSearch, + indices: action.payload.indices, }; const panelViewAndParameters = selectors.panelViewAndParameters(nextState); return { diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts index c372c98c6e060b..b864bb254a5fc1 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.test.ts @@ -664,4 +664,85 @@ describe('data state', () => { `); }); }); + describe('when the resolver tree response is complete, still use non-default indices', () => { + beforeEach(() => { + const { resolverTree } = mockTreeWithNoAncestorsAnd2Children({ + originID: 'a', + firstChildID: 'b', + secondChildID: 'c', + }); + const { schema, dataSource } = endpointSourceSchema(); + actions = [ + { + type: 'serverReturnedResolverData', + payload: { + result: resolverTree, + dataSource, + schema, + parameters: { + databaseDocumentID: '', + indices: ['someNonDefaultIndex'], + filters: {}, + }, + }, + }, + ]; + }); + it('should have an empty array for tree parameter indices, and a non empty array for event indices', () => { + const treeParameterIndices = selectors.treeParameterIndices(state()); + expect(treeParameterIndices.length).toBe(0); + const eventIndices = selectors.eventIndices(state()); + expect(eventIndices.length).toBe(1); + }); + }); + describe('when the resolver tree response is pending use the same indices the user is currently looking at data from', () => { + beforeEach(() => { + const { resolverTree } = mockTreeWithNoAncestorsAnd2Children({ + originID: 'a', + firstChildID: 'b', + secondChildID: 'c', + }); + const { schema, dataSource } = endpointSourceSchema(); + actions = [ + { + type: 'serverReturnedResolverData', + payload: { + result: resolverTree, + dataSource, + schema, + parameters: { + databaseDocumentID: '', + indices: ['defaultIndex'], + filters: {}, + }, + }, + }, + { + type: 'appReceivedNewExternalProperties', + payload: { + databaseDocumentID: '', + resolverComponentInstanceID: '', + locationSearch: '', + indices: ['someNonDefaultIndex', 'someOtherIndex'], + shouldUpdate: false, + filters: {}, + }, + }, + { + type: 'appRequestedResolverData', + payload: { + databaseDocumentID: '', + indices: ['someNonDefaultIndex', 'someOtherIndex'], + filters: {}, + }, + }, + ]; + }); + it('should have an empty array for tree parameter indices, and the same set of indices as the last tree response', () => { + const treeParameterIndices = selectors.treeParameterIndices(state()); + expect(treeParameterIndices.length).toBe(0); + const eventIndices = selectors.eventIndices(state()); + expect(eventIndices.length).toBe(1); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts index a39aa4f0cd9832..fb6fb6073d7cfd 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/data/selectors.ts @@ -63,6 +63,13 @@ export function resolverComponentInstanceID(state: DataState): string { return state.resolverComponentInstanceID ? state.resolverComponentInstanceID : ''; } +/** + * The indices resolver should use, passed in as external props. + */ +const currentIndices = (state: DataState): string[] => { + return state.indices; +}; + /** * The last NewResolverTree we received, if any. It may be stale (it might not be for the same databaseDocumentID that * we're currently interested in. @@ -71,6 +78,12 @@ const resolverTreeResponse = (state: DataState): NewResolverTree | undefined => return state.tree?.lastResponse?.successful ? state.tree?.lastResponse.result : undefined; }; +const lastResponseIndices = (state: DataState): string[] | undefined => { + return state.tree?.lastResponse?.successful + ? state.tree?.lastResponse?.parameters?.indices + : undefined; +}; + /** * If we received a NewResolverTree, return the schema associated with that tree, otherwise return undefined. * As of writing, this is only used for the info popover in the graph_controls panel @@ -336,10 +349,22 @@ export const timeRangeFilters = createSelector( /** * The indices to use for the requests with the backend. */ -export const treeParamterIndices = createSelector(treeParametersToFetch, (parameters) => { +export const treeParameterIndices = createSelector(treeParametersToFetch, (parameters) => { return parameters?.indices ?? []; }); +/** + * Panel requests should not use indices derived from the tree parameter selector, as this is only defined briefly while the resolver_tree_fetcher middleware is running. + * Instead, panel requests should use the indices used by the last good request, falling back to the indices passed as external props. + */ +export const eventIndices = createSelector( + lastResponseIndices, + currentIndices, + function eventIndices(lastIndices, current): string[] { + return lastIndices ?? current ?? []; + } +); + export const layout: (state: DataState) => IsometricTaxiLayout = createSelector( tree, originID, diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/current_related_event_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/current_related_event_fetcher.ts index 3b8389182e9902..33772dddd676ed 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/current_related_event_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/current_related_event_fetcher.ts @@ -32,7 +32,7 @@ export function CurrentRelatedEventFetcher( const state = api.getState(); const newParams = selectors.panelViewAndParameters(state); - const indices = selectors.treeParameterIndices(state); + const indices = selectors.eventIndices(state); const oldParams = last; last = newParams; diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/node_data_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/node_data_fetcher.ts index 696e7f921673bb..074fdf7535790e 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/node_data_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/node_data_fetcher.ts @@ -38,7 +38,7 @@ export function NodeDataFetcher( * This gets the visible nodes that we haven't already requested or received data for */ const newIDsToRequest: Set = selectors.newIDsToRequest(state)(Number.POSITIVE_INFINITY); - const indices = selectors.treeParameterIndices(state); + const indices = selectors.eventIndices(state); if (newIDsToRequest.size <= 0) { return; diff --git a/x-pack/plugins/security_solution/public/resolver/store/middleware/related_events_fetcher.ts b/x-pack/plugins/security_solution/public/resolver/store/middleware/related_events_fetcher.ts index fbce03caf64d8a..19a11e07a9d870 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/middleware/related_events_fetcher.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/middleware/related_events_fetcher.ts @@ -27,7 +27,7 @@ export function RelatedEventsFetcher( const newParams = selectors.panelViewAndParameters(state); const isLoadingMoreEvents = selectors.isLoadingMoreNodeEventsInCategory(state); - const indices = selectors.treeParameterIndices(state); + const indices = selectors.eventIndices(state); const oldParams = last; const timeRangeFilters = selectors.timeRangeFilters(state); diff --git a/x-pack/plugins/security_solution/public/resolver/store/selectors.ts b/x-pack/plugins/security_solution/public/resolver/store/selectors.ts index a845de57bbdc6d..4c088a8be4ed9b 100644 --- a/x-pack/plugins/security_solution/public/resolver/store/selectors.ts +++ b/x-pack/plugins/security_solution/public/resolver/store/selectors.ts @@ -80,9 +80,14 @@ export const treeRequestParametersToAbort = composeSelectors( */ export const treeParameterIndices = composeSelectors( dataStateSelector, - dataSelectors.treeParamterIndices + dataSelectors.treeParameterIndices ); +/** + * An array of indices to use for resolver panel requests. + */ +export const eventIndices = composeSelectors(dataStateSelector, dataSelectors.eventIndices); + export const resolverComponentInstanceID = composeSelectors( dataStateSelector, dataSelectors.resolverComponentInstanceID diff --git a/x-pack/plugins/security_solution/public/resolver/types.ts b/x-pack/plugins/security_solution/public/resolver/types.ts index d3ddc51429ccd4..e6a004938a267d 100644 --- a/x-pack/plugins/security_solution/public/resolver/types.ts +++ b/x-pack/plugins/security_solution/public/resolver/types.ts @@ -376,6 +376,8 @@ export interface DataState { */ readonly resolverComponentInstanceID?: string; + readonly indices: string[]; + /** * The `search` part of the URL. */ diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/index.test.tsx index 3783f5591c43e7..f57ce42e7e0794 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/index.test.tsx @@ -18,6 +18,7 @@ import { kibanaObservable, createSecuritySolutionStorageMock, } from '../../../common/mock'; +import { TimelineId } from '../../../../common/types/timeline'; import { createStore, State } from '../../../common/store'; import * as timelineActions from '../../store/timeline/actions'; @@ -43,7 +44,7 @@ describe('Flyout', () => { const { storage } = createSecuritySolutionStorageMock(); const props = { onAppLeave: jest.fn(), - timelineId: 'test', + timelineId: TimelineId.test, }; beforeEach(() => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/index.tsx index f7518c2c34f669..bd7c7fbd1941fd 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/index.tsx @@ -26,7 +26,7 @@ const Visible = styled.div<{ show?: boolean }>` Visible.displayName = 'Visible'; interface OwnProps { - timelineId: string; + timelineId: TimelineId; onAppLeave: (handler: AppLeaveHandler) => void; } diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.test.tsx index e16cec78cf13b2..4ccc7ef5b5bc53 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.test.tsx @@ -9,13 +9,14 @@ import { shallow } from 'enzyme'; import React from 'react'; import { TestProviders } from '../../../../common/mock'; +import { TimelineId } from '../../../../../common/types/timeline'; import { Pane } from '.'; describe('Pane', () => { test('renders correctly against snapshot', () => { const EmptyComponent = shallow( - + ); expect(EmptyComponent.find('Pane')).toMatchSnapshot(); diff --git a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.tsx index a4d85bd76b105d..e63ffedf3da7c3 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/flyout/pane/index.tsx @@ -11,12 +11,13 @@ import styled from 'styled-components'; import { useDispatch } from 'react-redux'; import { StatefulTimeline } from '../../timeline'; +import { TimelineId } from '../../../../../common/types/timeline'; import * as i18n from './translations'; import { timelineActions } from '../../../store/timeline'; import { focusActiveTimelineButton } from '../../timeline/helpers'; interface FlyoutPaneComponentProps { - timelineId: string; + timelineId: TimelineId; } const EuiFlyoutContainer = styled.div` diff --git a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.test.tsx index c0e1a54faa8ddf..1286208bff9e6a 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.test.tsx @@ -15,7 +15,6 @@ import { } from '../../../common/containers/use_full_screen'; import { mockTimelineModel, TestProviders } from '../../../common/mock'; import { TimelineId } from '../../../../common/types/timeline'; - import { GraphOverlay } from '.'; jest.mock('../../../common/hooks/use_selector', () => ({ @@ -28,6 +27,10 @@ jest.mock('../../../common/containers/use_full_screen', () => ({ useTimelineFullScreen: jest.fn(), })); +jest.mock('../../../resolver/view/use_resolver_query_params_cleaner'); +jest.mock('../../../resolver/view/use_state_syncing_actions'); +jest.mock('../../../resolver/view/use_sync_selected_node'); + describe('GraphOverlay', () => { beforeEach(() => { (useGlobalFullScreen as jest.Mock).mockReturnValue({ @@ -42,12 +45,11 @@ describe('GraphOverlay', () => { describe('when used in an events viewer (i.e. in the Detections view, or the Host > Events view)', () => { const isEventViewer = true; - const timelineId = 'used-as-an-events-viewer'; test('it has 100% width when isEventViewer is true and NOT in full screen mode', async () => { const wrapper = mount( - + ); @@ -69,7 +71,7 @@ describe('GraphOverlay', () => { const wrapper = mount( - + ); diff --git a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx index 1b3a0c21ef6832..9c9c56461609d9 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/graph_overlay/index.tsx @@ -20,7 +20,7 @@ import styled from 'styled-components'; import { FULL_SCREEN } from '../timeline/body/column_headers/translations'; import { EXIT_FULL_SCREEN } from '../../../common/components/exit_full_screen/translations'; -import { DEFAULT_INDEX_KEY, FULL_SCREEN_TOGGLED_CLASS_NAME } from '../../../../common/constants'; +import { FULL_SCREEN_TOGGLED_CLASS_NAME } from '../../../../common/constants'; import { useGlobalFullScreen, useTimelineFullScreen, @@ -30,6 +30,8 @@ import { TimelineId } from '../../../../common/types/timeline'; import { timelineSelectors } from '../../store/timeline'; import { timelineDefaults } from '../../store/timeline/defaults'; import { isFullScreen } from '../timeline/body/column_headers'; +import { useSourcererScope } from '../../../common/containers/sourcerer'; +import { SourcererScopeName } from '../../../common/store/sourcerer/model'; import { updateTimelineGraphEventId } from '../../../timelines/store/timeline/actions'; import { Resolver } from '../../../resolver/view'; import { @@ -38,8 +40,6 @@ import { endSelector, } from '../../../common/components/super_date_picker/selectors'; import * as i18n from './translations'; -import { useUiSetting$ } from '../../../common/lib/kibana'; -import { useSignalIndex } from '../../../detections/containers/detection_engine/alerts/use_signal_index'; const OverlayContainer = styled.div` ${({ $restrictWidth }: { $restrictWidth: boolean }) => @@ -61,14 +61,14 @@ const FullScreenButtonIcon = styled(EuiButtonIcon)` interface OwnProps { isEventViewer: boolean; - timelineId: string; + timelineId: TimelineId; } interface NavigationProps { fullScreen: boolean; globalFullScreen: boolean; onCloseOverlay: () => void; - timelineId: string; + timelineId: TimelineId; timelineFullScreen: boolean; toggleFullScreen: () => void; } @@ -169,16 +169,14 @@ const GraphOverlayComponent: React.FC = ({ isEventViewer, timelineId } globalFullScreen, ]); - const { signalIndexName } = useSignalIndex(); - const [siemDefaultIndices] = useUiSetting$(DEFAULT_INDEX_KEY); - const indices: string[] | null = useMemo(() => { - if (signalIndexName === null) { - return null; - } else { - return [...siemDefaultIndices, signalIndexName]; - } - }, [signalIndexName, siemDefaultIndices]); + let sourcereScope = SourcererScopeName.default; + if ([TimelineId.detectionsRulesDetailsPage, TimelineId.detectionsPage].includes(timelineId)) { + sourcereScope = SourcererScopeName.detections; + } else if (timelineId === TimelineId.active) { + sourcereScope = SourcererScopeName.timeline; + } + const { selectedPatterns } = useSourcererScope(sourcereScope); return ( = ({ isEventViewer, timelineId } - {graphEventId !== undefined && indices !== null ? ( + {graphEventId !== undefined ? ( diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/graph_tab_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/graph_tab_content/index.tsx index db4867e1abfe70..1678a92c4cdaa8 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/graph_tab_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/graph_tab_content/index.tsx @@ -9,10 +9,11 @@ import React, { useMemo } from 'react'; import { timelineSelectors } from '../../../store/timeline'; import { useShallowEqualSelector } from '../../../../common/hooks/use_selector'; +import { TimelineId } from '../../../../../common/types/timeline'; import { GraphOverlay } from '../../graph_overlay'; interface GraphTabContentProps { - timelineId: string; + timelineId: TimelineId; } const GraphTabContentComponent: React.FC = ({ timelineId }) => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx index 219d32f147b60c..e7422e32805a9b 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.test.tsx @@ -12,7 +12,7 @@ import useResizeObserver from 'use-resize-observer/polyfilled'; import { DragDropContextWrapper } from '../../../common/components/drag_and_drop/drag_drop_context_wrapper'; import '../../../common/mock/match_media'; import { mockBrowserFields, mockDocValueFields } from '../../../common/containers/source/mock'; - +import { TimelineId } from '../../../../common/types/timeline'; import { mockIndexNames, mockIndexPattern, TestProviders } from '../../../common/mock'; import { StatefulTimeline, Props as StatefulTimelineOwnProps } from './index'; @@ -55,7 +55,7 @@ jest.mock('../../../common/containers/sourcerer', () => { }); describe('StatefulTimeline', () => { const props: StatefulTimelineOwnProps = { - timelineId: 'timeline-test', + timelineId: TimelineId.test, }; beforeEach(() => { @@ -91,7 +91,7 @@ describe('StatefulTimeline', () => { ); expect( wrapper - .find(`[data-timeline-id="timeline-test"].${SELECTOR_TIMELINE_GLOBAL_CONTAINER}`) + .find(`[data-timeline-id="test"].${SELECTOR_TIMELINE_GLOBAL_CONTAINER}`) .first() .exists() ).toEqual(true); diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx index 9cb95daba685bc..c37fc93e33b08e 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/index.tsx @@ -18,7 +18,7 @@ import { isTab } from '../../../common/components/accessibility/helpers'; import { useSourcererScope } from '../../../common/containers/sourcerer'; import { SourcererScopeName } from '../../../common/store/sourcerer/model'; import { FlyoutHeader, FlyoutHeaderPanel } from '../flyout/header'; -import { TimelineType, TimelineTabs } from '../../../../common/types/timeline'; +import { TimelineType, TimelineTabs, TimelineId } from '../../../../common/types/timeline'; import { useDeepEqualSelector, useShallowEqualSelector } from '../../../common/hooks/use_selector'; import { activeTimeline } from '../../containers/active_timeline_context'; import { EVENTS_COUNT_BUTTON_CLASS_NAME, onTimelineTabKeyPressed } from './helpers'; @@ -35,7 +35,7 @@ const TimelineTemplateBadge = styled.div` `; export interface Props { - timelineId: string; + timelineId: TimelineId; } const TimelineSavingProgressComponent: React.FC = ({ timelineId }) => { diff --git a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx index 9f6bfcf7e320cc..ca70e4ae646869 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/timeline/tabs_content/index.tsx @@ -9,7 +9,7 @@ import { EuiBadge, EuiLoadingContent, EuiTabs, EuiTab } from '@elastic/eui'; import React, { lazy, memo, Suspense, useCallback, useEffect, useMemo } from 'react'; import { useDispatch } from 'react-redux'; import styled from 'styled-components'; -import { TimelineTabs } from '../../../../../common/types/timeline'; +import { TimelineTabs, TimelineId } from '../../../../../common/types/timeline'; import { useShallowEqualSelector, @@ -42,7 +42,7 @@ const NotesTabContent = lazy(() => import('../notes_tab_content')); const PinnedTabContent = lazy(() => import('../pinned_tab_content')); interface BasicTimelineTab { - timelineId: string; + timelineId: TimelineId; graphEventId?: string; } From d1653bc42583ba1ced34b59c7408f719713c08ab Mon Sep 17 00:00:00 2001 From: Jessica David Date: Wed, 10 Feb 2021 11:26:58 -0500 Subject: [PATCH 27/32] Adding new fields to the allowlist for alert telemetry (#90868) --- .../security_solution/common/endpoint/generate_data.ts | 2 ++ .../security_solution/server/lib/telemetry/sender.test.ts | 4 ++++ .../plugins/security_solution/server/lib/telemetry/sender.ts | 2 ++ 3 files changed, 8 insertions(+) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index ba64814cd1daf1..ffeaf853828f13 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -558,6 +558,8 @@ export class EndpointDocGenerator { version: '3.0.33', }, temp_file_path: 'C:/temp/fake_malware.exe', + quarantine_result: true, + quarantine_message: 'fake quarantine message', }, }, process: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts index 8af0df30630268..56e2f9c7c73043 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.test.ts @@ -47,6 +47,8 @@ describe('TelemetryEventsSender', () => { malware_classification: { key1: 'X', }, + quarantine_result: true, + quarantine_message: 'this file is bad', something_else: 'nope', }, }, @@ -79,6 +81,8 @@ describe('TelemetryEventsSender', () => { malware_classification: { key1: 'X', }, + quarantine_result: true, + quarantine_message: 'this file is bad', }, }, host: { diff --git a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts index 4e32410cdb6aec..a18604fb92a40a 100644 --- a/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts +++ b/x-pack/plugins/security_solution/server/lib/telemetry/sender.ts @@ -320,6 +320,8 @@ const allowlistEventFields: AllowlistFields = { Ext: { code_signature: true, malware_classification: true, + quarantine_result: true, + quarantine_message: true, }, }, host: { From c058d9b02422a87bbd4c263e51b3e8509ff205ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Kopyci=C5=84ski?= Date: Wed, 10 Feb 2021 17:48:42 +0100 Subject: [PATCH 28/32] [Asset Management] Migrate Osquery plugin to TS project references (#90916) --- tsconfig.refs.json | 1 + x-pack/plugins/osquery/kibana.json | 3 +-- x-pack/plugins/osquery/tsconfig.json | 34 ++++++++++++++++++++++++++++ x-pack/test/tsconfig.json | 1 + x-pack/tsconfig.json | 2 ++ 5 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/osquery/tsconfig.json diff --git a/tsconfig.refs.json b/tsconfig.refs.json index d5482a85856fef..4105f23fd5b3ea 100644 --- a/tsconfig.refs.json +++ b/tsconfig.refs.json @@ -87,6 +87,7 @@ { "path": "./x-pack/plugins/maps/tsconfig.json" }, { "path": "./x-pack/plugins/ml/tsconfig.json" }, { "path": "./x-pack/plugins/observability/tsconfig.json" }, + { "path": "./x-pack/plugins/osquery/tsconfig.json" }, { "path": "./x-pack/plugins/painless_lab/tsconfig.json" }, { "path": "./x-pack/plugins/reporting/tsconfig.json" }, { "path": "./x-pack/plugins/saved_objects_tagging/tsconfig.json" }, diff --git a/x-pack/plugins/osquery/kibana.json b/x-pack/plugins/osquery/kibana.json index f6e90b9460506f..8adb30f4271d06 100644 --- a/x-pack/plugins/osquery/kibana.json +++ b/x-pack/plugins/osquery/kibana.json @@ -14,8 +14,7 @@ "requiredBundles": [ "esUiShared", "kibanaUtils", - "kibanaReact", - "kibanaUtils" + "kibanaReact" ], "requiredPlugins": [ "data", diff --git a/x-pack/plugins/osquery/tsconfig.json b/x-pack/plugins/osquery/tsconfig.json new file mode 100644 index 00000000000000..61678337625835 --- /dev/null +++ b/x-pack/plugins/osquery/tsconfig.json @@ -0,0 +1,34 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + // add all the folders contains files to be compiled + "common/**/*", + "public/**/*", + "server/**/*" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + // add references to other TypeScript projects the plugin depends on + + // requiredPlugins from ./kibana.json + { "path": "../../../src/plugins/data/tsconfig.json" }, + { "path": "../../../src/plugins/navigation/tsconfig.json" }, + { "path": "../data_enhanced/tsconfig.json" }, + { "path": "../fleet/tsconfig.json" }, + + // optionalPlugins from ./kibana.json + { "path": "../../../src/plugins/home/tsconfig.json" }, + + // requiredBundles from ./kibana.json + { "path": "../../../src/plugins/es_ui_shared/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_react/tsconfig.json" }, + { "path": "../../../src/plugins/kibana_utils/tsconfig.json" }, + ] +} diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 4cbec2da21807d..6209503e756106 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -68,6 +68,7 @@ { "path": "../plugins/licensing/tsconfig.json" }, { "path": "../plugins/ml/tsconfig.json" }, { "path": "../plugins/observability/tsconfig.json" }, + { "path": "../plugins/osquery/tsconfig.json" }, { "path": "../plugins/painless_lab/tsconfig.json" }, { "path": "../plugins/runtime_fields/tsconfig.json" }, { "path": "../plugins/saved_objects_tagging/tsconfig.json" }, diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 00286ac47da6ea..5589c62010db1e 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -30,6 +30,7 @@ "plugins/maps_legacy_licensing/**/*", "plugins/ml/**/*", "plugins/observability/**/*", + "plugins/osquery/**/*", "plugins/reporting/**/*", "plugins/searchprofiler/**/*", "plugins/security_solution/cypress/**/*", @@ -133,6 +134,7 @@ { "path": "./plugins/maps/tsconfig.json" }, { "path": "./plugins/ml/tsconfig.json" }, { "path": "./plugins/observability/tsconfig.json" }, + { "path": "./plugins/osquery/tsconfig.json" }, { "path": "./plugins/painless_lab/tsconfig.json" }, { "path": "./plugins/saved_objects_tagging/tsconfig.json" }, { "path": "./plugins/searchprofiler/tsconfig.json" }, From 4881306419e401b774143063f9c29caff7c3ca9d Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Wed, 10 Feb 2021 17:22:37 +0000 Subject: [PATCH 29/32] [ML] Stops new line on enter key press for KQL query bars (#90960) --- .../components/exploration_query_bar/exploration_query_bar.tsx | 2 +- .../index_based/components/search_panel/search_panel.tsx | 2 +- .../components/explorer_query_bar/explorer_query_bar.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx index a017de8d43a71f..b079fc154713ed 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx @@ -150,7 +150,7 @@ export const ExplorationQueryBar: FC = ({ = ({ closePopover={() => setErrorMessage(undefined)} input={ = ({ closePopover={() => setErrorMessage(undefined)} input={ Date: Wed, 10 Feb 2021 12:19:13 -0600 Subject: [PATCH 30/32] Remove custom plot plugins when Canvas is unmounted (#90722) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/canvas/public/application.tsx | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugins/canvas/public/application.tsx b/x-pack/plugins/canvas/public/application.tsx index 0e5300eeb1b07a..66b02bdc164087 100644 --- a/x-pack/plugins/canvas/public/application.tsx +++ b/x-pack/plugins/canvas/public/application.tsx @@ -13,6 +13,8 @@ import { i18n } from '@kbn/i18n'; import { Provider } from 'react-redux'; import { BehaviorSubject } from 'rxjs'; +import { includes, remove } from 'lodash'; + import { AppMountParameters, CoreStart, CoreSetup, AppUpdater } from 'kibana/public'; import { CanvasStartDeps, CanvasSetupDeps } from './plugin'; @@ -39,6 +41,11 @@ import { initFunctions } from './functions'; // @ts-expect-error untyped local import { appUnload } from './state/actions/app'; +// @ts-expect-error Not going to convert +import { size } from '../canvas_plugin_src/renderers/plot/plugins/size'; +// @ts-expect-error Not going to convert +import { text } from '../canvas_plugin_src/renderers/plot/plugins/text'; + import './style/index.scss'; const { ReadOnlyBadge: strings } = CapabilitiesStrings; @@ -147,6 +154,17 @@ export const initializeCanvas = async ( export const teardownCanvas = (coreStart: CoreStart, startPlugins: CanvasStartDeps) => { destroyRegistries(); + // Canvas pollutes the jQuery plot plugins collection with custom plugins that only work in Canvas. + // Remove them when Canvas is unmounted. + // see: ../canvas_plugin_src/renderers/plot/plugins/index.ts + if (includes($.plot.plugins, size)) { + remove($.plot.plugins, size); + } + + if (includes($.plot.plugins, text)) { + remove($.plot.plugins, text); + } + // TODO: Not cleaning these up temporarily. // We have an issue where if requests are inflight, and you navigate away, // those requests could still be trying to act on the store and possibly require services. From 4f38d163bf9dda16634bf17d400cfbd6507657b6 Mon Sep 17 00:00:00 2001 From: Spencer Date: Wed, 10 Feb 2021 10:45:32 -0800 Subject: [PATCH 31/32] ignore CI Stats failures in flaky test jobs (#90999) Co-authored-by: spalger --- .ci/Jenkinsfile_flaky | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 33204d73964616..b9880c410fc686 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -29,16 +29,20 @@ kibanaPipeline(timeoutMinutes: 180) { catchErrors { print "Agent ${agentNumberInside} - ${agentExecutions} executions" - workers.functional('flaky-test-runner', { - if (!IS_XPACK) { - kibanaPipeline.buildOss() - if (CI_GROUP == '1') { - runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1") + withEnv([ + 'IGNORE_SHIP_CI_STATS_ERROR=true', + ]) { + workers.functional('flaky-test-runner', { + if (!IS_XPACK) { + kibanaPipeline.buildOss() + if (CI_GROUP == '1') { + runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1") + } + } else { + kibanaPipeline.buildXpack() } - } else { - kibanaPipeline.buildXpack() - } - }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() + }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))() + } } } } From 4ee5f094ce7f6a6c480e631fe2ffe562a2e3d86e Mon Sep 17 00:00:00 2001 From: Poff Poffenberger Date: Wed, 10 Feb 2021 12:48:32 -0600 Subject: [PATCH 32/32] [Time to Visualize] Add functional tests for adding visualizations from Visualize, Lens, and Maps and adjust capabilities for new modal (#89245) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/components/dashboard_picker.tsx | 1 + .../saved_object_save_modal_dashboard.tsx | 12 +- ...d_object_save_modal_dashboard_selector.tsx | 84 +++--- .../public/services/index.ts | 1 - .../public/services/kibana/capabilities.ts | 1 - .../components/visualize_listing.tsx | 1 + .../apps/visualize/_add_to_dashboard.ts | 147 +++++++++++ test/functional/apps/visualize/index.ts | 1 + test/functional/page_objects/index.ts | 2 + .../page_objects/time_to_visualize_page.ts | 101 ++++++++ .../functional/page_objects/visualize_page.ts | 1 + .../functional/apps/dashboard/sync_colors.ts | 4 +- .../functional/apps/lens/add_to_dashboard.ts | 243 ++++++++++++++++++ x-pack/test/functional/apps/lens/index.ts | 1 + .../apps/maps/embeddable/add_to_dashboard.js | 122 +++++++++ .../functional/apps/maps/embeddable/index.js | 1 + .../test/functional/page_objects/lens_page.ts | 16 +- 17 files changed, 676 insertions(+), 63 deletions(-) create mode 100644 test/functional/apps/visualize/_add_to_dashboard.ts create mode 100644 test/functional/page_objects/time_to_visualize_page.ts create mode 100644 x-pack/test/functional/apps/lens/add_to_dashboard.ts create mode 100644 x-pack/test/functional/apps/maps/embeddable/add_to_dashboard.js diff --git a/src/plugins/presentation_util/public/components/dashboard_picker.tsx b/src/plugins/presentation_util/public/components/dashboard_picker.tsx index 6667280d0a23be..83ccabe46cdc44 100644 --- a/src/plugins/presentation_util/public/components/dashboard_picker.tsx +++ b/src/plugins/presentation_util/public/components/dashboard_picker.tsx @@ -64,6 +64,7 @@ export function DashboardPicker(props: DashboardPickerProps) { return ( ( documentId || disableDashboardOptions ? null : 'existing' diff --git a/src/plugins/presentation_util/public/components/saved_object_save_modal_dashboard_selector.tsx b/src/plugins/presentation_util/public/components/saved_object_save_modal_dashboard_selector.tsx index 9f6fd5eabf5c5c..c2b5eac4dbb836 100644 --- a/src/plugins/presentation_util/public/components/saved_object_save_modal_dashboard_selector.tsx +++ b/src/plugins/presentation_util/public/components/saved_object_save_modal_dashboard_selector.tsx @@ -21,7 +21,6 @@ import { EuiSpacer, } from '@elastic/eui'; -import { pluginServices } from '../services'; import { DashboardPicker, DashboardPickerProps } from './dashboard_picker'; import './saved_object_save_modal_dashboard.scss'; @@ -37,9 +36,6 @@ export interface SaveModalDashboardSelectorProps { export function SaveModalDashboardSelector(props: SaveModalDashboardSelectorProps) { const { documentId, onSelectDashboard, dashboardOption, onChange, copyOnSave } = props; - const { capabilities } = pluginServices.getHooks(); - const { canCreateNewDashboards, canEditDashboards } = capabilities.useService(); - const isDisabled = !copyOnSave && !!documentId; return ( @@ -70,50 +66,44 @@ export function SaveModalDashboardSelector(props: SaveModalDashboardSelectorProp >
- {canEditDashboards() && ( - <> - {' '} - onChange('existing')} - disabled={isDisabled} - /> -
- -
- - - )} - {canCreateNewDashboards() && ( - <> - {' '} - onChange('new')} - disabled={isDisabled} + <> + onChange('existing')} + disabled={isDisabled} + /> +
+ - - - )} +
+ + + <> + onChange('new')} + disabled={isDisabled} + /> + + boolean; canCreateNewDashboards: () => boolean; - canEditDashboards: () => boolean; } export interface PresentationUtilServices { diff --git a/src/plugins/presentation_util/public/services/kibana/capabilities.ts b/src/plugins/presentation_util/public/services/kibana/capabilities.ts index a191e970591f46..546281d083f2fb 100644 --- a/src/plugins/presentation_util/public/services/kibana/capabilities.ts +++ b/src/plugins/presentation_util/public/services/kibana/capabilities.ts @@ -21,6 +21,5 @@ export const capabilitiesServiceFactory: CapabilitiesServiceFactory = ({ coreSta return { canAccessDashboards: () => Boolean(dashboard.show), canCreateNewDashboards: () => Boolean(dashboard.createNew), - canEditDashboards: () => !Boolean(dashboard.hideWriteControls), }; }; diff --git a/src/plugins/visualize/public/application/components/visualize_listing.tsx b/src/plugins/visualize/public/application/components/visualize_listing.tsx index bc766d63db5a78..1f1f8c0b5ac80e 100644 --- a/src/plugins/visualize/public/application/components/visualize_listing.tsx +++ b/src/plugins/visualize/public/application/components/visualize_listing.tsx @@ -149,6 +149,7 @@ export const VisualizeListing = () => { const calloutMessage = ( <> { + it('should allow new lens vizs be added to a new dashboard', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + + await PageObjects.lens.switchToVisualization('lnsMetric'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + await PageObjects.lens.save('New Lens from Modal', false, false, 'new'); + + await PageObjects.dashboard.waitForRenderComplete(); + + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + + await PageObjects.timeToVisualize.resetNewDashboard(); + }); + + it('should allow existing lens vizs be added to a new dashboard', async () => { + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('Artistpreviouslyknownaslens'); + await PageObjects.lens.clickVisualizeListItemTitle('Artistpreviouslyknownaslens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.assertMetric('Maximum of bytes', '19,986'); + + await PageObjects.lens.save('Artistpreviouslyknownaslens Copy', true, false, 'new'); + + await PageObjects.dashboard.waitForRenderComplete(); + + await PageObjects.lens.assertMetric('Maximum of bytes', '19,986'); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + + await PageObjects.timeToVisualize.resetNewDashboard(); + }); + + it('should allow new lens vizs be added to an existing dashboard', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + await dashboardAddPanel.clickOpenAddPanel(); + await dashboardAddPanel.filterEmbeddableNames('lnsXYvis'); + await find.clickByButtonText('lnsXYvis'); + await dashboardAddPanel.closeAddPanel(); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.dashboard.saveDashboard('My Very Cool Dashboard'); + await PageObjects.dashboard.gotoDashboardLandingPage(); + await listingTable.searchAndExpectItemsCount('dashboard', 'My Very Cool Dashboard', 1); + + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + + await PageObjects.lens.switchToVisualization('lnsMetric'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + await PageObjects.lens.save( + 'New Lens from Modal', + false, + false, + 'existing', + 'My Very Cool Dashboard' + ); + + await PageObjects.dashboard.waitForRenderComplete(); + + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(2); + }); + + it('should allow existing lens vizs be added to an existing dashboard', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + await dashboardAddPanel.clickOpenAddPanel(); + await dashboardAddPanel.filterEmbeddableNames('lnsXYvis'); + await find.clickByButtonText('lnsXYvis'); + await dashboardAddPanel.closeAddPanel(); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.dashboard.saveDashboard('My Wonderful Dashboard'); + await PageObjects.dashboard.gotoDashboardLandingPage(); + await listingTable.searchAndExpectItemsCount('dashboard', 'My Wonderful Dashboard', 1); + + await PageObjects.visualize.gotoVisualizationLandingPage(); + await listingTable.searchForItemWithName('Artistpreviouslyknownaslens'); + await PageObjects.lens.clickVisualizeListItemTitle('Artistpreviouslyknownaslens'); + await PageObjects.lens.goToTimeRange(); + await PageObjects.lens.assertMetric('Maximum of bytes', '19,986'); + + await PageObjects.lens.save( + 'Artistpreviouslyknownaslens Copy', + true, + false, + 'existing', + 'My Wonderful Dashboard' + ); + + await PageObjects.dashboard.waitForRenderComplete(); + + await PageObjects.lens.assertMetric('Maximum of bytes', '19,986'); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(2); + }); + + describe('Capabilities', function capabilitiesTests() { + describe('dashboard no-access privileges', () => { + before(async () => { + await PageObjects.common.navigateToApp('visualize'); + await security.testUser.setRoles(['test_logstash_reader', 'global_visualize_all'], true); + }); + + after(async () => { + await security.testUser.restoreDefaults(); + }); + + it('should not display dashboard flow prompt', async () => { + await PageObjects.common.navigateToApp('visualize'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.visualize.gotoLandingPage(); + + const hasPrompt = await testSubjects.exists('visualize-dashboard-flow-prompt'); + expect(hasPrompt).to.eql(false); + }); + + it('should not display add-to-dashboard options', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + + await PageObjects.lens.switchToVisualization('lnsMetric'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.click('lnsApp_saveButton'); + + const hasOptions = await testSubjects.exists('add-to-dashboard-options'); + expect(hasOptions).to.eql(false); + }); + }); + + describe('dashboard read-only privileges', () => { + before(async () => { + await security.testUser.setRoles( + ['test_logstash_reader', 'global_visualize_all', 'global_dashboard_read'], + true + ); + }); + + after(async () => { + await security.testUser.restoreDefaults(); + }); + + it('should not display dashboard flow prompt', async () => { + await PageObjects.common.navigateToApp('visualize'); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.visualize.gotoLandingPage(); + + const hasPrompt = await testSubjects.exists('visualize-dashboard-flow-prompt'); + expect(hasPrompt).to.eql(false); + }); + + it('should not display add-to-dashboard options', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.lens.goToTimeRange(); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + + await PageObjects.lens.switchToVisualization('lnsMetric'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.assertMetric('Average of bytes', '5,727.322'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.click('lnsApp_saveButton'); + + const hasOptions = await testSubjects.exists('add-to-dashboard-options'); + expect(hasOptions).to.eql(false); + }); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index 10b1f4d30145f8..31b7b665fb2f0f 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -29,6 +29,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { this.tags(['ciGroup4', 'skipFirefox']); loadTestFile(require.resolve('./smokescreen')); + loadTestFile(require.resolve('./add_to_dashboard')); loadTestFile(require.resolve('./table')); loadTestFile(require.resolve('./dashboard')); loadTestFile(require.resolve('./persistent_context')); diff --git a/x-pack/test/functional/apps/maps/embeddable/add_to_dashboard.js b/x-pack/test/functional/apps/maps/embeddable/add_to_dashboard.js new file mode 100644 index 00000000000000..9bbf6b1afab66d --- /dev/null +++ b/x-pack/test/functional/apps/maps/embeddable/add_to_dashboard.js @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +export default function ({ getPageObjects, getService }) { + const PageObjects = getPageObjects([ + 'common', + 'dashboard', + 'header', + 'maps', + 'timeToVisualize', + 'visualize', + ]); + + const listingTable = getService('listingTable'); + const testSubjects = getService('testSubjects'); + const security = getService('security'); + + describe('maps add-to-dashboard save flow', () => { + before(async () => { + await security.testUser.setRoles( + [ + 'test_logstash_reader', + 'global_maps_all', + 'geoshape_data_reader', + 'global_dashboard_all', + 'meta_for_geoshape_data_reader', + ], + false + ); + }); + + after(async () => { + await security.testUser.restoreDefaults(); + }); + + it('should allow new map be added to a new dashboard', async () => { + await PageObjects.maps.openNewMap(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.maps.waitForLayersToLoad(); + + await testSubjects.click('mapSaveButton'); + await PageObjects.timeToVisualize.saveFromModal('map 1', { addToDashboard: 'new' }); + + await PageObjects.dashboard.waitForRenderComplete(); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + + await PageObjects.timeToVisualize.resetNewDashboard(); + }); + + it('should allow existing maps be added to a new dashboard', async () => { + await PageObjects.maps.loadSavedMap('document example'); + + await testSubjects.click('mapSaveButton'); + await PageObjects.timeToVisualize.saveFromModal('document example copy', { + addToDashboard: 'new', + saveAsNew: true, + }); + + await PageObjects.dashboard.waitForRenderComplete(); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + + await PageObjects.timeToVisualize.resetNewDashboard(); + }); + + it('should allow new map be added to an existing dashboard', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + + await PageObjects.dashboard.saveDashboard('My Very Cool Dashboard'); + await PageObjects.dashboard.gotoDashboardLandingPage(); + await listingTable.searchAndExpectItemsCount('dashboard', 'My Very Cool Dashboard', 1); + + await PageObjects.maps.openNewMap(); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.maps.waitForLayersToLoad(); + + await testSubjects.click('mapSaveButton'); + await PageObjects.timeToVisualize.saveFromModal('My New Map 2', { + addToDashboard: 'existing', + dashboardId: 'My Very Cool Dashboard', + }); + + await PageObjects.dashboard.waitForRenderComplete(); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + }); + + it('should allow existing maps be added to an existing dashboard', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + + await PageObjects.dashboard.saveDashboard('My Wonderful Dashboard'); + await PageObjects.dashboard.gotoDashboardLandingPage(); + await listingTable.searchAndExpectItemsCount('dashboard', 'My Wonderful Dashboard', 1); + + await PageObjects.maps.loadSavedMap('document example'); + + await testSubjects.click('mapSaveButton'); + await PageObjects.timeToVisualize.saveFromModal('document example copy 2', { + addToDashboard: 'existing', + dashboardId: 'My Wonderful Dashboard', + saveAsNew: true, + }); + + await PageObjects.dashboard.waitForRenderComplete(); + + const panelCount = await PageObjects.dashboard.getPanelCount(); + expect(panelCount).to.eql(1); + }); + }); +} diff --git a/x-pack/test/functional/apps/maps/embeddable/index.js b/x-pack/test/functional/apps/maps/embeddable/index.js index 9fd4c9db703db9..552f830e2a379a 100644 --- a/x-pack/test/functional/apps/maps/embeddable/index.js +++ b/x-pack/test/functional/apps/maps/embeddable/index.js @@ -7,6 +7,7 @@ export default function ({ loadTestFile }) { describe('embeddable', function () { + loadTestFile(require.resolve('./add_to_dashboard')); loadTestFile(require.resolve('./save_and_return')); loadTestFile(require.resolve('./dashboard')); loadTestFile(require.resolve('./embeddable_library')); diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index add6979c2dde1f..dcb730f77725dc 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -17,7 +17,15 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont const find = getService('find'); const comboBox = getService('comboBox'); const browser = getService('browser'); - const PageObjects = getPageObjects(['header', 'timePicker', 'common', 'visualize', 'dashboard']); + + const PageObjects = getPageObjects([ + 'header', + 'timePicker', + 'common', + 'visualize', + 'dashboard', + 'timeToVisualize', + ]); return logWrapper('lensPage', log, { /** @@ -341,16 +349,16 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont title: string, saveAsNew?: boolean, redirectToOrigin?: boolean, - addToDashboard?: boolean, + addToDashboard?: 'new' | 'existing' | null, dashboardId?: string ) { await PageObjects.header.waitUntilLoadingHasFinished(); await testSubjects.click('lnsApp_saveButton'); - await PageObjects.visualize.setSaveModalValues(title, { + await PageObjects.timeToVisualize.setSaveModalValues(title, { saveAsNew, redirectToOrigin, - addToDashboard, + addToDashboard: addToDashboard ? addToDashboard : null, dashboardId, });