From 052dfe9f9a7132a9a13b709e862435065a44420e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 17 Jun 2020 12:23:31 -0700 Subject: [PATCH 01/60] [Ingest Manager] Replace `datasources` with `inputs` when generating agent config (#69226) * Adjust agent config generation, change `datasources` to `inputs` * Add dataset.type * Remove dead code * Revert "Add dataset.type" This reverts commit fbcf50cbe2b6a93536386c82d754a6a7c7f2b8e9. * Update endpoint policy test assertion --- .../common/services/config_to_yaml.ts | 5 +- .../datasource_to_agent_datasource.test.ts | 190 ------------------ .../datasource_to_agent_datasource.ts | 60 ------ .../datasources_to_agent_inputs.test.ts | 158 +++++++++++++++ .../services/datasources_to_agent_inputs.ts | 64 ++++++ .../ingest_manager/common/services/index.ts | 2 +- .../common/types/models/agent_config.ts | 38 ++-- .../ingest_manager/services/index.ts | 2 +- .../server/services/agent_config.test.ts | 6 +- .../server/services/agent_config.ts | 6 +- .../ingest_manager/server/types/index.tsx | 2 +- .../apps/endpoint/policy_details.ts | 144 +++++-------- 12 files changed, 303 insertions(+), 374 deletions(-) delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts diff --git a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts index 9dfd76b9ddd211..a3bef72e8db5a3 100644 --- a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts +++ b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts @@ -11,10 +11,11 @@ const CONFIG_KEYS_ORDER = [ 'name', 'revision', 'type', - 'outputs', 'settings', - 'datasources', + 'outputs', + 'inputs', 'enabled', + 'use_output', 'package', 'input', ]; diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts deleted file mode 100644 index d319ba2beddf94..00000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, DatasourceInput } from '../types'; -import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; - -describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { - const mockDatasource: Datasource = { - id: 'some-uuid', - name: 'mock-datasource', - description: '', - created_at: '', - created_by: '', - updated_at: '', - updated_by: '', - config_id: '', - enabled: true, - output_id: '', - namespace: 'default', - inputs: [], - revision: 1, - }; - - const mockInput: DatasourceInput = { - type: 'test-logs', - enabled: true, - vars: { - inputVar: { value: 'input-value' }, - inputVar2: { value: undefined }, - inputVar3: { - type: 'yaml', - value: 'testField: test', - }, - inputVar4: { value: '' }, - }, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - vars: { - fooVar: { value: 'foo-value' }, - fooVar2: { value: [1, 2] }, - }, - agent_stream: { - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - vars: { - barVar: { value: 'bar-value' }, - barVar2: { value: [1, 2] }, - barVar3: { - type: 'yaml', - value: - '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', - }, - barVar4: { - type: 'yaml', - value: '', - }, - barVar5: { - type: 'yaml', - value: 'testField: test\n invalidSpacing: foo', - }, - }, - }, - ], - }; - - it('returns agent datasource config for datasource with no inputs', () => { - expect(storedDatasourceToAgentDatasource(mockDatasource)).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - package: { - name: 'mock-package', - title: 'Mock package', - version: '0.0.0', - }, - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - package: { - name: 'mock-package', - version: '0.0.0', - }, - inputs: [], - }); - }); - - it('returns agent datasource config with flattened input and package stream', () => { - expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled streams', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [ - { - ...mockInput, - streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], - }, - ], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled inputs', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [{ ...mockInput, enabled: false }], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - }); -}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts deleted file mode 100644 index 2a8b687675bf9e..00000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, FullAgentConfigDatasource } from '../types'; -import { DEFAULT_OUTPUT } from '../constants'; - -export const storedDatasourceToAgentDatasource = ( - datasource: Datasource -): FullAgentConfigDatasource => { - const { id, name, namespace, enabled, package: pkg, inputs } = datasource; - - const fullDatasource: FullAgentConfigDatasource = { - id: id || name, - name, - namespace, - enabled, - use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now - inputs: inputs - .filter((input) => input.enabled) - .map((input) => { - const fullInput = { - ...input, - ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - streams: input.streams - .filter((stream) => stream.enabled) - .map((stream) => { - const fullStream = { - ...stream, - ...stream.agent_stream, - ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - }; - delete fullStream.agent_stream; - delete fullStream.vars; - delete fullStream.config; - return fullStream; - }), - }; - delete fullInput.vars; - delete fullInput.config; - return fullInput; - }), - }; - - if (pkg) { - fullDatasource.package = { - name: pkg.name, - version: pkg.version, - }; - } - - return fullDatasource; -}; diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts new file mode 100644 index 00000000000000..df94168ec88d02 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, DatasourceInput } from '../types'; +import { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; + +describe('Ingest Manager - storedDatasourcesToAgentInputs', () => { + const mockDatasource: Datasource = { + id: 'some-uuid', + name: 'mock-datasource', + description: '', + created_at: '', + created_by: '', + updated_at: '', + updated_by: '', + config_id: '', + enabled: true, + output_id: '', + namespace: 'default', + inputs: [], + revision: 1, + }; + + const mockInput: DatasourceInput = { + type: 'test-logs', + enabled: true, + vars: { + inputVar: { value: 'input-value' }, + inputVar2: { value: undefined }, + inputVar3: { + type: 'yaml', + value: 'testField: test', + }, + inputVar4: { value: '' }, + }, + streams: [ + { + id: 'test-logs-foo', + enabled: true, + dataset: 'foo', + vars: { + fooVar: { value: 'foo-value' }, + fooVar2: { value: [1, 2] }, + }, + agent_stream: { + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + }, + { + id: 'test-logs-bar', + enabled: true, + dataset: 'bar', + vars: { + barVar: { value: 'bar-value' }, + barVar2: { value: [1, 2] }, + barVar3: { + type: 'yaml', + value: + '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', + }, + barVar4: { + type: 'yaml', + value: '', + }, + barVar5: { + type: 'yaml', + value: 'testField: test\n invalidSpacing: foo', + }, + }, + }, + ], + }; + + it('returns no inputs for datasource with no inputs, or only disabled inputs', () => { + expect(storedDatasourcesToAgentInputs([mockDatasource])).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + package: { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + }, + }, + ]) + ).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [{ ...mockInput, enabled: false }], + }, + ]) + ).toEqual([]); + }); + + it('returns agent inputs', () => { + expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + { + id: 'test-logs-bar', + dataset: { name: 'bar' }, + }, + ], + }, + ]); + }); + + it('returns agent inputs without disabled streams', () => { + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [ + { + ...mockInput, + streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], + }, + ], + }, + ]) + ).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + ], + }, + ]); + }); +}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts new file mode 100644 index 00000000000000..d5a752e817b4f2 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, FullAgentConfigInput, FullAgentConfigInputStream } from '../types'; +import { DEFAULT_OUTPUT } from '../constants'; + +export const storedDatasourcesToAgentInputs = ( + datasources: Datasource[] +): FullAgentConfigInput[] => { + const fullInputs: FullAgentConfigInput[] = []; + + datasources.forEach((datasource) => { + if (!datasource.enabled || !datasource.inputs || !datasource.inputs.length) { + return; + } + datasource.inputs.forEach((input) => { + if (!input.enabled) { + return; + } + + const fullInput: FullAgentConfigInput = { + id: datasource.id || datasource.name, + name: datasource.name, + type: input.type, + dataset: { namespace: datasource.namespace || 'default' }, + use_output: DEFAULT_OUTPUT.name, + ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + streams: input.streams + .filter((stream) => stream.enabled) + .map((stream) => { + const fullStream: FullAgentConfigInputStream = { + id: stream.id, + dataset: { name: stream.dataset }, + ...stream.agent_stream, + ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + }; + if (stream.processors) { + fullStream.processors = stream.processors; + } + return fullStream; + }), + }; + + if (datasource.package) { + fullInput.package = { + name: datasource.package.name, + version: datasource.package.version, + }; + } + + fullInputs.push(fullInput); + }); + }); + + return fullInputs; +}; diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index c595c9a52f66f8..e53d97972fa2f3 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -7,7 +7,7 @@ import * as AgentStatusKueryHelper from './agent_status'; export * from './routes'; export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config'; -export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; +export { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; export { configToYaml } from './config_to_yaml'; export { AgentStatusKueryHelper }; export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 7547f56237eec1..36b3176ffa4151 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -3,12 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { - Datasource, - DatasourcePackage, - DatasourceInput, - DatasourceInputStream, -} from './datasource'; +import { Datasource, DatasourcePackage, DatasourceInputStream } from './datasource'; import { Output } from './output'; export enum AgentConfigStatus { @@ -35,23 +30,22 @@ export interface AgentConfig extends NewAgentConfig { export type AgentConfigSOAttributes = Omit; -export type FullAgentConfigDatasource = Pick< - Datasource, - 'id' | 'name' | 'namespace' | 'enabled' -> & { - package?: Pick; - use_output: string; - inputs: Array< - Omit & { - streams: Array< - Omit & { - [key: string]: any; - } - >; - } - >; +export type FullAgentConfigInputStream = Pick & { + dataset: { name: string }; + [key: string]: any; }; +export interface FullAgentConfigInput { + id: string; + name: string; + type: string; + dataset: { namespace: string }; + use_output: string; + package?: Pick; + streams: FullAgentConfigInputStream[]; + [key: string]: any; +} + export interface FullAgentConfig { id: string; outputs: { @@ -59,7 +53,7 @@ export interface FullAgentConfig { [key: string]: any; }; }; - datasources: FullAgentConfigDatasource[]; + inputs: FullAgentConfigInput[]; revision?: number; settings?: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 6dbc8d67caaee6..ece7aef2c247fe 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -19,7 +19,7 @@ export { settingsRoutesService, appRoutesService, packageToConfigDatasourceInputs, - storedDatasourceToAgentDatasource, + storedDatasourcesToAgentInputs, configToYaml, AgentStatusKueryHelper, } from '../../../../common'; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts index 17758f6e3d7f12..c46e648ad088a3 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -59,7 +59,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -88,7 +88,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -118,7 +118,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index 18d5d8dedfb1fa..9e0386de747630 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -20,7 +20,7 @@ import { AgentConfigStatus, ListWithKuery, } from '../types'; -import { DeleteAgentConfigResponse, storedDatasourceToAgentDatasource } from '../../common'; +import { DeleteAgentConfigResponse, storedDatasourcesToAgentInputs } from '../../common'; import { listAgents } from './agents'; import { datasourceService } from './datasource'; import { outputService } from './output'; @@ -375,9 +375,7 @@ class AgentConfigService { {} as FullAgentConfig['outputs'] ), }, - datasources: (config.datasources as Datasource[]) - .filter((datasource) => datasource.enabled) - .map((ds) => storedDatasourceToAgentDatasource(ds)), + inputs: storedDatasourcesToAgentInputs(config.datasources as Datasource[]), revision: config.revision, ...(config.monitoring_enabled && config.monitoring_enabled.length > 0 ? { diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 2218d967fa8aa1..2b543490ca8dab 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -20,7 +20,7 @@ export { Datasource, NewDatasource, DatasourceSOAttributes, - FullAgentConfigDatasource, + FullAgentConfigInput, FullAgentConfig, AgentConfig, AgentConfigSOAttributes, diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 25fb477b5a99aa..036f82a591fb3f 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -99,107 +99,71 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { ); expect(agentFullConfig).to.eql({ - datasources: [ + inputs: [ { - enabled: true, id: policyInfo.datasource.id, - inputs: [ - { - enabled: true, - policy: { - linux: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - file: false, - network: true, - process: true, - }, - logging: { - file: 'info', - stdout: 'debug', + dataset: { namespace: 'default' }, + name: 'Protect East Coast', + package: { + name: 'endpoint', + version: policyInfo.packageInfo.version, + }, + policy: { + linux: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', }, + kernel: { connect: true, process: true }, }, - mac: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - file: false, - network: true, - process: true, - }, - logging: { - file: 'info', - stdout: 'debug', - }, - malware: { - mode: 'detect', + }, + events: { file: false, network: true, process: true }, + logging: { file: 'info', stdout: 'debug' }, + }, + mac: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', }, + kernel: { connect: true, process: true }, }, - windows: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - dll_and_driver_load: true, - dns: true, - file: false, - network: true, - process: true, - registry: true, - security: true, - }, - logging: { - file: 'info', - stdout: 'debug', - }, - malware: { - mode: 'prevent', + }, + events: { file: false, network: true, process: true }, + logging: { file: 'info', stdout: 'debug' }, + malware: { mode: 'detect' }, + }, + windows: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', }, + kernel: { connect: true, process: true }, }, }, - streams: [], - type: 'endpoint', + events: { + dll_and_driver_load: true, + dns: true, + file: false, + network: true, + process: true, + registry: true, + security: true, + }, + logging: { file: 'info', stdout: 'debug' }, + malware: { mode: 'prevent' }, }, - ], - name: 'Protect East Coast', - namespace: 'default', - package: { - name: 'endpoint', - version: policyInfo.packageInfo.version, }, + streams: [], + type: 'endpoint', use_output: 'default', }, ], From 1dd5db2cf09c91a9888b8364bc35285e21085986 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Wed, 17 Jun 2020 21:25:37 +0200 Subject: [PATCH 02/60] [ML] Add Anomaly Swimlane Embeddable to the dashboard from the Anomaly Explorer page (#68784) * [ML] WIP attach swimlane embeddable to dashboard from the explorer page * [ML] fix deps * [ML] getDefaultPanelTitle * [ML] fix TS issue * [ML] DashboardService * [ML] unit tests * [ML] redirect to the dashboard * [ML] swimlane_panel * [ML] Anomaly Timeline panel * [ML] swimlane container * [ML] fix ts * [ML] Add multiple swimlanes * [ML] fix SwimlaneType usage * [ML] disable edit button on update * [ML] fix i18n translation key * [ML] use ViewMode enum * [ML] use navigateToUrl * [ML] TODO for edit dashboard * [ML] check kibana dashboard capabilities * [ML] mlApiServicesProvider * [ML] mlResultsServiceProvider * [ML] fix alignment * [ML] labels and tooltips * [ML] fix ts issue for proxyHttpStart * [ML] canEditDashboards check * [ML] fix TS * [ML] update add_to_dashboard_control.tsx * [ML] add form label, disable control on empty swimlanes selection * [ML] resolve PR review comments * [ML] e2e test * [ML] increase panel padding * [ML] position in row * [ML] update e2e * [ML] add data-test-subj for search box * [ML] PR remarks --- src/plugins/dashboard/public/index.ts | 4 +- src/plugins/dashboard/public/url_generator.ts | 7 + x-pack/package.json | 1 + x-pack/plugins/ml/public/application/app.tsx | 2 + .../loading_indicator/{index.js => index.ts} | 0 ...ing_indicator.js => loading_indicator.tsx} | 11 +- .../contexts/kibana/kibana_context.ts | 4 +- .../application/contexts/ml/ml_context.ts | 1 + .../explorer/add_to_dashboard_control.tsx | 321 +++ .../application/explorer/anomaly_timeline.tsx | 392 +++ ...d.js => explorer_no_influencers_found.tsx} | 23 +- .../public/application/explorer/explorer.js | 297 +-- .../explorer/explorer_constants.ts | 10 +- .../explorer/explorer_swimlane.tsx | 4 +- .../reducers/explorer_reducer/state.ts | 7 +- .../explorer/select_limit/select_limit.tsx | 2 +- .../explorer/swimlane_container.tsx | 61 + .../services/dashboard_service.test.ts | 174 ++ .../application/services/dashboard_service.ts | 136 + .../application/services/http_service.ts | 6 +- .../services/ml_api_service/index.ts | 1191 +++++---- .../services/ml_api_service/results.ts | 16 +- .../services/results_service/index.ts | 53 +- .../results_service/result_service_rx.ts | 940 +++---- .../results_service/results_service.d.ts | 86 +- .../results_service/results_service.js | 2361 +++++++++-------- .../anomaly_swimlane_embeddable.tsx | 17 +- .../anomaly_swimlane_embeddable_factory.ts | 6 +- .../anomaly_swimlane_initializer.tsx | 10 +- .../anomaly_swimlane_setup_flyout.tsx | 13 +- .../explorer_swimlane_container.tsx | 18 +- .../swimlane_input_resolver.ts | 4 +- x-pack/plugins/ml/public/index.ts | 4 +- x-pack/plugins/ml/public/plugin.ts | 14 +- .../ml/anomaly_detection/anomaly_explorer.ts | 7 + x-pack/test/functional/apps/ml/index.ts | 1 + .../services/ml/anomaly_explorer.ts | 33 + .../functional/services/ml/test_resources.ts | 37 +- .../services/ml/test_resources_data.ts | 19 + yarn.lock | 5 + 40 files changed, 3638 insertions(+), 2660 deletions(-) rename x-pack/plugins/ml/public/application/components/loading_indicator/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/components/loading_indicator/{loading_indicator.js => loading_indicator.tsx} (70%) create mode 100644 x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx create mode 100644 x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx rename x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/{explorer_no_influencers_found.js => explorer_no_influencers_found.tsx} (78%) create mode 100644 x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx create mode 100644 x-pack/plugins/ml/public/application/services/dashboard_service.test.ts create mode 100644 x-pack/plugins/ml/public/application/services/dashboard_service.ts diff --git a/src/plugins/dashboard/public/index.ts b/src/plugins/dashboard/public/index.ts index 28606b7dd9784c..17968dd0281e6c 100644 --- a/src/plugins/dashboard/public/index.ts +++ b/src/plugins/dashboard/public/index.ts @@ -32,8 +32,10 @@ export { export { DashboardConstants, createDashboardEditUrl } from './dashboard_constants'; export { DashboardStart, DashboardUrlGenerator } from './plugin'; -export { DASHBOARD_APP_URL_GENERATOR } from './url_generator'; +export { DASHBOARD_APP_URL_GENERATOR, createDashboardUrlGenerator } from './url_generator'; export { addEmbeddableToDashboardUrl } from './url_utils/url_helper'; +export { SavedObjectDashboard } from './saved_dashboards'; +export { SavedDashboardPanel } from './types'; export function plugin(initializerContext: PluginInitializerContext) { return new DashboardPlugin(initializerContext); diff --git a/src/plugins/dashboard/public/url_generator.ts b/src/plugins/dashboard/public/url_generator.ts index d6805b2d94119e..188de7fd857be8 100644 --- a/src/plugins/dashboard/public/url_generator.ts +++ b/src/plugins/dashboard/public/url_generator.ts @@ -28,6 +28,7 @@ import { import { setStateToKbnUrl } from '../../kibana_utils/public'; import { UrlGeneratorsDefinition, UrlGeneratorState } from '../../share/public'; import { SavedObjectLoader } from '../../saved_objects/public'; +import { ViewMode } from '../../embeddable/public'; export const STATE_STORAGE_KEY = '_a'; export const GLOBAL_STATE_STORAGE_KEY = '_g'; @@ -73,6 +74,11 @@ export type DashboardAppLinkGeneratorState = UrlGeneratorState<{ * true is default */ preserveSavedFilters?: boolean; + + /** + * View mode of the dashboard. + */ + viewMode?: ViewMode; }>; export const createDashboardUrlGenerator = ( @@ -123,6 +129,7 @@ export const createDashboardUrlGenerator = ( cleanEmptyKeys({ query: state.query, filters: filters?.filter((f) => !esFilters.isFilterPinned(f)), + viewMode: state.viewMode, }), { useHash }, `${appBasePath}#/${hash}` diff --git a/x-pack/package.json b/x-pack/package.json index e24d75cc0d9680..b40a1e43642516 100644 --- a/x-pack/package.json +++ b/x-pack/package.json @@ -61,6 +61,7 @@ "@types/d3-shape": "^1.3.1", "@types/d3-time": "^1.0.10", "@types/d3-time-format": "^2.1.1", + "@types/dragselect": "^1.13.1", "@types/elasticsearch": "^5.0.33", "@types/fancy-log": "^1.3.1", "@types/file-saver": "^2.0.0", diff --git a/x-pack/plugins/ml/public/application/app.tsx b/x-pack/plugins/ml/public/application/app.tsx index 4b6ff8c64822b9..b871d857f7fded 100644 --- a/x-pack/plugins/ml/public/application/app.tsx +++ b/x-pack/plugins/ml/public/application/app.tsx @@ -35,6 +35,8 @@ const App: FC = ({ coreStart, deps }) => { }; const services = { appName: 'ML', + kibanaVersion: deps.kibanaVersion, + share: deps.share, data: deps.data, security: deps.security, licenseManagement: deps.licenseManagement, diff --git a/x-pack/plugins/ml/public/application/components/loading_indicator/index.js b/x-pack/plugins/ml/public/application/components/loading_indicator/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/components/loading_indicator/index.js rename to x-pack/plugins/ml/public/application/components/loading_indicator/index.ts diff --git a/x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.js b/x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.tsx similarity index 70% rename from x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.js rename to x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.tsx index 20f4fb86b5372c..364b23a27eaf7c 100644 --- a/x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.js +++ b/x-pack/plugins/ml/public/application/components/loading_indicator/loading_indicator.tsx @@ -4,12 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import PropTypes from 'prop-types'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiLoadingChart, EuiSpacer } from '@elastic/eui'; -export function LoadingIndicator({ height, label }) { +export const LoadingIndicator: FC<{ height?: number; label?: string }> = ({ height, label }) => { height = height ? +height : 100; return (
-
{label}
+
{label}
)}
); -} -LoadingIndicator.propTypes = { - height: PropTypes.number, - label: PropTypes.string, }; diff --git a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts index c65d872212ad67..2a156b5716ad4d 100644 --- a/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts @@ -12,13 +12,15 @@ import { } from '../../../../../../../src/plugins/kibana_react/public'; import { SecurityPluginSetup } from '../../../../../security/public'; import { LicenseManagementUIPluginSetup } from '../../../../../license_management/public'; +import { SharePluginStart } from '../../../../../../../src/plugins/share/public'; interface StartPlugins { data: DataPublicPluginStart; security?: SecurityPluginSetup; licenseManagement?: LicenseManagementUIPluginSetup; + share: SharePluginStart; } -export type StartServices = CoreStart & StartPlugins; +export type StartServices = CoreStart & StartPlugins & { kibanaVersion: string }; // eslint-disable-next-line react-hooks/rules-of-hooks export const useMlKibana = () => useKibana(); export type MlKibanaReactContextValue = KibanaReactContextValue; diff --git a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts index f8abd48ce85620..07d5a153664b75 100644 --- a/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts +++ b/x-pack/plugins/ml/public/application/contexts/ml/ml_context.ts @@ -14,6 +14,7 @@ export interface MlContextValue { currentSavedSearch: SavedSearchSavedObject | null; indexPatterns: IndexPatternsContract; kibanaConfig: any; // IUiSettingsClient; + kibanaVersion: string; } export type SavedSearchQuery = object; diff --git a/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx b/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx new file mode 100644 index 00000000000000..cb11a33ccfd76a --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/add_to_dashboard_control.tsx @@ -0,0 +1,321 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useMemo, useState, useEffect } from 'react'; +import { debounce } from 'lodash'; +import { + EuiFormRow, + EuiCheckboxGroup, + EuiInMemoryTableProps, + EuiModal, + EuiModalHeader, + EuiModalHeaderTitle, + EuiOverlayMask, + EuiSpacer, + EuiButtonEmpty, + EuiButton, + EuiModalFooter, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiModalBody } from '@elastic/eui'; +import { EuiInMemoryTable } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useMlKibana } from '../contexts/kibana'; +import { SavedObjectDashboard } from '../../../../../../src/plugins/dashboard/public'; +import { + ANOMALY_SWIMLANE_EMBEDDABLE_TYPE, + getDefaultPanelTitle, +} from '../../embeddables/anomaly_swimlane/anomaly_swimlane_embeddable'; +import { useDashboardService } from '../services/dashboard_service'; +import { SWIMLANE_TYPE, SwimlaneType } from './explorer_constants'; +import { JobId } from '../../../common/types/anomaly_detection_jobs'; + +export interface DashboardItem { + id: string; + title: string; + description: string | undefined; + attributes: SavedObjectDashboard; +} + +export type EuiTableProps = EuiInMemoryTableProps; + +function getDefaultEmbeddablepaPanelConfig(jobIds: JobId[]) { + return { + type: ANOMALY_SWIMLANE_EMBEDDABLE_TYPE, + title: getDefaultPanelTitle(jobIds), + }; +} + +interface AddToDashboardControlProps { + jobIds: JobId[]; + viewBy: string; + limit: number; + onClose: (callback?: () => Promise) => void; +} + +/** + * Component for attaching anomaly swimlane embeddable to dashboards. + */ +export const AddToDashboardControl: FC = ({ + onClose, + jobIds, + viewBy, + limit, +}) => { + const { + notifications: { toasts }, + services: { + application: { navigateToUrl }, + }, + } = useMlKibana(); + + useEffect(() => { + fetchDashboards(); + + return () => { + fetchDashboards.cancel(); + }; + }, []); + + const dashboardService = useDashboardService(); + + const [isLoading, setIsLoading] = useState(false); + const [selectedSwimlanes, setSelectedSwimlanes] = useState<{ [key in SwimlaneType]: boolean }>({ + [SWIMLANE_TYPE.OVERALL]: true, + [SWIMLANE_TYPE.VIEW_BY]: false, + }); + const [dashboardItems, setDashboardItems] = useState([]); + const [selectedItems, setSelectedItems] = useState([]); + + const fetchDashboards = useCallback( + debounce(async (query?: string) => { + try { + const response = await dashboardService.fetchDashboards(query); + const items: DashboardItem[] = response.savedObjects.map((savedObject) => { + return { + id: savedObject.id, + title: savedObject.attributes.title, + description: savedObject.attributes.description, + attributes: savedObject.attributes, + }; + }); + setDashboardItems(items); + } catch (e) { + toasts.danger({ + body: e, + }); + } + setIsLoading(false); + }, 500), + [] + ); + + const search: EuiTableProps['search'] = useMemo(() => { + return { + onChange: ({ queryText }) => { + setIsLoading(true); + fetchDashboards(queryText); + }, + box: { + incremental: true, + 'data-test-subj': 'mlDashboardsSearchBox', + }, + }; + }, []); + + const addSwimlaneToDashboardCallback = useCallback(async () => { + const swimlanes = Object.entries(selectedSwimlanes) + .filter(([, isSelected]) => isSelected) + .map(([swimlaneType]) => swimlaneType); + + for (const selectedDashboard of selectedItems) { + const panelsData = swimlanes.map((swimlaneType) => { + const config = getDefaultEmbeddablepaPanelConfig(jobIds); + if (swimlaneType === SWIMLANE_TYPE.VIEW_BY) { + return { + ...config, + embeddableConfig: { + jobIds, + swimlaneType, + viewBy, + limit, + }, + }; + } + return { + ...config, + embeddableConfig: { + jobIds, + swimlaneType, + }, + }; + }); + + try { + await dashboardService.attachPanels( + selectedDashboard.id, + selectedDashboard.attributes, + panelsData + ); + toasts.success({ + title: ( + + ), + toastLifeTimeMs: 3000, + }); + } catch (e) { + toasts.danger({ + body: e, + }); + } + } + }, [selectedSwimlanes, selectedItems]); + + const columns: EuiTableProps['columns'] = [ + { + field: 'title', + name: i18n.translate('xpack.ml.explorer.dashboardsTable.titleColumnHeader', { + defaultMessage: 'Title', + }), + sortable: true, + truncateText: true, + }, + { + field: 'description', + name: i18n.translate('xpack.ml.explorer.dashboardsTable.descriptionColumnHeader', { + defaultMessage: 'Description', + }), + truncateText: true, + }, + ]; + + const swimlaneTypeOptions = [ + { + id: SWIMLANE_TYPE.OVERALL, + label: i18n.translate('xpack.ml.explorer.overallLabel', { + defaultMessage: 'Overall', + }), + }, + { + id: SWIMLANE_TYPE.VIEW_BY, + label: i18n.translate('xpack.ml.explorer.viewByFieldLabel', { + defaultMessage: 'View by {viewByField}, up to {limit} rows', + values: { viewByField: viewBy, limit }, + }), + }, + ]; + + const selection: EuiTableProps['selection'] = { + onSelectionChange: setSelectedItems, + }; + + const noSwimlaneSelected = Object.values(selectedSwimlanes).every((isSelected) => !isSelected); + + return ( + + + + + + + + + + } + > + { + const newSelection = { + ...selectedSwimlanes, + [optionId]: !selectedSwimlanes[optionId as SwimlaneType], + }; + setSelectedSwimlanes(newSelection); + }} + data-test-subj="mlAddToDashboardSwimlaneTypeSelector" + /> + + + + + + } + data-test-subj="mlDashboardSelectionContainer" + > + + + + + + + + { + onClose(async () => { + const selectedDashboardId = selectedItems[0].id; + await addSwimlaneToDashboardCallback(); + await navigateToUrl( + await dashboardService.getDashboardEditUrl(selectedDashboardId) + ); + }); + }} + data-test-subj="mlAddAndEditDashboardButton" + > + + + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx new file mode 100644 index 00000000000000..b4d32e2af64b8e --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/anomaly_timeline.tsx @@ -0,0 +1,392 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useMemo, useRef, useState } from 'react'; +import { isEqual } from 'lodash'; +import DragSelect from 'dragselect'; +import { + EuiPanel, + EuiPopover, + EuiContextMenuPanel, + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiSelect, + EuiTitle, + EuiSpacer, + EuiContextMenuItem, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { DRAG_SELECT_ACTION, VIEW_BY_JOB_LABEL } from './explorer_constants'; +import { AddToDashboardControl } from './add_to_dashboard_control'; +import { useMlKibana } from '../contexts/kibana'; +import { TimeBuckets } from '../util/time_buckets'; +import { UI_SETTINGS } from '../../../../../../src/plugins/data/common'; +import { SelectLimit } from './select_limit'; +import { + ALLOW_CELL_RANGE_SELECTION, + dragSelect$, + explorerService, +} from './explorer_dashboard_service'; +import { ExplorerState } from './reducers/explorer_reducer'; +import { hasMatchingPoints } from './has_matching_points'; +import { ExplorerNoInfluencersFound } from './components/explorer_no_influencers_found/explorer_no_influencers_found'; +import { LoadingIndicator } from '../components/loading_indicator'; +import { SwimlaneContainer } from './swimlane_container'; +import { OverallSwimlaneData } from './explorer_utils'; + +function mapSwimlaneOptionsToEuiOptions(options: string[]) { + return options.map((option) => ({ + value: option, + text: option, + })); +} + +interface AnomalyTimelineProps { + explorerState: ExplorerState; + setSelectedCells: (cells?: any) => void; +} + +export const AnomalyTimeline: FC = React.memo( + ({ explorerState, setSelectedCells }) => { + const { + services: { + uiSettings, + application: { capabilities }, + }, + } = useMlKibana(); + + const [isMenuOpen, setIsMenuOpen] = useState(false); + const [isAddDashboardsActive, setIsAddDashboardActive] = useState(false); + + const isSwimlaneSelectActive = useRef(false); + // make sure dragSelect is only available if the mouse pointer is actually over a swimlane + const disableDragSelectOnMouseLeave = useRef(true); + + const canEditDashboards = capabilities.dashboard?.createNew ?? false; + + const timeBuckets = useMemo(() => { + return new TimeBuckets({ + 'histogram:maxBars': uiSettings.get(UI_SETTINGS.HISTOGRAM_MAX_BARS), + 'histogram:barTarget': uiSettings.get(UI_SETTINGS.HISTOGRAM_BAR_TARGET), + dateFormat: uiSettings.get('dateFormat'), + 'dateFormat:scaled': uiSettings.get('dateFormat:scaled'), + }); + }, [uiSettings]); + + const dragSelect = useMemo( + () => + new DragSelect({ + selectorClass: 'ml-swimlane-selector', + selectables: document.querySelectorAll('.sl-cell'), + callback(elements) { + if (elements.length > 1 && !ALLOW_CELL_RANGE_SELECTION) { + elements = [elements[0]]; + } + + if (elements.length > 0) { + dragSelect$.next({ + action: DRAG_SELECT_ACTION.NEW_SELECTION, + elements, + }); + } + + disableDragSelectOnMouseLeave.current = true; + }, + onDragStart(e) { + let target = e.target as HTMLElement; + while (target && target !== document.body && !target.classList.contains('sl-cell')) { + target = target.parentNode as HTMLElement; + } + if (ALLOW_CELL_RANGE_SELECTION && target !== document.body) { + dragSelect$.next({ + action: DRAG_SELECT_ACTION.DRAG_START, + }); + disableDragSelectOnMouseLeave.current = false; + } + }, + onElementSelect() { + if (ALLOW_CELL_RANGE_SELECTION) { + dragSelect$.next({ + action: DRAG_SELECT_ACTION.ELEMENT_SELECT, + }); + } + }, + }), + [] + ); + + const { + filterActive, + filteredFields, + maskAll, + overallSwimlaneData, + selectedCells, + viewByLoadedForTimeFormatted, + viewBySwimlaneData, + viewBySwimlaneDataLoading, + viewBySwimlaneFieldName, + viewBySwimlaneOptions, + swimlaneLimit, + selectedJobs, + } = explorerState; + + const setSwimlaneSelectActive = useCallback((active: boolean) => { + if (isSwimlaneSelectActive.current && !active && disableDragSelectOnMouseLeave.current) { + dragSelect.stop(); + isSwimlaneSelectActive.current = active; + return; + } + if (!isSwimlaneSelectActive.current && active) { + dragSelect.start(); + dragSelect.clearSelection(); + dragSelect.setSelectables(document.querySelectorAll('.sl-cell')); + isSwimlaneSelectActive.current = active; + } + }, []); + const onSwimlaneEnterHandler = () => setSwimlaneSelectActive(true); + const onSwimlaneLeaveHandler = () => setSwimlaneSelectActive(false); + + // Listens to render updates of the swimlanes to update dragSelect + const swimlaneRenderDoneListener = useCallback(() => { + dragSelect.clearSelection(); + dragSelect.setSelectables(document.querySelectorAll('.sl-cell')); + }, []); + + // Listener for click events in the swimlane to load corresponding anomaly data. + const swimlaneCellClick = useCallback((selectedCellsUpdate: any) => { + // If selectedCells is an empty object we clear any existing selection, + // otherwise we save the new selection in AppState and update the Explorer. + if (Object.keys(selectedCellsUpdate).length === 0) { + setSelectedCells(); + } else { + setSelectedCells(selectedCellsUpdate); + } + }, []); + + const showOverallSwimlane = + overallSwimlaneData !== null && + overallSwimlaneData.laneLabels && + overallSwimlaneData.laneLabels.length > 0; + + const showViewBySwimlane = + viewBySwimlaneData !== null && + viewBySwimlaneData.laneLabels && + viewBySwimlaneData.laneLabels.length > 0; + + const menuItems = useMemo(() => { + const items = []; + if (canEditDashboards) { + items.push( + + + + ); + } + return items; + }, [canEditDashboards]); + + return ( + <> + + + + +

+ +

+ + + {viewBySwimlaneOptions.length > 0 && ( + <> + + + + + } + display={'columnCompressed'} + > + explorerService.setViewBySwimlaneFieldName(e.target.value)} + /> + + + + + + + } + display={'columnCompressed'} + > + + + + +
+ {viewByLoadedForTimeFormatted && ( + + )} + {viewByLoadedForTimeFormatted === undefined && ( + + )} + {filterActive === true && viewBySwimlaneFieldName === VIEW_BY_JOB_LABEL && ( + + )} +
+ + + )} + + {menuItems.length > 0 && ( + + + } + isOpen={isMenuOpen} + closePopover={setIsMenuOpen.bind(null, false)} + panelPaddingSize="none" + anchorPosition="downLeft" + > + + + + )} + + + + +
+ {showOverallSwimlane && ( + explorerService.setSwimlaneContainerWidth(width)} + /> + )} +
+ + {viewBySwimlaneOptions.length > 0 && ( + <> + {showViewBySwimlane && ( + <> + +
+ explorerService.setSwimlaneContainerWidth(width)} + /> +
+ + )} + + {viewBySwimlaneDataLoading && } + + {!showViewBySwimlane && + !viewBySwimlaneDataLoading && + typeof viewBySwimlaneFieldName === 'string' && ( + + )} + + )} + + {isAddDashboardsActive && selectedJobs && ( + { + setIsAddDashboardActive(false); + if (callback) { + await callback(); + } + }} + jobIds={selectedJobs.map(({ id }) => id)} + viewBy={viewBySwimlaneFieldName!} + limit={swimlaneLimit} + /> + )} + + ); + }, + (prevProps, nextProps) => { + return isEqual(prevProps.explorerState, nextProps.explorerState); + } +); diff --git a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.js b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx similarity index 78% rename from x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.js rename to x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx index 5f54c383e76ad7..639c0f7b785043 100644 --- a/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.js +++ b/x-pack/plugins/ml/public/application/explorer/components/explorer_no_influencers_found/explorer_no_influencers_found.tsx @@ -4,20 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -/* - * React component for rendering EuiEmptyPrompt when no influencers were found. - */ - -import PropTypes from 'prop-types'; -import React from 'react'; +import React, { FC } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiEmptyPrompt } from '@elastic/eui'; -export const ExplorerNoInfluencersFound = ({ - viewBySwimlaneFieldName, - showFilterMessage = false, -}) => ( +/* + * React component for rendering EuiEmptyPrompt when no influencers were found. + */ +export const ExplorerNoInfluencersFound: FC<{ + viewBySwimlaneFieldName: string; + showFilterMessage?: boolean; +}> = ({ viewBySwimlaneFieldName, showFilterMessage = false }) => ( ); - -ExplorerNoInfluencersFound.propTypes = { - viewBySwimlaneFieldName: PropTypes.string.isRequired, - showFilterMessage: PropTypes.bool, -}; diff --git a/x-pack/plugins/ml/public/application/explorer/explorer.js b/x-pack/plugins/ml/public/application/explorer/explorer.js index 1a5a9a9d828623..71c96840d1b579 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer.js +++ b/x-pack/plugins/ml/public/application/explorer/explorer.js @@ -9,10 +9,9 @@ */ import PropTypes from 'prop-types'; -import React, { createRef } from 'react'; +import React from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import DragSelect from 'dragselect/dist/ds.min.js'; import { Subject } from 'rxjs'; import { takeUntil } from 'rxjs/operators'; @@ -26,34 +25,23 @@ import { EuiPageBody, EuiPageHeader, EuiPageHeaderSection, - EuiSelect, EuiSpacer, EuiTitle, } from '@elastic/eui'; import { AnnotationFlyout } from '../components/annotations/annotation_flyout'; import { AnnotationsTable } from '../components/annotations/annotations_table'; -import { - ExplorerNoInfluencersFound, - ExplorerNoJobsFound, - ExplorerNoResultsFound, -} from './components'; -import { ExplorerSwimlane } from './explorer_swimlane'; -import { getTimeBucketsFromCache } from '../util/time_buckets'; +import { ExplorerNoJobsFound, ExplorerNoResultsFound } from './components'; import { DatePickerWrapper } from '../components/navigation_menu/date_picker_wrapper'; import { InfluencersList } from '../components/influencers_list'; -import { - ALLOW_CELL_RANGE_SELECTION, - dragSelect$, - explorerService, -} from './explorer_dashboard_service'; +import { explorerService } from './explorer_dashboard_service'; import { AnomalyResultsViewSelector } from '../components/anomaly_results_view_selector'; import { LoadingIndicator } from '../components/loading_indicator/loading_indicator'; import { NavigationMenu } from '../components/navigation_menu'; import { CheckboxShowCharts } from '../components/controls/checkbox_showcharts'; import { JobSelector } from '../components/job_selector'; import { SelectInterval } from '../components/controls/select_interval/select_interval'; -import { SelectLimit, limit$ } from './select_limit/select_limit'; +import { limit$ } from './select_limit/select_limit'; import { SelectSeverity } from '../components/controls/select_severity/select_severity'; import { ExplorerQueryBar, @@ -67,14 +55,9 @@ import { escapeParens, escapeDoubleQuotes, } from './explorer_utils'; -import { getSwimlaneContainerWidth } from './legacy_utils'; +import { AnomalyTimeline } from './anomaly_timeline'; -import { - DRAG_SELECT_ACTION, - FILTER_ACTION, - SWIMLANE_TYPE, - VIEW_BY_JOB_LABEL, -} from './explorer_constants'; +import { FILTER_ACTION } from './explorer_constants'; // Explorer Charts import { ExplorerChartsContainer } from './explorer_charts/explorer_charts_container'; @@ -82,17 +65,7 @@ import { ExplorerChartsContainer } from './explorer_charts/explorer_charts_conta // Anomalies Table import { AnomaliesTable } from '../components/anomalies_table/anomalies_table'; -import { ResizeChecker } from '../../../../../../src/plugins/kibana_utils/public'; import { getTimefilter, getToastNotifications } from '../util/dependency_cache'; -import { MlTooltipComponent } from '../components/chart_tooltip'; -import { hasMatchingPoints } from './has_matching_points'; - -function mapSwimlaneOptionsToEuiOptions(options) { - return options.map((option) => ({ - value: option, - text: option, - })); -} const ExplorerPage = ({ children, @@ -105,9 +78,8 @@ const ExplorerPage = ({ queryString, filterIconTriggeredQuery, updateLanguage, - resizeRef, }) => ( -
+
@@ -171,108 +143,18 @@ export class Explorer extends React.Component { state = { filterIconTriggeredQuery: undefined, language: DEFAULT_QUERY_LANG }; _unsubscribeAll = new Subject(); - // make sure dragSelect is only available if the mouse pointer is actually over a swimlane - disableDragSelectOnMouseLeave = true; - - dragSelect = new DragSelect({ - selectorClass: 'ml-swimlane-selector', - selectables: document.getElementsByClassName('sl-cell'), - callback(elements) { - if (elements.length > 1 && !ALLOW_CELL_RANGE_SELECTION) { - elements = [elements[0]]; - } - - if (elements.length > 0) { - dragSelect$.next({ - action: DRAG_SELECT_ACTION.NEW_SELECTION, - elements, - }); - } - - this.disableDragSelectOnMouseLeave = true; - }, - onDragStart(e) { - let target = e.target; - while (target && target !== document.body && !target.classList.contains('sl-cell')) { - target = target.parentNode; - } - if (ALLOW_CELL_RANGE_SELECTION && target !== document.body) { - dragSelect$.next({ - action: DRAG_SELECT_ACTION.DRAG_START, - }); - this.disableDragSelectOnMouseLeave = false; - } - }, - onElementSelect() { - if (ALLOW_CELL_RANGE_SELECTION) { - dragSelect$.next({ - action: DRAG_SELECT_ACTION.ELEMENT_SELECT, - }); - } - }, - }); - - // Listens to render updates of the swimlanes to update dragSelect - swimlaneRenderDoneListener = () => { - this.dragSelect.clearSelection(); - this.dragSelect.setSelectables(document.getElementsByClassName('sl-cell')); - }; - - resizeRef = createRef(); - resizeChecker = undefined; - resizeHandler = () => { - explorerService.setSwimlaneContainerWidth(getSwimlaneContainerWidth()); - }; componentDidMount() { limit$.pipe(takeUntil(this._unsubscribeAll)).subscribe(explorerService.setSwimlaneLimit); - - // Required to redraw the time series chart when the container is resized. - this.resizeChecker = new ResizeChecker(this.resizeRef.current); - this.resizeChecker.on('resize', this.resizeHandler); - - this.timeBuckets = getTimeBucketsFromCache(); } componentWillUnmount() { this._unsubscribeAll.next(); this._unsubscribeAll.complete(); - this.resizeChecker.destroy(); - } - - resetCache() { - this.anomaliesTablePreviousArgs = null; } viewByChangeHandler = (e) => explorerService.setViewBySwimlaneFieldName(e.target.value); - isSwimlaneSelectActive = false; - onSwimlaneEnterHandler = () => this.setSwimlaneSelectActive(true); - onSwimlaneLeaveHandler = () => this.setSwimlaneSelectActive(false); - setSwimlaneSelectActive = (active) => { - if (this.isSwimlaneSelectActive && !active && this.disableDragSelectOnMouseLeave) { - this.dragSelect.stop(); - this.isSwimlaneSelectActive = active; - return; - } - if (!this.isSwimlaneSelectActive && active) { - this.dragSelect.start(); - this.dragSelect.clearSelection(); - this.dragSelect.setSelectables(document.getElementsByClassName('sl-cell')); - this.isSwimlaneSelectActive = active; - } - }; - - // Listener for click events in the swimlane to load corresponding anomaly data. - swimlaneCellClick = (selectedCells) => { - // If selectedCells is an empty object we clear any existing selection, - // otherwise we save the new selection in AppState and update the Explorer. - if (Object.keys(selectedCells).length === 0) { - this.props.setSelectedCells(); - } else { - this.props.setSelectedCells(selectedCells); - } - }; // Escape regular parens from fieldName as that portion of the query is not wrapped in double quotes // and will cause a syntax error when called with getKqlQueryValues applyFilter = (fieldName, fieldValue, action) => { @@ -339,24 +221,16 @@ export class Explorer extends React.Component { annotationsData, chartsData, filterActive, - filteredFields, filterPlaceHolder, indexPattern, influencers, loading, - maskAll, noInfluencersConfigured, overallSwimlaneData, queryString, selectedCells, selectedJobs, - swimlaneContainerWidth, tableData, - viewByLoadedForTimeFormatted, - viewBySwimlaneData, - viewBySwimlaneDataLoading, - viewBySwimlaneFieldName, - viewBySwimlaneOptions, } = this.props.explorerState; const jobSelectorProps = { @@ -378,7 +252,6 @@ export class Explorer extends React.Component { indexPattern={indexPattern} queryString={queryString} updateLanguage={this.updateLanguage} - resizeRef={this.resizeRef} > + ); @@ -399,7 +272,7 @@ export class Explorer extends React.Component { if (noJobsFound && hasResults === false) { return ( - + ); @@ -408,15 +281,6 @@ export class Explorer extends React.Component { const mainColumnWidthClassName = noInfluencersConfigured === true ? 'col-xs-12' : 'col-xs-10'; const mainColumnClasses = `column ${mainColumnWidthClassName}`; - const showOverallSwimlane = - overallSwimlaneData !== null && - overallSwimlaneData.laneLabels && - overallSwimlaneData.laneLabels.length > 0; - const showViewBySwimlane = - viewBySwimlaneData !== null && - viewBySwimlaneData.laneLabels && - viewBySwimlaneData.laneLabels.length > 0; - const timefilter = getTimefilter(); const bounds = timefilter.getActiveBounds(); @@ -431,7 +295,6 @@ export class Explorer extends React.Component { indexPattern={indexPattern} queryString={queryString} updateLanguage={this.updateLanguage} - resizeRef={this.resizeRef} >
{noInfluencersConfigured && ( @@ -462,142 +325,12 @@ export class Explorer extends React.Component { )}
- -

- -

- - -
- {showOverallSwimlane && ( - - {(tooltipService) => ( - - )} - - )} -
- - {viewBySwimlaneOptions.length > 0 && ( - <> - - - - - - - - - - - - - -
- {viewByLoadedForTimeFormatted && ( - - )} - {viewByLoadedForTimeFormatted === undefined && ( - - )} - {filterActive === true && viewBySwimlaneFieldName === VIEW_BY_JOB_LABEL && ( - - )} -
- - - - - {showViewBySwimlane && ( - <> - -
- - {(tooltipService) => ( - - )} - -
- - )} - - {viewBySwimlaneDataLoading && } - - {!showViewBySwimlane && - !viewBySwimlaneDataLoading && - viewBySwimlaneFieldName !== null && ( - - )} - - )} + + + {annotationsData.length > 0 && ( <> diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts index 1cfd29e2f60d22..d1adf8c7ad744c 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts +++ b/x-pack/plugins/ml/public/application/explorer/explorer_constants.ts @@ -37,10 +37,12 @@ export const FILTER_ACTION = { REMOVE: '-', }; -export enum SWIMLANE_TYPE { - OVERALL = 'overall', - VIEW_BY = 'viewBy', -} +export const SWIMLANE_TYPE = { + OVERALL: 'overall', + VIEW_BY: 'viewBy', +} as const; + +export type SwimlaneType = typeof SWIMLANE_TYPE[keyof typeof SWIMLANE_TYPE]; export const CHART_TYPE = { EVENT_DISTRIBUTION: 'event_distribution', diff --git a/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx b/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx index 18b5de1d51f9c5..4e6dcdcc5129ca 100644 --- a/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx +++ b/x-pack/plugins/ml/public/application/explorer/explorer_swimlane.tsx @@ -22,7 +22,7 @@ import { numTicksForDateFormat } from '../util/chart_utils'; import { getSeverityColor } from '../../../common/util/anomaly_utils'; import { mlEscape } from '../util/string_utils'; import { ALLOW_CELL_RANGE_SELECTION, dragSelect$ } from './explorer_dashboard_service'; -import { DRAG_SELECT_ACTION } from './explorer_constants'; +import { DRAG_SELECT_ACTION, SwimlaneType } from './explorer_constants'; import { EMPTY_FIELD_VALUE_LABEL } from '../timeseriesexplorer/components/entity_control/entity_control'; import { TimeBuckets as TimeBucketsClass } from '../util/time_buckets'; import { @@ -58,7 +58,7 @@ export interface ExplorerSwimlaneProps { timeBuckets: InstanceType; swimlaneCellClick?: Function; swimlaneData: OverallSwimlaneData; - swimlaneType: string; + swimlaneType: SwimlaneType; selection?: { lanes: any[]; type: string; diff --git a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts index 0a2dbf5bcff35f..4e1a2af9b13a60 100644 --- a/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts +++ b/x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/state.ts @@ -16,8 +16,9 @@ import { AnomaliesTableData, ExplorerJob, AppStateSelectedCells, - SwimlaneData, TimeRangeBounds, + OverallSwimlaneData, + SwimlaneData, } from '../../explorer_utils'; export interface ExplorerState { @@ -35,7 +36,7 @@ export interface ExplorerState { loading: boolean; maskAll: boolean; noInfluencersConfigured: boolean; - overallSwimlaneData: SwimlaneData; + overallSwimlaneData: SwimlaneData | OverallSwimlaneData; queryString: string; selectedCells: AppStateSelectedCells | undefined; selectedJobs: ExplorerJob[] | null; @@ -45,7 +46,7 @@ export interface ExplorerState { tableData: AnomaliesTableData; tableQueryString: string; viewByLoadedForTimeFormatted: string | null; - viewBySwimlaneData: SwimlaneData; + viewBySwimlaneData: SwimlaneData | OverallSwimlaneData; viewBySwimlaneDataLoading: boolean; viewBySwimlaneFieldName?: string; viewBySwimlaneOptions: string[]; diff --git a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx b/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx index 7f7a8fc5a70bd0..7a2df1a0f05350 100644 --- a/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx +++ b/x-pack/plugins/ml/public/application/explorer/select_limit/select_limit.tsx @@ -36,5 +36,5 @@ export const SelectLimit = () => { setLimit(parseInt(e.target.value, 10)); } - return ; + return ; }; diff --git a/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx new file mode 100644 index 00000000000000..57d1fd81000b7e --- /dev/null +++ b/x-pack/plugins/ml/public/application/explorer/swimlane_container.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useCallback, useState } from 'react'; +import { EuiResizeObserver, EuiText } from '@elastic/eui'; + +import { throttle } from 'lodash'; +import { + ExplorerSwimlane, + ExplorerSwimlaneProps, +} from '../../application/explorer/explorer_swimlane'; + +import { MlTooltipComponent } from '../../application/components/chart_tooltip'; + +const RESIZE_THROTTLE_TIME_MS = 500; + +export const SwimlaneContainer: FC< + Omit & { + onResize: (width: number) => void; + } +> = ({ children, onResize, ...props }) => { + const [chartWidth, setChartWidth] = useState(0); + + const resizeHandler = useCallback( + throttle((e: { width: number; height: number }) => { + const labelWidth = 200; + setChartWidth(e.width - labelWidth); + onResize(e.width); + }, RESIZE_THROTTLE_TIME_MS), + [] + ); + + return ( + + {(resizeRef) => ( +
{ + resizeRef(el); + }} + > +
+ + + {(tooltipService) => ( + + )} + + +
+
+ )} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts b/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts new file mode 100644 index 00000000000000..6cab23eb187c7c --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/dashboard_service.test.ts @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { dashboardServiceProvider } from './dashboard_service'; +import { savedObjectsServiceMock } from '../../../../../../src/core/public/mocks'; +import { SavedObjectDashboard } from '../../../../../../src/plugins/dashboard/public/saved_dashboards'; +import { + DashboardUrlGenerator, + SavedDashboardPanel, +} from '../../../../../../src/plugins/dashboard/public'; + +jest.mock('@elastic/eui', () => { + return { + htmlIdGenerator: jest.fn(() => { + return jest.fn(() => 'test-panel-id'); + }), + }; +}); + +describe('DashboardService', () => { + const mockSavedObjectClient = savedObjectsServiceMock.createStartContract().client; + const dashboardUrlGenerator = ({ + createUrl: jest.fn(), + } as unknown) as DashboardUrlGenerator; + const dashboardService = dashboardServiceProvider( + mockSavedObjectClient, + '8.0.0', + dashboardUrlGenerator + ); + + test('should fetch dashboard', () => { + // act + dashboardService.fetchDashboards('test'); + // assert + expect(mockSavedObjectClient.find).toHaveBeenCalledWith({ + type: 'dashboard', + perPage: 10, + search: `test*`, + searchFields: ['title^3', 'description'], + }); + }); + + test('should attach panel to the dashboard', () => { + // act + dashboardService.attachPanels( + 'test-dashboard', + ({ + title: 'ML Test', + hits: 0, + description: '', + panelsJSON: JSON.stringify([ + { + version: '8.0.0', + type: 'ml_anomaly_swimlane', + gridData: { x: 0, y: 0, w: 24, h: 15, i: 'i63c960b1-ab1b-11ea-809d-f5c60c43347f' }, + panelIndex: 'i63c960b1-ab1b-11ea-809d-f5c60c43347f', + embeddableConfig: { + title: 'Panel test!', + jobIds: ['cw_multi_1'], + swimlaneType: 'overall', + }, + title: 'Panel test!', + }, + { + version: '8.0.0', + type: 'ml_anomaly_swimlane', + gridData: { x: 24, y: 0, w: 24, h: 15, i: '0aa334bd-8308-4ded-9462-80dbd37680ee' }, + panelIndex: '0aa334bd-8308-4ded-9462-80dbd37680ee', + embeddableConfig: { + title: 'ML anomaly swimlane for fb_population_1', + jobIds: ['fb_population_1'], + limit: 5, + swimlaneType: 'overall', + }, + title: 'ML anomaly swimlane for fb_population_1', + }, + { + version: '8.0.0', + gridData: { x: 0, y: 15, w: 24, h: 15, i: 'abd36eb7-4774-4216-891e-12100752b46d' }, + panelIndex: 'abd36eb7-4774-4216-891e-12100752b46d', + embeddableConfig: {}, + panelRefName: 'panel_2', + }, + ]), + optionsJSON: '{"hidePanelTitles":false,"useMargins":true}', + version: 1, + timeRestore: false, + kibanaSavedObjectMeta: { + searchSourceJSON: '{"query":{"language":"kuery","query":""},"filter":[]}', + }, + } as unknown) as SavedObjectDashboard, + [{ title: 'Test title', type: 'test-panel', embeddableConfig: { testConfig: '' } }] + ); + // assert + expect(mockSavedObjectClient.update).toHaveBeenCalledWith('dashboard', 'test-dashboard', { + title: 'ML Test', + hits: 0, + description: '', + panelsJSON: JSON.stringify([ + { + version: '8.0.0', + type: 'ml_anomaly_swimlane', + gridData: { x: 0, y: 0, w: 24, h: 15, i: 'i63c960b1-ab1b-11ea-809d-f5c60c43347f' }, + panelIndex: 'i63c960b1-ab1b-11ea-809d-f5c60c43347f', + embeddableConfig: { + title: 'Panel test!', + jobIds: ['cw_multi_1'], + swimlaneType: 'overall', + }, + title: 'Panel test!', + }, + { + version: '8.0.0', + type: 'ml_anomaly_swimlane', + gridData: { x: 24, y: 0, w: 24, h: 15, i: '0aa334bd-8308-4ded-9462-80dbd37680ee' }, + panelIndex: '0aa334bd-8308-4ded-9462-80dbd37680ee', + embeddableConfig: { + title: 'ML anomaly swimlane for fb_population_1', + jobIds: ['fb_population_1'], + limit: 5, + swimlaneType: 'overall', + }, + title: 'ML anomaly swimlane for fb_population_1', + }, + { + version: '8.0.0', + gridData: { x: 0, y: 15, w: 24, h: 15, i: 'abd36eb7-4774-4216-891e-12100752b46d' }, + panelIndex: 'abd36eb7-4774-4216-891e-12100752b46d', + embeddableConfig: {}, + panelRefName: 'panel_2', + }, + { + panelIndex: 'test-panel-id', + embeddableConfig: { testConfig: '' }, + title: 'Test title', + type: 'test-panel', + version: '8.0.0', + gridData: { h: 15, i: 'test-panel-id', w: 24, x: 24, y: 15 }, + }, + ]), + optionsJSON: '{"hidePanelTitles":false,"useMargins":true}', + version: 1, + timeRestore: false, + kibanaSavedObjectMeta: { + searchSourceJSON: '{"query":{"language":"kuery","query":""},"filter":[]}', + }, + }); + }); + + test('should generate edit url to the dashboard', () => { + dashboardService.getDashboardEditUrl('test-id'); + expect(dashboardUrlGenerator.createUrl).toHaveBeenCalledWith({ + dashboardId: 'test-id', + useHash: false, + viewMode: 'edit', + }); + }); + + test('should find the panel positioned at the end', () => { + expect( + dashboardService.getLastPanel([ + { gridData: { y: 15, x: 7 } }, + { gridData: { y: 17, x: 9 } }, + { gridData: { y: 15, x: 1 } }, + { gridData: { y: 17, x: 10 } }, + { gridData: { y: 15, x: 22 } }, + { gridData: { y: 17, x: 9 } }, + ] as SavedDashboardPanel[]) + ).toEqual({ gridData: { y: 17, x: 10 } }); + }); +}); diff --git a/x-pack/plugins/ml/public/application/services/dashboard_service.ts b/x-pack/plugins/ml/public/application/services/dashboard_service.ts new file mode 100644 index 00000000000000..7f2bb71d18eb98 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/dashboard_service.ts @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectsClientContract } from 'kibana/public'; +import { htmlIdGenerator } from '@elastic/eui'; +import { useMemo } from 'react'; +import { + DASHBOARD_APP_URL_GENERATOR, + DashboardUrlGenerator, + SavedDashboardPanel, + SavedObjectDashboard, +} from '../../../../../../src/plugins/dashboard/public'; +import { useMlKibana } from '../contexts/kibana'; +import { ViewMode } from '../../../../../../src/plugins/embeddable/public'; + +export type DashboardService = ReturnType; + +export function dashboardServiceProvider( + savedObjectClient: SavedObjectsClientContract, + kibanaVersion: string, + dashboardUrlGenerator: DashboardUrlGenerator +) { + const generateId = htmlIdGenerator(); + const DEFAULT_PANEL_WIDTH = 24; + const DEFAULT_PANEL_HEIGHT = 15; + + return { + /** + * Fetches dashboards + */ + async fetchDashboards(query?: string) { + return await savedObjectClient.find({ + type: 'dashboard', + perPage: 10, + search: query ? `${query}*` : '', + searchFields: ['title^3', 'description'], + }); + }, + /** + * Resolves the last positioned panel from the collection. + */ + getLastPanel(panels: SavedDashboardPanel[]): SavedDashboardPanel | null { + return panels.length > 0 + ? panels.reduce((prev, current) => + prev.gridData.y >= current.gridData.y + ? prev.gridData.y === current.gridData.y + ? prev.gridData.x > current.gridData.x + ? prev + : current + : prev + : current + ) + : null; + }, + /** + * Attaches embeddable panels to the dashboard + */ + async attachPanels( + dashboardId: string, + dashboardAttributes: SavedObjectDashboard, + panelsData: Array> + ) { + const panels = JSON.parse(dashboardAttributes.panelsJSON) as SavedDashboardPanel[]; + const version = kibanaVersion; + const rowWidth = DEFAULT_PANEL_WIDTH * 2; + + for (const panelData of panelsData) { + const panelIndex = generateId(); + const lastPanel = this.getLastPanel(panels); + + const xOffset = lastPanel ? lastPanel.gridData.w + lastPanel.gridData.x : 0; + const availableRowSpace = rowWidth - xOffset; + const xPosition = availableRowSpace - DEFAULT_PANEL_WIDTH >= 0 ? xOffset : 0; + + panels.push({ + panelIndex, + embeddableConfig: panelData.embeddableConfig as { [key: string]: any }, + title: panelData.title, + type: panelData.type, + version, + gridData: { + h: DEFAULT_PANEL_HEIGHT, + i: panelIndex, + w: DEFAULT_PANEL_WIDTH, + x: xPosition, + y: lastPanel + ? xPosition > 0 + ? lastPanel.gridData.y + : lastPanel.gridData.y + lastPanel.gridData.h + : 0, + }, + }); + } + + await savedObjectClient.update('dashboard', dashboardId, { + ...dashboardAttributes, + panelsJSON: JSON.stringify(panels), + }); + }, + /** + * Generates dashboard url with edit mode + */ + async getDashboardEditUrl(dashboardId: string) { + return await dashboardUrlGenerator.createUrl({ + dashboardId, + useHash: false, + viewMode: ViewMode.EDIT, + }); + }, + }; +} + +/** + * Hook to use {@link DashboardService} in react components + */ +export function useDashboardService(): DashboardService { + const { + services: { + savedObjects: { client: savedObjectClient }, + kibanaVersion, + share: { urlGenerators }, + }, + } = useMlKibana(); + return useMemo( + () => + dashboardServiceProvider( + savedObjectClient, + kibanaVersion, + urlGenerators.getUrlGenerator(DASHBOARD_APP_URL_GENERATOR) + ), + [savedObjectClient, kibanaVersion] + ); +} diff --git a/x-pack/plugins/ml/public/application/services/http_service.ts b/x-pack/plugins/ml/public/application/services/http_service.ts index 7144411c2885d2..bd927dc0e30111 100644 --- a/x-pack/plugins/ml/public/application/services/http_service.ts +++ b/x-pack/plugins/ml/public/application/services/http_service.ts @@ -37,6 +37,8 @@ function getFetchOptions( /** * Function for making HTTP requests to Kibana's backend. * Wrapper for Kibana's HttpHandler. + * + * @deprecated use {@link HttpService} instead */ export async function http(options: HttpFetchOptionsWithPath): Promise { const { path, fetchOptions } = getFetchOptions(options); @@ -46,6 +48,8 @@ export async function http(options: HttpFetchOptionsWithPath): Promise { /** * Function for making HTTP requests to Kibana's backend which returns an Observable * with request cancellation support. + * + * @deprecated use {@link HttpService} instead */ export function http$(options: HttpFetchOptionsWithPath): Observable { const { path, fetchOptions } = getFetchOptions(options); @@ -55,7 +59,7 @@ export function http$(options: HttpFetchOptionsWithPath): Observable { /** * Creates an Observable from Kibana's HttpHandler. */ -export function fromHttpHandler(input: string, init?: RequestInit): Observable { +function fromHttpHandler(input: string, init?: RequestInit): Observable { return new Observable((subscriber) => { const controller = new AbortController(); const signal = controller.signal; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index fdaa3c2ffe79ec..6d32fca6a645c1 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -5,12 +5,13 @@ */ import { Observable } from 'rxjs'; -import { http, http$ } from '../http_service'; +import { HttpStart } from 'kibana/public'; +import { HttpService } from '../http_service'; import { annotations } from './annotations'; import { dataFrameAnalytics } from './data_frame_analytics'; import { filters } from './filters'; -import { results } from './results'; +import { resultsApiProvider } from './results'; import { jobs } from './jobs'; import { fileDatavisualizer } from './datavisualizer'; import { MlServerDefaults, MlServerLimits } from '../../../../common/types/ml_server_info'; @@ -28,6 +29,7 @@ import { import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types'; import { FieldRequestConfig } from '../../datavisualizer/index_based/common'; import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; +import { getHttp } from '../../util/dependency_cache'; export interface MlInfoResponse { defaults: MlServerDefaults; @@ -87,327 +89,330 @@ export function basePath() { return '/api/ml'; } -export const ml = { - getJobs(obj?: { jobId?: string }) { - const jobId = obj && obj.jobId ? `/${obj.jobId}` : ''; - return http({ - path: `${basePath()}/anomaly_detectors${jobId}`, - }); - }, - - getJobStats(obj: { jobId?: string }) { - const jobId = obj && obj.jobId ? `/${obj.jobId}` : ''; - return http({ - path: `${basePath()}/anomaly_detectors${jobId}/_stats`, - }); - }, - - addJob({ jobId, job }: { jobId: string; job: Job }) { - const body = JSON.stringify(job); - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}`, - method: 'PUT', - body, - }); - }, - - openJob({ jobId }: { jobId: string }) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/_open`, - method: 'POST', - }); - }, - - closeJob({ jobId }: { jobId: string }) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/_close`, - method: 'POST', - }); - }, - - forceCloseJob({ jobId }: { jobId: string }) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/_close?force=true`, - method: 'POST', - }); - }, - - deleteJob({ jobId }: { jobId: string }) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}`, - method: 'DELETE', - }); - }, - - forceDeleteJob({ jobId }: { jobId: string }) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}?force=true`, - method: 'DELETE', - }); - }, - - updateJob({ jobId, job }: { jobId: string; job: Job }) { - const body = JSON.stringify(job); - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/_update`, - method: 'POST', - body, - }); - }, - - estimateBucketSpan(obj: BucketSpanEstimatorData) { - const body = JSON.stringify(obj); - return http({ - path: `${basePath()}/validate/estimate_bucket_span`, - method: 'POST', - body, - }); - }, - - validateJob(payload: { - job: Job; - duration: { - start?: number; - end?: number; - }; - fields?: any[]; - }) { - const body = JSON.stringify(payload); - return http({ - path: `${basePath()}/validate/job`, - method: 'POST', - body, - }); - }, - - validateCardinality$(job: CombinedJob): Observable { - const body = JSON.stringify(job); - return http$({ - path: `${basePath()}/validate/cardinality`, - method: 'POST', - body, - }); - }, - - getDatafeeds(obj: { datafeedId: string }) { - const datafeedId = obj && obj.datafeedId ? `/${obj.datafeedId}` : ''; - return http({ - path: `${basePath()}/datafeeds${datafeedId}`, - }); - }, - - getDatafeedStats(obj: { datafeedId: string }) { - const datafeedId = obj && obj.datafeedId ? `/${obj.datafeedId}` : ''; - return http({ - path: `${basePath()}/datafeeds${datafeedId}/_stats`, - }); - }, - - addDatafeed({ datafeedId, datafeedConfig }: { datafeedId: string; datafeedConfig: Datafeed }) { - const body = JSON.stringify(datafeedConfig); - return http({ - path: `${basePath()}/datafeeds/${datafeedId}`, - method: 'PUT', - body, - }); - }, - - updateDatafeed({ datafeedId, datafeedConfig }: { datafeedId: string; datafeedConfig: Datafeed }) { - const body = JSON.stringify(datafeedConfig); - return http({ - path: `${basePath()}/datafeeds/${datafeedId}/_update`, - method: 'POST', - body, - }); - }, - - deleteDatafeed({ datafeedId }: { datafeedId: string }) { - return http({ - path: `${basePath()}/datafeeds/${datafeedId}`, - method: 'DELETE', - }); - }, - - forceDeleteDatafeed({ datafeedId }: { datafeedId: string }) { - return http({ - path: `${basePath()}/datafeeds/${datafeedId}?force=true`, - method: 'DELETE', - }); - }, - - startDatafeed({ datafeedId, start, end }: { datafeedId: string; start: number; end: number }) { - const body = JSON.stringify({ - ...(start !== undefined ? { start } : {}), - ...(end !== undefined ? { end } : {}), - }); - - return http({ - path: `${basePath()}/datafeeds/${datafeedId}/_start`, - method: 'POST', - body, - }); - }, - - stopDatafeed({ datafeedId }: { datafeedId: string }) { - return http({ - path: `${basePath()}/datafeeds/${datafeedId}/_stop`, - method: 'POST', - }); - }, - - forceStopDatafeed({ datafeedId }: { datafeedId: string }) { - return http({ - path: `${basePath()}/datafeeds/${datafeedId}/_stop?force=true`, - method: 'POST', - }); - }, - - datafeedPreview({ datafeedId }: { datafeedId: string }) { - return http({ - path: `${basePath()}/datafeeds/${datafeedId}/_preview`, - method: 'GET', - }); - }, - - validateDetector({ detector }: { detector: Detector }) { - const body = JSON.stringify(detector); - return http({ - path: `${basePath()}/anomaly_detectors/_validate/detector`, - method: 'POST', - body, - }); - }, - - forecast({ jobId, duration }: { jobId: string; duration?: string }) { - const body = JSON.stringify({ - ...(duration !== undefined ? { duration } : {}), - }); - - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/_forecast`, - method: 'POST', - body, - }); - }, - - overallBuckets({ - jobId, - topN, - bucketSpan, - start, - end, - }: { - jobId: string; - topN: string; - bucketSpan: string; - start: number; - end: number; - }) { - const body = JSON.stringify({ topN, bucketSpan, start, end }); - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/results/overall_buckets`, - method: 'POST', - body, - }); - }, - - hasPrivileges(obj: any) { - const body = JSON.stringify(obj); - return http({ - path: `${basePath()}/_has_privileges`, - method: 'POST', - body, - }); - }, - - checkMlCapabilities() { - return http({ - path: `${basePath()}/ml_capabilities`, - method: 'GET', - }); - }, - - checkManageMLCapabilities() { - return http({ - path: `${basePath()}/ml_capabilities`, - method: 'GET', - }); - }, - - getNotificationSettings() { - return http({ - path: `${basePath()}/notification_settings`, - method: 'GET', - }); - }, - - getFieldCaps({ index, fields }: { index: string; fields: string[] }) { - const body = JSON.stringify({ - ...(index !== undefined ? { index } : {}), - ...(fields !== undefined ? { fields } : {}), - }); - - return http({ - path: `${basePath()}/indices/field_caps`, - method: 'POST', - body, - }); - }, - - recognizeIndex({ indexPatternTitle }: { indexPatternTitle: string }) { - return http({ - path: `${basePath()}/modules/recognize/${indexPatternTitle}`, - method: 'GET', - }); - }, - - listDataRecognizerModules() { - return http({ - path: `${basePath()}/modules/get_module`, - method: 'GET', - }); - }, - - getDataRecognizerModule({ moduleId }: { moduleId: string }) { - return http({ - path: `${basePath()}/modules/get_module/${moduleId}`, - method: 'GET', - }); - }, - - dataRecognizerModuleJobsExist({ moduleId }: { moduleId: string }) { - return http({ - path: `${basePath()}/modules/jobs_exist/${moduleId}`, - method: 'GET', - }); +/** + * Temp solution to allow {@link ml} service to use http from + * the dependency_cache. + */ +const proxyHttpStart = new Proxy(({} as unknown) as HttpStart, { + get(obj, prop: keyof HttpStart) { + try { + return getHttp()[prop]; + } catch (e) { + // eslint-disable-next-line no-console + console.error(e); + } }, - - setupDataRecognizerConfig({ - moduleId, - prefix, - groups, - indexPatternName, - query, - useDedicatedIndex, - startDatafeed, - start, - end, - jobOverrides, - estimateModelMemory, - }: { - moduleId: string; - prefix?: string; - groups?: string[]; - indexPatternName?: string; - query?: any; - useDedicatedIndex?: boolean; - startDatafeed?: boolean; - start?: number; - end?: number; - jobOverrides?: Array>; - estimateModelMemory?: boolean; - }) { - const body = JSON.stringify({ +}); + +export type MlApiServices = ReturnType; + +export const ml = mlApiServicesProvider(new HttpService(proxyHttpStart)); + +export function mlApiServicesProvider(httpService: HttpService) { + const { http } = httpService; + return { + getJobs(obj?: { jobId?: string }) { + const jobId = obj && obj.jobId ? `/${obj.jobId}` : ''; + return httpService.http({ + path: `${basePath()}/anomaly_detectors${jobId}`, + }); + }, + + getJobStats(obj: { jobId?: string }) { + const jobId = obj && obj.jobId ? `/${obj.jobId}` : ''; + return httpService.http({ + path: `${basePath()}/anomaly_detectors${jobId}/_stats`, + }); + }, + + addJob({ jobId, job }: { jobId: string; job: Job }) { + const body = JSON.stringify(job); + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}`, + method: 'PUT', + body, + }); + }, + + openJob({ jobId }: { jobId: string }) { + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}/_open`, + method: 'POST', + }); + }, + + closeJob({ jobId }: { jobId: string }) { + return http({ + path: `${basePath()}/anomaly_detectors/${jobId}/_close`, + method: 'POST', + }); + }, + + forceCloseJob({ jobId }: { jobId: string }) { + return http({ + path: `${basePath()}/anomaly_detectors/${jobId}/_close?force=true`, + method: 'POST', + }); + }, + + deleteJob({ jobId }: { jobId: string }) { + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}`, + method: 'DELETE', + }); + }, + + forceDeleteJob({ jobId }: { jobId: string }) { + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}?force=true`, + method: 'DELETE', + }); + }, + + updateJob({ jobId, job }: { jobId: string; job: Job }) { + const body = JSON.stringify(job); + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}/_update`, + method: 'POST', + body, + }); + }, + + estimateBucketSpan(obj: BucketSpanEstimatorData) { + const body = JSON.stringify(obj); + return httpService.http({ + path: `${basePath()}/validate/estimate_bucket_span`, + method: 'POST', + body, + }); + }, + + validateJob(payload: { + job: Job; + duration: { + start?: number; + end?: number; + }; + fields?: any[]; + }) { + const body = JSON.stringify(payload); + return httpService.http({ + path: `${basePath()}/validate/job`, + method: 'POST', + body, + }); + }, + + validateCardinality$(job: CombinedJob): Observable { + const body = JSON.stringify(job); + return httpService.http$({ + path: `${basePath()}/validate/cardinality`, + method: 'POST', + body, + }); + }, + + getDatafeeds(obj: { datafeedId: string }) { + const datafeedId = obj && obj.datafeedId ? `/${obj.datafeedId}` : ''; + return httpService.http({ + path: `${basePath()}/datafeeds${datafeedId}`, + }); + }, + + getDatafeedStats(obj: { datafeedId: string }) { + const datafeedId = obj && obj.datafeedId ? `/${obj.datafeedId}` : ''; + return httpService.http({ + path: `${basePath()}/datafeeds${datafeedId}/_stats`, + }); + }, + + addDatafeed({ datafeedId, datafeedConfig }: { datafeedId: string; datafeedConfig: Datafeed }) { + const body = JSON.stringify(datafeedConfig); + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}`, + method: 'PUT', + body, + }); + }, + + updateDatafeed({ + datafeedId, + datafeedConfig, + }: { + datafeedId: string; + datafeedConfig: Datafeed; + }) { + const body = JSON.stringify(datafeedConfig); + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}/_update`, + method: 'POST', + body, + }); + }, + + deleteDatafeed({ datafeedId }: { datafeedId: string }) { + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}`, + method: 'DELETE', + }); + }, + + forceDeleteDatafeed({ datafeedId }: { datafeedId: string }) { + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}?force=true`, + method: 'DELETE', + }); + }, + + startDatafeed({ datafeedId, start, end }: { datafeedId: string; start: number; end: number }) { + const body = JSON.stringify({ + ...(start !== undefined ? { start } : {}), + ...(end !== undefined ? { end } : {}), + }); + + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}/_start`, + method: 'POST', + body, + }); + }, + + stopDatafeed({ datafeedId }: { datafeedId: string }) { + return http({ + path: `${basePath()}/datafeeds/${datafeedId}/_stop`, + method: 'POST', + }); + }, + + forceStopDatafeed({ datafeedId }: { datafeedId: string }) { + return http({ + path: `${basePath()}/datafeeds/${datafeedId}/_stop?force=true`, + method: 'POST', + }); + }, + + datafeedPreview({ datafeedId }: { datafeedId: string }) { + return httpService.http({ + path: `${basePath()}/datafeeds/${datafeedId}/_preview`, + method: 'GET', + }); + }, + + validateDetector({ detector }: { detector: Detector }) { + const body = JSON.stringify(detector); + return httpService.http({ + path: `${basePath()}/anomaly_detectors/_validate/detector`, + method: 'POST', + body, + }); + }, + + forecast({ jobId, duration }: { jobId: string; duration?: string }) { + const body = JSON.stringify({ + ...(duration !== undefined ? { duration } : {}), + }); + + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}/_forecast`, + method: 'POST', + body, + }); + }, + + overallBuckets({ + jobId, + topN, + bucketSpan, + start, + end, + }: { + jobId: string; + topN: string; + bucketSpan: string; + start: number; + end: number; + }) { + const body = JSON.stringify({ topN, bucketSpan, start, end }); + return httpService.http({ + path: `${basePath()}/anomaly_detectors/${jobId}/results/overall_buckets`, + method: 'POST', + body, + }); + }, + + hasPrivileges(obj: any) { + const body = JSON.stringify(obj); + return httpService.http({ + path: `${basePath()}/_has_privileges`, + method: 'POST', + body, + }); + }, + + checkMlCapabilities() { + return httpService.http({ + path: `${basePath()}/ml_capabilities`, + method: 'GET', + }); + }, + + checkManageMLCapabilities() { + return httpService.http({ + path: `${basePath()}/ml_capabilities`, + method: 'GET', + }); + }, + + getNotificationSettings() { + return httpService.http({ + path: `${basePath()}/notification_settings`, + method: 'GET', + }); + }, + + getFieldCaps({ index, fields }: { index: string; fields: string[] }) { + const body = JSON.stringify({ + ...(index !== undefined ? { index } : {}), + ...(fields !== undefined ? { fields } : {}), + }); + + return httpService.http({ + path: `${basePath()}/indices/field_caps`, + method: 'POST', + body, + }); + }, + + recognizeIndex({ indexPatternTitle }: { indexPatternTitle: string }) { + return httpService.http({ + path: `${basePath()}/modules/recognize/${indexPatternTitle}`, + method: 'GET', + }); + }, + + listDataRecognizerModules() { + return httpService.http({ + path: `${basePath()}/modules/get_module`, + method: 'GET', + }); + }, + + getDataRecognizerModule({ moduleId }: { moduleId: string }) { + return httpService.http({ + path: `${basePath()}/modules/get_module/${moduleId}`, + method: 'GET', + }); + }, + + dataRecognizerModuleJobsExist({ moduleId }: { moduleId: string }) { + return httpService.http({ + path: `${basePath()}/modules/jobs_exist/${moduleId}`, + method: 'GET', + }); + }, + + setupDataRecognizerConfig({ + moduleId, prefix, groups, indexPatternName, @@ -418,37 +423,41 @@ export const ml = { end, jobOverrides, estimateModelMemory, - }); - - return http({ - path: `${basePath()}/modules/setup/${moduleId}`, - method: 'POST', - body, - }); - }, - - getVisualizerFieldStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - interval, - fields, - maxExamples, - }: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - interval?: string; - fields?: FieldRequestConfig[]; - maxExamples?: number; - }) { - const body = JSON.stringify({ + }: { + moduleId: string; + prefix?: string; + groups?: string[]; + indexPatternName?: string; + query?: any; + useDedicatedIndex?: boolean; + startDatafeed?: boolean; + start?: number; + end?: number; + jobOverrides?: Array>; + estimateModelMemory?: boolean; + }) { + const body = JSON.stringify({ + prefix, + groups, + indexPatternName, + query, + useDedicatedIndex, + startDatafeed, + start, + end, + jobOverrides, + estimateModelMemory, + }); + + return httpService.http({ + path: `${basePath()}/modules/setup/${moduleId}`, + method: 'POST', + body, + }); + }, + + getVisualizerFieldStats({ + indexPatternTitle, query, timeFieldName, earliest, @@ -457,35 +466,37 @@ export const ml = { interval, fields, maxExamples, - }); - - return http({ - path: `${basePath()}/data_visualizer/get_field_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); - }, - - getVisualizerOverallStats({ - indexPatternTitle, - query, - timeFieldName, - earliest, - latest, - samplerShardSize, - aggregatableFields, - nonAggregatableFields, - }: { - indexPatternTitle: string; - query: any; - timeFieldName?: string; - earliest?: number; - latest?: number; - samplerShardSize?: number; - aggregatableFields: string[]; - nonAggregatableFields: string[]; - }) { - const body = JSON.stringify({ + }: { + indexPatternTitle: string; + query: any; + timeFieldName?: string; + earliest?: number; + latest?: number; + samplerShardSize?: number; + interval?: string; + fields?: FieldRequestConfig[]; + maxExamples?: number; + }) { + const body = JSON.stringify({ + query, + timeFieldName, + earliest, + latest, + samplerShardSize, + interval, + fields, + maxExamples, + }); + + return httpService.http({ + path: `${basePath()}/data_visualizer/get_field_stats/${indexPatternTitle}`, + method: 'POST', + body, + }); + }, + + getVisualizerOverallStats({ + indexPatternTitle, query, timeFieldName, earliest, @@ -493,204 +504,230 @@ export const ml = { samplerShardSize, aggregatableFields, nonAggregatableFields, - }); - - return http({ - path: `${basePath()}/data_visualizer/get_overall_stats/${indexPatternTitle}`, - method: 'POST', - body, - }); - }, - - /** - * Gets a list of calendars - * @param obj - * @returns {Promise} - */ - calendars(obj?: { calendarId?: CalendarId; calendarIds?: CalendarId[] }) { - const { calendarId, calendarIds } = obj || {}; - let calendarIdsPathComponent = ''; - if (calendarId) { - calendarIdsPathComponent = `/${calendarId}`; - } else if (calendarIds) { - calendarIdsPathComponent = `/${calendarIds.join(',')}`; - } - return http({ - path: `${basePath()}/calendars${calendarIdsPathComponent}`, - method: 'GET', - }); - }, - - addCalendar(obj: Calendar) { - const body = JSON.stringify(obj); - return http({ - path: `${basePath()}/calendars`, - method: 'PUT', - body, - }); - }, - - updateCalendar(obj: UpdateCalendar) { - const calendarId = obj && obj.calendarId ? `/${obj.calendarId}` : ''; - const body = JSON.stringify(obj); - return http({ - path: `${basePath()}/calendars${calendarId}`, - method: 'PUT', - body, - }); - }, - - deleteCalendar({ calendarId }: { calendarId?: string }) { - return http({ - path: `${basePath()}/calendars/${calendarId}`, - method: 'DELETE', - }); - }, - - mlNodeCount() { - return http<{ count: number }>({ - path: `${basePath()}/ml_node_count`, - method: 'GET', - }); - }, - - mlInfo() { - return http({ - path: `${basePath()}/info`, - method: 'GET', - }); - }, - - calculateModelMemoryLimit$({ - analysisConfig, - indexPattern, - query, - timeFieldName, - earliestMs, - latestMs, - }: { - analysisConfig: AnalysisConfig; - indexPattern: string; - query: any; - timeFieldName: string; - earliestMs: number; - latestMs: number; - }) { - const body = JSON.stringify({ + }: { + indexPatternTitle: string; + query: any; + timeFieldName?: string; + earliest?: number; + latest?: number; + samplerShardSize?: number; + aggregatableFields: string[]; + nonAggregatableFields: string[]; + }) { + const body = JSON.stringify({ + query, + timeFieldName, + earliest, + latest, + samplerShardSize, + aggregatableFields, + nonAggregatableFields, + }); + + return httpService.http({ + path: `${basePath()}/data_visualizer/get_overall_stats/${indexPatternTitle}`, + method: 'POST', + body, + }); + }, + + /** + * Gets a list of calendars + * @param obj + * @returns {Promise} + */ + calendars(obj?: { calendarId?: CalendarId; calendarIds?: CalendarId[] }) { + const { calendarId, calendarIds } = obj || {}; + let calendarIdsPathComponent = ''; + if (calendarId) { + calendarIdsPathComponent = `/${calendarId}`; + } else if (calendarIds) { + calendarIdsPathComponent = `/${calendarIds.join(',')}`; + } + return httpService.http({ + path: `${basePath()}/calendars${calendarIdsPathComponent}`, + method: 'GET', + }); + }, + + addCalendar(obj: Calendar) { + const body = JSON.stringify(obj); + return httpService.http({ + path: `${basePath()}/calendars`, + method: 'PUT', + body, + }); + }, + + updateCalendar(obj: UpdateCalendar) { + const calendarId = obj && obj.calendarId ? `/${obj.calendarId}` : ''; + const body = JSON.stringify(obj); + return httpService.http({ + path: `${basePath()}/calendars${calendarId}`, + method: 'PUT', + body, + }); + }, + + deleteCalendar({ calendarId }: { calendarId?: string }) { + return httpService.http({ + path: `${basePath()}/calendars/${calendarId}`, + method: 'DELETE', + }); + }, + + mlNodeCount() { + return httpService.http<{ count: number }>({ + path: `${basePath()}/ml_node_count`, + method: 'GET', + }); + }, + + mlInfo() { + return httpService.http({ + path: `${basePath()}/info`, + method: 'GET', + }); + }, + + calculateModelMemoryLimit$({ analysisConfig, indexPattern, query, timeFieldName, earliestMs, latestMs, - }); - - return http$<{ modelMemoryLimit: string }>({ - path: `${basePath()}/validate/calculate_model_memory_limit`, - method: 'POST', - body, - }); - }, - - getCardinalityOfFields({ - index, - fieldNames, - query, - timeFieldName, - earliestMs, - latestMs, - }: { - index: string; - fieldNames: string[]; - query: any; - timeFieldName: string; - earliestMs: number; - latestMs: number; - }) { - const body = JSON.stringify({ index, fieldNames, query, timeFieldName, earliestMs, latestMs }); - - return http({ - path: `${basePath()}/fields_service/field_cardinality`, - method: 'POST', - body, - }); - }, - - getTimeFieldRange({ - index, - timeFieldName, - query, - }: { - index: string; - timeFieldName?: string; - query: any; - }) { - const body = JSON.stringify({ index, timeFieldName, query }); - - return http({ - path: `${basePath()}/fields_service/time_field_range`, - method: 'POST', - body, - }); - }, - - esSearch(obj: any) { - const body = JSON.stringify(obj); - return http({ - path: `${basePath()}/es_search`, - method: 'POST', - body, - }); - }, - - esSearch$(obj: any) { - const body = JSON.stringify(obj); - return http$({ - path: `${basePath()}/es_search`, - method: 'POST', - body, - }); - }, - - getIndices() { - const tempBasePath = '/api'; - return http>({ - path: `${tempBasePath}/index_management/indices`, - method: 'GET', - }); - }, - - getModelSnapshots(jobId: string, snapshotId?: string) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots${ - snapshotId !== undefined ? `/${snapshotId}` : '' - }`, - }); - }, - - updateModelSnapshot( - jobId: string, - snapshotId: string, - body: { description?: string; retain?: boolean } - ) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots/${snapshotId}/_update`, - method: 'POST', - body: JSON.stringify(body), - }); - }, - - deleteModelSnapshot(jobId: string, snapshotId: string) { - return http({ - path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots/${snapshotId}`, - method: 'DELETE', - }); - }, - - annotations, - dataFrameAnalytics, - filters, - results, - jobs, - fileDatavisualizer, -}; + }: { + analysisConfig: AnalysisConfig; + indexPattern: string; + query: any; + timeFieldName: string; + earliestMs: number; + latestMs: number; + }) { + const body = JSON.stringify({ + analysisConfig, + indexPattern, + query, + timeFieldName, + earliestMs, + latestMs, + }); + + return httpService.http$<{ modelMemoryLimit: string }>({ + path: `${basePath()}/validate/calculate_model_memory_limit`, + method: 'POST', + body, + }); + }, + + getCardinalityOfFields({ + index, + fieldNames, + query, + timeFieldName, + earliestMs, + latestMs, + }: { + index: string; + fieldNames: string[]; + query: any; + timeFieldName: string; + earliestMs: number; + latestMs: number; + }) { + const body = JSON.stringify({ + index, + fieldNames, + query, + timeFieldName, + earliestMs, + latestMs, + }); + + return httpService.http({ + path: `${basePath()}/fields_service/field_cardinality`, + method: 'POST', + body, + }); + }, + + getTimeFieldRange({ + index, + timeFieldName, + query, + }: { + index: string; + timeFieldName?: string; + query: any; + }) { + const body = JSON.stringify({ index, timeFieldName, query }); + + return httpService.http({ + path: `${basePath()}/fields_service/time_field_range`, + method: 'POST', + body, + }); + }, + + esSearch(obj: any) { + const body = JSON.stringify(obj); + return httpService.http({ + path: `${basePath()}/es_search`, + method: 'POST', + body, + }); + }, + + esSearch$(obj: any) { + const body = JSON.stringify(obj); + return httpService.http$({ + path: `${basePath()}/es_search`, + method: 'POST', + body, + }); + }, + + getIndices() { + const tempBasePath = '/api'; + return httpService.http>({ + path: `${tempBasePath}/index_management/indices`, + method: 'GET', + }); + }, + + getModelSnapshots(jobId: string, snapshotId?: string) { + return http({ + path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots${ + snapshotId !== undefined ? `/${snapshotId}` : '' + }`, + }); + }, + + updateModelSnapshot( + jobId: string, + snapshotId: string, + body: { description?: string; retain?: boolean } + ) { + return http({ + path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots/${snapshotId}/_update`, + method: 'POST', + body: JSON.stringify(body), + }); + }, + + deleteModelSnapshot(jobId: string, snapshotId: string) { + return http({ + path: `${basePath()}/anomaly_detectors/${jobId}/model_snapshots/${snapshotId}`, + method: 'DELETE', + }); + }, + + annotations, + dataFrameAnalytics, + filters, + results: resultsApiProvider(httpService), + jobs, + fileDatavisualizer, + }; +} diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts index 830e6fab4163a3..521fd306847eba 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/results.ts @@ -5,14 +5,14 @@ */ // Service for obtaining data for the ML Results dashboards. -import { http, http$ } from '../http_service'; +import { HttpService } from '../http_service'; import { basePath } from './index'; import { JobId } from '../../../../common/types/anomaly_detection_jobs'; import { PartitionFieldsDefinition } from '../results_service/result_service_rx'; -export const results = { +export const resultsApiProvider = (httpService: HttpService) => ({ getAnomaliesTableData( jobIds: string[], criteriaFields: string[], @@ -40,7 +40,7 @@ export const results = { influencersFilterQuery, }); - return http$({ + return httpService.http$({ path: `${basePath()}/results/anomalies_table_data`, method: 'POST', body, @@ -53,7 +53,7 @@ export const results = { earliestMs, latestMs, }); - return http({ + return httpService.http({ path: `${basePath()}/results/max_anomaly_score`, method: 'POST', body, @@ -62,7 +62,7 @@ export const results = { getCategoryDefinition(jobId: string, categoryId: string) { const body = JSON.stringify({ jobId, categoryId }); - return http({ + return httpService.http({ path: `${basePath()}/results/category_definition`, method: 'POST', body, @@ -75,7 +75,7 @@ export const results = { categoryIds, maxExamples, }); - return http({ + return httpService.http({ path: `${basePath()}/results/category_examples`, method: 'POST', body, @@ -90,10 +90,10 @@ export const results = { latestMs: number ) { const body = JSON.stringify({ jobId, searchTerm, criteriaFields, earliestMs, latestMs }); - return http$({ + return httpService.http$({ path: `${basePath()}/results/partition_fields_values`, method: 'POST', body, }); }, -}; +}); diff --git a/x-pack/plugins/ml/public/application/services/results_service/index.ts b/x-pack/plugins/ml/public/application/services/results_service/index.ts index cc02248f4d5a9a..6c508422e70632 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/index.ts @@ -4,47 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - getMetricData, - getModelPlotOutput, - getRecordsForCriteria, - getScheduledEventsByBucket, - fetchPartitionFieldsValues, -} from './result_service_rx'; -import { - getEventDistributionData, - getEventRateData, - getInfluencerValueMaxScoreByTime, - getOverallBucketScores, - getRecordInfluencers, - getRecordMaxScoreByTime, - getRecords, - getRecordsForDetector, - getRecordsForInfluencer, - getScoresByBucket, - getTopInfluencers, - getTopInfluencerValues, -} from './results_service'; - -export const mlResultsService = { - getScoresByBucket, - getScheduledEventsByBucket, - getTopInfluencers, - getTopInfluencerValues, - getOverallBucketScores, - getInfluencerValueMaxScoreByTime, - getRecordInfluencers, - getRecordsForInfluencer, - getRecordsForDetector, - getRecords, - getRecordsForCriteria, - getMetricData, - getEventRateData, - getEventDistributionData, - getModelPlotOutput, - getRecordMaxScoreByTime, - fetchPartitionFieldsValues, -}; +import { resultsServiceRxProvider } from './result_service_rx'; +import { resultsServiceProvider } from './results_service'; +import { ml, MlApiServices } from '../ml_api_service'; export type MlResultsService = typeof mlResultsService; @@ -57,3 +19,12 @@ export interface CriteriaField { fieldName: string; fieldValue: any; } + +export const mlResultsService = mlResultsServiceProvider(ml); + +export function mlResultsServiceProvider(mlApiServices: MlApiServices) { + return { + ...resultsServiceProvider(mlApiServices), + ...resultsServiceRxProvider(mlApiServices), + }; +} diff --git a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts index a21d0caaedd339..1bcbd8dbcdd639 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/result_service_rx.ts @@ -17,7 +17,7 @@ import _ from 'lodash'; import { Dictionary } from '../../../../common/types/common'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; import { JobId } from '../../../../common/types/anomaly_detection_jobs'; -import { ml } from '../ml_api_service'; +import { MlApiServices } from '../ml_api_service'; import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; import { CriteriaField } from './index'; @@ -46,524 +46,528 @@ export type PartitionFieldsDefinition = { [field in FieldTypes]: FieldDefinition; }; -export function getMetricData( - index: string, - entityFields: any[], - query: object | undefined, - metricFunction: string, // ES aggregation name - metricFieldName: string, - timeFieldName: string, - earliestMs: number, - latestMs: number, - interval: string -): Observable { - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, entity fields, - // plus any additional supplied query. - const shouldCriteria: object[] = []; - const mustCriteria: object[] = [ - { - range: { - [timeFieldName]: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - ...(query ? [query] : []), - ]; - - entityFields.forEach((entity) => { - if (entity.fieldValue.length !== 0) { - mustCriteria.push({ - term: { - [entity.fieldName]: entity.fieldValue, - }, - }); - } else { - // Add special handling for blank entity field values, checking for either - // an empty string or the field not existing. - shouldCriteria.push({ - bool: { - must: [ - { - term: { - [entity.fieldName]: '', - }, - }, - ], - }, - }); - shouldCriteria.push({ - bool: { - must_not: [ - { - exists: { field: entity.fieldName }, - }, - ], - }, - }); - } - }); - - const body: any = { - query: { - bool: { - must: mustCriteria, - }, - }, - size: 0, - _source: { - excludes: [], - }, - aggs: { - byTime: { - date_histogram: { - field: timeFieldName, - interval, - min_doc_count: 0, - }, - }, - }, - }; - - if (shouldCriteria.length > 0) { - body.query.bool.should = shouldCriteria; - body.query.bool.minimum_should_match = shouldCriteria.length / 2; - } - - if (metricFieldName !== undefined && metricFieldName !== '') { - body.aggs.byTime.aggs = {}; - - const metricAgg: any = { - [metricFunction]: { - field: metricFieldName, - }, - }; - - if (metricFunction === 'percentiles') { - metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; - } - body.aggs.byTime.aggs.metric = metricAgg; - } - - return ml.esSearch$({ index, body }).pipe( - map((resp: any) => { - const obj: MetricData = { success: true, results: {} }; - const dataByTime = resp?.aggregations?.byTime?.buckets ?? []; - dataByTime.forEach((dataForTime: any) => { - if (metricFunction === 'count') { - obj.results[dataForTime.key] = dataForTime.doc_count; - } else { - const value = dataForTime?.metric?.value; - const values = dataForTime?.metric?.values; - if (dataForTime.doc_count === 0) { - obj.results[dataForTime.key] = null; - } else if (value !== undefined) { - obj.results[dataForTime.key] = value; - } else if (values !== undefined) { - // Percentiles agg currently returns NaN rather than null when none of the docs in the - // bucket contain the field used in the aggregation - // (see elasticsearch issue https://github.com/elastic/elasticsearch/issues/29066). - // Store as null, so values can be handled in the same manner downstream as other aggs - // (min, mean, max) which return null. - const medianValues = values[ML_MEDIAN_PERCENTS]; - obj.results[dataForTime.key] = !isNaN(medianValues) ? medianValues : null; - } else { - obj.results[dataForTime.key] = null; - } - } - }); - - return obj; - }) - ); -} - export interface ModelPlotOutput extends ResultResponse { results: Record; } -export function getModelPlotOutput( - jobId: string, - detectorIndex: number, - criteriaFields: any[], - earliestMs: number, - latestMs: number, - interval: string, - aggType?: { min: any; max: any } -): Observable { - const obj: ModelPlotOutput = { - success: true, - results: {}, - }; +export interface RecordsForCriteria extends ResultResponse { + records: any[]; +} - // if an aggType object has been passed in, use it. - // otherwise default to min and max aggs for the upper and lower bounds - const modelAggs = - aggType === undefined - ? { max: 'max', min: 'min' } - : { - max: aggType.max, - min: aggType.min, - }; +export interface ScheduledEventsByBucket extends ResultResponse { + events: Record; +} - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the job ID and time range. - const mustCriteria: object[] = [ - { - term: { job_id: jobId }, - }, - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - ]; - - // Add in term queries for each of the specified criteria. - _.each(criteriaFields, (criteria) => { - mustCriteria.push({ - term: { - [criteria.fieldName]: criteria.fieldValue, - }, - }); - }); - - // Add criteria for the detector index. Results from jobs created before 6.1 will not - // contain a detector_index field, so use a should criteria with a 'not exists' check. - const shouldCriteria = [ - { - term: { detector_index: detectorIndex }, - }, - { - bool: { - must_not: [ - { - exists: { field: 'detector_index' }, +export function resultsServiceRxProvider(mlApiServices: MlApiServices) { + return { + getMetricData( + index: string, + entityFields: any[], + query: object | undefined, + metricFunction: string, // ES aggregation name + metricFieldName: string, + timeFieldName: string, + earliestMs: number, + latestMs: number, + interval: string + ): Observable { + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, entity fields, + // plus any additional supplied query. + const shouldCriteria: object[] = []; + const mustCriteria: object[] = [ + { + range: { + [timeFieldName]: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, }, - ], - }, - }, - ]; + }, + ...(query ? [query] : []), + ]; + + entityFields.forEach((entity) => { + if (entity.fieldValue.length !== 0) { + mustCriteria.push({ + term: { + [entity.fieldName]: entity.fieldValue, + }, + }); + } else { + // Add special handling for blank entity field values, checking for either + // an empty string or the field not existing. + shouldCriteria.push({ + bool: { + must: [ + { + term: { + [entity.fieldName]: '', + }, + }, + ], + }, + }); + shouldCriteria.push({ + bool: { + must_not: [ + { + exists: { field: entity.fieldName }, + }, + ], + }, + }); + } + }); - return ml - .esSearch$({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { + const body: any = { query: { bool: { - filter: [ - { - query_string: { - query: 'result_type:model_plot', - analyze_wildcard: true, - }, - }, - { - bool: { - must: mustCriteria, - should: shouldCriteria, - minimum_should_match: 1, - }, - }, - ], + must: mustCriteria, }, }, + size: 0, + _source: { + excludes: [], + }, aggs: { - times: { + byTime: { date_histogram: { - field: 'timestamp', + field: timeFieldName, interval, min_doc_count: 0, }, - aggs: { - actual: { - avg: { - field: 'actual', - }, - }, - modelUpper: { - [modelAggs.max]: { - field: 'model_upper', - }, - }, - modelLower: { - [modelAggs.min]: { - field: 'model_lower', - }, - }, - }, }, }, - }, - }) - .pipe( - map((resp) => { - const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []); - _.each(aggregationsByTime, (dataForTime: any) => { - const time = dataForTime.key; - const modelUpper: number | undefined = _.get(dataForTime, ['modelUpper', 'value']); - const modelLower: number | undefined = _.get(dataForTime, ['modelLower', 'value']); - const actual = _.get(dataForTime, ['actual', 'value']); - - obj.results[time] = { - actual, - modelUpper: - modelUpper === undefined || isFinite(modelUpper) === false ? null : modelUpper, - modelLower: - modelLower === undefined || isFinite(modelLower) === false ? null : modelLower, - }; - }); + }; - return obj; - }) - ); -} + if (shouldCriteria.length > 0) { + body.query.bool.should = shouldCriteria; + body.query.bool.minimum_should_match = shouldCriteria.length / 2; + } -export interface RecordsForCriteria extends ResultResponse { - records: any[]; -} + if (metricFieldName !== undefined && metricFieldName !== '') { + body.aggs.byTime.aggs = {}; -// Queries Elasticsearch to obtain the record level results matching the given criteria, -// for the specified job(s), time range, and record score threshold. -// criteriaFields parameter must be an array, with each object in the array having 'fieldName' -// 'fieldValue' properties. -// Pass an empty array or ['*'] to search over all job IDs. -export function getRecordsForCriteria( - jobIds: string[] | undefined, - criteriaFields: CriteriaField[], - threshold: any, - earliestMs: number, - latestMs: number, - maxResults: number | undefined -): Observable { - const obj: RecordsForCriteria = { success: true, records: [] }; - - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, record score, plus any specified job IDs. - const boolCriteria: any[] = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - { - range: { - record_score: { - gte: threshold, - }, - }, - }, - ]; + const metricAgg: any = { + [metricFunction]: { + field: metricFieldName, + }, + }; - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; + if (metricFunction === 'percentiles') { + metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; + } + body.aggs.byTime.aggs.metric = metricAgg; } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - // Add in term queries for each of the specified criteria. - _.each(criteriaFields, (criteria) => { - boolCriteria.push({ - term: { - [criteria.fieldName]: criteria.fieldValue, - }, - }); - }); - - return ml - .esSearch$({ - index: ML_RESULTS_INDEX_PATTERN, - rest_total_hits_as_int: true, - size: maxResults !== undefined ? maxResults : 100, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: false, - }, - }, - { - bool: { - must: boolCriteria, - }, - }, - ], - }, - }, - sort: [{ record_score: { order: 'desc' } }], - }, - }) - .pipe( - map((resp) => { - if (resp.hits.total !== 0) { - _.each(resp.hits.hits, (hit: any) => { - obj.records.push(hit._source); + + return mlApiServices.esSearch$({ index, body }).pipe( + map((resp: any) => { + const obj: MetricData = { success: true, results: {} }; + const dataByTime = resp?.aggregations?.byTime?.buckets ?? []; + dataByTime.forEach((dataForTime: any) => { + if (metricFunction === 'count') { + obj.results[dataForTime.key] = dataForTime.doc_count; + } else { + const value = dataForTime?.metric?.value; + const values = dataForTime?.metric?.values; + if (dataForTime.doc_count === 0) { + obj.results[dataForTime.key] = null; + } else if (value !== undefined) { + obj.results[dataForTime.key] = value; + } else if (values !== undefined) { + // Percentiles agg currently returns NaN rather than null when none of the docs in the + // bucket contain the field used in the aggregation + // (see elasticsearch issue https://github.com/elastic/elasticsearch/issues/29066). + // Store as null, so values can be handled in the same manner downstream as other aggs + // (min, mean, max) which return null. + const medianValues = values[ML_MEDIAN_PERCENTS]; + obj.results[dataForTime.key] = !isNaN(medianValues) ? medianValues : null; + } else { + obj.results[dataForTime.key] = null; + } + } }); - } - return obj; - }) - ); -} -export interface ScheduledEventsByBucket extends ResultResponse { - events: Record; -} + return obj; + }) + ); + }, -// Obtains a list of scheduled events by job ID and time. -// Pass an empty array or ['*'] to search over all job IDs. -// Returned response contains a events property, which will only -// contains keys for jobs which have scheduled events for the specified time range. -export function getScheduledEventsByBucket( - jobIds: string[] | undefined, - earliestMs: number, - latestMs: number, - interval: string, - maxJobs: number, - maxEvents: number -): Observable { - const obj: ScheduledEventsByBucket = { - success: true, - events: {}, - }; + getModelPlotOutput( + jobId: string, + detectorIndex: number, + criteriaFields: any[], + earliestMs: number, + latestMs: number, + interval: string, + aggType?: { min: any; max: any } + ): Observable { + const obj: ModelPlotOutput = { + success: true, + results: {}, + }; + + // if an aggType object has been passed in, use it. + // otherwise default to min and max aggs for the upper and lower bounds + const modelAggs = + aggType === undefined + ? { max: 'max', min: 'min' } + : { + max: aggType.max, + min: aggType.min, + }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the job ID and time range. + const mustCriteria: object[] = [ + { + term: { job_id: jobId }, + }, + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }, + ]; - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the time range plus any specified job IDs. - const boolCriteria: any[] = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', + // Add in term queries for each of the specified criteria. + _.each(criteriaFields, (criteria) => { + mustCriteria.push({ + term: { + [criteria.fieldName]: criteria.fieldValue, + }, + }); + }); + + // Add criteria for the detector index. Results from jobs created before 6.1 will not + // contain a detector_index field, so use a should criteria with a 'not exists' check. + const shouldCriteria = [ + { + term: { detector_index: detectorIndex }, }, - }, - }, - { - exists: { field: 'scheduled_events' }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - jobIdFilterStr += `${i > 0 ? ' OR ' : ''}job_id:${jobId}`; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - return ml - .esSearch$({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { + { bool: { - filter: [ + must_not: [ { - query_string: { - query: 'result_type:bucket', - analyze_wildcard: false, - }, - }, - { - bool: { - must: boolCriteria, - }, + exists: { field: 'detector_index' }, }, ], }, }, - aggs: { - jobs: { - terms: { - field: 'job_id', - min_doc_count: 1, - size: maxJobs, + ]; + + return mlApiServices + .esSearch$({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:model_plot', + analyze_wildcard: true, + }, + }, + { + bool: { + must: mustCriteria, + should: shouldCriteria, + minimum_should_match: 1, + }, + }, + ], + }, }, aggs: { times: { date_histogram: { field: 'timestamp', interval, - min_doc_count: 1, + min_doc_count: 0, }, aggs: { - events: { - terms: { - field: 'scheduled_events', - size: maxEvents, + actual: { + avg: { + field: 'actual', + }, + }, + modelUpper: { + [modelAggs.max]: { + field: 'model_upper', + }, + }, + modelLower: { + [modelAggs.min]: { + field: 'model_lower', }, }, }, }, }, }, + }) + .pipe( + map((resp) => { + const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []); + _.each(aggregationsByTime, (dataForTime: any) => { + const time = dataForTime.key; + const modelUpper: number | undefined = _.get(dataForTime, ['modelUpper', 'value']); + const modelLower: number | undefined = _.get(dataForTime, ['modelLower', 'value']); + const actual = _.get(dataForTime, ['actual', 'value']); + + obj.results[time] = { + actual, + modelUpper: + modelUpper === undefined || isFinite(modelUpper) === false ? null : modelUpper, + modelLower: + modelLower === undefined || isFinite(modelLower) === false ? null : modelLower, + }; + }); + + return obj; + }) + ); + }, + + // Queries Elasticsearch to obtain the record level results matching the given criteria, + // for the specified job(s), time range, and record score threshold. + // criteriaFields parameter must be an array, with each object in the array having 'fieldName' + // 'fieldValue' properties. + // Pass an empty array or ['*'] to search over all job IDs. + getRecordsForCriteria( + jobIds: string[] | undefined, + criteriaFields: CriteriaField[], + threshold: any, + earliestMs: number, + latestMs: number, + maxResults: number | undefined + ): Observable { + const obj: RecordsForCriteria = { success: true, records: [] }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, record score, plus any specified job IDs. + const boolCriteria: any[] = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, }, - }, - }) - .pipe( - map((resp) => { - const dataByJobId = _.get(resp, ['aggregations', 'jobs', 'buckets'], []); - _.each(dataByJobId, (dataForJob: any) => { - const jobId: string = dataForJob.key; - const resultsForTime: Record = {}; - const dataByTime = _.get(dataForJob, ['times', 'buckets'], []); - _.each(dataByTime, (dataForTime: any) => { - const time: string = dataForTime.key; - const events: object[] = _.get(dataForTime, ['events', 'buckets']); - resultsForTime[time] = _.map(events, 'key'); - }); - obj.events[jobId] = resultsForTime; + { + range: { + record_score: { + gte: threshold, + }, + }, + }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, }); + } - return obj; - }) - ); -} + // Add in term queries for each of the specified criteria. + _.each(criteriaFields, (criteria) => { + boolCriteria.push({ + term: { + [criteria.fieldName]: criteria.fieldValue, + }, + }); + }); -export function fetchPartitionFieldsValues( - jobId: JobId, - searchTerm: Dictionary, - criteriaFields: Array<{ fieldName: string; fieldValue: any }>, - earliestMs: number, - latestMs: number -) { - return ml.results.fetchPartitionFieldsValues( - jobId, - searchTerm, - criteriaFields, - earliestMs, - latestMs - ); + return mlApiServices + .esSearch$({ + index: ML_RESULTS_INDEX_PATTERN, + rest_total_hits_as_int: true, + size: maxResults !== undefined ? maxResults : 100, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], + }, + }, + sort: [{ record_score: { order: 'desc' } }], + }, + }) + .pipe( + map((resp) => { + if (resp.hits.total !== 0) { + _.each(resp.hits.hits, (hit: any) => { + obj.records.push(hit._source); + }); + } + return obj; + }) + ); + }, + + // Obtains a list of scheduled events by job ID and time. + // Pass an empty array or ['*'] to search over all job IDs. + // Returned response contains a events property, which will only + // contains keys for jobs which have scheduled events for the specified time range. + getScheduledEventsByBucket( + jobIds: string[] | undefined, + earliestMs: number, + latestMs: number, + interval: string, + maxJobs: number, + maxEvents: number + ): Observable { + const obj: ScheduledEventsByBucket = { + success: true, + events: {}, + }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the time range plus any specified job IDs. + const boolCriteria: any[] = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }, + { + exists: { field: 'scheduled_events' }, + }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + jobIdFilterStr += `${i > 0 ? ' OR ' : ''}job_id:${jobId}`; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, + }); + } + + return mlApiServices + .esSearch$({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:bucket', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], + }, + }, + aggs: { + jobs: { + terms: { + field: 'job_id', + min_doc_count: 1, + size: maxJobs, + }, + aggs: { + times: { + date_histogram: { + field: 'timestamp', + interval, + min_doc_count: 1, + }, + aggs: { + events: { + terms: { + field: 'scheduled_events', + size: maxEvents, + }, + }, + }, + }, + }, + }, + }, + }, + }) + .pipe( + map((resp) => { + const dataByJobId = _.get(resp, ['aggregations', 'jobs', 'buckets'], []); + _.each(dataByJobId, (dataForJob: any) => { + const jobId: string = dataForJob.key; + const resultsForTime: Record = {}; + const dataByTime = _.get(dataForJob, ['times', 'buckets'], []); + _.each(dataByTime, (dataForTime: any) => { + const time: string = dataForTime.key; + const events: object[] = _.get(dataForTime, ['events', 'buckets']); + resultsForTime[time] = _.map(events, 'key'); + }); + obj.events[jobId] = resultsForTime; + }); + + return obj; + }) + ); + }, + + fetchPartitionFieldsValues( + jobId: JobId, + searchTerm: Dictionary, + criteriaFields: Array<{ fieldName: string; fieldValue: any }>, + earliestMs: number, + latestMs: number + ) { + return mlApiServices.results.fetchPartitionFieldsValues( + jobId, + searchTerm, + criteriaFields, + earliestMs, + latestMs + ); + }, + }; } diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts index 4af08994432bdb..1b2c01ab73fcef 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.d.ts @@ -4,43 +4,49 @@ * you may not use this file except in compliance with the Elastic License. */ -export function getScoresByBucket( - jobIds: string[], - earliestMs: number, - latestMs: number, - interval: string | number, - maxResults: number -): Promise; -export function getTopInfluencers(): Promise; -export function getTopInfluencerValues(): Promise; -export function getOverallBucketScores( - jobIds: any, - topN: any, - earliestMs: any, - latestMs: any, - interval?: any -): Promise; -export function getInfluencerValueMaxScoreByTime( - jobIds: string[], - influencerFieldName: string, - influencerFieldValues: string[], - earliestMs: number, - latestMs: number, - interval: string, - maxResults: number, - influencersFilterQuery: any -): Promise; -export function getRecordInfluencers(): Promise; -export function getRecordsForInfluencer(): Promise; -export function getRecordsForDetector(): Promise; -export function getRecords(): Promise; -export function getEventRateData( - index: string, - query: any, - timeFieldName: string, - earliestMs: number, - latestMs: number, - interval: string | number -): Promise; -export function getEventDistributionData(): Promise; -export function getRecordMaxScoreByTime(): Promise; +import { MlApiServices } from '../ml_api_service'; + +export function resultsServiceProvider( + mlApiServices: MlApiServices +): { + getScoresByBucket( + jobIds: string[], + earliestMs: number, + latestMs: number, + interval: string | number, + maxResults: number + ): Promise; + getTopInfluencers(): Promise; + getTopInfluencerValues(): Promise; + getOverallBucketScores( + jobIds: any, + topN: any, + earliestMs: any, + latestMs: any, + interval?: any + ): Promise; + getInfluencerValueMaxScoreByTime( + jobIds: string[], + influencerFieldName: string, + influencerFieldValues: string[], + earliestMs: number, + latestMs: number, + interval: string, + maxResults: number, + influencersFilterQuery: any + ): Promise; + getRecordInfluencers(): Promise; + getRecordsForInfluencer(): Promise; + getRecordsForDetector(): Promise; + getRecords(): Promise; + getEventRateData( + index: string, + query: any, + timeFieldName: string, + earliestMs: number, + latestMs: number, + interval: string | number + ): Promise; + getEventDistributionData(): Promise; + getRecordMaxScoreByTime(): Promise; +}; diff --git a/x-pack/plugins/ml/public/application/services/results_service/results_service.js b/x-pack/plugins/ml/public/application/services/results_service/results_service.js index 4fccc4d789370c..9e3fed189b6f48 100644 --- a/x-pack/plugins/ml/public/application/services/results_service/results_service.js +++ b/x-pack/plugins/ml/public/application/services/results_service/results_service.js @@ -4,1322 +4,1331 @@ * you may not use this file except in compliance with the Elastic License. */ -// Service for carrying out Elasticsearch queries to obtain data for the -// Ml Results dashboards. import _ from 'lodash'; -// import d3 from 'd3'; import { ML_MEDIAN_PERCENTS } from '../../../../common/util/job_utils'; import { escapeForElasticsearchQuery } from '../../util/string_utils'; import { ML_RESULTS_INDEX_PATTERN } from '../../../../common/constants/index_patterns'; -import { ml } from '../ml_api_service'; - -// Obtains the maximum bucket anomaly scores by job ID and time. -// Pass an empty array or ['*'] to search over all job IDs. -// Returned response contains a results property, with a key for job -// which has results for the specified time range. -export function getScoresByBucket(jobIds, earliestMs, latestMs, interval, maxResults) { - return new Promise((resolve, reject) => { - const obj = { - success: true, - results: {}, - }; - - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the time range plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; - } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:bucket', - analyze_wildcard: false, - }, - }, - { - bool: { - must: boolCriteria, - }, +/** + * Service for carrying out Elasticsearch queries to obtain data for the Ml Results dashboards. + */ +export function resultsServiceProvider(mlApiServices) { + const SAMPLER_TOP_TERMS_SHARD_SIZE = 20000; + const ENTITY_AGGREGATION_SIZE = 10; + const AGGREGATION_MIN_DOC_COUNT = 1; + const CARDINALITY_PRECISION_THRESHOLD = 100; + + return { + // Obtains the maximum bucket anomaly scores by job ID and time. + // Pass an empty array or ['*'] to search over all job IDs. + // Returned response contains a results property, with a key for job + // which has results for the specified time range. + getScoresByBucket(jobIds, earliestMs, latestMs, interval, maxResults) { + return new Promise((resolve, reject) => { + const obj = { + success: true, + results: {}, + }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the time range plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, - ], + }, }, - }, - aggs: { - jobId: { - terms: { - field: 'job_id', - size: maxResults !== undefined ? maxResults : 5, - order: { - anomalyScore: 'desc', - }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, }, - aggs: { - anomalyScore: { - max: { - field: 'anomaly_score', + }); + } + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:bucket', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, }, - byTime: { - date_histogram: { - field: 'timestamp', - interval: interval, - min_doc_count: 1, - extended_bounds: { - min: earliestMs, - max: latestMs, + aggs: { + jobId: { + terms: { + field: 'job_id', + size: maxResults !== undefined ? maxResults : 5, + order: { + anomalyScore: 'desc', + }, }, - }, - aggs: { - anomalyScore: { - max: { - field: 'anomaly_score', + aggs: { + anomalyScore: { + max: { + field: 'anomaly_score', + }, + }, + byTime: { + date_histogram: { + field: 'timestamp', + interval: interval, + min_doc_count: 1, + extended_bounds: { + min: earliestMs, + max: latestMs, + }, + }, + aggs: { + anomalyScore: { + max: { + field: 'anomaly_score', + }, + }, + }, }, }, }, }, }, + }) + .then((resp) => { + const dataByJobId = _.get(resp, ['aggregations', 'jobId', 'buckets'], []); + _.each(dataByJobId, (dataForJob) => { + const jobId = dataForJob.key; + + const resultsForTime = {}; + + const dataByTime = _.get(dataForJob, ['byTime', 'buckets'], []); + _.each(dataByTime, (dataForTime) => { + const value = _.get(dataForTime, ['anomalyScore', 'value']); + if (value !== undefined) { + const time = dataForTime.key; + resultsForTime[time] = _.get(dataForTime, ['anomalyScore', 'value']); + } + }); + obj.results[jobId] = resultsForTime; + }); + + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + + // Obtains the top influencers, by maximum influencer score, for the specified index, time range and job ID(s). + // Pass an empty array or ['*'] to search over all job IDs. + // An optional array of influencers may be supplied, with each object in the array having 'fieldName' + // and 'fieldValue' properties, to limit data to the supplied list of influencers. + // Returned response contains an influencers property, with a key for each of the influencer field names, + // whose value is an array of objects containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys. + getTopInfluencers( + jobIds, + earliestMs, + latestMs, + maxFieldValues = 10, + influencers = [], + influencersFilterQuery + ) { + return new Promise((resolve, reject) => { + const obj = { success: true, influencers: {} }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the time range plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, }, - }, - }, - }) - .then((resp) => { - const dataByJobId = _.get(resp, ['aggregations', 'jobId', 'buckets'], []); - _.each(dataByJobId, (dataForJob) => { - const jobId = dataForJob.key; - - const resultsForTime = {}; - - const dataByTime = _.get(dataForJob, ['byTime', 'buckets'], []); - _.each(dataByTime, (dataForTime) => { - const value = _.get(dataForTime, ['anomalyScore', 'value']); - if (value !== undefined) { - const time = dataForTime.key; - resultsForTime[time] = _.get(dataForTime, ['anomalyScore', 'value']); + { + range: { + influencer_score: { + gt: 0, + }, + }, + }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; }); - obj.results[jobId] = resultsForTime; - }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, + }); + } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} + if (influencersFilterQuery !== undefined) { + boolCriteria.push(influencersFilterQuery); + } -// Obtains the top influencers, by maximum influencer score, for the specified index, time range and job ID(s). -// Pass an empty array or ['*'] to search over all job IDs. -// An optional array of influencers may be supplied, with each object in the array having 'fieldName' -// and 'fieldValue' properties, to limit data to the supplied list of influencers. -// Returned response contains an influencers property, with a key for each of the influencer field names, -// whose value is an array of objects containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys. -export function getTopInfluencers( - jobIds, - earliestMs, - latestMs, - maxFieldValues = 10, - influencers = [], - influencersFilterQuery -) { - return new Promise((resolve, reject) => { - const obj = { success: true, influencers: {} }; - - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the time range plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - { - range: { - influencer_score: { - gt: 0, - }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; + // Add a should query to filter for each of the specified influencers. + if (influencers.length > 0) { + boolCriteria.push({ + bool: { + should: influencers.map((influencer) => { + return { + bool: { + must: [ + { term: { influencer_field_name: influencer.fieldName } }, + { term: { influencer_field_value: influencer.fieldValue } }, + ], + }, + }; + }), + minimum_should_match: 1, + }, + }); } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - if (influencersFilterQuery !== undefined) { - boolCriteria.push(influencersFilterQuery); - } - - // Add a should query to filter for each of the specified influencers. - if (influencers.length > 0) { - boolCriteria.push({ - bool: { - should: influencers.map((influencer) => { - return { - bool: { - must: [ - { term: { influencer_field_name: influencer.fieldName } }, - { term: { influencer_field_value: influencer.fieldValue } }, - ], - }, - }; - }), - minimum_should_match: 1, - }, - }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:influencer', - analyze_wildcard: false, - }, - }, - { + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { bool: { - must: boolCriteria, - }, - }, - ], - }, - }, - aggs: { - influencerFieldNames: { - terms: { - field: 'influencer_field_name', - size: 5, - order: { - maxAnomalyScore: 'desc', - }, - }, - aggs: { - maxAnomalyScore: { - max: { - field: 'influencer_score', + filter: [ + { + query_string: { + query: 'result_type:influencer', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, }, - influencerFieldValues: { - terms: { - field: 'influencer_field_value', - size: maxFieldValues, - order: { - maxAnomalyScore: 'desc', - }, - }, - aggs: { - maxAnomalyScore: { - max: { - field: 'influencer_score', + aggs: { + influencerFieldNames: { + terms: { + field: 'influencer_field_name', + size: 5, + order: { + maxAnomalyScore: 'desc', }, }, - sumAnomalyScore: { - sum: { - field: 'influencer_score', + aggs: { + maxAnomalyScore: { + max: { + field: 'influencer_score', + }, + }, + influencerFieldValues: { + terms: { + field: 'influencer_field_value', + size: maxFieldValues, + order: { + maxAnomalyScore: 'desc', + }, + }, + aggs: { + maxAnomalyScore: { + max: { + field: 'influencer_score', + }, + }, + sumAnomalyScore: { + sum: { + field: 'influencer_score', + }, + }, + }, }, }, }, }, }, - }, - }, - }, - }) - .then((resp) => { - const fieldNameBuckets = _.get( - resp, - ['aggregations', 'influencerFieldNames', 'buckets'], - [] - ); - _.each(fieldNameBuckets, (nameBucket) => { - const fieldName = nameBucket.key; - const fieldValues = []; - - const fieldValueBuckets = _.get(nameBucket, ['influencerFieldValues', 'buckets'], []); - _.each(fieldValueBuckets, (valueBucket) => { - const fieldValueResult = { - influencerFieldValue: valueBucket.key, - maxAnomalyScore: valueBucket.maxAnomalyScore.value, - sumAnomalyScore: valueBucket.sumAnomalyScore.value, - }; - fieldValues.push(fieldValueResult); - }); - - obj.influencers[fieldName] = fieldValues; - }); - - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} + }) + .then((resp) => { + const fieldNameBuckets = _.get( + resp, + ['aggregations', 'influencerFieldNames', 'buckets'], + [] + ); + _.each(fieldNameBuckets, (nameBucket) => { + const fieldName = nameBucket.key; + const fieldValues = []; + + const fieldValueBuckets = _.get(nameBucket, ['influencerFieldValues', 'buckets'], []); + _.each(fieldValueBuckets, (valueBucket) => { + const fieldValueResult = { + influencerFieldValue: valueBucket.key, + maxAnomalyScore: valueBucket.maxAnomalyScore.value, + sumAnomalyScore: valueBucket.sumAnomalyScore.value, + }; + fieldValues.push(fieldValueResult); + }); + + obj.influencers[fieldName] = fieldValues; + }); -// Obtains the top influencer field values, by maximum anomaly score, for a -// particular index, field name and job ID(s). -// Pass an empty array or ['*'] to search over all job IDs. -// Returned response contains a results property, which is an array of objects -// containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys. -export function getTopInfluencerValues( - jobIds, - influencerFieldName, - earliestMs, - latestMs, - maxResults -) { - return new Promise((resolve, reject) => { - const obj = { success: true, results: [] }; - - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the time range plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; - } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: `result_type:influencer AND influencer_field_name: ${escapeForElasticsearchQuery( - influencerFieldName - )}`, - analyze_wildcard: false, - }, + }, + + // Obtains the top influencer field values, by maximum anomaly score, for a + // particular index, field name and job ID(s). + // Pass an empty array or ['*'] to search over all job IDs. + // Returned response contains a results property, which is an array of objects + // containing influencerFieldValue, maxAnomalyScore and sumAnomalyScore keys. + getTopInfluencerValues(jobIds, influencerFieldName, earliestMs, latestMs, maxResults) { + return new Promise((resolve, reject) => { + const obj = { success: true, results: [] }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the time range plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, - { - bool: { - must: boolCriteria, - }, - }, - ], + }, }, - }, - aggs: { - influencerFieldValues: { - terms: { - field: 'influencer_field_value', - size: maxResults !== undefined ? maxResults : 2, - order: { - maxAnomalyScore: 'desc', - }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, }, - aggs: { - maxAnomalyScore: { - max: { - field: 'influencer_score', + }); + } + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: `result_type:influencer AND influencer_field_name: ${escapeForElasticsearchQuery( + influencerFieldName + )}`, + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, }, - sumAnomalyScore: { - sum: { - field: 'influencer_score', + aggs: { + influencerFieldValues: { + terms: { + field: 'influencer_field_value', + size: maxResults !== undefined ? maxResults : 2, + order: { + maxAnomalyScore: 'desc', + }, + }, + aggs: { + maxAnomalyScore: { + max: { + field: 'influencer_score', + }, + }, + sumAnomalyScore: { + sum: { + field: 'influencer_score', + }, + }, + }, }, }, }, - }, - }, - }, - }) - .then((resp) => { - const buckets = _.get(resp, ['aggregations', 'influencerFieldValues', 'buckets'], []); - _.each(buckets, (bucket) => { - const result = { - influencerFieldValue: bucket.key, - maxAnomalyScore: bucket.maxAnomalyScore.value, - sumAnomalyScore: bucket.sumAnomalyScore.value, - }; - obj.results.push(result); - }); + }) + .then((resp) => { + const buckets = _.get(resp, ['aggregations', 'influencerFieldValues', 'buckets'], []); + _.each(buckets, (bucket) => { + const result = { + influencerFieldValue: bucket.key, + maxAnomalyScore: bucket.maxAnomalyScore.value, + sumAnomalyScore: bucket.sumAnomalyScore.value, + }; + obj.results.push(result); + }); - resolve(obj); - }) - .catch((resp) => { - reject(resp); + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); }); - }); -} - -// Obtains the overall bucket scores for the specified job ID(s). -// Pass ['*'] to search over all job IDs. -// Returned response contains a results property as an object of max score by time. -export function getOverallBucketScores(jobIds, topN, earliestMs, latestMs, interval) { - return new Promise((resolve, reject) => { - const obj = { success: true, results: {} }; - - ml.overallBuckets({ - jobId: jobIds, - topN: topN, - bucketSpan: interval, - start: earliestMs, - end: latestMs, - }) - .then((resp) => { - const dataByTime = _.get(resp, ['overall_buckets'], []); - _.each(dataByTime, (dataForTime) => { - const value = _.get(dataForTime, ['overall_score']); - if (value !== undefined) { - obj.results[dataForTime.timestamp] = value; - } - }); + }, + + // Obtains the overall bucket scores for the specified job ID(s). + // Pass ['*'] to search over all job IDs. + // Returned response contains a results property as an object of max score by time. + getOverallBucketScores(jobIds, topN, earliestMs, latestMs, interval) { + return new Promise((resolve, reject) => { + const obj = { success: true, results: {} }; + + mlApiServices + .overallBuckets({ + jobId: jobIds, + topN: topN, + bucketSpan: interval, + start: earliestMs, + end: latestMs, + }) + .then((resp) => { + const dataByTime = _.get(resp, ['overall_buckets'], []); + _.each(dataByTime, (dataForTime) => { + const value = _.get(dataForTime, ['overall_score']); + if (value !== undefined) { + obj.results[dataForTime.timestamp] = value; + } + }); - resolve(obj); - }) - .catch((resp) => { - reject(resp); + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); }); - }); -} - -// Obtains the maximum score by influencer_field_value and by time for the specified job ID(s) -// (pass an empty array or ['*'] to search over all job IDs), and specified influencer field -// values (pass an empty array to search over all field values). -// Returned response contains a results property with influencer field values keyed -// against max score by time. -export function getInfluencerValueMaxScoreByTime( - jobIds, - influencerFieldName, - influencerFieldValues, - earliestMs, - latestMs, - interval, - maxResults, - influencersFilterQuery -) { - return new Promise((resolve, reject) => { - const obj = { success: true, results: {} }; - - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the time range plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', + }, + + // Obtains the maximum score by influencer_field_value and by time for the specified job ID(s) + // (pass an empty array or ['*'] to search over all job IDs), and specified influencer field + // values (pass an empty array to search over all field values). + // Returned response contains a results property with influencer field values keyed + // against max score by time. + getInfluencerValueMaxScoreByTime( + jobIds, + influencerFieldName, + influencerFieldValues, + earliestMs, + latestMs, + interval, + maxResults, + influencersFilterQuery + ) { + return new Promise((resolve, reject) => { + const obj = { success: true, results: {} }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the time range plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, }, - }, - }, - { - range: { - influencer_score: { - gt: 0, + { + range: { + influencer_score: { + gt: 0, + }, + }, }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; - } - jobIdFilterStr += `job_id:${jobId}`; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } + ]; - if (influencersFilterQuery !== undefined) { - boolCriteria.push(influencersFilterQuery); - } + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += `job_id:${jobId}`; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, + }); + } - if (influencerFieldValues && influencerFieldValues.length > 0) { - let influencerFilterStr = ''; - _.each(influencerFieldValues, (value, i) => { - if (i > 0) { - influencerFilterStr += ' OR '; + if (influencersFilterQuery !== undefined) { + boolCriteria.push(influencersFilterQuery); } - if (value.trim().length > 0) { - influencerFilterStr += `influencer_field_value:${escapeForElasticsearchQuery(value)}`; - } else { - // Wrap whitespace influencer field values in quotes for the query_string query. - influencerFilterStr += `influencer_field_value:"${value}"`; + + if (influencerFieldValues && influencerFieldValues.length > 0) { + let influencerFilterStr = ''; + _.each(influencerFieldValues, (value, i) => { + if (i > 0) { + influencerFilterStr += ' OR '; + } + if (value.trim().length > 0) { + influencerFilterStr += `influencer_field_value:${escapeForElasticsearchQuery(value)}`; + } else { + // Wrap whitespace influencer field values in quotes for the query_string query. + influencerFilterStr += `influencer_field_value:"${value}"`; + } + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: influencerFilterStr, + }, + }); } - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: influencerFilterStr, - }, - }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: `result_type:influencer AND influencer_field_name: ${escapeForElasticsearchQuery( - influencerFieldName - )}`, - analyze_wildcard: false, + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: `result_type:influencer AND influencer_field_name: ${escapeForElasticsearchQuery( + influencerFieldName + )}`, + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], + }, + }, + aggs: { + influencerFieldValues: { + terms: { + field: 'influencer_field_value', + size: maxResults !== undefined ? maxResults : 10, + order: { + maxAnomalyScore: 'desc', + }, + }, + aggs: { + maxAnomalyScore: { + max: { + field: 'influencer_score', + }, + }, + byTime: { + date_histogram: { + field: 'timestamp', + interval, + min_doc_count: 1, + }, + aggs: { + maxAnomalyScore: { + max: { + field: 'influencer_score', + }, + }, + }, + }, + }, }, }, - { + }, + }) + .then((resp) => { + const fieldValueBuckets = _.get( + resp, + ['aggregations', 'influencerFieldValues', 'buckets'], + [] + ); + _.each(fieldValueBuckets, (valueBucket) => { + const fieldValue = valueBucket.key; + const fieldValues = {}; + + const timeBuckets = _.get(valueBucket, ['byTime', 'buckets'], []); + _.each(timeBuckets, (timeBucket) => { + const time = timeBucket.key; + const score = timeBucket.maxAnomalyScore.value; + fieldValues[time] = score; + }); + + obj.results[fieldValue] = fieldValues; + }); + + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + + // Queries Elasticsearch to obtain record level results containing the influencers + // for the specified job(s), record score threshold, and time range. + // Pass an empty array or ['*'] to search over all job IDs. + // Returned response contains a records property, with each record containing + // only the fields job_id, detector_index, record_score and influencers. + getRecordInfluencers(jobIds, threshold, earliestMs, latestMs, maxResults) { + return new Promise((resolve, reject) => { + const obj = { success: true, records: [] }; + + // Build the criteria to use in the bool filter part of the request. + // Adds criteria for the existence of the nested influencers field, time range, + // record score, plus any specified job IDs. + const boolCriteria = [ + { + nested: { + path: 'influencers', + query: { bool: { - must: boolCriteria, + must: [ + { + exists: { field: 'influencers' }, + }, + ], }, }, - ], + }, }, - }, - aggs: { - influencerFieldValues: { - terms: { - field: 'influencer_field_value', - size: maxResults !== undefined ? maxResults : 10, - order: { - maxAnomalyScore: 'desc', + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, }, - aggs: { - maxAnomalyScore: { - max: { - field: 'influencer_score', - }, + }, + { + range: { + record_score: { + gte: threshold, }, - byTime: { - date_histogram: { - field: 'timestamp', - interval, - min_doc_count: 1, - }, - aggs: { - maxAnomalyScore: { - max: { - field: 'influencer_score', + }, + }, + ]; + + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, + }); + } + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: maxResults !== undefined ? maxResults : 100, + rest_total_hits_as_int: true, + body: { + _source: ['job_id', 'detector_index', 'influencers', 'record_score'], + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, + }, }, - }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, }, + sort: [{ record_score: { order: 'desc' } }], }, - }, - }, - }, - }) - .then((resp) => { - const fieldValueBuckets = _.get( - resp, - ['aggregations', 'influencerFieldValues', 'buckets'], - [] - ); - _.each(fieldValueBuckets, (valueBucket) => { - const fieldValue = valueBucket.key; - const fieldValues = {}; - - const timeBuckets = _.get(valueBucket, ['byTime', 'buckets'], []); - _.each(timeBuckets, (timeBucket) => { - const time = timeBucket.key; - const score = timeBucket.maxAnomalyScore.value; - fieldValues[time] = score; + }) + .then((resp) => { + if (resp.hits.total !== 0) { + _.each(resp.hits.hits, (hit) => { + obj.records.push(hit._source); + }); + } + resolve(obj); + }) + .catch((resp) => { + reject(resp); }); + }); + }, + + // Queries Elasticsearch to obtain the record level results containing the specified influencer(s), + // for the specified job(s), time range, and record score threshold. + // influencers parameter must be an array, with each object in the array having 'fieldName' + // 'fieldValue' properties. The influencer array uses 'should' for the nested bool query, + // so this returns record level results which have at least one of the influencers. + // Pass an empty array or ['*'] to search over all job IDs. + getRecordsForInfluencer( + jobIds, + influencers, + threshold, + earliestMs, + latestMs, + maxResults, + influencersFilterQuery + ) { + return new Promise((resolve, reject) => { + const obj = { success: true, records: [] }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, record score, plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }, + { + range: { + record_score: { + gte: threshold, + }, + }, + }, + ]; - obj.results[fieldValue] = fieldValues; - }); + if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { + let jobIdFilterStr = ''; + _.each(jobIds, (jobId, i) => { + if (i > 0) { + jobIdFilterStr += ' OR '; + } + jobIdFilterStr += 'job_id:'; + jobIdFilterStr += jobId; + }); + boolCriteria.push({ + query_string: { + analyze_wildcard: false, + query: jobIdFilterStr, + }, + }); + } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} + if (influencersFilterQuery !== undefined) { + boolCriteria.push(influencersFilterQuery); + } -// Queries Elasticsearch to obtain record level results containing the influencers -// for the specified job(s), record score threshold, and time range. -// Pass an empty array or ['*'] to search over all job IDs. -// Returned response contains a records property, with each record containing -// only the fields job_id, detector_index, record_score and influencers. -export function getRecordInfluencers(jobIds, threshold, earliestMs, latestMs, maxResults) { - return new Promise((resolve, reject) => { - const obj = { success: true, records: [] }; - - // Build the criteria to use in the bool filter part of the request. - // Adds criteria for the existence of the nested influencers field, time range, - // record score, plus any specified job IDs. - const boolCriteria = [ - { - nested: { - path: 'influencers', - query: { + // Add a nested query to filter for each of the specified influencers. + if (influencers.length > 0) { + boolCriteria.push({ bool: { - must: [ - { - exists: { field: 'influencers' }, + should: influencers.map((influencer) => { + return { + nested: { + path: 'influencers', + query: { + bool: { + must: [ + { + match: { + 'influencers.influencer_field_name': influencer.fieldName, + }, + }, + { + match: { + 'influencers.influencer_field_values': influencer.fieldValue, + }, + }, + ], + }, + }, + }, + }; + }), + minimum_should_match: 1, + }, + }); + } + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: maxResults !== undefined ? maxResults : 100, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, - ], + }, + sort: [{ record_score: { order: 'desc' } }], + }, + }) + .then((resp) => { + if (resp.hits.total !== 0) { + _.each(resp.hits.hits, (hit) => { + obj.records.push(hit._source); + }); + } + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); + }); + }, + + // Queries Elasticsearch to obtain the record level results for the specified job and detector, + // time range, record score threshold, and whether to only return results containing influencers. + // An additional, optional influencer field name and value may also be provided. + getRecordsForDetector( + jobId, + detectorIndex, + checkForInfluencers, + influencerFieldName, + influencerFieldValue, + threshold, + earliestMs, + latestMs, + maxResults + ) { + return new Promise((resolve, reject) => { + const obj = { success: true, records: [] }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, record score, plus any specified job IDs. + const boolCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, }, }, - }, - }, - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', + { + term: { job_id: jobId }, }, - }, - }, - { - range: { - record_score: { - gte: threshold, + { + term: { detector_index: detectorIndex }, }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; - } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: maxResults !== undefined ? maxResults : 100, - rest_total_hits_as_int: true, - body: { - _source: ['job_id', 'detector_index', 'influencers', 'record_score'], - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: false, - }, + { + range: { + record_score: { + gte: threshold, }, - { + }, + }, + ]; + + // Add a nested query to filter for the specified influencer field name and value. + if (influencerFieldName && influencerFieldValue) { + boolCriteria.push({ + nested: { + path: 'influencers', + query: { bool: { - must: boolCriteria, + must: [ + { + match: { + 'influencers.influencer_field_name': influencerFieldName, + }, + }, + { + match: { + 'influencers.influencer_field_values': influencerFieldValue, + }, + }, + ], }, }, - ], - }, - }, - sort: [{ record_score: { order: 'desc' } }], - }, - }) - .then((resp) => { - if (resp.hits.total !== 0) { - _.each(resp.hits.hits, (hit) => { - obj.records.push(hit._source); + }, }); - } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} - -// Queries Elasticsearch to obtain the record level results containing the specified influencer(s), -// for the specified job(s), time range, and record score threshold. -// influencers parameter must be an array, with each object in the array having 'fieldName' -// 'fieldValue' properties. The influencer array uses 'should' for the nested bool query, -// so this returns record level results which have at least one of the influencers. -// Pass an empty array or ['*'] to search over all job IDs. -export function getRecordsForInfluencer( - jobIds, - influencers, - threshold, - earliestMs, - latestMs, - maxResults, - influencersFilterQuery -) { - return new Promise((resolve, reject) => { - const obj = { success: true, records: [] }; - - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, record score, plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - { - range: { - record_score: { - gte: threshold, - }, - }, - }, - ]; - - if (jobIds && jobIds.length > 0 && !(jobIds.length === 1 && jobIds[0] === '*')) { - let jobIdFilterStr = ''; - _.each(jobIds, (jobId, i) => { - if (i > 0) { - jobIdFilterStr += ' OR '; - } - jobIdFilterStr += 'job_id:'; - jobIdFilterStr += jobId; - }); - boolCriteria.push({ - query_string: { - analyze_wildcard: false, - query: jobIdFilterStr, - }, - }); - } - - if (influencersFilterQuery !== undefined) { - boolCriteria.push(influencersFilterQuery); - } - - // Add a nested query to filter for each of the specified influencers. - if (influencers.length > 0) { - boolCriteria.push({ - bool: { - should: influencers.map((influencer) => { - return { + } else { + if (checkForInfluencers === true) { + boolCriteria.push({ nested: { path: 'influencers', query: { bool: { must: [ { - match: { - 'influencers.influencer_field_name': influencer.fieldName, - }, - }, - { - match: { - 'influencers.influencer_field_values': influencer.fieldValue, - }, + exists: { field: 'influencers' }, }, ], }, }, }, - }; - }), - minimum_should_match: 1, - }, - }); - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: maxResults !== undefined ? maxResults : 100, - rest_total_hits_as_int: true, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: false, - }, - }, - { - bool: { - must: boolCriteria, - }, - }, - ], - }, - }, - sort: [{ record_score: { order: 'desc' } }], - }, - }) - .then((resp) => { - if (resp.hits.total !== 0) { - _.each(resp.hits.hits, (hit) => { - obj.records.push(hit._source); - }); + }); + } } - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} -// Queries Elasticsearch to obtain the record level results for the specified job and detector, -// time range, record score threshold, and whether to only return results containing influencers. -// An additional, optional influencer field name and value may also be provided. -export function getRecordsForDetector( - jobId, - detectorIndex, - checkForInfluencers, - influencerFieldName, - influencerFieldValue, - threshold, - earliestMs, - latestMs, - maxResults -) { - return new Promise((resolve, reject) => { - const obj = { success: true, records: [] }; - - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, record score, plus any specified job IDs. - const boolCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - { - term: { job_id: jobId }, - }, - { - term: { detector_index: detectorIndex }, - }, - { - range: { - record_score: { - gte: threshold, - }, - }, - }, - ]; - - // Add a nested query to filter for the specified influencer field name and value. - if (influencerFieldName && influencerFieldValue) { - boolCriteria.push({ - nested: { - path: 'influencers', - query: { - bool: { - must: [ - { - match: { - 'influencers.influencer_field_name': influencerFieldName, - }, - }, - { - match: { - 'influencers.influencer_field_values': influencerFieldValue, - }, + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: maxResults !== undefined ? maxResults : 100, + rest_total_hits_as_int: true, + body: { + query: { + bool: { + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: false, + }, + }, + { + bool: { + must: boolCriteria, + }, + }, + ], }, - ], + }, + sort: [{ record_score: { order: 'desc' } }], }, - }, - }, + }) + .then((resp) => { + if (resp.hits.total !== 0) { + _.each(resp.hits.hits, (hit) => { + obj.records.push(hit._source); + }); + } + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); }); - } else { - if (checkForInfluencers === true) { - boolCriteria.push({ - nested: { - path: 'influencers', - query: { - bool: { - must: [ - { - exists: { field: 'influencers' }, - }, - ], + }, + + // Queries Elasticsearch to obtain all the record level results for the specified job(s), time range, + // and record score threshold. + // Pass an empty array or ['*'] to search over all job IDs. + // Returned response contains a records property, which is an array of the matching results. + getRecords(jobIds, threshold, earliestMs, latestMs, maxResults) { + return this.getRecordsForInfluencer(jobIds, [], threshold, earliestMs, latestMs, maxResults); + }, + + // Queries Elasticsearch to obtain event rate data i.e. the count + // of documents over time. + // index can be a String, or String[], of index names to search. + // Extra query object can be supplied, or pass null if no additional query. + // Returned response contains a results property, which is an object + // of document counts against time (epoch millis). + getEventRateData(index, query, timeFieldName, earliestMs, latestMs, interval) { + return new Promise((resolve, reject) => { + const obj = { success: true, results: {} }; + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, entity fields, + // plus any additional supplied query. + const mustCriteria = [ + { + range: { + [timeFieldName]: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, }, }, - }); - } - } - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: maxResults !== undefined ? maxResults : 100, - rest_total_hits_as_int: true, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: false, + ]; + + if (query) { + mustCriteria.push(query); + } + + mlApiServices + .esSearch({ + index, + rest_total_hits_as_int: true, + size: 0, + body: { + query: { + bool: { + must: mustCriteria, }, }, - { - bool: { - must: boolCriteria, + _source: { + excludes: [], + }, + aggs: { + eventRate: { + date_histogram: { + field: timeFieldName, + interval: interval, + min_doc_count: 0, + extended_bounds: { + min: earliestMs, + max: latestMs, + }, + }, }, }, - ], - }, - }, - sort: [{ record_score: { order: 'desc' } }], - }, - }) - .then((resp) => { - if (resp.hits.total !== 0) { - _.each(resp.hits.hits, (hit) => { - obj.records.push(hit._source); + }, + }) + .then((resp) => { + const dataByTimeBucket = _.get(resp, ['aggregations', 'eventRate', 'buckets'], []); + _.each(dataByTimeBucket, (dataForTime) => { + const time = dataForTime.key; + obj.results[time] = dataForTime.doc_count; + }); + obj.total = resp.hits.total; + + resolve(obj); + }) + .catch((resp) => { + reject(resp); }); - } - resolve(obj); - }) - .catch((resp) => { - reject(resp); }); - }); -} - -// Queries Elasticsearch to obtain all the record level results for the specified job(s), time range, -// and record score threshold. -// Pass an empty array or ['*'] to search over all job IDs. -// Returned response contains a records property, which is an array of the matching results. -export function getRecords(jobIds, threshold, earliestMs, latestMs, maxResults) { - return this.getRecordsForInfluencer(jobIds, [], threshold, earliestMs, latestMs, maxResults); -} + }, -// Queries Elasticsearch to obtain event rate data i.e. the count -// of documents over time. -// index can be a String, or String[], of index names to search. -// Extra query object can be supplied, or pass null if no additional query. -// Returned response contains a results property, which is an object -// of document counts against time (epoch millis). -export function getEventRateData(index, query, timeFieldName, earliestMs, latestMs, interval) { - return new Promise((resolve, reject) => { - const obj = { success: true, results: {} }; - - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, entity fields, - // plus any additional supplied query. - const mustCriteria = [ - { - range: { - [timeFieldName]: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - ]; - - if (query) { - mustCriteria.push(query); - } + // Queries Elasticsearch to obtain event distribution i.e. the count + // of entities over time. + // index can be a String, or String[], of index names to search. + // Extra query object can be supplied, or pass null if no additional query. + // Returned response contains a results property, which is an object + // of document counts against time (epoch millis). - ml.esSearch({ + getEventDistributionData( index, - rest_total_hits_as_int: true, - size: 0, - body: { - query: { - bool: { - must: mustCriteria, - }, - }, - _source: { - excludes: [], - }, - aggs: { - eventRate: { - date_histogram: { - field: timeFieldName, - interval: interval, - min_doc_count: 0, - extended_bounds: { - min: earliestMs, - max: latestMs, - }, + splitField, + filterField = null, + query, + metricFunction, // ES aggregation name + metricFieldName, + timeFieldName, + earliestMs, + latestMs, + interval + ) { + return new Promise((resolve, reject) => { + if (splitField === undefined) { + return resolve([]); + } + + // Build the criteria to use in the bool filter part of the request. + // Add criteria for the time range, entity fields, + // plus any additional supplied query. + const mustCriteria = []; + + mustCriteria.push({ + range: { + [timeFieldName]: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, }, - }, - }, - }) - .then((resp) => { - const dataByTimeBucket = _.get(resp, ['aggregations', 'eventRate', 'buckets'], []); - _.each(dataByTimeBucket, (dataForTime) => { - const time = dataForTime.key; - obj.results[time] = dataForTime.doc_count; }); - obj.total = resp.hits.total; - resolve(obj); - }) - .catch((resp) => { - reject(resp); - }); - }); -} + if (query) { + mustCriteria.push(query); + } -// Queries Elasticsearch to obtain event distribution i.e. the count -// of entities over time. -// index can be a String, or String[], of index names to search. -// Extra query object can be supplied, or pass null if no additional query. -// Returned response contains a results property, which is an object -// of document counts against time (epoch millis). -const SAMPLER_TOP_TERMS_SHARD_SIZE = 20000; -const ENTITY_AGGREGATION_SIZE = 10; -const AGGREGATION_MIN_DOC_COUNT = 1; -const CARDINALITY_PRECISION_THRESHOLD = 100; -export function getEventDistributionData( - index, - splitField, - filterField = null, - query, - metricFunction, // ES aggregation name - metricFieldName, - timeFieldName, - earliestMs, - latestMs, - interval -) { - return new Promise((resolve, reject) => { - if (splitField === undefined) { - return resolve([]); - } - - // Build the criteria to use in the bool filter part of the request. - // Add criteria for the time range, entity fields, - // plus any additional supplied query. - const mustCriteria = []; - - mustCriteria.push({ - range: { - [timeFieldName]: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }); - - if (query) { - mustCriteria.push(query); - } - - if (filterField !== null) { - mustCriteria.push({ - term: { - [filterField.fieldName]: filterField.fieldValue, - }, - }); - } - - const body = { - query: { - // using function_score and random_score to get a random sample of documents. - // otherwise all documents would have the same score and the sampler aggregation - // would pick the first N documents instead of a random set. - function_score: { - query: { - bool: { - must: mustCriteria, + if (filterField !== null) { + mustCriteria.push({ + term: { + [filterField.fieldName]: filterField.fieldValue, }, - }, - functions: [ - { - random_score: { - // static seed to get same randomized results on every request - seed: 10, - field: '_seq_no', + }); + } + + const body = { + query: { + // using function_score and random_score to get a random sample of documents. + // otherwise all documents would have the same score and the sampler aggregation + // would pick the first N documents instead of a random set. + function_score: { + query: { + bool: { + must: mustCriteria, + }, }, + functions: [ + { + random_score: { + // static seed to get same randomized results on every request + seed: 10, + field: '_seq_no', + }, + }, + ], }, - ], - }, - }, - size: 0, - _source: { - excludes: [], - }, - aggs: { - sample: { - sampler: { - shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, + }, + size: 0, + _source: { + excludes: [], }, aggs: { - byTime: { - date_histogram: { - field: timeFieldName, - interval: interval, - min_doc_count: AGGREGATION_MIN_DOC_COUNT, + sample: { + sampler: { + shard_size: SAMPLER_TOP_TERMS_SHARD_SIZE, }, aggs: { - entities: { - terms: { - field: splitField.fieldName, - size: ENTITY_AGGREGATION_SIZE, + byTime: { + date_histogram: { + field: timeFieldName, + interval: interval, min_doc_count: AGGREGATION_MIN_DOC_COUNT, }, + aggs: { + entities: { + terms: { + field: splitField.fieldName, + size: ENTITY_AGGREGATION_SIZE, + min_doc_count: AGGREGATION_MIN_DOC_COUNT, + }, + }, + }, }, }, }, }, - }, - }, - }; - - if (metricFieldName !== undefined && metricFieldName !== '') { - body.aggs.sample.aggs.byTime.aggs.entities.aggs = {}; - - const metricAgg = { - [metricFunction]: { - field: metricFieldName, - }, - }; - - if (metricFunction === 'percentiles') { - metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; - } - - if (metricFunction === 'cardinality') { - metricAgg[metricFunction].precision_threshold = CARDINALITY_PRECISION_THRESHOLD; - } - body.aggs.sample.aggs.byTime.aggs.entities.aggs.metric = metricAgg; - } - - ml.esSearch({ - index, - body, - rest_total_hits_as_int: true, - }) - .then((resp) => { - // Because of the sampling, results of metricFunctions which use sum or count - // can be significantly skewed. Taking into account totalHits we calculate a - // a factor to normalize results for these metricFunctions. - const totalHits = _.get(resp, ['hits', 'total'], 0); - const successfulShards = _.get(resp, ['_shards', 'successful'], 0); - - let normalizeFactor = 1; - if (totalHits > successfulShards * SAMPLER_TOP_TERMS_SHARD_SIZE) { - normalizeFactor = totalHits / (successfulShards * SAMPLER_TOP_TERMS_SHARD_SIZE); + }; + + if (metricFieldName !== undefined && metricFieldName !== '') { + body.aggs.sample.aggs.byTime.aggs.entities.aggs = {}; + + const metricAgg = { + [metricFunction]: { + field: metricFieldName, + }, + }; + + if (metricFunction === 'percentiles') { + metricAgg[metricFunction].percents = [ML_MEDIAN_PERCENTS]; + } + + if (metricFunction === 'cardinality') { + metricAgg[metricFunction].precision_threshold = CARDINALITY_PRECISION_THRESHOLD; + } + body.aggs.sample.aggs.byTime.aggs.entities.aggs.metric = metricAgg; } - const dataByTime = _.get(resp, ['aggregations', 'sample', 'byTime', 'buckets'], []); - const data = dataByTime.reduce((d, dataForTime) => { - const date = +dataForTime.key; - const entities = _.get(dataForTime, ['entities', 'buckets'], []); - entities.forEach((entity) => { - let value = metricFunction === 'count' ? entity.doc_count : entity.metric.value; - - if ( - metricFunction === 'count' || - metricFunction === 'cardinality' || - metricFunction === 'sum' - ) { - value = value * normalizeFactor; + mlApiServices + .esSearch({ + index, + body, + rest_total_hits_as_int: true, + }) + .then((resp) => { + // Because of the sampling, results of metricFunctions which use sum or count + // can be significantly skewed. Taking into account totalHits we calculate a + // a factor to normalize results for these metricFunctions. + const totalHits = _.get(resp, ['hits', 'total'], 0); + const successfulShards = _.get(resp, ['_shards', 'successful'], 0); + + let normalizeFactor = 1; + if (totalHits > successfulShards * SAMPLER_TOP_TERMS_SHARD_SIZE) { + normalizeFactor = totalHits / (successfulShards * SAMPLER_TOP_TERMS_SHARD_SIZE); } - d.push({ - date, - entity: entity.key, - value, - }); + const dataByTime = _.get(resp, ['aggregations', 'sample', 'byTime', 'buckets'], []); + const data = dataByTime.reduce((d, dataForTime) => { + const date = +dataForTime.key; + const entities = _.get(dataForTime, ['entities', 'buckets'], []); + entities.forEach((entity) => { + let value = metricFunction === 'count' ? entity.doc_count : entity.metric.value; + + if ( + metricFunction === 'count' || + metricFunction === 'cardinality' || + metricFunction === 'sum' + ) { + value = value * normalizeFactor; + } + + d.push({ + date, + entity: entity.key, + value, + }); + }); + return d; + }, []); + resolve(data); + }) + .catch((resp) => { + reject(resp); }); - return d; - }, []); - resolve(data); - }) - .catch((resp) => { - reject(resp); }); - }); -} - -// Queries Elasticsearch to obtain the max record score over time for the specified job, -// criteria, time range, and aggregation interval. -// criteriaFields parameter must be an array, with each object in the array having 'fieldName' -// 'fieldValue' properties. -export function getRecordMaxScoreByTime(jobId, criteriaFields, earliestMs, latestMs, interval) { - return new Promise((resolve, reject) => { - const obj = { - success: true, - results: {}, - }; - - // Build the criteria to use in the bool filter part of the request. - const mustCriteria = [ - { - range: { - timestamp: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }, - { term: { job_id: jobId } }, - ]; - - _.each(criteriaFields, (criteria) => { - mustCriteria.push({ - term: { - [criteria.fieldName]: criteria.fieldValue, - }, - }); - }); - - ml.esSearch({ - index: ML_RESULTS_INDEX_PATTERN, - size: 0, - body: { - query: { - bool: { - filter: [ - { - query_string: { - query: 'result_type:record', - analyze_wildcard: true, - }, + }, + + // Queries Elasticsearch to obtain the max record score over time for the specified job, + // criteria, time range, and aggregation interval. + // criteriaFields parameter must be an array, with each object in the array having 'fieldName' + // 'fieldValue' properties. + getRecordMaxScoreByTime(jobId, criteriaFields, earliestMs, latestMs, interval) { + return new Promise((resolve, reject) => { + const obj = { + success: true, + results: {}, + }; + + // Build the criteria to use in the bool filter part of the request. + const mustCriteria = [ + { + range: { + timestamp: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', }, - { + }, + }, + { term: { job_id: jobId } }, + ]; + + _.each(criteriaFields, (criteria) => { + mustCriteria.push({ + term: { + [criteria.fieldName]: criteria.fieldValue, + }, + }); + }); + + mlApiServices + .esSearch({ + index: ML_RESULTS_INDEX_PATTERN, + size: 0, + body: { + query: { bool: { - must: mustCriteria, + filter: [ + { + query_string: { + query: 'result_type:record', + analyze_wildcard: true, + }, + }, + { + bool: { + must: mustCriteria, + }, + }, + ], }, }, - ], - }, - }, - aggs: { - times: { - date_histogram: { - field: 'timestamp', - interval: interval, - min_doc_count: 1, - }, - aggs: { - recordScore: { - max: { - field: 'record_score', + aggs: { + times: { + date_histogram: { + field: 'timestamp', + interval: interval, + min_doc_count: 1, + }, + aggs: { + recordScore: { + max: { + field: 'record_score', + }, + }, + }, }, }, }, - }, - }, - }, - }) - .then((resp) => { - const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []); - _.each(aggregationsByTime, (dataForTime) => { - const time = dataForTime.key; - obj.results[time] = { - score: _.get(dataForTime, ['recordScore', 'value']), - }; - }); + }) + .then((resp) => { + const aggregationsByTime = _.get(resp, ['aggregations', 'times', 'buckets'], []); + _.each(aggregationsByTime, (dataForTime) => { + const time = dataForTime.key; + obj.results[time] = { + score: _.get(dataForTime, ['recordScore', 'value']), + }; + }); - resolve(obj); - }) - .catch((resp) => { - reject(resp); + resolve(obj); + }) + .catch((resp) => { + reject(resp); + }); }); - }); + }, + }; } diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx index b53b08e5f6146f..b4b25db452bdb7 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable.tsx @@ -7,6 +7,7 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { CoreStart } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; import { Subject } from 'rxjs'; import { Embeddable, @@ -25,12 +26,19 @@ import { RefreshInterval, TimeRange, } from '../../../../../../src/plugins/data/common'; +import { SwimlaneType } from '../../application/explorer/explorer_constants'; export const ANOMALY_SWIMLANE_EMBEDDABLE_TYPE = 'ml_anomaly_swimlane'; +export const getDefaultPanelTitle = (jobIds: JobId[]) => + i18n.translate('xpack.ml.swimlaneEmbeddable.title', { + defaultMessage: 'ML anomaly swimlane for {jobIds}', + values: { jobIds: jobIds.join(', ') }, + }); + export interface AnomalySwimlaneEmbeddableCustomInput { jobIds: JobId[]; - swimlaneType: string; + swimlaneType: SwimlaneType; viewBy?: string; limit?: number; @@ -43,9 +51,12 @@ export interface AnomalySwimlaneEmbeddableCustomInput { export type AnomalySwimlaneEmbeddableInput = EmbeddableInput & AnomalySwimlaneEmbeddableCustomInput; -export interface AnomalySwimlaneEmbeddableOutput extends EmbeddableOutput { +export type AnomalySwimlaneEmbeddableOutput = EmbeddableOutput & + AnomalySwimlaneEmbeddableCustomOutput; + +export interface AnomalySwimlaneEmbeddableCustomOutput { jobIds: JobId[]; - swimlaneType: string; + swimlaneType: SwimlaneType; viewBy?: string; limit?: number; } diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts index e86d738d8b8093..09091b21e49b6d 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_embeddable_factory.ts @@ -23,8 +23,9 @@ import { MlStartDependencies } from '../../plugin'; import { HttpService } from '../../application/services/http_service'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; import { ExplorerService } from '../../application/services/explorer_service'; -import { mlResultsService } from '../../application/services/results_service'; +import { mlResultsServiceProvider } from '../../application/services/results_service'; import { resolveAnomalySwimlaneUserInput } from './anomaly_swimlane_setup_flyout'; +import { mlApiServicesProvider } from '../../application/services/ml_api_service'; export class AnomalySwimlaneEmbeddableFactory implements EmbeddableFactoryDefinition { @@ -64,8 +65,7 @@ export class AnomalySwimlaneEmbeddableFactory const explorerService = new ExplorerService( pluginsStart.data.query.timefilter.timefilter, coreStart.uiSettings, - // TODO mlResultsService to use DI - mlResultsService + mlResultsServiceProvider(mlApiServicesProvider(httpService)) ); return [coreStart, pluginsStart, { anomalyDetectorService, explorerService }]; diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx index 00d47c0d897c74..4c93b9ef232391 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_initializer.tsx @@ -20,7 +20,7 @@ import { } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; -import { SWIMLANE_TYPE } from '../../application/explorer/explorer_constants'; +import { SWIMLANE_TYPE, SwimlaneType } from '../../application/explorer/explorer_constants'; import { AnomalySwimlaneEmbeddableInput } from './anomaly_swimlane_embeddable'; export interface AnomalySwimlaneInitializerProps { @@ -31,7 +31,7 @@ export interface AnomalySwimlaneInitializerProps { >; onCreate: (swimlaneProps: { panelTitle: string; - swimlaneType: string; + swimlaneType: SwimlaneType; viewBy?: string; limit?: number; }) => void; @@ -51,8 +51,8 @@ export const AnomalySwimlaneInitializer: FC = ( initialInput, }) => { const [panelTitle, setPanelTitle] = useState(defaultTitle); - const [swimlaneType, setSwimlaneType] = useState( - (initialInput?.swimlaneType ?? SWIMLANE_TYPE.OVERALL) as SWIMLANE_TYPE + const [swimlaneType, setSwimlaneType] = useState( + initialInput?.swimlaneType ?? SWIMLANE_TYPE.OVERALL ); const [viewBySwimlaneFieldName, setViewBySwimlaneFieldName] = useState(initialInput?.viewBy); const [limit, setLimit] = useState(initialInput?.limit ?? 5); @@ -135,7 +135,7 @@ export const AnomalySwimlaneInitializer: FC = ( })} options={swimlaneTypeOptions} idSelected={swimlaneType} - onChange={(id) => setSwimlaneType(id as SWIMLANE_TYPE)} + onChange={(id) => setSwimlaneType(id as SwimlaneType)} /> diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx index 83f9833109bf45..54f50d2d3da326 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/anomaly_swimlane_setup_flyout.tsx @@ -6,7 +6,6 @@ import React from 'react'; import { IUiSettingsClient, OverlayStart } from 'kibana/public'; -import { i18n } from '@kbn/i18n'; import moment from 'moment'; import { VIEW_BY_JOB_LABEL } from '../../application/explorer/explorer_constants'; import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public'; @@ -14,7 +13,10 @@ import { AnomalySwimlaneInitializer } from './anomaly_swimlane_initializer'; import { JobSelectorFlyout } from '../../application/components/job_selector/job_selector_flyout'; import { AnomalyDetectorService } from '../../application/services/anomaly_detector_service'; import { getInitialGroupsMap } from '../../application/components/job_selector/job_selector'; -import { AnomalySwimlaneEmbeddableInput } from './anomaly_swimlane_embeddable'; +import { + AnomalySwimlaneEmbeddableInput, + getDefaultPanelTitle, +} from './anomaly_swimlane_embeddable'; export async function resolveAnomalySwimlaneUserInput( { @@ -52,12 +54,7 @@ export async function resolveAnomalySwimlaneUserInput( reject(); }} onSelectionConfirmed={async ({ jobIds, groups }) => { - const title = - input?.title ?? - i18n.translate('xpack.ml.swimlaneEmbeddable.title', { - defaultMessage: 'ML anomaly swimlane for {jobIds}', - values: { jobIds: jobIds.join(', ') }, - }); + const title = input?.title ?? getDefaultPanelTitle(jobIds); const jobs = await anomalyDetectorService.getJobs$(jobIds).toPromise(); diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx index e5d8584683c55b..0bba9b59f7bf73 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/explorer_swimlane_container.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, useState } from 'react'; +import React, { FC, useCallback, useState } from 'react'; import { EuiCallOut, EuiFlexGroup, @@ -28,6 +28,7 @@ import { } from './anomaly_swimlane_embeddable'; import { MlTooltipComponent } from '../../application/components/chart_tooltip'; import { useSwimlaneInputResolver } from './swimlane_input_resolver'; +import { SwimlaneType } from '../../application/explorer/explorer_constants'; const RESIZE_THROTTLE_TIME_MS = 500; @@ -54,10 +55,13 @@ export const ExplorerSwimlaneContainer: FC = ({ chartWidth ); - const onResize = throttle((e: { width: number; height: number }) => { - const labelWidth = 200; - setChartWidth(e.width - labelWidth); - }, RESIZE_THROTTLE_TIME_MS); + const onResize = useCallback( + throttle((e: { width: number; height: number }) => { + const labelWidth = 200; + setChartWidth(e.width - labelWidth); + }, RESIZE_THROTTLE_TIME_MS), + [] + ); if (error) { return ( @@ -91,14 +95,14 @@ export const ExplorerSwimlaneContainer: FC = ({ {chartWidth > 0 && swimlaneData && swimlaneType ? ( - + {(tooltipService) => ( )} diff --git a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts index e704582d5d61ab..3829bbce5e5c96 100644 --- a/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts +++ b/x-pack/plugins/ml/public/embeddables/anomaly_swimlane/swimlane_input_resolver.ts @@ -24,7 +24,7 @@ import { AnomalySwimlaneServices, } from './anomaly_swimlane_embeddable'; import { MlStartDependencies } from '../../plugin'; -import { SWIMLANE_TYPE } from '../../application/explorer/explorer_constants'; +import { SWIMLANE_TYPE, SwimlaneType } from '../../application/explorer/explorer_constants'; import { Filter } from '../../../../../../src/plugins/data/common/es_query/filters'; import { Query } from '../../../../../../src/plugins/data/common/query'; import { esKuery, UI_SETTINGS } from '../../../../../../src/plugins/data/public'; @@ -55,7 +55,7 @@ export function useSwimlaneInputResolver( const [{ uiSettings }, , { explorerService, anomalyDetectorService }] = services; const [swimlaneData, setSwimlaneData] = useState(); - const [swimlaneType, setSwimlaneType] = useState(); + const [swimlaneType, setSwimlaneType] = useState(); const [error, setError] = useState(); const chartWidth$ = useMemo(() => new Subject(), []); diff --git a/x-pack/plugins/ml/public/index.ts b/x-pack/plugins/ml/public/index.ts index a9ffb1a5bf5792..5a956651c86d8b 100755 --- a/x-pack/plugins/ml/public/index.ts +++ b/x-pack/plugins/ml/public/index.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializer } from 'kibana/public'; +import { PluginInitializer, PluginInitializerContext } from 'kibana/public'; import './index.scss'; import { MlPlugin, @@ -19,7 +19,7 @@ export const plugin: PluginInitializer< MlPluginStart, MlSetupDependencies, MlStartDependencies -> = () => new MlPlugin(); +> = (initializerContext: PluginInitializerContext) => new MlPlugin(initializerContext); export { MlPluginSetup, MlPluginStart }; export * from './shared'; diff --git a/x-pack/plugins/ml/public/plugin.ts b/x-pack/plugins/ml/public/plugin.ts index fe9f602bc3637e..be2ebb3caa4161 100644 --- a/x-pack/plugins/ml/public/plugin.ts +++ b/x-pack/plugins/ml/public/plugin.ts @@ -5,7 +5,13 @@ */ import { i18n } from '@kbn/i18n'; -import { Plugin, CoreStart, CoreSetup, AppMountParameters } from 'kibana/public'; +import { + Plugin, + CoreStart, + CoreSetup, + AppMountParameters, + PluginInitializerContext, +} from 'kibana/public'; import { ManagementSetup } from 'src/plugins/management/public'; import { SharePluginStart } from 'src/plugins/share/public'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; @@ -38,9 +44,13 @@ export interface MlSetupDependencies { home: HomePublicPluginSetup; embeddable: EmbeddableSetup; uiActions: UiActionsSetup; + kibanaVersion: string; + share: SharePluginStart; } export class MlPlugin implements Plugin { + constructor(private initializerContext: PluginInitializerContext) {} + setup(core: CoreSetup, pluginsSetup: MlSetupDependencies) { core.application.register({ id: PLUGIN_ID, @@ -53,6 +63,7 @@ export class MlPlugin implements Plugin { category: DEFAULT_APP_CATEGORIES.kibana, mount: async (params: AppMountParameters) => { const [coreStart, pluginsStart] = await core.getStartServices(); + const kibanaVersion = this.initializerContext.env.packageInfo.version; const { renderApp } = await import('./application/app'); return renderApp( coreStart, @@ -67,6 +78,7 @@ export class MlPlugin implements Plugin { home: pluginsSetup.home, embeddable: pluginsSetup.embeddable, uiActions: pluginsSetup.uiActions, + kibanaVersion, }, { element: params.element, diff --git a/x-pack/test/functional/apps/ml/anomaly_detection/anomaly_explorer.ts b/x-pack/test/functional/apps/ml/anomaly_detection/anomaly_explorer.ts index 56d00a4e11390a..c23abead458f17 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection/anomaly_explorer.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection/anomaly_explorer.ts @@ -61,6 +61,7 @@ export default function ({ getService }: FtrProviderContext) { before(async () => { await esArchiver.loadIfNeeded('ml/farequote'); await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp'); + await ml.testResources.createMLTestDashboardIfNeeded(); await ml.testResources.setKibanaTimeZoneToUTC(); await ml.securityUI.loginAsMlPowerUser(); @@ -125,6 +126,12 @@ export default function ({ getService }: FtrProviderContext) { it('anomalies table is not empty', async () => { await ml.anomaliesTable.assertTableNotEmpty(); }); + + // should be the last step because it navigates away from the Anomaly Explorer page + it('should allow to attach anomaly swimlane embeddable to the dashboard', async () => { + await ml.anomalyExplorer.openAddToDashboardControl(); + await ml.anomalyExplorer.addAndEditSwimlaneInDashboard('ML Test'); + }); }); } }); diff --git a/x-pack/test/functional/apps/ml/index.ts b/x-pack/test/functional/apps/ml/index.ts index 92e836e0c4c1bf..2d8aac3b8dddf8 100644 --- a/x-pack/test/functional/apps/ml/index.ts +++ b/x-pack/test/functional/apps/ml/index.ts @@ -22,6 +22,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await ml.securityCommon.cleanMlRoles(); await ml.testResources.deleteSavedSearches(); + await ml.testResources.deleteDashboards(); await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); await ml.testResources.deleteIndexPatternByTitle('ft_ecommerce'); diff --git a/x-pack/test/functional/services/ml/anomaly_explorer.ts b/x-pack/test/functional/services/ml/anomaly_explorer.ts index 6ec72c76bb9cf0..7c479a4234673e 100644 --- a/x-pack/test/functional/services/ml/anomaly_explorer.ts +++ b/x-pack/test/functional/services/ml/anomaly_explorer.ts @@ -66,5 +66,38 @@ export function MachineLearningAnomalyExplorerProvider({ getService }: FtrProvid async assertSwimlaneViewByExists() { await testSubjects.existOrFail('mlAnomalyExplorerSwimlaneViewBy'); }, + + async openAddToDashboardControl() { + await testSubjects.click('mlAnomalyTimelinePanelMenu'); + await testSubjects.click('mlAnomalyTimelinePanelAddToDashboardButton'); + await testSubjects.existOrFail('mlAddToDashboardModal'); + }, + + async addAndEditSwimlaneInDashboard(dashboardTitle: string) { + await this.filterWithSearchString(dashboardTitle); + await testSubjects.isDisplayed('mlDashboardSelectionTable > checkboxSelectAll'); + await testSubjects.click('mlDashboardSelectionTable > checkboxSelectAll'); + expect(await testSubjects.isChecked('mlDashboardSelectionTable > checkboxSelectAll')).to.be( + true + ); + await testSubjects.clickWhenNotDisabled('mlAddAndEditDashboardButton'); + const embeddable = await testSubjects.find('mlAnomalySwimlaneEmbeddableWrapper'); + const swimlane = await embeddable.findByClassName('ml-swimlanes'); + expect(await swimlane.isDisplayed()).to.eql( + true, + 'Anomaly swimlane should be displayed in dashboard' + ); + }, + + async waitForDashboardsToLoad() { + await testSubjects.existOrFail('~mlDashboardSelectionTable', { timeout: 60 * 1000 }); + }, + + async filterWithSearchString(filter: string) { + await this.waitForDashboardsToLoad(); + const searchBarInput = await testSubjects.find('mlDashboardsSearchBox'); + await searchBarInput.clearValueWithKeyboard(); + await searchBarInput.type(filter); + }, }; } diff --git a/x-pack/test/functional/services/ml/test_resources.ts b/x-pack/test/functional/services/ml/test_resources.ts index 4e3d1d9d86271a..9927c987bbea5d 100644 --- a/x-pack/test/functional/services/ml/test_resources.ts +++ b/x-pack/test/functional/services/ml/test_resources.ts @@ -5,7 +5,7 @@ */ import { ProvidedType } from '@kbn/test/types/ftr'; -import { savedSearches } from './test_resources_data'; +import { savedSearches, dashboards } from './test_resources_data'; import { COMMON_REQUEST_HEADERS } from './common'; import { FtrProviderContext } from '../../ftr_provider_context'; @@ -137,6 +137,20 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider return createResponse.id; }, + async createDashboard(title: string, body: object): Promise { + log.debug(`Creating dashboard with title '${title}'`); + + const createResponse = await supertest + .post(`/api/saved_objects/${SavedObjectType.DASHBOARD}`) + .set(COMMON_REQUEST_HEADERS) + .send(body) + .expect(200) + .then((res: any) => res.body); + + log.debug(` > Created with id '${createResponse.id}'`); + return createResponse.id; + }, + async createSavedSearchIfNeeded(savedSearch: any): Promise { const title = savedSearch.requestBody.attributes.title; const savedSearchId = await this.getSavedSearchId(title); @@ -181,6 +195,21 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider await this.createSavedSearchIfNeeded(savedSearches.farequoteFilter); }, + async createMLTestDashboardIfNeeded() { + await this.createDashboardIfNeeded(dashboards.mlTestDashboard); + }, + + async createDashboardIfNeeded(dashboard: any) { + const title = dashboard.requestBody.attributes.title; + const dashboardId = await this.getDashboardId(title); + if (dashboardId !== undefined) { + log.debug(`Dashboard with title '${title}' already exists. Nothing to create.`); + return dashboardId; + } else { + return await this.createDashboard(title, dashboard.requestBody); + } + }, + async createSavedSearchFarequoteLuceneIfNeeded() { await this.createSavedSearchIfNeeded(savedSearches.farequoteLucene); }, @@ -285,6 +314,12 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider } }, + async deleteDashboards() { + for (const dashboard of Object.values(dashboards)) { + await this.deleteDashboardByTitle(dashboard.requestBody.attributes.title); + } + }, + async assertSavedObjectExistsByTitle(title: string, objectType: SavedObjectType) { await retry.waitForWithTimeout( `${objectType} with title '${title}' to exist`, diff --git a/x-pack/test/functional/services/ml/test_resources_data.ts b/x-pack/test/functional/services/ml/test_resources_data.ts index dd600077182f93..2ab1f4de542284 100644 --- a/x-pack/test/functional/services/ml/test_resources_data.ts +++ b/x-pack/test/functional/services/ml/test_resources_data.ts @@ -247,3 +247,22 @@ export const savedSearches = { }, }, }; + +export const dashboards = { + mlTestDashboard: { + requestBody: { + attributes: { + title: 'ML Test', + hits: 0, + description: '', + panelsJSON: '[]', + optionsJSON: '{"hidePanelTitles":false,"useMargins":true}', + version: 1, + timeRestore: false, + kibanaSavedObjectMeta: { + searchSourceJSON: '{"query":{"language":"kuery","query":""},"filter":[]}', + }, + }, + }, + }, +}; diff --git a/yarn.lock b/yarn.lock index 2b2eab444d04a1..c9bbbd8a999f8b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4803,6 +4803,11 @@ resolved "https://registry.yarnpkg.com/@types/delete-empty/-/delete-empty-2.0.0.tgz#1647ae9e68f708a6ba778531af667ec55bc61964" integrity sha512-sq+kwx8zA9BSugT9N+Jr8/uWjbHMZ+N/meJEzRyT3gmLq/WMtx/iSIpvdpmBUi/cvXl6Kzpvve8G2ESkabFwmg== +"@types/dragselect@^1.13.1": + version "1.13.1" + resolved "https://registry.yarnpkg.com/@types/dragselect/-/dragselect-1.13.1.tgz#f19b7b41063a7c9d5963194c83c3c364e84d46ee" + integrity sha512-3m0fvSM0cSs0DXvprytV/ZY92hNX3jJuEb/vkdqU+4QMzV2jxYKgBFTuaT2fflqbmfzUqHHIkGP55WIuigElQw== + "@types/elasticsearch@^5.0.33": version "5.0.33" resolved "https://registry.yarnpkg.com/@types/elasticsearch/-/elasticsearch-5.0.33.tgz#b0fd37dc674f498223b6d68c313bdfd71f4d812b" From b434cac29a0fe3b592c90eb8d3651b0aa4585aaf Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Wed, 17 Jun 2020 14:57:45 -0500 Subject: [PATCH 03/60] [docker] add spaces settings (#69019) Co-authored-by: Elastic Machine --- .../os_packages/docker_generator/resources/bin/kibana-docker | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker index d1fb544de733c3..745a3d1f0c8307 100755 --- a/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker +++ b/src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker @@ -235,6 +235,8 @@ kibana_vars=( xpack.security.session.lifespan xpack.security.loginAssistanceMessage xpack.security.loginHelp + xpack.spaces.enabled + xpack.spaces.maxSpaces telemetry.allowChangingOptInStatus telemetry.enabled telemetry.optIn From f7266d3b7b3502e31ae27b824524818ba8b98597 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Wed, 17 Jun 2020 16:17:30 -0600 Subject: [PATCH 04/60] [Maps] layer wizard select re-design (#69313) * [Maps] layer wizard select re-design * review feedback * tslint * add unit test * use smaller gutters * review feedback --- x-pack/plugins/maps/common/constants.ts | 6 + .../classes/layers/layer_wizard_registry.ts | 2 + .../observability_layer_wizard.tsx | 2 + .../security/security_layer_wizard.tsx | 2 + .../upload_layer_wizard.tsx | 1 + .../ems_boundaries_layer_wizard.tsx | 2 + .../ems_base_map_layer_wizard.tsx | 2 + .../clusters_layer_wizard.tsx | 2 + .../heatmap_layer_wizard.tsx | 3 +- .../point_2_point_layer_wizard.tsx | 2 + .../es_documents_layer_wizard.tsx | 3 +- .../kibana_regionmap_layer_wizard.tsx | 2 + .../kibana_base_map_layer_wizard.tsx | 2 + .../layer_wizard.tsx | 2 + .../sources/wms_source/wms_layer_wizard.tsx | 2 + .../sources/xyz_tms_source/layer_wizard.tsx | 2 + .../public/connected_components/_index.scss | 1 - .../add_layer_panel/_index.scss | 12 -- .../layer_wizard_select.test.tsx.snap | 83 ++++++++++ .../flyout_body/layer_wizard_select.test.tsx | 59 +++++++ .../flyout_body/layer_wizard_select.tsx | 151 +++++++++++++++--- .../gis_map/_gis_map.scss | 4 +- 22 files changed, 304 insertions(+), 43 deletions(-) delete mode 100644 x-pack/plugins/maps/public/connected_components/add_layer_panel/_index.scss create mode 100644 x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/__snapshots__/layer_wizard_select.test.tsx.snap create mode 100644 x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.test.tsx diff --git a/x-pack/plugins/maps/common/constants.ts b/x-pack/plugins/maps/common/constants.ts index ad99780a7d32fb..edb395633827f6 100644 --- a/x-pack/plugins/maps/common/constants.ts +++ b/x-pack/plugins/maps/common/constants.ts @@ -227,3 +227,9 @@ export enum INITIAL_LOCATION { FIXED_LOCATION = 'FIXED_LOCATION', BROWSER_LOCATION = 'BROWSER_LOCATION', } + +export enum LAYER_WIZARD_CATEGORY { + ELASTICSEARCH = 'ELASTICSEARCH', + REFERENCE = 'REFERENCE', + SOLUTIONS = 'SOLUTIONS', +} diff --git a/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts b/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts index 2bdeb6446cf288..a255ffb00e312b 100644 --- a/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts +++ b/x-pack/plugins/maps/public/classes/layers/layer_wizard_registry.ts @@ -7,6 +7,7 @@ import { ReactElement } from 'react'; import { LayerDescriptor } from '../../../common/descriptor_types'; +import { LAYER_WIZARD_CATEGORY } from '../../../common/constants'; export type RenderWizardArguments = { previewLayers: (layerDescriptors: LayerDescriptor[], isIndexingSource?: boolean) => void; @@ -20,6 +21,7 @@ export type RenderWizardArguments = { }; export type LayerWizard = { + categories: LAYER_WIZARD_CATEGORY[]; checkVisibility?: () => Promise; description: string; icon: string; diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/observability_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/observability_layer_wizard.tsx index db97c08596e065..ddb07a9facee7b 100644 --- a/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/observability_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/observability/observability_layer_wizard.tsx @@ -6,12 +6,14 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; +import { LAYER_WIZARD_CATEGORY } from '../../../../../common/constants'; import { LayerWizard, RenderWizardArguments } from '../../layer_wizard_registry'; import { ObservabilityLayerTemplate } from './observability_layer_template'; import { APM_INDEX_PATTERN_ID } from './create_layer_descriptor'; import { getIndexPatternService } from '../../../../kibana_services'; export const ObservabilityLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH, LAYER_WIZARD_CATEGORY.SOLUTIONS], checkVisibility: async () => { try { await getIndexPatternService().get(APM_INDEX_PATTERN_ID); diff --git a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/security_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/security_layer_wizard.tsx index cece00fa373503..f51aa5b40aa80b 100644 --- a/x-pack/plugins/maps/public/classes/layers/solution_layers/security/security_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/layers/solution_layers/security/security_layer_wizard.tsx @@ -6,11 +6,13 @@ import React from 'react'; import { i18n } from '@kbn/i18n'; +import { LAYER_WIZARD_CATEGORY } from '../../../../../common/constants'; import { LayerWizard, RenderWizardArguments } from '../../layer_wizard_registry'; import { getSecurityIndexPatterns } from './security_index_pattern_utils'; import { SecurityLayerTemplate } from './security_layer_template'; export const SecurityLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH, LAYER_WIZARD_CATEGORY.SOLUTIONS], checkVisibility: async () => { const indexPatterns = await getSecurityIndexPatterns(); return indexPatterns.length > 0; diff --git a/x-pack/plugins/maps/public/classes/sources/client_file_source/upload_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/client_file_source/upload_layer_wizard.tsx index 3f4ec0d3f12685..0a224f75b981d6 100644 --- a/x-pack/plugins/maps/public/classes/sources/client_file_source/upload_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/client_file_source/upload_layer_wizard.tsx @@ -22,6 +22,7 @@ import { GeojsonFileSource } from './geojson_file_source'; import { VectorLayer } from '../../layers/vector_layer/vector_layer'; export const uploadLayerWizardConfig: LayerWizard = { + categories: [], description: i18n.translate('xpack.maps.source.geojsonFileDescription', { defaultMessage: 'Index GeoJSON data in Elasticsearch', }), diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_boundaries_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_boundaries_layer_wizard.tsx index 7eec84ef5bb2e1..c53a7a4facb0c3 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_boundaries_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_boundaries_layer_wizard.tsx @@ -13,8 +13,10 @@ import { EMSFileSource, sourceTitle } from './ems_file_source'; // @ts-ignore import { getIsEmsEnabled } from '../../../kibana_services'; import { EMSFileSourceDescriptor } from '../../../../common/descriptor_types'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const emsBoundariesLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], checkVisibility: () => { return getIsEmsEnabled(); }, diff --git a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_base_map_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_base_map_layer_wizard.tsx index 60e67b1ae70534..49d262cbad1a10 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_base_map_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_tms_source/ems_base_map_layer_wizard.tsx @@ -13,8 +13,10 @@ import { VectorTileLayer } from '../../layers/vector_tile_layer/vector_tile_laye // @ts-ignore import { TileServiceSelect } from './tile_service_select'; import { getIsEmsEnabled } from '../../../kibana_services'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const emsBaseMapLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], checkVisibility: () => { return getIsEmsEnabled(); }, diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/clusters_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/clusters_layer_wizard.tsx index b9d5faa8e18f1b..715c16b22dc51a 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/clusters_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/clusters_layer_wizard.tsx @@ -23,6 +23,7 @@ import { COUNT_PROP_NAME, COLOR_MAP_TYPE, FIELD_ORIGIN, + LAYER_WIZARD_CATEGORY, RENDER_AS, VECTOR_STYLES, STYLE_TYPE, @@ -30,6 +31,7 @@ import { import { COLOR_GRADIENTS } from '../../styles/color_utils'; export const clustersLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], description: i18n.translate('xpack.maps.source.esGridClustersDescription', { defaultMessage: 'Geospatial data grouped in grids with metrics for each gridded cell', }), diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/heatmap_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/heatmap_layer_wizard.tsx index 79252c7febf8c9..92a0f1006ea439 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/heatmap_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/heatmap_layer_wizard.tsx @@ -14,9 +14,10 @@ import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_re // @ts-ignore import { HeatmapLayer } from '../../layers/heatmap_layer/heatmap_layer'; import { ESGeoGridSourceDescriptor } from '../../../../common/descriptor_types'; -import { RENDER_AS } from '../../../../common/constants'; +import { LAYER_WIZARD_CATEGORY, RENDER_AS } from '../../../../common/constants'; export const heatmapLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], description: i18n.translate('xpack.maps.source.esGridHeatmapDescription', { defaultMessage: 'Geospatial data grouped in grids to show density', }), diff --git a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/point_2_point_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/point_2_point_layer_wizard.tsx index 5169af9bdddf2f..ae7414b827c8d8 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/point_2_point_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/point_2_point_layer_wizard.tsx @@ -14,6 +14,7 @@ import { VectorStyle } from '../../styles/vector/vector_style'; import { FIELD_ORIGIN, COUNT_PROP_NAME, + LAYER_WIZARD_CATEGORY, VECTOR_STYLES, STYLE_TYPE, } from '../../../../common/constants'; @@ -24,6 +25,7 @@ import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_re import { ColorDynamicOptions, SizeDynamicOptions } from '../../../../common/descriptor_types'; export const point2PointLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], description: i18n.translate('xpack.maps.source.pewPewDescription', { defaultMessage: 'Aggregated data paths between the source and destination', }), diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx index 888de2e7297cba..4598b1467229d4 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_documents_layer_wizard.tsx @@ -13,7 +13,7 @@ import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_re import { ESSearchSource, sourceTitle } from './es_search_source'; import { BlendedVectorLayer } from '../../layers/blended_vector_layer/blended_vector_layer'; import { VectorLayer } from '../../layers/vector_layer/vector_layer'; -import { SCALING_TYPES } from '../../../../common/constants'; +import { LAYER_WIZARD_CATEGORY, SCALING_TYPES } from '../../../../common/constants'; export function createDefaultLayerDescriptor(sourceConfig: unknown, mapColors: string[]) { const sourceDescriptor = ESSearchSource.createDescriptor(sourceConfig); @@ -24,6 +24,7 @@ export function createDefaultLayerDescriptor(sourceConfig: unknown, mapColors: s } export const esDocumentsLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], description: i18n.translate('xpack.maps.source.esSearchDescription', { defaultMessage: 'Vector data from a Kibana index pattern', }), diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx index ca78aaefe404f7..c8a1c346646e01 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/kibana_regionmap_source/kibana_regionmap_layer_wizard.tsx @@ -13,8 +13,10 @@ import { VectorLayer } from '../../layers/vector_layer/vector_layer'; // @ts-ignore import { CreateSourceEditor } from './create_source_editor'; import { getKibanaRegionList } from '../../../meta'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const kibanaRegionMapLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], checkVisibility: async () => { const regions = getKibanaRegionList(); return regions.length > 0; diff --git a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx index 84d2e5e74fa9a1..9f63372a785119 100644 --- a/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/kibana_tilemap_source/kibana_base_map_layer_wizard.tsx @@ -13,8 +13,10 @@ import { CreateSourceEditor } from './create_source_editor'; import { KibanaTilemapSource, sourceTitle } from './kibana_tilemap_source'; import { TileLayer } from '../../layers/tile_layer/tile_layer'; import { getKibanaTileMap } from '../../../meta'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const kibanaBasemapLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], checkVisibility: async () => { const tilemap = getKibanaTileMap(); // @ts-ignore diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx index c29302a2058b21..067c7f5a47ca35 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/layer_wizard.tsx @@ -13,8 +13,10 @@ import { import { MVTSingleLayerVectorSource, sourceTitle } from './mvt_single_layer_vector_source'; import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_registry'; import { TiledVectorLayer } from '../../layers/tiled_vector_layer/tiled_vector_layer'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const mvtVectorSourceWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], description: i18n.translate('xpack.maps.source.mvtVectorSourceWizard', { defaultMessage: 'Vector source wizard', }), diff --git a/x-pack/plugins/maps/public/classes/sources/wms_source/wms_layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/wms_source/wms_layer_wizard.tsx index 62eeef234f4141..b3950baf8dbeb0 100644 --- a/x-pack/plugins/maps/public/classes/sources/wms_source/wms_layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/wms_source/wms_layer_wizard.tsx @@ -12,8 +12,10 @@ import { WMSCreateSourceEditor } from './wms_create_source_editor'; import { sourceTitle, WMSSource } from './wms_source'; import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_registry'; import { TileLayer } from '../../layers/tile_layer/tile_layer'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const wmsLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], description: i18n.translate('xpack.maps.source.wmsDescription', { defaultMessage: 'Maps from OGC Standard WMS', }), diff --git a/x-pack/plugins/maps/public/classes/sources/xyz_tms_source/layer_wizard.tsx b/x-pack/plugins/maps/public/classes/sources/xyz_tms_source/layer_wizard.tsx index b99b17c1d22d47..48c526855d3a4f 100644 --- a/x-pack/plugins/maps/public/classes/sources/xyz_tms_source/layer_wizard.tsx +++ b/x-pack/plugins/maps/public/classes/sources/xyz_tms_source/layer_wizard.tsx @@ -10,8 +10,10 @@ import { XYZTMSEditor, XYZTMSSourceConfig } from './xyz_tms_editor'; import { XYZTMSSource, sourceTitle } from './xyz_tms_source'; import { LayerWizard, RenderWizardArguments } from '../../layers/layer_wizard_registry'; import { TileLayer } from '../../layers/tile_layer/tile_layer'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; export const tmsLayerWizardConfig: LayerWizard = { + categories: [LAYER_WIZARD_CATEGORY.REFERENCE], description: i18n.translate('xpack.maps.source.ems_xyzDescription', { defaultMessage: 'Tile map service configured in interface', }), diff --git a/x-pack/plugins/maps/public/connected_components/_index.scss b/x-pack/plugins/maps/public/connected_components/_index.scss index 6de2a51590700a..bd8070e8c36fdb 100644 --- a/x-pack/plugins/maps/public/connected_components/_index.scss +++ b/x-pack/plugins/maps/public/connected_components/_index.scss @@ -1,5 +1,4 @@ @import 'gis_map/gis_map'; -@import 'add_layer_panel/index'; @import 'layer_panel/index'; @import 'widget_overlay/index'; @import 'toolbar_overlay/index'; diff --git a/x-pack/plugins/maps/public/connected_components/add_layer_panel/_index.scss b/x-pack/plugins/maps/public/connected_components/add_layer_panel/_index.scss deleted file mode 100644 index 4e60b8d4b7c4b0..00000000000000 --- a/x-pack/plugins/maps/public/connected_components/add_layer_panel/_index.scss +++ /dev/null @@ -1,12 +0,0 @@ -.mapLayerAddpanel__card { - // EUITODO: Fix horizontal layout so it works with any size icon - .euiCard__content { - // sass-lint:disable-block no-important - padding-top: 0 !important; - } - - .euiCard__top + .euiCard__content { - // sass-lint:disable-block no-important - padding-top: 2px !important; - } -} diff --git a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/__snapshots__/layer_wizard_select.test.tsx.snap b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/__snapshots__/layer_wizard_select.test.tsx.snap new file mode 100644 index 00000000000000..ef11f9958d8db6 --- /dev/null +++ b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/__snapshots__/layer_wizard_select.test.tsx.snap @@ -0,0 +1,83 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`LayerWizardSelect Should render layer select after layer wizards are loaded 1`] = ` + + + + + + + Elasticsearch + + + Solutions + + + + + + + + + + } + onClick={[Function]} + title="wizard 2" + /> + + + +`; + +exports[`LayerWizardSelect Should render loading screen before layer wizards are loaded 1`] = ` +
+ + } + layout="horizontal" + title="" + /> +
+`; diff --git a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.test.tsx b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.test.tsx new file mode 100644 index 00000000000000..e802c5259e5eda --- /dev/null +++ b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.test.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +jest.mock('../../../classes/layers/layer_wizard_registry', () => ({})); + +import React from 'react'; +import { shallow } from 'enzyme'; +import { LayerWizardSelect } from './layer_wizard_select'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; + +const defaultProps = { + onSelect: () => {}, +}; + +describe('LayerWizardSelect', () => { + beforeAll(() => { + require('../../../classes/layers/layer_wizard_registry').getLayerWizards = async () => { + return [ + { + categories: [LAYER_WIZARD_CATEGORY.ELASTICSEARCH], + description: 'mock wizard without icon', + renderWizard: () => { + return
; + }, + title: 'wizard 1', + }, + { + categories: [LAYER_WIZARD_CATEGORY.SOLUTIONS], + description: 'mock wizard with icon', + icon: 'logoObservability', + renderWizard: () => { + return
; + }, + title: 'wizard 2', + }, + ]; + }; + }); + + test('Should render layer select after layer wizards are loaded', async () => { + const component = shallow(); + + // Ensure all promises resolve + await new Promise((resolve) => process.nextTick(resolve)); + // Ensure the state changes are reflected + component.update(); + + expect(component).toMatchSnapshot(); + }); + + test('Should render loading screen before layer wizards are loaded', () => { + const component = shallow(); + + expect(component).toMatchSnapshot(); + }); +}); diff --git a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx index b0c50133ceabb8..f0195bc5dee2f2 100644 --- a/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx +++ b/x-pack/plugins/maps/public/connected_components/add_layer_panel/flyout_body/layer_wizard_select.tsx @@ -5,26 +5,63 @@ */ import _ from 'lodash'; -import React, { Component, Fragment } from 'react'; -import { EuiSpacer, EuiCard, EuiIcon } from '@elastic/eui'; -import { EuiLoadingContent } from '@elastic/eui'; +import React, { Component } from 'react'; +import { + EuiCard, + EuiIcon, + EuiFlexGrid, + EuiFlexItem, + EuiLoadingContent, + EuiFacetGroup, + EuiFacetButton, + EuiSpacer, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; import { getLayerWizards, LayerWizard } from '../../../classes/layers/layer_wizard_registry'; +import { LAYER_WIZARD_CATEGORY } from '../../../../common/constants'; interface Props { onSelect: (layerWizard: LayerWizard) => void; } interface State { - layerWizards: LayerWizard[]; + activeCategories: LAYER_WIZARD_CATEGORY[]; hasLoadedWizards: boolean; + layerWizards: LayerWizard[]; + selectedCategory: LAYER_WIZARD_CATEGORY | null; +} + +function getCategoryLabel(category: LAYER_WIZARD_CATEGORY): string { + if (category === LAYER_WIZARD_CATEGORY.ELASTICSEARCH) { + return i18n.translate('xpack.maps.layerWizardSelect.elasticsearchCategoryLabel', { + defaultMessage: 'Elasticsearch', + }); + } + + if (category === LAYER_WIZARD_CATEGORY.REFERENCE) { + return i18n.translate('xpack.maps.layerWizardSelect.referenceCategoryLabel', { + defaultMessage: 'Reference', + }); + } + + if (category === LAYER_WIZARD_CATEGORY.SOLUTIONS) { + return i18n.translate('xpack.maps.layerWizardSelect.solutionsCategoryLabel', { + defaultMessage: 'Solutions', + }); + } + + throw new Error(`Unexpected category: ${category}`); } export class LayerWizardSelect extends Component { private _isMounted: boolean = false; state = { - layerWizards: [], + activeCategories: [], hasLoadedWizards: false, + layerWizards: [], + selectedCategory: null, }; componentDidMount() { @@ -38,9 +75,57 @@ export class LayerWizardSelect extends Component { async _loadLayerWizards() { const layerWizards = await getLayerWizards(); + const activeCategories: LAYER_WIZARD_CATEGORY[] = []; + layerWizards.forEach((layerWizard: LayerWizard) => { + layerWizard.categories.forEach((category: LAYER_WIZARD_CATEGORY) => { + if (!activeCategories.includes(category)) { + activeCategories.push(category); + } + }); + }); + if (this._isMounted) { - this.setState({ layerWizards, hasLoadedWizards: true }); + this.setState({ + activeCategories, + layerWizards, + hasLoadedWizards: true, + }); + } + } + + _filterByCategory(category: LAYER_WIZARD_CATEGORY | null) { + this.setState({ selectedCategory: category }); + } + + _renderCategoryFacets() { + if (this.state.activeCategories.length === 0) { + return null; } + + const facets = this.state.activeCategories.map((category: LAYER_WIZARD_CATEGORY) => { + return ( + this._filterByCategory(category)} + > + {getCategoryLabel(category)} + + ); + }); + + return ( + + this._filterByCategory(null)} + > + + + {facets} + + ); } render() { @@ -51,27 +136,41 @@ export class LayerWizardSelect extends Component {
); } - return this.state.layerWizards.map((layerWizard: LayerWizard) => { - const icon = layerWizard.icon ? : undefined; - const onClick = () => { - this.props.onSelect(layerWizard); - }; + const wizardCards = this.state.layerWizards + .filter((layerWizard: LayerWizard) => { + return this.state.selectedCategory + ? layerWizard.categories.includes(this.state.selectedCategory!) + : true; + }) + .map((layerWizard: LayerWizard) => { + const icon = layerWizard.icon ? : undefined; - return ( - - - - - ); - }); + const onClick = () => { + this.props.onSelect(layerWizard); + }; + + return ( + + + + ); + }); + + return ( + <> + {this._renderCategoryFacets()} + + + {wizardCards} + + + ); } } diff --git a/x-pack/plugins/maps/public/connected_components/gis_map/_gis_map.scss b/x-pack/plugins/maps/public/connected_components/gis_map/_gis_map.scss index 85168d970c6dec..2180573ef4583d 100644 --- a/x-pack/plugins/maps/public/connected_components/gis_map/_gis_map.scss +++ b/x-pack/plugins/maps/public/connected_components/gis_map/_gis_map.scss @@ -9,11 +9,11 @@ overflow: hidden; > * { - width: $euiSizeXXL * 11; + width: $euiSizeXXL * 12; } &-isVisible { - width: $euiSizeXXL * 11; + width: $euiSizeXXL * 12; transition: width $euiAnimSpeedNormal $euiAnimSlightResistance; } } From 789851eb1a2d794b17fdcdbfc474679ddd493d0b Mon Sep 17 00:00:00 2001 From: spalger Date: Wed, 17 Jun 2020 16:46:16 -0700 Subject: [PATCH 05/60] skip flaky suite (#52854) --- test/functional/apps/dashboard/dashboard_snapshots.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/functional/apps/dashboard/dashboard_snapshots.js b/test/functional/apps/dashboard/dashboard_snapshots.js index 787e839aa08a5c..20bc30c889d651 100644 --- a/test/functional/apps/dashboard/dashboard_snapshots.js +++ b/test/functional/apps/dashboard/dashboard_snapshots.js @@ -28,7 +28,8 @@ export default function ({ getService, getPageObjects, updateBaselines }) { const dashboardPanelActions = getService('dashboardPanelActions'); const dashboardAddPanel = getService('dashboardAddPanel'); - describe('dashboard snapshots', function describeIndexTests() { + // FLAKY: https://github.com/elastic/kibana/issues/52854 + describe.skip('dashboard snapshots', function describeIndexTests() { before(async function () { await esArchiver.load('dashboard/current/kibana'); await kibanaServer.uiSettings.replace({ From 2642d658c4a94fc77017ad425a442cd65baa2521 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Wed, 17 Jun 2020 19:18:15 -0500 Subject: [PATCH 06/60] [APM] License feature tracking for service maps (#69455) Use the license feature API to register the service maps feature and track its usage when the API endpoint is accessed. Fixes #64850. --- x-pack/plugins/apm/server/feature.ts | 3 ++ x-pack/plugins/apm/server/plugin.ts | 39 +++++++++++++------ .../server/routes/create_api/index.test.ts | 3 +- .../plugins/apm/server/routes/service_map.ts | 5 +++ x-pack/plugins/apm/server/routes/typings.ts | 5 ++- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/apm/server/feature.ts b/x-pack/plugins/apm/server/feature.ts index 60a7be9391eea3..80f722bae08686 100644 --- a/x-pack/plugins/apm/server/feature.ts +++ b/x-pack/plugins/apm/server/feature.ts @@ -70,3 +70,6 @@ export const APM_FEATURE = { }, }, }; + +export const APM_SERVICE_MAPS_FEATURE_NAME = 'APM service maps'; +export const APM_SERVICE_MAPS_LICENSE_TYPE = 'platinum'; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index f0a05dfc0df30c..eb781ee0783075 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -14,7 +14,7 @@ import { import { Observable, combineLatest } from 'rxjs'; import { map, take } from 'rxjs/operators'; import { ObservabilityPluginSetup } from '../../observability/server'; -import { SecurityPluginSetup } from '../../security/public'; +import { SecurityPluginSetup } from '../../security/server'; import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; import { TaskManagerSetupContract } from '../../task_manager/server'; import { AlertingPlugin } from '../../alerts/server'; @@ -28,11 +28,19 @@ import { APMConfig, mergeConfigs, APMXPackConfig } from '.'; import { HomeServerPluginSetup } from '../../../../src/plugins/home/server'; import { CloudSetup } from '../../cloud/server'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; -import { LicensingPluginSetup } from '../../licensing/public'; +import { + LicensingPluginSetup, + LicensingPluginStart, +} from '../../licensing/server'; import { registerApmAlerts } from './lib/alerts/register_apm_alerts'; import { createApmTelemetry } from './lib/apm_telemetry'; + import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; -import { APM_FEATURE } from './feature'; +import { + APM_FEATURE, + APM_SERVICE_MAPS_FEATURE_NAME, + APM_SERVICE_MAPS_LICENSE_TYPE, +} from './feature'; import { apmIndices, apmTelemetry } from './saved_objects'; import { createElasticCloudInstructions } from './tutorial/elastic_cloud'; import { MlPluginSetup } from '../../ml/server'; @@ -120,16 +128,25 @@ export class APMPlugin implements Plugin { elasticCloud: createElasticCloudInstructions(plugins.cloud), }; }); + plugins.features.registerFeature(APM_FEATURE); + plugins.licensing.featureUsage.register( + APM_SERVICE_MAPS_FEATURE_NAME, + APM_SERVICE_MAPS_LICENSE_TYPE + ); - createApmApi().init(core, { - config$: mergedConfig$, - logger: this.logger!, - plugins: { - observability: plugins.observability, - security: plugins.security, - ml: plugins.ml, - }, + core.getStartServices().then(([_coreStart, pluginsStart]) => { + createApmApi().init(core, { + config$: mergedConfig$, + logger: this.logger!, + plugins: { + licensing: (pluginsStart as { licensing: LicensingPluginStart }) + .licensing, + observability: plugins.observability, + security: plugins.security, + ml: plugins.ml, + }, + }); }); return { diff --git a/x-pack/plugins/apm/server/routes/create_api/index.test.ts b/x-pack/plugins/apm/server/routes/create_api/index.test.ts index 3d3e26f680e0d2..f5db936c00d3a7 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.test.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.test.ts @@ -9,6 +9,7 @@ import { CoreSetup, Logger } from 'src/core/server'; import { Params } from '../typings'; import { BehaviorSubject } from 'rxjs'; import { APMConfig } from '../..'; +import { LicensingPluginStart } from '../../../../licensing/server'; const getCoreMock = () => { const get = jest.fn(); @@ -40,7 +41,7 @@ const getCoreMock = () => { logger: ({ error: jest.fn(), } as unknown) as Logger, - plugins: {}, + plugins: { licensing: {} as LicensingPluginStart }, }, }; }; diff --git a/x-pack/plugins/apm/server/routes/service_map.ts b/x-pack/plugins/apm/server/routes/service_map.ts index df0403be7b9759..3937c18b3fe5e0 100644 --- a/x-pack/plugins/apm/server/routes/service_map.ts +++ b/x-pack/plugins/apm/server/routes/service_map.ts @@ -15,6 +15,7 @@ import { getServiceMap } from '../lib/service_map/get_service_map'; import { getServiceMapServiceNodeInfo } from '../lib/service_map/get_service_map_service_node_info'; import { createRoute } from './create_route'; import { rangeRt } from './default_api_types'; +import { APM_SERVICE_MAPS_FEATURE_NAME } from '../feature'; export const serviceMapRoute = createRoute(() => ({ path: '/api/apm/service-map', @@ -35,6 +36,10 @@ export const serviceMapRoute = createRoute(() => ({ throw Boom.forbidden(invalidLicenseMessage); } + context.plugins.licensing.featureUsage.notifyUsage( + APM_SERVICE_MAPS_FEATURE_NAME + ); + const setup = await setupRequest(context, request); const { query: { serviceName, environment }, diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index bc31cb7a582af2..f30a9d18d7aeab 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -14,10 +14,11 @@ import { import { PickByValue, Optional } from 'utility-types'; import { Observable } from 'rxjs'; import { Server } from 'hapi'; +import { LicensingPluginStart } from '../../../licensing/server'; import { ObservabilityPluginSetup } from '../../../observability/server'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { FetchOptions } from '../../public/services/rest/callApi'; -import { SecurityPluginSetup } from '../../../security/public'; +import { SecurityPluginSetup } from '../../../security/server'; import { MlPluginSetup } from '../../../ml/server'; import { APMConfig } from '..'; @@ -66,6 +67,7 @@ export type APMRequestHandlerContext< config: APMConfig; logger: Logger; plugins: { + licensing: LicensingPluginStart; observability?: ObservabilityPluginSetup; security?: SecurityPluginSetup; ml?: MlPluginSetup; @@ -114,6 +116,7 @@ export interface ServerAPI { config$: Observable; logger: Logger; plugins: { + licensing: LicensingPluginStart; observability?: ObservabilityPluginSetup; security?: SecurityPluginSetup; ml?: MlPluginSetup; From dea2768545080cc779cac30a2d2f9d85b7a50f0f Mon Sep 17 00:00:00 2001 From: MadameSheema Date: Thu, 18 Jun 2020 08:22:50 +0200 Subject: [PATCH 07/60] fixes screenshots upload (#69392) --- vars/kibanaPipeline.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index ca9af5d2346cd2..4a5c8dff8a2301 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -98,7 +98,7 @@ def withGcsArtifactUpload(workerName, closure) { def uploadPrefix = "kibana-ci-artifacts/jobs/${env.JOB_NAME}/${BUILD_NUMBER}/${workerName}" def ARTIFACT_PATTERNS = [ 'target/kibana-*', - 'target/kibana-siem/**/*.png', + 'target/kibana-security-solution/**/*.png', 'target/junit/**/*', 'test/**/screenshots/**/*.png', 'test/functional/failure_debug/html/*.html', From 9f9a5c69f94a03cc41c738480088161dd814adae Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Thu, 18 Jun 2020 09:53:16 +0300 Subject: [PATCH 08/60] Remove unused deps (#69243) * remove unused deps * move types in devDeps --- package.json | 6 ++---- yarn.lock | 28 ---------------------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/package.json b/package.json index e91c5e96b78ab4..06dfb4cdfe3872 100644 --- a/package.json +++ b/package.json @@ -144,7 +144,6 @@ "@kbn/test-subj-selector": "0.2.1", "@kbn/ui-framework": "1.0.0", "@kbn/ui-shared-deps": "1.0.0", - "@types/tar": "^4.0.3", "JSONStream": "1.3.5", "abortcontroller-polyfill": "^1.4.0", "accept": "3.0.2", @@ -390,9 +389,10 @@ "@types/styled-components": "^5.1.0", "@types/supertest": "^2.0.5", "@types/supertest-as-promised": "^2.0.38", + "@types/tar": "^4.0.3", + "@types/testing-library__dom": "^6.10.0", "@types/testing-library__react": "^9.1.2", "@types/testing-library__react-hooks": "^3.1.0", - "@types/testing-library__dom": "^6.10.0", "@types/type-detect": "^4.0.1", "@types/uuid": "^3.4.4", "@types/vinyl-fs": "^2.4.11", @@ -486,12 +486,10 @@ "prettier": "^2.0.5", "proxyquire": "1.8.0", "react-popper-tooltip": "^2.10.1", - "react-textarea-autosize": "^7.1.2", "regenerate": "^1.4.0", "sass-lint": "^1.12.1", "selenium-webdriver": "^4.0.0-alpha.7", "simple-git": "1.116.0", - "simplebar-react": "^2.1.0", "sinon": "^7.4.2", "strip-ansi": "^3.0.1", "supertest": "^3.1.0", diff --git a/yarn.lock b/yarn.lock index c9bbbd8a999f8b..256c8642a02ae0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -25637,14 +25637,6 @@ react-textarea-autosize@^7.1.0: "@babel/runtime" "^7.1.2" prop-types "^15.6.0" -react-textarea-autosize@^7.1.2: - version "7.1.2" - resolved "https://registry.yarnpkg.com/react-textarea-autosize/-/react-textarea-autosize-7.1.2.tgz#70fdb333ef86bcca72717e25e623e90c336e2cda" - integrity sha512-uH3ORCsCa3C6LHxExExhF4jHoXYCQwE5oECmrRsunlspaDAbS4mGKNlWZqjLfInWtFQcf0o1n1jC/NGXFdUBCg== - dependencies: - "@babel/runtime" "^7.1.2" - prop-types "^15.6.0" - react-tiny-virtual-list@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/react-tiny-virtual-list/-/react-tiny-virtual-list-2.2.0.tgz#eafb6fcf764e4ed41150ff9752cdaad8b35edf4a" @@ -27892,14 +27884,6 @@ simplebar-react@^1.0.0-alpha.6: prop-types "^15.6.1" simplebar "^4.2.0" -simplebar-react@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/simplebar-react/-/simplebar-react-2.1.0.tgz#57d524f4253579d81ac30db00acf7886b17bf826" - integrity sha512-UIMFPNkn6o57v058vPOiYbnbpc1CUZwPKLmQaDMvEJdgm+btZ2umFA6meXfiqFEQUjDE6Vq4ePnL7Fr6nzJd8w== - dependencies: - prop-types "^15.6.1" - simplebar "^5.1.0" - simplebar@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/simplebar/-/simplebar-4.2.0.tgz#97e5c1c85d05cc04f8c92939e4da71dd087e325c" @@ -27912,18 +27896,6 @@ simplebar@^4.2.0: lodash.throttle "^4.1.1" resize-observer-polyfill "^1.5.1" -simplebar@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/simplebar/-/simplebar-5.1.0.tgz#15437ace314ec888accd7d8f24ada672e9bb2717" - integrity sha512-bdi1SleK1YOSnfeUjo5UQXt/79zNjsCJVEfzrm6photmGi2aU6x0l7rX4KAGcrtj5AwsWPBVXgDyYAqbbpnuRg== - dependencies: - can-use-dom "^0.1.0" - core-js "^3.0.1" - lodash.debounce "^4.0.8" - lodash.memoize "^4.1.2" - lodash.throttle "^4.1.1" - resize-observer-polyfill "^1.5.1" - simplicial-complex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/simplicial-complex/-/simplicial-complex-1.0.0.tgz#6c33a4ed69fcd4d91b7bcadd3b30b63683eae241" From f27162a2133f71ce0b6050a864a05383e3049706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Thu, 18 Jun 2020 09:35:48 +0200 Subject: [PATCH 09/60] [Observability] Creates "Add data" links in all Observability app headers (#69016) * Add data options to all Observability app headers * Updated failing snapshot * [Uptime] Update snapshot Co-authored-by: Elastic Machine --- .../components/app/ServiceDetails/index.tsx | 24 ++++++++++++++++++- .../ServiceOverview.test.tsx.snap | 2 +- .../shared/Links/SetupInstructionsLink.tsx | 10 +++++--- .../infra/public/pages/logs/page_content.tsx | 18 +++++++++++++- .../infra/public/pages/metrics/index.tsx | 20 +++++++++++++++- .../__snapshots__/page_header.test.tsx.snap | 23 ++++++++++++++++++ .../uptime/public/pages/page_header.tsx | 16 +++++++++++++ 7 files changed, 106 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx index 0dbde5ea86a187..2d52ad88d20dca 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ServiceDetails/index.tsx @@ -4,8 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiFlexGroup, EuiFlexItem, EuiTitle } from '@elastic/eui'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiTitle, + EuiButtonEmpty, +} from '@elastic/eui'; import React from 'react'; +import { i18n } from '@kbn/i18n'; import { ApmHeader } from '../../shared/ApmHeader'; import { ServiceDetailTabs } from './ServiceDetailTabs'; import { ServiceIntegrations } from './ServiceIntegrations'; @@ -33,6 +39,12 @@ export function ServiceDetails({ tab }: Props) { const isAlertingAvailable = isAlertingPluginEnabled && (canReadAlerts || canSaveAlerts); + const { core } = useApmPluginContext(); + + const ADD_DATA_LABEL = i18n.translate('xpack.apm.addDataButtonLabel', { + defaultMessage: 'Add data', + }); + return (
@@ -53,6 +65,16 @@ export function ServiceDetails({ tab }: Props) { /> )} + + + {ADD_DATA_LABEL} + + diff --git a/x-pack/plugins/apm/public/components/app/ServiceOverview/__test__/__snapshots__/ServiceOverview.test.tsx.snap b/x-pack/plugins/apm/public/components/app/ServiceOverview/__test__/__snapshots__/ServiceOverview.test.tsx.snap index e89acca55d4fe5..241ba8c2444961 100644 --- a/x-pack/plugins/apm/public/components/app/ServiceOverview/__test__/__snapshots__/ServiceOverview.test.tsx.snap +++ b/x-pack/plugins/apm/public/components/app/ServiceOverview/__test__/__snapshots__/ServiceOverview.test.tsx.snap @@ -114,7 +114,7 @@ NodeList [ - Setup Instructions + Setup instructions diff --git a/x-pack/plugins/apm/public/components/shared/Links/SetupInstructionsLink.tsx b/x-pack/plugins/apm/public/components/shared/Links/SetupInstructionsLink.tsx index e85605e42981cb..a5bcec1501ad3f 100644 --- a/x-pack/plugins/apm/public/components/shared/Links/SetupInstructionsLink.tsx +++ b/x-pack/plugins/apm/public/components/shared/Links/SetupInstructionsLink.tsx @@ -12,10 +12,14 @@ import { useApmPluginContext } from '../../../hooks/useApmPluginContext'; const SETUP_INSTRUCTIONS_LABEL = i18n.translate( 'xpack.apm.setupInstructionsButtonLabel', { - defaultMessage: 'Setup Instructions', + defaultMessage: 'Setup instructions', } ); +const ADD_DATA_LABEL = i18n.translate('xpack.apm.addDataButtonLabel', { + defaultMessage: 'Add data', +}); + // renders a filled button or a link as a kibana link to setup instructions export function SetupInstructionsLink({ buttonFill = false, @@ -30,8 +34,8 @@ export function SetupInstructionsLink({ {SETUP_INSTRUCTIONS_LABEL} ) : ( - - {SETUP_INSTRUCTIONS_LABEL} + + {ADD_DATA_LABEL} )} diff --git a/x-pack/plugins/infra/public/pages/logs/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/page_content.tsx index 14c53557ba2c74..78b7f86993cbde 100644 --- a/x-pack/plugins/infra/public/pages/logs/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/page_content.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup, EuiFlexItem, EuiButtonEmpty } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; import { Route, Switch } from 'react-router-dom'; @@ -32,6 +32,8 @@ export const LogsPageContent: React.FunctionComponent = () => { const { initialize } = useLogSourceContext(); + const kibana = useKibana(); + useMount(() => { initialize(); }); @@ -88,6 +90,16 @@ export const LogsPageContent: React.FunctionComponent = () => { + + + {ADD_DATA_LABEL} + + @@ -123,3 +135,7 @@ const settingsTabTitle = i18n.translate('xpack.infra.logs.index.settingsTabTitle }); const feedbackLinkUrl = 'https://discuss.elastic.co/c/logs'; + +const ADD_DATA_LABEL = i18n.translate('xpack.infra.logsHeaderAddDataButtonLabel', { + defaultMessage: 'Add data', +}); diff --git a/x-pack/plugins/infra/public/pages/metrics/index.tsx b/x-pack/plugins/infra/public/pages/metrics/index.tsx index 35a6cadc786f61..05296fbf6b0a32 100644 --- a/x-pack/plugins/infra/public/pages/metrics/index.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/index.tsx @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { Route, RouteComponentProps, Switch } from 'react-router-dom'; -import { EuiErrorBoundary, EuiFlexItem, EuiFlexGroup } from '@elastic/eui'; +import { EuiErrorBoundary, EuiFlexItem, EuiFlexGroup, EuiButtonEmpty } from '@elastic/eui'; import { DocumentTitle } from '../../components/document_title'; import { HelpCenterContent } from '../../components/help_center_content'; import { RoutedTabs } from '../../components/navigation/routed_tabs'; @@ -32,9 +32,15 @@ import { WaffleFiltersProvider } from './inventory_view/hooks/use_waffle_filters import { InventoryAlertDropdown } from '../../components/alerting/inventory/alert_dropdown'; import { MetricsAlertDropdown } from '../../alerting/metric_threshold/components/alert_dropdown'; +const ADD_DATA_LABEL = i18n.translate('xpack.infra.metricsHeaderAddDataButtonLabel', { + defaultMessage: 'Add data', +}); + export const InfrastructurePage = ({ match }: RouteComponentProps) => { const uiCapabilities = useKibana().services.application?.capabilities; + const kibana = useKibana(); + return ( @@ -102,6 +108,18 @@ export const InfrastructurePage = ({ match }: RouteComponentProps) => { + + + {ADD_DATA_LABEL} + + diff --git a/x-pack/plugins/uptime/public/pages/__tests__/__snapshots__/page_header.test.tsx.snap b/x-pack/plugins/uptime/public/pages/__tests__/__snapshots__/page_header.test.tsx.snap index dbdfd6b27e69f5..fcf68ad97c8cec 100644 --- a/x-pack/plugins/uptime/public/pages/__tests__/__snapshots__/page_header.test.tsx.snap +++ b/x-pack/plugins/uptime/public/pages/__tests__/__snapshots__/page_header.test.tsx.snap @@ -104,6 +104,29 @@ Array [
+
+ +
) : null; + const kibana = useKibana(); + const extraLinkComponents = !extraLinks ? null : ( @@ -64,6 +71,15 @@ export const PageHeader = React.memo( + + + {ADD_DATA_LABEL} + + ); From daf20daf2dc40999e60c18413807bc933bf47d04 Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 18 Jun 2020 11:32:13 +0200 Subject: [PATCH 10/60] Beats Management: Pre-migration cleanup (#69155) * adapt KibanaDatabaseAdapter to use core ES service * remove unused `exposeStaticDir` method * create empty KP plugin * remove unused and broken tests * delete unused wallaby config file * delete unused utils * delete unused d.ts and move formsy definition to new plugin * fix findNonExistentItems * remove dead code and useless exports from common package * delete non migratable test suite + remove unused test adapter * remove yet another unused adapter * restore awaits in KibanaDatabaseAdapter --- .../common/config_block_validation.ts | 57 ---- .../common/config_schemas_translations_map.ts | 2 +- .../beats_management/common/domain_types.ts | 2 - .../beats_management/common/io_ts_types.ts | 2 +- .../beats_management/common/return_types.ts | 11 - .../adapters/beats/memory_beats_adapter.ts | 118 --------- .../memory_tags_adapter.ts | 75 ------ .../lib/adapters/database/__tests__/kibana.ts | 40 --- .../database/__tests__/test_contract.ts | 58 ---- .../lib/adapters/database/adapter_types.ts | 11 +- .../database/kibana_database_adapter.ts | 56 ++-- .../lib/adapters/events/adapter_types.ts | 12 - .../elasticsearch_beat_events_adapter.ts | 21 -- .../lib/adapters/framework/adapter_types.ts | 6 +- .../framework/hapi_framework_adapter.ts | 148 ----------- .../framework/integration_tests/kibana.ts | 38 --- .../integration_tests/test_contract.ts | 29 -- .../framework/kibana_framework_adapter.ts | 12 - .../lib/adapters/tags/memory_tags_adapter.ts | 50 ---- .../adapters/tokens/memory_tokens_adapter.ts | 49 ---- .../server/lib/beat_events.ts | 4 +- .../beats_management/server/lib/beats.ts | 10 +- .../server/lib/compose/kibana.ts | 6 +- .../server/lib/compose/testing.ts | 51 ---- .../beats_management/server/lib/framework.ts | 1 - .../__tests__/beats_assignments.test.ts | 249 ------------------ .../server/rest_api/__tests__/data.json | 178 ------------- .../server/rest_api/__tests__/test_harnes.ts | 102 ------- .../beats_management/server/utils/README.md | 1 - .../server/utils/find_non_existent_items.ts | 19 -- .../server/utils/helper_types.ts | 13 - .../utils/index_templates/beats_template.json | 119 --------- .../server/utils/polyfills.ts | 17 -- .../server/utils/wrap_request.ts | 36 --- .../plugins/beats_management/types/eui.d.ts | 16 -- .../plugins/beats_management/wallaby.js | 62 ----- .../plugins/beats_management/server/index.ts | 8 +- .../plugins/beats_management/server/plugin.ts | 39 +++ .../beats_management/types/formsy.d.ts | 0 39 files changed, 83 insertions(+), 1645 deletions(-) delete mode 100644 x-pack/legacy/plugins/beats_management/common/config_block_validation.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/beats/memory_beats_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/configuration_blocks/memory_tags_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/kibana.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/test_contract.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/events/adapter_types.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/events/elasticsearch_beat_events_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/hapi_framework_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/kibana.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/test_contract.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/tags/memory_tags_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/adapters/tokens/memory_tokens_adapter.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/lib/compose/testing.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/beats_assignments.test.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/data.json delete mode 100644 x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/test_harnes.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/README.md delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/find_non_existent_items.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/helper_types.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/index_templates/beats_template.json delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/polyfills.ts delete mode 100644 x-pack/legacy/plugins/beats_management/server/utils/wrap_request.ts delete mode 100644 x-pack/legacy/plugins/beats_management/types/eui.d.ts delete mode 100644 x-pack/legacy/plugins/beats_management/wallaby.js create mode 100644 x-pack/plugins/beats_management/server/plugin.ts rename x-pack/{legacy => }/plugins/beats_management/types/formsy.d.ts (100%) diff --git a/x-pack/legacy/plugins/beats_management/common/config_block_validation.ts b/x-pack/legacy/plugins/beats_management/common/config_block_validation.ts deleted file mode 100644 index f3d1b9164e9768..00000000000000 --- a/x-pack/legacy/plugins/beats_management/common/config_block_validation.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import * as t from 'io-ts'; -import { PathReporter } from 'io-ts/lib/PathReporter'; -import { isLeft } from 'fp-ts/lib/Either'; -import { configBlockSchemas } from './config_schemas'; -import { ConfigurationBlock, createConfigurationBlockInterface } from './domain_types'; - -export const validateConfigurationBlocks = (configurationBlocks: ConfigurationBlock[]) => { - const validationMap = { - isHosts: t.array(t.string), - isString: t.string, - isPeriod: t.string, - isPath: t.string, - isPaths: t.array(t.string), - isYaml: t.string, - }; - - for (const [index, block] of configurationBlocks.entries()) { - const blockSchema = configBlockSchemas.find((s) => s.id === block.type); - if (!blockSchema) { - throw new Error( - `Invalid config type of ${block.type} used in 'configuration_blocks' at index ${index}` - ); - } - - const interfaceConfig = blockSchema.configs.reduce((props, config) => { - if (config.options) { - props[config.id] = t.keyof( - Object.fromEntries(config.options.map((opt) => [opt.value, null])) as Record - ); - } else if (config.validation) { - props[config.id] = validationMap[config.validation]; - } - - return props; - }, {} as t.Props); - - const runtimeInterface = createConfigurationBlockInterface( - t.literal(blockSchema.id), - t.interface(interfaceConfig) - ); - - const validationResults = runtimeInterface.decode(block); - - if (isLeft(validationResults)) { - throw new Error( - `configuration_blocks validation error, configuration_blocks at index ${index} is invalid. ${ - PathReporter.report(validationResults)[0] - }` - ); - } - } -}; diff --git a/x-pack/legacy/plugins/beats_management/common/config_schemas_translations_map.ts b/x-pack/legacy/plugins/beats_management/common/config_schemas_translations_map.ts index 1aec3e80817088..7cae2a85dc4ca8 100644 --- a/x-pack/legacy/plugins/beats_management/common/config_schemas_translations_map.ts +++ b/x-pack/legacy/plugins/beats_management/common/config_schemas_translations_map.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import { ConfigBlockSchema } from './domain_types'; -export const supportedConfigLabelsMap = new Map([ +const supportedConfigLabelsMap = new Map([ [ 'filebeatInputConfig.paths.ui.label', i18n.translate('xpack.beatsManagement.filebeatInputConfig.pathsLabel', { diff --git a/x-pack/legacy/plugins/beats_management/common/domain_types.ts b/x-pack/legacy/plugins/beats_management/common/domain_types.ts index b4a9ac8a074798..32e1d81451c652 100644 --- a/x-pack/legacy/plugins/beats_management/common/domain_types.ts +++ b/x-pack/legacy/plugins/beats_management/common/domain_types.ts @@ -7,8 +7,6 @@ import * as t from 'io-ts'; import { configBlockSchemas } from './config_schemas'; import { DateFromString } from './io_ts_types'; -export const OutputTypesArray = ['elasticsearch', 'logstash', 'kafka', 'redis']; - // Here we create the runtime check for a generic, unknown beat config type. // We can also pass in optional params to create spacific runtime checks that // can be used to validate blocs on the API and UI diff --git a/x-pack/legacy/plugins/beats_management/common/io_ts_types.ts b/x-pack/legacy/plugins/beats_management/common/io_ts_types.ts index d77ad922986995..7d71ea5ad82562 100644 --- a/x-pack/legacy/plugins/beats_management/common/io_ts_types.ts +++ b/x-pack/legacy/plugins/beats_management/common/io_ts_types.ts @@ -7,7 +7,7 @@ import * as t from 'io-ts'; import { isRight } from 'fp-ts/lib/Either'; -export class DateFromStringType extends t.Type { +class DateFromStringType extends t.Type { // eslint-disable-next-line public readonly _tag: 'DateFromISOStringType' = 'DateFromISOStringType'; constructor() { diff --git a/x-pack/legacy/plugins/beats_management/common/return_types.ts b/x-pack/legacy/plugins/beats_management/common/return_types.ts index a7125795a5c7d5..7e0e39e12e60aa 100644 --- a/x-pack/legacy/plugins/beats_management/common/return_types.ts +++ b/x-pack/legacy/plugins/beats_management/common/return_types.ts @@ -34,11 +34,6 @@ export interface ReturnTypeBulkCreate extends BaseReturnType { }>; } -// delete -export interface ReturnTypeDelete extends BaseReturnType { - action: 'deleted'; -} - export interface ReturnTypeBulkDelete extends BaseReturnType { results: Array<{ success: boolean; @@ -84,12 +79,6 @@ export interface ReturnTypeBulkGet extends BaseReturnType { items: T[]; } -// action -- e.g. validate config block. Like ES simulate endpoint -export interface ReturnTypeAction extends BaseReturnType { - result: { - [key: string]: any; - }; -} // e.g. // { // result: { diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/beats/memory_beats_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/beats/memory_beats_adapter.ts deleted file mode 100644 index afae87c4901588..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/beats/memory_beats_adapter.ts +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { intersection, omit } from 'lodash'; - -import { CMBeat } from '../../../../common/domain_types'; -import { FrameworkUser } from '../framework/adapter_types'; -import { BeatsTagAssignment, CMBeatsAdapter } from './adapter_types'; - -export class MemoryBeatsAdapter implements CMBeatsAdapter { - private beatsDB: CMBeat[]; - - constructor(beatsDB: CMBeat[]) { - this.beatsDB = beatsDB; - } - - public async get(user: FrameworkUser, id: string) { - return this.beatsDB.find((beat) => beat.id === id) || null; - } - - public async insert(user: FrameworkUser, beat: CMBeat) { - this.beatsDB.push(beat); - } - - public async update(user: FrameworkUser, beat: CMBeat) { - const beatIndex = this.beatsDB.findIndex((b) => b.id === beat.id); - - this.beatsDB[beatIndex] = { - ...this.beatsDB[beatIndex], - ...beat, - }; - } - - public async getWithIds(user: FrameworkUser, beatIds: string[]) { - return this.beatsDB.filter((beat) => beatIds.includes(beat.id)); - } - - public async getAllWithTags(user: FrameworkUser, tagIds: string[]): Promise { - return this.beatsDB.filter((beat) => intersection(tagIds, beat.tags || []).length !== 0); - } - - public async getBeatWithToken( - user: FrameworkUser, - enrollmentToken: string - ): Promise { - return this.beatsDB.find((beat) => enrollmentToken === beat.enrollment_token) || null; - } - - public async getAll(user: FrameworkUser) { - return this.beatsDB.map((beat: any) => omit(beat, ['access_token'])); - } - - public async removeTagsFromBeats( - user: FrameworkUser, - removals: BeatsTagAssignment[] - ): Promise { - const beatIds = removals.map((r) => r.beatId); - - const response = this.beatsDB - .filter((beat) => beatIds.includes(beat.id)) - .map((beat) => { - const tagData = removals.find((r) => r.beatId === beat.id); - if (tagData) { - if (beat.tags) { - beat.tags = beat.tags.filter((tag) => tag !== tagData.tag); - } - } - return beat; - }); - - return response.map((item: CMBeat, resultIdx: number) => ({ - idxInRequest: removals[resultIdx].idxInRequest, - result: 'updated', - status: 200, - })); - } - - public async assignTagsToBeats( - user: FrameworkUser, - assignments: BeatsTagAssignment[] - ): Promise { - const beatIds = assignments.map((r) => r.beatId); - - this.beatsDB - .filter((beat) => beatIds.includes(beat.id)) - .map((beat) => { - // get tags that need to be assigned to this beat - const tags = assignments - .filter((a) => a.beatId === beat.id) - .map((t: BeatsTagAssignment) => t.tag); - - if (tags.length > 0) { - if (!beat.tags) { - beat.tags = []; - } - const nonExistingTags = tags.filter((t: string) => beat.tags && !beat.tags.includes(t)); - - if (nonExistingTags.length > 0) { - beat.tags = beat.tags.concat(nonExistingTags); - } - } - return beat; - }); - - return assignments.map((item: BeatsTagAssignment, resultIdx: number) => ({ - idxInRequest: assignments[resultIdx].idxInRequest, - result: 'updated', - status: 200, - })); - } - - public setDB(beatsDB: CMBeat[]) { - this.beatsDB = beatsDB; - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/configuration_blocks/memory_tags_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/configuration_blocks/memory_tags_adapter.ts deleted file mode 100644 index ea8a75c92fad27..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/configuration_blocks/memory_tags_adapter.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import Chance from 'chance'; // eslint-disable-line -import { ConfigurationBlock } from '../../../../common/domain_types'; -import { FrameworkUser } from '../framework/adapter_types'; -import { ConfigurationBlockAdapter } from './adapter_types'; - -const chance = new Chance(); - -export class MemoryConfigurationBlockAdapter implements ConfigurationBlockAdapter { - private db: ConfigurationBlock[] = []; - - constructor(db: ConfigurationBlock[]) { - this.db = db.map((config) => { - if (config.id === undefined) { - config.id = chance.word(); - } - return config as ConfigurationBlock & { id: string }; - }); - } - - public async getByIds(user: FrameworkUser, ids: string[]) { - return this.db.filter((block) => ids.includes(block.id)); - } - public async delete(user: FrameworkUser, blockIds: string[]) { - this.db = this.db.filter((block) => !blockIds.includes(block.id)); - return blockIds.map((id) => ({ - id, - success: true, - })); - } - public async deleteForTags( - user: FrameworkUser, - tagIds: string[] - ): Promise<{ success: boolean; reason?: string }> { - this.db = this.db.filter((block) => !tagIds.includes(block.tag)); - return { - success: true, - }; - } - - public async getForTags(user: FrameworkUser, tagIds: string[], page?: number, size?: number) { - const results = this.db.filter((block) => tagIds.includes(block.id)); - return { - page: 0, - total: results.length, - blocks: results, - }; - } - - public async create(user: FrameworkUser, blocks: ConfigurationBlock[]) { - return blocks.map((block) => { - const existingIndex = this.db.findIndex((t) => t.id === block.id); - if (existingIndex !== -1) { - this.db[existingIndex] = block; - } else { - this.db.push(block); - } - return block.id; - }); - } - - public setDB(db: ConfigurationBlock[]) { - this.db = db.map((block) => { - if (block.id === undefined) { - block.id = chance.word(); - } - return block; - }); - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/kibana.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/kibana.ts deleted file mode 100644 index 460fc412e94910..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/kibana.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -// file.skip - -// @ts-ignore -import { createLegacyEsTestCluster } from '@kbn/test'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { Root } from 'src/core/server/root'; -// @ts-ignore -import * as kbnTestServer from '../../../../../../../../../src/test_utils/kbn_server'; -import { DatabaseKbnESPlugin } from '../adapter_types'; -import { KibanaDatabaseAdapter } from '../kibana_database_adapter'; -import { contractTests } from './test_contract'; -const es = createLegacyEsTestCluster({}); - -let legacyServer: any; -let rootServer: Root; -contractTests('Kibana Database Adapter', { - before: async () => { - await es.start(); - - rootServer = kbnTestServer.createRootWithCorePlugins({ - server: { maxPayloadBytes: 100 }, - }); - - await rootServer.setup(); - legacyServer = kbnTestServer.getKbnServer(rootServer); - return await legacyServer.plugins.elasticsearch.waitUntilReady(); - }, - after: async () => { - await rootServer.shutdown(); - return await es.cleanup(); - }, - adapterSetup: () => { - return new KibanaDatabaseAdapter(legacyServer.plugins.elasticsearch as DatabaseKbnESPlugin); - }, -}); diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/test_contract.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/test_contract.ts deleted file mode 100644 index 369c2e10562118..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/__tests__/test_contract.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { DatabaseAdapter } from '../adapter_types'; - -interface ContractConfig { - before?(): Promise; - after?(): Promise; - adapterSetup(): DatabaseAdapter; -} - -export const contractTests = (testName: string, config: ContractConfig) => { - describe.skip(testName, () => { - let database: DatabaseAdapter; - beforeAll(async () => { - jest.setTimeout(100000); // 1 second - - if (config.before) { - await config.before(); - } - }); - afterAll(async () => config.after && (await config.after())); - beforeEach(async () => { - database = config.adapterSetup(); - }); - - it('Unauthorized users cant query', async () => { - const params = { - id: `beat:foo`, - ignore: [404], - index: '.management-beats', - }; - let ranWithoutError = false; - try { - await database.get({ kind: 'unauthenticated' }, params); - ranWithoutError = true; - } catch (e) { - expect(e).not.toEqual(null); - } - expect(ranWithoutError).toEqual(false); - }); - - it('Should query ES', async () => { - const params = { - id: `beat:foo`, - ignore: [404], - index: '.management-beats', - }; - const response = await database.get({ kind: 'internal' }, params); - - expect(response).not.toEqual(undefined); - // @ts-ignore - expect(response.found).toEqual(undefined); - }); - }); -}; diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/adapter_types.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/adapter_types.ts index 0a06c3dcc6412d..90519840af213b 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/adapter_types.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/adapter_types.ts @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { FrameworkRequest, FrameworkUser } from '../framework/adapter_types'; +import { FrameworkUser } from '../framework/adapter_types'; export interface DatabaseAdapter { get( @@ -39,15 +39,6 @@ export interface DatabaseAdapter { putTemplate(name: string, template: any): Promise; } -export interface DatabaseKbnESCluster { - callWithInternalUser(esMethod: string, options: {}): Promise; - callWithRequest(req: FrameworkRequest, esMethod: string, options: {}): Promise; -} - -export interface DatabaseKbnESPlugin { - getCluster(clusterName: string): DatabaseKbnESCluster; -} - export interface DatabaseSearchParams extends DatabaseGenericParams { analyzer?: string; analyzeWildcard?: boolean; diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/kibana_database_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/kibana_database_adapter.ts index 1ca3bcae8bfca6..baccbe416f3980 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/kibana_database_adapter.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/adapters/database/kibana_database_adapter.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { ElasticsearchServiceStart, IClusterClient } from 'src/core/server'; import { FrameworkUser } from '../framework/adapter_types'; import { internalAuthData } from './../framework/adapter_types'; import { @@ -15,8 +16,6 @@ import { DatabaseGetDocumentResponse, DatabaseGetParams, DatabaseIndexDocumentParams, - DatabaseKbnESCluster, - DatabaseKbnESPlugin, DatabaseMGetParams, DatabaseMGetResponse, DatabaseSearchParams, @@ -24,75 +23,67 @@ import { } from './adapter_types'; export class KibanaDatabaseAdapter implements DatabaseAdapter { - private es: DatabaseKbnESCluster; + private es: IClusterClient; - constructor(kbnElasticSearch: DatabaseKbnESPlugin) { - this.es = kbnElasticSearch.getCluster('admin'); + constructor(elasticsearch: ElasticsearchServiceStart) { + this.es = elasticsearch.legacy.client; } public async get( user: FrameworkUser, params: DatabaseGetParams ): Promise> { - const result = await this.callWithUser(user, 'get', params); - return result; - // todo + return await this.callWithUser(user, 'get', params); } public async mget( user: FrameworkUser, params: DatabaseMGetParams ): Promise> { - const result = await this.callWithUser(user, 'mget', params); - return result; - // todo + return await this.callWithUser(user, 'mget', params); } public async bulk(user: FrameworkUser, params: DatabaseBulkIndexDocumentsParams): Promise { - const result = await this.callWithUser(user, 'bulk', params); - return result; + return await this.callWithUser(user, 'bulk', params); } public async create( user: FrameworkUser, params: DatabaseCreateDocumentParams ): Promise { - const result = await this.callWithUser(user, 'create', params); - return result; + return await this.callWithUser(user, 'create', params); } + public async index(user: FrameworkUser, params: DatabaseIndexDocumentParams): Promise { - const result = await this.callWithUser(user, 'index', params); - return result; + return await this.callWithUser(user, 'index', params); } + public async delete( user: FrameworkUser, params: DatabaseDeleteDocumentParams ): Promise { - const result = await this.callWithUser(user, 'delete', params); - return result; + return await this.callWithUser(user, 'delete', params); } public async deleteByQuery( user: FrameworkUser, params: DatabaseSearchParams ): Promise { - const result = await this.callWithUser(user, 'deleteByQuery', params); - return result; + return await this.callWithUser(user, 'deleteByQuery', params); } public async search( user: FrameworkUser, params: DatabaseSearchParams ): Promise> { - const result = await this.callWithUser(user, 'search', params); - return result; + return await this.callWithUser(user, 'search', params); } public async searchAll( user: FrameworkUser, params: DatabaseSearchParams ): Promise> { - const result = await this.callWithUser(user, 'search', { + return await this.callWithUser(user, 'search', { scroll: '1m', ...params, body: { @@ -100,29 +91,24 @@ export class KibanaDatabaseAdapter implements DatabaseAdapter { ...params.body, }, }); - return result; } public async putTemplate(name: string, template: any): Promise { - const result = await this.callWithUser({ kind: 'internal' }, 'indices.putTemplate', { + return await this.callWithUser({ kind: 'internal' }, 'indices.putTemplate', { name, body: template, }); - - return result; } private callWithUser(user: FrameworkUser, esMethod: string, options: any = {}): any { if (user.kind === 'authenticated') { - return this.es.callWithRequest( - { + return this.es + .asScoped({ headers: user[internalAuthData], - } as any, - esMethod, - options - ); + }) + .callAsCurrentUser(esMethod, options); } else if (user.kind === 'internal') { - return this.es.callWithInternalUser(esMethod, options); + return this.es.callAsInternalUser(esMethod, options); } else { throw new Error('Invalid user type'); } diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/adapter_types.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/adapter_types.ts deleted file mode 100644 index 4cb38bb3d057b6..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/adapter_types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { BeatEvent } from '../../../../common/domain_types'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { FrameworkUser } from '../../../../../../../plugins/beats_management/public/lib/adapters/framework/adapter_types'; - -export interface BeatEventsAdapter { - bulkInsert(user: FrameworkUser, beatId: string, events: BeatEvent[]): Promise; -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/elasticsearch_beat_events_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/elasticsearch_beat_events_adapter.ts deleted file mode 100644 index b5056140c8b860..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/events/elasticsearch_beat_events_adapter.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { BeatEvent } from '../../../../common/domain_types'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { FrameworkUser } from '../../../../../../../plugins/beats_management/public/lib/adapters/framework/adapter_types'; -import { DatabaseAdapter } from '../database/adapter_types'; -import { BeatEventsAdapter } from './adapter_types'; - -export class ElasticsearchBeatEventsAdapter implements BeatEventsAdapter { - // @ts-ignore - constructor(private readonly database: DatabaseAdapter) {} - - // eslint-disable-next-line - public bulkInsert = async (user: FrameworkUser, beatId: string, events: BeatEvent[]) => { - // await this.database.putTemplate(INDEX_NAMES.EVENTS_TODAY, beatsIndexTemplate); - }; -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/adapter_types.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/adapter_types.ts index 80599f38d982ab..e2703cb5786dd8 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/adapter_types.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/adapter_types.ts @@ -8,6 +8,7 @@ import { Lifecycle, ResponseToolkit } from 'hapi'; import * as t from 'io-ts'; +import { CoreSetup, CoreStart } from 'src/core/server'; import { SecurityPluginSetup } from '../../../../../../../plugins/security/server'; import { LicenseType } from '../../../../common/constants/security'; @@ -33,7 +34,6 @@ export interface BackendFrameworkAdapter { log(text: string): void; on(event: 'xpack.status.green' | 'elasticsearch.status.green', cb: () => void): void; getSetting(settingPath: string): any; - exposeStaticDir(urlPath: string, dir: string): void; registerRoute( route: FrameworkRouteOptions ): void; @@ -42,8 +42,12 @@ export interface BackendFrameworkAdapter { export interface KibanaLegacyServer { newPlatform: { setup: { + core: CoreSetup; plugins: { security: SecurityPluginSetup }; }; + start: { + core: CoreStart; + }; }; plugins: { xpack_main: { diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/hapi_framework_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/hapi_framework_adapter.ts deleted file mode 100644 index 90500e02835116..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/hapi_framework_adapter.ts +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { LicenseType } from './../../../../common/constants/security'; -import { KibanaServerRequest } from './adapter_types'; -import { - BackendFrameworkAdapter, - FrameworkInfo, - FrameworkRequest, - FrameworkResponse, - FrameworkRouteOptions, - internalAuthData, - internalUser, -} from './adapter_types'; - -interface TestSettings { - enrollmentTokensTtlInSeconds: number; - encryptionKey: string; -} - -export class HapiBackendFrameworkAdapter implements BackendFrameworkAdapter { - public info: null | FrameworkInfo = null; - public readonly internalUser = internalUser; - - private settings: TestSettings; - private server: any; - - constructor( - settings: TestSettings = { - encryptionKey: 'something_who_cares', - enrollmentTokensTtlInSeconds: 10 * 60, // 10 minutes - }, - hapiServer?: any, - license: LicenseType = 'trial', - securityEnabled: boolean = true, - licenseActive: boolean = true - ) { - this.server = hapiServer; - this.settings = settings; - const now = new Date(); - - this.info = { - kibana: { - version: 'unknown', - }, - license: { - type: license, - expired: !licenseActive, - expiry_date_in_millis: new Date(now.getFullYear(), now.getMonth() + 1, 1).getTime(), - }, - security: { - enabled: securityEnabled, - available: securityEnabled, - }, - watcher: { - enabled: true, - available: true, - }, - }; - } - public log(text: string) { - this.server.log(text); - } - public on(event: 'xpack.status.green', cb: () => void) { - cb(); - } - public getSetting(settingPath: string) { - switch (settingPath) { - case 'xpack.beats.enrollmentTokensTtlInSeconds': - return this.settings.enrollmentTokensTtlInSeconds; - case 'xpack.beats.encryptionKey': - return this.settings.encryptionKey; - } - } - - public exposeStaticDir(urlPath: string, dir: string): void { - if (!this.server) { - throw new Error('Must pass a hapi server into the adapter to use exposeStaticDir'); - } - this.server.route({ - handler: { - directory: { - path: dir, - }, - }, - method: 'GET', - path: urlPath, - }); - } - - public registerRoute< - RouteRequest extends FrameworkRequest, - RouteResponse extends FrameworkResponse - >(route: FrameworkRouteOptions) { - if (!this.server) { - throw new Error('Must pass a hapi server into the adapter to use registerRoute'); - } - const wrappedHandler = (licenseRequired: string[]) => (request: any, h: any) => { - return route.handler(this.wrapRequest(request), h); - }; - - this.server.route({ - handler: wrappedHandler(route.licenseRequired || []), - method: route.method, - path: route.path, - config: { - ...route.config, - auth: false, - }, - }); - } - - public async injectRequstForTesting({ method, url, headers, payload }: any) { - return await this.server.inject({ method, url, headers, payload }); - } - - private wrapRequest( - req: InternalRequest - ): FrameworkRequest { - const { params, payload, query, headers, info } = req; - - const isAuthenticated = headers.authorization != null; - - return { - user: isAuthenticated - ? { - kind: 'authenticated', - [internalAuthData]: headers, - username: 'elastic', - roles: ['superuser'], - full_name: null, - email: null, - enabled: true, - } - : { - kind: 'unauthenticated', - }, - headers, - info, - params, - payload, - query, - }; - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/kibana.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/kibana.ts deleted file mode 100644 index 4f0ba01b860825..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/kibana.ts +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -// file.skip - -import { camelCase } from 'lodash'; -// @ts-ignore -import * as kbnTestServer from '../../../../../../../../../src/test_utils/kbn_server'; -// @ts-ignore -import { TestKbnServerConfig } from '../../../../../../../test_utils/kbn_server_config'; -import { CONFIG_PREFIX } from '../../../../../common/constants/plugin'; -import { PLUGIN } from './../../../../../common/constants/plugin'; -import { KibanaBackendFrameworkAdapter } from './../kibana_framework_adapter'; -import { contractTests } from './test_contract'; - -let kbnServer: any; -let kbn: any; -let esServer: any; -contractTests('Kibana Framework Adapter', { - async before() { - const servers = kbnTestServer.createTestServers({ - adjustTimeout: (t: number) => jest.setTimeout(t), - settings: TestKbnServerConfig, - }); - esServer = await servers.startES(); - kbn = await servers.startKibana(); - kbnServer = kbn.kbnServer; - }, - async after() { - await kbn.stop(); - await esServer.stop(); - }, - adapterSetup: () => { - return new KibanaBackendFrameworkAdapter(camelCase(PLUGIN.ID), kbnServer.server, CONFIG_PREFIX); - }, -}); diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/test_contract.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/test_contract.ts deleted file mode 100644 index 8e21f8cf78ad7c..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/integration_tests/test_contract.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { BackendFrameworkAdapter } from '../adapter_types'; - -interface ContractConfig { - before(): Promise; - after(): Promise; - adapterSetup(): BackendFrameworkAdapter; -} - -export const contractTests = (testName: string, config: ContractConfig) => { - describe(testName, () => { - let frameworkAdapter: BackendFrameworkAdapter; - beforeAll(config.before); - afterAll(config.after); - beforeEach(async () => { - frameworkAdapter = config.adapterSetup(); - }); - - it('Should have tests here', () => { - expect(frameworkAdapter.info).toHaveProperty('server'); - - expect(frameworkAdapter).toHaveProperty('server'); - }); - }); -}; diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/kibana_framework_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/kibana_framework_adapter.ts index 1bf9bbb22b3525..3b29e50e4465b9 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/kibana_framework_adapter.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/adapters/framework/kibana_framework_adapter.ts @@ -68,18 +68,6 @@ export class KibanaBackendFrameworkAdapter implements BackendFrameworkAdapter { this.server.log(text); } - public exposeStaticDir(urlPath: string, dir: string): void { - this.server.route({ - handler: { - directory: { - path: dir, - }, - }, - method: 'GET', - path: urlPath, - }); - } - public registerRoute< RouteRequest extends FrameworkRequest, RouteResponse extends FrameworkResponse diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/tags/memory_tags_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/tags/memory_tags_adapter.ts deleted file mode 100644 index 66a6c7ebebc2c5..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/tags/memory_tags_adapter.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { BeatTag } from '../../../../common/domain_types'; -import { FrameworkUser } from '../framework/adapter_types'; -import { CMTagsAdapter } from './adapter_types'; - -export class MemoryTagsAdapter implements CMTagsAdapter { - private tagsDB: BeatTag[] = []; - - constructor(tagsDB: BeatTag[]) { - this.tagsDB = tagsDB; - } - - public async getAll(user: FrameworkUser) { - return this.tagsDB; - } - public async delete(user: FrameworkUser, tagIds: string[]) { - this.tagsDB = this.tagsDB.filter((tag) => !tagIds.includes(tag.id)); - - return true; - } - public async getTagsWithIds(user: FrameworkUser, tagIds: string[]) { - return this.tagsDB.filter((tag) => tagIds.includes(tag.id)); - } - - public async upsertTag(user: FrameworkUser, tag: BeatTag) { - const existingTagIndex = this.tagsDB.findIndex((t) => t.id === tag.id); - if (existingTagIndex !== -1) { - this.tagsDB[existingTagIndex] = tag; - } else { - this.tagsDB.push(tag); - } - return tag.id; - } - - public async getWithoutConfigTypes( - user: FrameworkUser, - blockTypes: string[] - ): Promise { - return this.tagsDB.filter((tag) => tag.hasConfigurationBlocksTypes.includes(blockTypes[0])); - } - - public setDB(tagsDB: BeatTag[]) { - this.tagsDB = tagsDB; - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/adapters/tokens/memory_tokens_adapter.ts b/x-pack/legacy/plugins/beats_management/server/lib/adapters/tokens/memory_tokens_adapter.ts deleted file mode 100644 index 431263c808b45d..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/adapters/tokens/memory_tokens_adapter.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { FrameworkAuthenticatedUser, FrameworkUser } from '../framework/adapter_types'; -import { CMTokensAdapter, TokenEnrollmentData } from './adapter_types'; - -export class MemoryTokensAdapter implements CMTokensAdapter { - private tokenDB: TokenEnrollmentData[]; - - constructor(tokenDB: TokenEnrollmentData[]) { - this.tokenDB = tokenDB; - } - - public async deleteEnrollmentToken(user: FrameworkUser, enrollmentToken: string) { - const index = this.tokenDB.findIndex((token) => token.token === enrollmentToken); - - if (index > -1) { - this.tokenDB.splice(index, 1); - } - } - - public async getEnrollmentToken( - user: FrameworkUser, - tokenString: string - ): Promise { - return new Promise((resolve) => { - return resolve(this.tokenDB.find((token) => token.token === tokenString)); - }); - } - - public async insertTokens(user: FrameworkAuthenticatedUser, tokens: TokenEnrollmentData[]) { - tokens.forEach((token) => { - const existingIndex = this.tokenDB.findIndex((t) => t.token === token.token); - if (existingIndex !== -1) { - this.tokenDB[existingIndex] = token; - } else { - this.tokenDB.push(token); - } - }); - return tokens; - } - - public setDB(tokenDB: TokenEnrollmentData[]) { - this.tokenDB = tokenDB; - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/beat_events.ts b/x-pack/legacy/plugins/beats_management/server/lib/beat_events.ts index f4cb3cb424f6f0..54782783f94ca1 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/beat_events.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/beat_events.ts @@ -6,13 +6,11 @@ import { PathReporter } from 'io-ts/lib/PathReporter'; import { isLeft } from 'fp-ts/lib/Either'; import { BeatEvent, RuntimeBeatEvent } from '../../common/domain_types'; -import { BeatEventsAdapter } from './adapters/events/adapter_types'; import { FrameworkUser } from './adapters/framework/adapter_types'; import { CMBeatsDomain } from './beats'; export class BeatEventsLib { - // @ts-ignore - constructor(private readonly adapter: BeatEventsAdapter, private readonly beats: CMBeatsDomain) {} + constructor(private readonly beats: CMBeatsDomain) {} public log = async ( user: FrameworkUser, diff --git a/x-pack/legacy/plugins/beats_management/server/lib/beats.ts b/x-pack/legacy/plugins/beats_management/server/lib/beats.ts index 3b9c4d35d8331a..6b7053f40550b7 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/beats.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/beats.ts @@ -7,7 +7,6 @@ import { uniq } from 'lodash'; import moment from 'moment'; import { CMBeat } from '../../common/domain_types'; -import { findNonExistentItems } from '../utils/find_non_existent_items'; import { BeatsRemovalReturn, BeatsTagAssignment, @@ -249,3 +248,12 @@ function addToResultsToResponse(key: string, response: any, assignmentResults: a }); return response; } + +export function findNonExistentItems(items: Array<{ id: string }>, requestedItems: string[]) { + return requestedItems.reduce((nonExistentItems: string[], requestedItem: string, idx: number) => { + if (items.findIndex((item) => item && item.id === requestedItem) === -1) { + nonExistentItems.push(requestedItems[idx]); + } + return nonExistentItems; + }, []); +} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/compose/kibana.ts b/x-pack/legacy/plugins/beats_management/server/lib/compose/kibana.ts index 2bda2fe85d62ff..b6a645ded81647 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/compose/kibana.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/compose/kibana.ts @@ -9,9 +9,7 @@ import { PLUGIN } from '../../../common/constants'; import { CONFIG_PREFIX } from '../../../common/constants/plugin'; import { ElasticsearchBeatsAdapter } from '../adapters/beats/elasticsearch_beats_adapter'; import { ElasticsearchConfigurationBlockAdapter } from '../adapters/configuration_blocks/elasticsearch_configuration_block_adapter'; -import { DatabaseKbnESPlugin } from '../adapters/database/adapter_types'; import { KibanaDatabaseAdapter } from '../adapters/database/kibana_database_adapter'; -import { ElasticsearchBeatEventsAdapter } from '../adapters/events/elasticsearch_beat_events_adapter'; import { KibanaLegacyServer } from '../adapters/framework/adapter_types'; import { KibanaBackendFrameworkAdapter } from '../adapters/framework/kibana_framework_adapter'; import { ElasticsearchTagsAdapter } from '../adapters/tags/elasticsearch_tags_adapter'; @@ -28,7 +26,7 @@ export function compose(server: KibanaLegacyServer): CMServerLibs { const framework = new BackendFrameworkLib( new KibanaBackendFrameworkAdapter(camelCase(PLUGIN.ID), server, CONFIG_PREFIX) ); - const database = new KibanaDatabaseAdapter(server.plugins.elasticsearch as DatabaseKbnESPlugin); + const database = new KibanaDatabaseAdapter(server.newPlatform.start.core.elasticsearch); const beatsAdapter = new ElasticsearchBeatsAdapter(database); const configAdapter = new ElasticsearchConfigurationBlockAdapter(database); @@ -46,7 +44,7 @@ export function compose(server: KibanaLegacyServer): CMServerLibs { tokens, framework, }); - const beatEvents = new BeatEventsLib(new ElasticsearchBeatEventsAdapter(database), beats); + const beatEvents = new BeatEventsLib(beats); const libs: CMServerLibs = { beatEvents, diff --git a/x-pack/legacy/plugins/beats_management/server/lib/compose/testing.ts b/x-pack/legacy/plugins/beats_management/server/lib/compose/testing.ts deleted file mode 100644 index b5fe6195fc7c70..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/lib/compose/testing.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { MemoryBeatsAdapter } from '../adapters/beats/memory_beats_adapter'; -import { MemoryConfigurationBlockAdapter } from '../adapters/configuration_blocks/memory_tags_adapter'; -import { HapiBackendFrameworkAdapter } from '../adapters/framework/hapi_framework_adapter'; -import { MemoryTagsAdapter } from '../adapters/tags/memory_tags_adapter'; -import { MemoryTokensAdapter } from '../adapters/tokens/memory_tokens_adapter'; -import { BeatEventsLib } from '../beat_events'; -import { CMBeatsDomain } from '../beats'; -import { ConfigurationBlocksLib } from '../configuration_blocks'; -import { BackendFrameworkLib } from '../framework'; -import { CMTagsDomain } from '../tags'; -import { CMTokensDomain } from '../tokens'; -import { CMServerLibs } from '../types'; - -export function compose(server: any): CMServerLibs { - const framework = new BackendFrameworkLib(new HapiBackendFrameworkAdapter(undefined, server)); - - const beatsAdapter = new MemoryBeatsAdapter(server.beatsDB || []); - const configAdapter = new MemoryConfigurationBlockAdapter(server.configsDB || []); - const tags = new CMTagsDomain( - new MemoryTagsAdapter(server.tagsDB || []), - configAdapter, - beatsAdapter - ); - const configurationBlocks = new ConfigurationBlocksLib(configAdapter, tags); - const tokens = new CMTokensDomain(new MemoryTokensAdapter(server.tokensDB || []), { - framework, - }); - const beats = new CMBeatsDomain(beatsAdapter, { - tags, - tokens, - framework, - }); - const beatEvents = new BeatEventsLib({} as any, beats); - - const libs: CMServerLibs = { - beatEvents, - framework, - beats, - tags, - tokens, - configurationBlocks, - }; - - return libs; -} diff --git a/x-pack/legacy/plugins/beats_management/server/lib/framework.ts b/x-pack/legacy/plugins/beats_management/server/lib/framework.ts index 1a6f84a6979c6f..96a06929073e5c 100644 --- a/x-pack/legacy/plugins/beats_management/server/lib/framework.ts +++ b/x-pack/legacy/plugins/beats_management/server/lib/framework.ts @@ -16,7 +16,6 @@ import { export class BackendFrameworkLib { public log = this.adapter.log; public on = this.adapter.on.bind(this.adapter); - public exposeStaticDir = this.adapter.exposeStaticDir; public internalUser = this.adapter.internalUser; constructor(private readonly adapter: BackendFrameworkAdapter) { this.validateConfig(); diff --git a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/beats_assignments.test.ts b/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/beats_assignments.test.ts deleted file mode 100644 index 156304443431d1..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/beats_assignments.test.ts +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { CMServerLibs } from '../../lib/types'; -import { HapiBackendFrameworkAdapter } from './../../lib/adapters/framework/hapi_framework_adapter'; -import { testHarnes } from './test_harnes'; - -describe('assign_tags_to_beats', () => { - let serverLibs: CMServerLibs; - - beforeAll(async () => { - jest.setTimeout(100000); // 1 second - - serverLibs = await testHarnes.getServerLibs(); - }); - beforeEach(async () => await testHarnes.loadData()); - - it('should add a single tag to a single beat', async () => { - const { result, statusCode } = await ((serverLibs.framework as any) - .adapter as HapiBackendFrameworkAdapter).injectRequstForTesting({ - method: 'POST', - url: '/api/beats/agents_tags/assignments', - headers: { - 'kbn-xsrf': 'xxx', - authorization: 'loggedin', - }, - payload: { - assignments: [{ beatId: 'bar', tag: 'production' }], - }, - }); - - expect(statusCode).toEqual(200); - expect(result.results).toEqual([{ success: true, result: { message: 'updated' } }]); - }); - - it('should not re-add an existing tag to a beat', async () => { - const { result, statusCode } = await ((serverLibs.framework as any) - .adapter as HapiBackendFrameworkAdapter).injectRequstForTesting({ - method: 'POST', - url: '/api/beats/agents_tags/assignments', - headers: { - 'kbn-xsrf': 'xxx', - authorization: 'loggedin', - }, - payload: { - assignments: [{ beatId: 'foo', tag: 'production' }], - }, - }); - - expect(statusCode).toEqual(200); - - expect(result.results).toEqual([{ success: true, result: { message: 'updated' } }]); - - const beat = await serverLibs.beats.getById( - { - kind: 'internal', - }, - 'foo' - ); - expect(beat!.tags).toEqual(['production', 'qa']); // as - }); - - it('should add a single tag to a multiple beats', async () => { - const { result, statusCode } = await ((serverLibs.framework as any) - .adapter as HapiBackendFrameworkAdapter).injectRequstForTesting({ - method: 'POST', - url: '/api/beats/agents_tags/assignments', - headers: { - 'kbn-xsrf': 'xxx', - authorization: 'loggedin', - }, - payload: { - assignments: [ - { beatId: 'foo', tag: 'development' }, - { beatId: 'bar', tag: 'development' }, - ], - }, - }); - - expect(statusCode).toEqual(200); - - expect(result.results).toEqual([ - { success: true, result: { message: 'updated' } }, - { success: true, result: { message: 'updated' } }, - ]); - - let beat; - - beat = await serverLibs.beats.getById( - { - kind: 'internal', - }, - 'foo' - ); - expect(beat!.tags).toEqual(['production', 'qa', 'development']); // as beat 'foo' already had 'production' and 'qa' tags attached to it - - // Beat bar - beat = await serverLibs.beats.getById( - { - kind: 'internal', - }, - 'bar' - ); - - expect(beat!.tags).toEqual(['development']); - }); - - it('should add multiple tags to a single beat', async () => { - const { result, statusCode } = await ((serverLibs.framework as any) - .adapter as HapiBackendFrameworkAdapter).injectRequstForTesting({ - method: 'POST', - url: '/api/beats/agents_tags/assignments', - headers: { - 'kbn-xsrf': 'xxx', - authorization: 'loggedin', - }, - payload: { - assignments: [ - { beatId: 'bar', tag: 'development' }, - { beatId: 'bar', tag: 'production' }, - ], - }, - }); - - expect(statusCode).toEqual(200); - - expect(result.results).toEqual([ - { success: true, result: { message: 'updated' } }, - { success: true, result: { message: 'updated' } }, - ]); - - const beat = await serverLibs.beats.getById( - { - kind: 'internal', - }, - 'bar' - ); - - expect(beat!.tags).toEqual(['development', 'production']); - }); - - // it('should add multiple tags to a multiple beats', async () => { - // const { body: apiResponse } = await supertest - // .post('/api/beats/agents_tags/assignments') - // .set('kbn-xsrf', 'xxx') - // .send({ - // assignments: [{ beatId: 'foo', tag: 'development' }, { beatId: 'bar', tag: 'production' }], - // }) - // .expect(200); - - // expect(apiResponse.assignments).to.eql([ - // { status: 200, result: 'updated' }, - // { status: 200, result: 'updated' }, - // ]); - - // let esResponse; - // let beat; - - // // Beat foo - // esResponse = await es.get({ - // index: ES_INDEX_NAME, - // type: ES_TYPE_NAME, - // id: `beat:foo`, - // }); - - // beat = esResponse._source.beat; - // expect(beat.tags).to.eql(['production', 'qa', 'development']); // as beat 'foo' already had 'production' and 'qa' tags attached to it - - // // Beat bar - // esResponse = await es.get({ - // index: ES_INDEX_NAME, - // type: ES_TYPE_NAME, - // id: `beat:bar`, - // }); - - // beat = esResponse._source.beat; - // expect(beat.tags).to.eql(['production']); - // }); - - // it('should return errors for non-existent beats', async () => { - // const nonExistentBeatId = chance.word(); - - // const { body: apiResponse } = await supertest - // .post('/api/beats/agents_tags/assignments') - // .set('kbn-xsrf', 'xxx') - // .send({ - // assignments: [{ beatId: nonExistentBeatId, tag: 'production' }], - // }) - // .expect(200); - - // expect(apiResponse.assignments).to.eql([ - // { status: 404, result: `Beat ${nonExistentBeatId} not found` }, - // ]); - // }); - - // it('should return errors for non-existent tags', async () => { - // const nonExistentTag = chance.word(); - - // const { body: apiResponse } = await supertest - // .post('/api/beats/agents_tags/assignments') - // .set('kbn-xsrf', 'xxx') - // .send({ - // assignments: [{ beatId: 'bar', tag: nonExistentTag }], - // }) - // .expect(200); - - // expect(apiResponse.assignments).to.eql([ - // { status: 404, result: `Tag ${nonExistentTag} not found` }, - // ]); - - // const esResponse = await es.get({ - // index: ES_INDEX_NAME, - // type: ES_TYPE_NAME, - // id: `beat:bar`, - // }); - - // const beat = esResponse._source.beat; - // expect(beat).to.not.have.property('tags'); - // }); - - // it('should return errors for non-existent beats and tags', async () => { - // const nonExistentBeatId = chance.word(); - // const nonExistentTag = chance.word(); - - // const { body: apiResponse } = await supertest - // .post('/api/beats/agents_tags/assignments') - // .set('kbn-xsrf', 'xxx') - // .send({ - // assignments: [{ beatID: nonExistentBeatId, tag: nonExistentTag }], - // }) - // .expect(200); - - // expect(apiResponse.assignments).to.eql([ - // { status: 404, result: `Beat ${nonExistentBeatId} and tag ${nonExistentTag} not found` }, - // ]); - - // const esResponse = await es.get({ - // index: ES_INDEX_NAME, - // type: ES_TYPE_NAME, - // id: `beat:bar`, - // }); - - // const beat = esResponse._source.beat; - // expect(beat).to.not.have.property('tags'); - // }); -}); diff --git a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/data.json b/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/data.json deleted file mode 100644 index 4ee5a4a7e2d55e..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/data.json +++ /dev/null @@ -1,178 +0,0 @@ -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "beat:qux", - "source": { - "type": "beat", - "beat": { - "type": "filebeat", - "active": true, - "host_ip": "1.2.3.4", - "host_name": "foo.bar.com", - "id": "qux", - "name": "qux_filebeat", - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjcmVhdGVkIjoiMjAxOC0wNi0zMFQwMzo0MjoxNS4yMzBaIiwiaWF0IjoxNTMwMzMwMTM1fQ.SSsX2Byyo1B1bGxV8C3G4QldhE5iH87EY_1r21-bwbI" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "beat:baz", - "source": { - "type": "beat", - "beat": { - "type": "metricbeat", - "active": true, - "host_ip": "22.33.11.44", - "host_name": "baz.bar.com", - "id": "baz", - "name": "baz_metricbeat", - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjcmVhdGVkIjoiMjAxOC0wNi0zMFQwMzo0MjoxNS4yMzBaIiwiaWF0IjoxNTMwMzMwMTM1fQ.SSsX2Byyo1B1bGxV8C3G4QldhE5iH87EY_1r21-bwbI" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "beat:foo", - "source": { - "type": "beat", - "beat": { - "type": "metricbeat", - "active": true, - "host_ip": "1.2.3.4", - "host_name": "foo.bar.com", - "id": "foo", - "name": "foo_metricbeat", - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjcmVhdGVkIjoiMjAxOC0wNi0zMFQwMzo0MjoxNS4yMzBaIiwiaWF0IjoxNTMwMzMwMTM1fQ.SSsX2Byyo1B1bGxV8C3G4QldhE5iH87EY_1r21-bwbI", - "verified_on": "2018-05-15T16:25:38.924Z", - "tags": [ - "production", - "qa" - ] - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "beat:bar", - "source": { - "type": "beat", - "beat": { - "type": "filebeat", - "active": true, - "host_ip": "11.22.33.44", - "host_name": "foo.com", - "id": "bar", - "name": "bar_filebeat", - "access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjcmVhdGVkIjoiMjAxOC0wNi0zMFQwMzo0MjoxNS4yMzBaIiwiaWF0IjoxNTMwMzMwMTM1fQ.SSsX2Byyo1B1bGxV8C3G4QldhE5iH87EY_1r21-bwbI" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "tag:production", - "source": { - "type": "tag", - "tag": { - "color": "blue" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "tag:development", - "source": { - "type": "tag", - "tag": { - "color": "red" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "tag:qa", - "source": { - "type": "tag", - "tag": { - "color": "green" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "configuration_block:SDfsdfIBdsfsf50zbta", - "source": { - "type": "configuration_block", - "configuration_block": { - "type": "output", - "description": "some description", - "tag": "production", - "last_updated": "2018-05-15T16:25:38.924Z", - "config": "{ \"username\": \"some-username\", \"hosts\": [\"localhost:11211\"] }" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "configuration_block:W0tpsmIBdsfsf50zbta", - "source": { - "type": "configuration_block", - "configuration_block": { - "type": "metricbeat.modules", - "tag": "production", - "last_updated": "2018-05-15T16:25:38.924Z", - "config": "{ \"module\": \"memcached\", \"hosts\": [\"localhost:11211\"] }" - } - } - } -} - -{ - "value": { - "index": ".management-beats", - "type": "_doc", - "id": "configuration_block:W0tpsmIBdwcYyG50zbta", - "source": { - "type": "configuration_block", - "configuration_block": { - "type": "metricbeat.modules", - "tag": "qa", - "last_updated": "2018-05-15T16:25:38.924Z", - "config": "{\"module\": \"memcached\", \"node.namespace\": \"node\", \"hosts\": [\"localhost:4949\"] }" - } - } - } -} \ No newline at end of file diff --git a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/test_harnes.ts b/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/test_harnes.ts deleted file mode 100644 index 590ce0bd7b287e..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/rest_api/__tests__/test_harnes.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { badRequest } from 'boom'; -import { readFile } from 'fs'; -// @ts-ignore -import Hapi from 'hapi'; -import { resolve } from 'path'; -import { promisify } from 'util'; -import { BeatTag, CMBeat } from '../../../common/domain_types'; -import { TokenEnrollmentData } from '../../lib/adapters/tokens/adapter_types'; -import { compose } from '../../lib/compose/testing'; -import { CMServerLibs } from '../../lib/types'; -import { initManagementServer } from './../../management_server'; - -const readFileAsync = promisify(readFile); -let serverLibs: CMServerLibs; - -export const testHarnes = { - description: 'API Development Tests', - loadData: async () => { - if (!serverLibs) { - throw new Error('Server libs not composed yet...'); - } - const contents = await readFileAsync(resolve(__dirname, './data.json'), 'utf8'); - const database = contents.split(/\n\n/); - - // @ts-ignore the private access - serverLibs.beats.adapter.setDB( - database.reduce((inserts: CMBeat[], source) => { - const type = 'beat'; - const data = JSON.parse(source); - - if (data.value.source.type === type) { - inserts.push({ - id: data.value.id.substring(data.value.id.indexOf(':') + 1), - ...data.value.source[type], - }); - } - return inserts; - }, []) - ); - - // @ts-ignore the private access - serverLibs.tags.adapter.setDB( - database.reduce((inserts: BeatTag[], source) => { - const type = 'tag'; - const data = JSON.parse(source); - - if (data.value.source.type === type) { - inserts.push({ - id: data.value.id.substring(data.value.id.indexOf(':') + 1), - ...data.value.source[type], - }); - } - return inserts; - }, []) - ); - - // @ts-ignore the private access - serverLibs.tokens.adapter.setDB( - database.reduce((inserts: TokenEnrollmentData[], source) => { - const type = 'token'; - const data = JSON.parse(source); - - if (data.value.source.type === type) { - inserts.push({ - id: data.value.id.substring(data.value.id.indexOf(':') + 1), - ...data.value.source[type], - }); - } - return inserts; - }, []) - ); - }, - getServerLibs: async () => { - if (!serverLibs) { - const server = new Hapi.Server({ port: 111111 }); - const versionHeader = 'kbn-version'; - const xsrfHeader = 'kbn-xsrf'; - - server.ext('onPostAuth', (req: any, h: any) => { - const isSafeMethod = req.method === 'get' || req.method === 'head'; - const hasVersionHeader = versionHeader in req.headers; - const hasXsrfHeader = xsrfHeader in req.headers; - - if (!isSafeMethod && !hasVersionHeader && !hasXsrfHeader) { - throw badRequest(`Request must contain a ${xsrfHeader} header.`); - } - - return h.continue; - }); - - serverLibs = compose(server); - initManagementServer(serverLibs); - } - return serverLibs; - }, -}; diff --git a/x-pack/legacy/plugins/beats_management/server/utils/README.md b/x-pack/legacy/plugins/beats_management/server/utils/README.md deleted file mode 100644 index 8a6a27aa29867c..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/README.md +++ /dev/null @@ -1 +0,0 @@ -Utils should be data processing functions and other tools.... all in all utils is basicly everything that is not an adaptor, or presenter and yet too much to put in a lib. \ No newline at end of file diff --git a/x-pack/legacy/plugins/beats_management/server/utils/find_non_existent_items.ts b/x-pack/legacy/plugins/beats_management/server/utils/find_non_existent_items.ts deleted file mode 100644 index 0e9b4f0b6fa5e0..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/find_non_existent_items.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -interface RandomItem { - id: string; - [key: string]: any; -} - -export function findNonExistentItems(items: RandomItem[], requestedItems: any) { - return requestedItems.reduce((nonExistentItems: string[], requestedItem: string, idx: number) => { - if (items.findIndex((item: RandomItem) => item && item.id === requestedItem) === -1) { - nonExistentItems.push(requestedItems[idx]); - } - return nonExistentItems; - }, []); -} diff --git a/x-pack/legacy/plugins/beats_management/server/utils/helper_types.ts b/x-pack/legacy/plugins/beats_management/server/utils/helper_types.ts deleted file mode 100644 index 96f7b7bc79b626..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/helper_types.ts +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export type InterfaceExcept = Pick>; - -export function arrayFromEnum(e: any): T[] { - return Object.keys(e) - .filter((key) => isNaN(+key)) - .map((name) => e[name]) as T[]; -} diff --git a/x-pack/legacy/plugins/beats_management/server/utils/index_templates/beats_template.json b/x-pack/legacy/plugins/beats_management/server/utils/index_templates/beats_template.json deleted file mode 100644 index ba3a0aba6c2567..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/index_templates/beats_template.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "index_patterns": [".management-beats"], - "version": 66000, - "settings": { - "index": { - "number_of_shards": 1, - "auto_expand_replicas": "0-1", - "codec": "best_compression" - } - }, - "mappings": { - "_doc": { - "dynamic": "strict", - "properties": { - "type": { - "type": "keyword" - }, - "configuration_block": { - "properties": { - "id": { - "type": "keyword" - }, - "type": { - "type": "keyword" - }, - "tag": { - "type": "keyword" - }, - "description": { - "type": "text" - }, - "config": { - "type": "keyword" - }, - "last_updated": { - "type": "date" - } - } - }, - "enrollment_token": { - "properties": { - "token": { - "type": "keyword" - }, - "expires_on": { - "type": "date" - } - } - }, - "tag": { - "properties": { - "id": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "color": { - "type": "keyword" - }, - "hasConfigurationBlocksTypes": { - "type": "keyword" - } - } - }, - "beat": { - "properties": { - "id": { - "type": "keyword" - }, - "config_status": { - "type": "keyword" - }, - "active": { - "type": "boolean" - }, - "last_checkin": { - "type": "date" - }, - "enrollment_token": { - "type": "keyword" - }, - "access_token": { - "type": "keyword" - }, - "verified_on": { - "type": "date" - }, - "type": { - "type": "keyword" - }, - "version": { - "type": "keyword" - }, - "host_ip": { - "type": "ip" - }, - "host_name": { - "type": "keyword" - }, - "ephemeral_id": { - "type": "keyword" - }, - "tags": { - "type": "keyword" - }, - "metadata": { - "dynamic": "true", - "type": "object" - }, - "name": { - "type": "keyword" - } - } - } - } - } - } -} diff --git a/x-pack/legacy/plugins/beats_management/server/utils/polyfills.ts b/x-pack/legacy/plugins/beats_management/server/utils/polyfills.ts deleted file mode 100644 index 5291e2c72be7d9..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/polyfills.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export const entries = (obj: any) => { - const ownProps = Object.keys(obj); - let i = ownProps.length; - const resArray = new Array(i); // preallocate the Array - - while (i--) { - resArray[i] = [ownProps[i], obj[ownProps[i]]]; - } - - return resArray; -}; diff --git a/x-pack/legacy/plugins/beats_management/server/utils/wrap_request.ts b/x-pack/legacy/plugins/beats_management/server/utils/wrap_request.ts deleted file mode 100644 index 57cf70a99a296c..00000000000000 --- a/x-pack/legacy/plugins/beats_management/server/utils/wrap_request.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { - FrameworkRequest, - internalAuthData, - KibanaServerRequest, -} from '../lib/adapters/framework/adapter_types'; - -export function wrapRequest( - req: InternalRequest -): FrameworkRequest { - const { params, payload, query, headers, info } = req; - - const isAuthenticated = headers.authorization != null; - - return { - // @ts-ignore -- partial applucation, adapter adds other user data - user: isAuthenticated - ? { - kind: 'authenticated', - [internalAuthData]: headers, - } - : { - kind: 'unauthenticated', - }, - headers, - info, - params, - payload, - query, - }; -} diff --git a/x-pack/legacy/plugins/beats_management/types/eui.d.ts b/x-pack/legacy/plugins/beats_management/types/eui.d.ts deleted file mode 100644 index 636d0a2f7b51e1..00000000000000 --- a/x-pack/legacy/plugins/beats_management/types/eui.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -/** - * /!\ These type definitions are temporary until the upstream @elastic/eui - * package includes them. - */ - -import * as eui from '@elastic/eui'; -import { Moment } from 'moment'; -import { ChangeEventHandler, MouseEventHandler, ReactType, Ref, FC } from 'react'; - -declare module '@elastic/eui' {} diff --git a/x-pack/legacy/plugins/beats_management/wallaby.js b/x-pack/legacy/plugins/beats_management/wallaby.js deleted file mode 100644 index 823f63b15bcb35..00000000000000 --- a/x-pack/legacy/plugins/beats_management/wallaby.js +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -const path = require('path'); -process.env.NODE_PATH = path.resolve(__dirname, '..', '..', '..', 'node_modules'); - -module.exports = function (wallaby) { - return { - debug: true, - files: [ - './tsconfig.json', - //'plugins/beats/public/**/*.+(js|jsx|ts|tsx|json|snap|css|less|sass|scss|jpg|jpeg|gif|png|svg)', - 'server/**/*.+(js|jsx|ts|tsx|json|snap|css|less|sass|scss|jpg|jpeg|gif|png|svg)', - 'common/**/*.+(js|jsx|ts|tsx|json|snap|css|less|sass|scss|jpg|jpeg|gif|png|svg)', - 'public/**/*.+(js|jsx|ts|tsx|json|snap|css|less|sass|scss|jpg|jpeg|gif|png|svg)', - '!**/*.test.ts', - ], - - tests: ['**/*.test.ts', '**/*.test.tsx'], - env: { - type: 'node', - runner: 'node', - }, - testFramework: { - type: 'jest', - //path: jestPath, - }, - compilers: { - '**/*.ts?(x)': wallaby.compilers.typeScript({ - typescript: require('typescript'), // eslint-disable-line - }), - '**/*.js': wallaby.compilers.babel({ - babelrc: false, - presets: [require.resolve('@kbn/babel-preset/node_preset')], - }), - }, - - setup: (wallaby) => { - const path = require('path'); - - const kibanaDirectory = path.resolve(wallaby.localProjectDir, '..', '..', '..'); - wallaby.testFramework.configure({ - rootDir: wallaby.localProjectDir, - moduleNameMapper: { - '^ui/(.*)': `${kibanaDirectory}/src/legacy/ui/public/$1`, - // eslint-disable-next-line - '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': `${kibanaDirectory}/src/dev/jest/mocks/file_mock.js`, - '\\.(css|less|scss)$': `${kibanaDirectory}/src/dev/jest/mocks/style_mock.js`, - }, - testURL: 'http://localhost', - setupFiles: [`${kibanaDirectory}/x-pack/dev-tools/jest/setup/enzyme.js`], - snapshotSerializers: [`${kibanaDirectory}/node_modules/enzyme-to-json/serializer`], - transform: { - '^.+\\.js$': `${kibanaDirectory}/src/dev/jest/babel_transform.js`, - //"^.+\\.tsx?$": `${kibanaDirectory}/src/dev/jest/ts_transform.js`, - }, - }); - }, - }; -}; diff --git a/x-pack/plugins/beats_management/server/index.ts b/x-pack/plugins/beats_management/server/index.ts index 607fb0ab2725d3..ad19087f5ac9ff 100644 --- a/x-pack/plugins/beats_management/server/index.ts +++ b/x-pack/plugins/beats_management/server/index.ts @@ -4,7 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ +import { PluginInitializer } from '../../../../src/core/server'; import { beatsManagementConfigSchema } from '../common'; +import { BeatsManagementPlugin } from './plugin'; export const config = { schema: beatsManagementConfigSchema, @@ -16,8 +18,4 @@ export const config = { }, }; -export const plugin = () => ({ - setup() {}, - start() {}, - stop() {}, -}); +export const plugin: PluginInitializer<{}, {}> = (context) => new BeatsManagementPlugin(context); diff --git a/x-pack/plugins/beats_management/server/plugin.ts b/x-pack/plugins/beats_management/server/plugin.ts new file mode 100644 index 00000000000000..a82dbcb4a3a6ed --- /dev/null +++ b/x-pack/plugins/beats_management/server/plugin.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + CoreSetup, + CoreStart, + Plugin, + PluginInitializerContext, +} from '../../../../src/core/server'; +import { SecurityPluginSetup } from '../../security/server'; +import { LicensingPluginStart } from '../../licensing/server'; +import { BeatsManagementConfigType } from '../common'; + +interface SetupDeps { + security?: SecurityPluginSetup; +} + +interface StartDeps { + licensing: LicensingPluginStart; +} + +export class BeatsManagementPlugin implements Plugin<{}, {}, SetupDeps, StartDeps> { + constructor( + private readonly initializerContext: PluginInitializerContext + ) {} + + public async setup(core: CoreSetup, plugins: SetupDeps) { + this.initializerContext.config.create(); + + return {}; + } + + public async start(core: CoreStart, { licensing }: StartDeps) { + return {}; + } +} diff --git a/x-pack/legacy/plugins/beats_management/types/formsy.d.ts b/x-pack/plugins/beats_management/types/formsy.d.ts similarity index 100% rename from x-pack/legacy/plugins/beats_management/types/formsy.d.ts rename to x-pack/plugins/beats_management/types/formsy.d.ts From 7dd4fa2618accbb26f56ddb2b18aade7b5d1f6df Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 18 Jun 2020 11:37:41 +0200 Subject: [PATCH 11/60] Add section about marble testing to `TESTING.md` (#68749) * Add section about marble testing * improve `callServerAPI` example * review comments * add comment on abort observable anti-pattern --- src/core/TESTING.md | 276 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 276 insertions(+) diff --git a/src/core/TESTING.md b/src/core/TESTING.md index bed41ab583496e..a62922d9b5d64b 100644 --- a/src/core/TESTING.md +++ b/src/core/TESTING.md @@ -29,6 +29,14 @@ This document outlines best practices and patterns for testing Kibana Plugins. - [Testing dependencies usages](#testing-dependencies-usages) - [Testing components consuming the dependencies](#testing-components-consuming-the-dependencies) - [Testing optional plugin dependencies](#testing-optional-plugin-dependencies) + - [RXJS testing](#rxjs-testing) + - [Testing RXJS observables with marble](#rxjs-testing-with-marble) + - [Precondition](#preconditions-2) + - [Examples](#example-5) + - [Testing an interval based observable](#testing-an-interval-based-observable) + - [Testing observable completion](#testing-observable-completion) + - [Testing observable errors](#testing-observable-errors) + - [Testing promise based observables](#testing-promise-based-observables) ## Strategy @@ -1087,3 +1095,271 @@ describe('Plugin', () => { }); }); ``` + +## RXJS testing + +### Testing RXJS observables with marble + +Testing observable based APIs can be challenging, specially when asynchronous operators or sources are used, +or when trying to assert against emission's timing. + +Fortunately, RXJS comes with it's own `marble` testing module to greatly facilitate that kind of testing. + +See [the official doc](https://rxjs-dev.firebaseapp.com/guide/testing/marble-testing) for more information about marble testing. + +### Preconditions + +The following examples all assume that the following snippet is included in every test file: + +```typescript +import { TestScheduler } from 'rxjs/testing'; + +const getTestScheduler = () => + new TestScheduler((actual, expected) => { + expect(actual).toEqual(expected); + }); +``` + +`getTestScheduler` creates a `TestScheduler` that is wired on `jest`'s `expect` statement when comparing an observable's time frame. + +### Examples + +#### Testing an interval based observable + +Here is a very basic example of an interval-based API: + +```typescript +class FooService { + setup() { + return { + getUpdate$: () => { + return interval(100).pipe(map((count) => `update-${count + 1}`)); + }, + }; + } +} +``` + +If we were to be adding a test that asserts the correct behavior of this API without using marble testing, it +would probably be something like: + +```typescript +it('getUpdate$ emits updates every 100ms', async () => { + const service = new FooService(); + const { getUpdate$ } = service.setup(); + expect(await getUpdate$().pipe(take(3), toArray()).toPromise()).toEqual([ + 'update-1', + 'update-2', + 'update-3', + ]); +}); +``` + +Note that if we are able to test the correct value of each emission, we don't have any way to assert that +the interval of 100ms was respected. Even using a subscription based test to try to do so would result +in potential flakiness, as the subscription execution could trigger on the `101ms` time frame for example. + +It also may be important to note: +- as we need to convert the observable to a promise and wait for the result, the test is `async` +- we need to perform observable transformation (`take` + `toArray`) in the test to have an usable value to assert against. + +Marble testing would allow to get rid of these limitations. An equivalent and improved marble test could be: + +```typescript + describe('getUpdate$', () => { + it('emits updates every 100ms', () => { + getTestScheduler().run(({ expectObservable }) => { + const { getUpdate$ } = service.setup(); + expectObservable(getUpdate$(), '301ms !').toBe('100ms a 99ms b 99ms c', { + a: 'update-1', + b: 'update-2', + c: 'update-3', + }); + }); + }); + }); +``` + +Notes: +- the test is now synchronous +- the second parameter of `expectObservable` (`'301ms !'`) is used to perform manual unsubscription to the observable, as + `interval` never ends. +- an emission is considered a time frame, meaning that after the initial `a` emission, we are at the frame `101`, not `100` + which is why we are then only using a `99ms` gap between a->b and b->c. + +#### Testing observable completion + +Let's 'improve' our `getUpdate$` API by allowing the consumer to manually terminate the observable chain using +a new `abort$` option: + +```typescript +class FooService { + setup() { + return { + // note: using an abortion observable is usually an anti-pattern, as unsubscribing from the observable + // is, most of the time, a better solution. This is only used for the example purpose. + getUpdate$: ({ abort$ = EMPTY }: { abort$?: Observable } = {}) => { + return interval(100).pipe( + takeUntil(abort$), + map((count) => `update-${count + 1}`) + ); + }, + }; + } +} +``` + +We would then add a test to assert than this new option usage is respected: + +```typescript +it('getUpdate$ completes when `abort$` emits', () => { + const service = new FooService(); + getTestScheduler().run(({ expectObservable, hot }) => { + const { getUpdate$ } = service.setup(); + const abort$ = hot('149ms a', { a: undefined }); + expectObservable(getUpdate$({ abort$ })).toBe('100ms a 48ms |', { + a: 'update-1', + }); + }); +}); +``` + +Notes: + - the `|` symbol represents the completion of the observable. + - we are here using the `hot` testing utility to create the `abort$` observable to ensure correct emission timing. + +#### Testing observable errors + +Testing errors thrown by the observable is very close to the previous examples and is done using +the third parameter of `expectObservable`. + +Say we have a service in charge of processing data from an observable and returning the results in a new observable: + +```typescript +interface SomeDataType { + id: string; +} + +class BarService { + setup() { + return { + processDataStream: (data$: Observable) => { + return data$.pipe( + map((data) => { + if (data.id === 'invalid') { + throw new Error(`invalid data: '${data.id}'`); + } + return { + ...data, + processed: 'additional-data', + }; + }) + ); + }, + }; + } +} +``` + +We could write a test that asserts the service properly emit processed results until an invalid data is encountered: + +```typescript +it('processDataStream throw an error when processing invalid data', () => { + getTestScheduler().run(({ expectObservable, hot }) => { + const service = new BarService(); + const { processDataStream } = service.setup(); + + const data = hot('--a--b--(c|)', { + a: { id: 'a' }, + b: { id: 'invalid' }, + c: { id: 'c' }, + }); + + expectObservable(processDataStream(data)).toBe( + '--a--#', + { + a: { id: 'a', processed: 'additional-data' }, + }, + `'[Error: invalid data: 'invalid']'` + ); + }); +}); +``` + +Notes: + - the `-` symbol represents one virtual time frame. + - the `#` symbol represents an error. + - when throwing custom `Error` classes, the assertion can be against an error instance, but this doesn't work + with base errors. + +#### Testing promise based observables + +In some cases, the observable we want to test is based on a Promise (like `of(somePromise).pipe(...)`). This can occur +when using promise-based services, such as core's `http`, for instance. + +```typescript +export const callServerAPI = ( + http: HttpStart, + body: Record, + { abort$ }: { abort$: Observable } +): Observable => { + let controller: AbortController | undefined; + if (abort$) { + controller = new AbortController(); + abort$.subscribe(() => { + controller!.abort(); + }); + } + return from( + http.post('/api/endpoint', { + body, + signal: controller?.signal, + }) + ).pipe( + takeUntil(abort$ ?? EMPTY), + map((response) => response.results) + ); +}; +``` + +Testing that kind of promise based observable does not work out of the box with marble testing, as the asynchronous promise resolution +is not handled by the test scheduler's 'sandbox'. + +Fortunately, there are workarounds for this problem. The most common one being to mock the promise-returning API to return +an observable instead for testing, as `of(observable)` also works and returns the input observable. + +Note that when doing so, the test suite must also include tests using a real promise value to ensure correct behavior in real situation. + +```typescript + +// NOTE: test scheduler do not properly work with promises because of their asynchronous nature. +// we are cheating here by having `http.post` return an observable instead of a promise. +// this still allows more finely grained testing about timing, and asserting that the method +// works properly when `post` returns a real promise is handled in other tests of this suite + +it('callServerAPI result observable emits when the response is received', () => { + const http = httpServiceMock.createStartContract(); + getTestScheduler().run(({ expectObservable, hot }) => { + // need to cast the observable as `any` because http.post.mockReturnValue expects a promise, see previous comment + http.post.mockReturnValue(hot('---(a|)', { a: { someData: 'foo' } }) as any); + + const results = callServerAPI(http, { query: 'term' }, {}); + + expectObservable(results).toBe('---(a|)', { + a: { someData: 'foo' }, + }); + }); +}); + +it('completes without returning results if aborted$ emits before the response', () => { + const http = httpServiceMock.createStartContract(); + getTestScheduler().run(({ expectObservable, hot }) => { + // need to cast the observable as `any` because http.post.mockReturnValue expects a promise, see previous comment + http.post.mockReturnValue(hot('---(a|)', { a: { someData: 'foo' } }) as any); + const aborted$ = hot('-(a|)', { a: undefined }); + const results = callServerAPI(http, { query: 'term' }, { aborted$ }); + + expectObservable(results).toBe('-|'); + }); +}); +``` \ No newline at end of file From d2006ea8a050448ec3d835a9b0c7a56763379c2c Mon Sep 17 00:00:00 2001 From: Pierre Gayvallet Date: Thu, 18 Jun 2020 11:39:25 +0200 Subject: [PATCH 12/60] savedObjects: add `score` to repository.find results (#68894) * add `score` to repository.find results * update generated doc * fix FTR result set * remove score from exports * fix FTR for find API * fix label * fix tsdoc --- .../core/server/kibana-plugin-core-server.md | 1 + ...in-core-server.savedobjectsfindresponse.md | 2 +- ....savedobjectsfindresponse.saved_objects.md | 2 +- ...ugin-core-server.savedobjectsfindresult.md | 19 +++++++++++++++++++ ...ore-server.savedobjectsfindresult.score.md | 13 +++++++++++++ .../saved_objects/saved_objects_client.ts | 6 ++++-- src/core/server/index.ts | 1 + .../get_sorted_objects_for_export.test.ts | 13 +++++++++++++ .../export/get_sorted_objects_for_export.ts | 7 +++++-- .../routes/integration_tests/find.test.ts | 2 ++ .../service/lib/repository.test.js | 8 +++++--- .../saved_objects/service/lib/repository.ts | 8 ++++++-- .../service/saved_objects_client.ts | 13 ++++++++++++- src/core/server/server.api.md | 7 ++++++- .../server/lib/find_all.test.ts | 5 +++-- .../server/lib/find_relationships.test.ts | 1 + .../apis/saved_objects/find.js | 2 ++ .../apis/saved_objects_management/find.ts | 1 + .../actions/server/actions_client.test.ts | 1 + .../alerts/server/alerts_client.test.ts | 1 + .../case/server/routes/api/utils.test.ts | 15 ++++++++++++--- .../plugins/case/server/routes/api/utils.ts | 4 ++-- ...ypted_saved_objects_client_wrapper.test.ts | 6 ++++++ .../routes/__mocks__/request_responses.ts | 2 ++ .../signals/__mocks__/es_results.ts | 2 +- .../spaces_saved_objects_client.test.ts | 4 ++-- 26 files changed, 123 insertions(+), 23 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.score.md diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 147a72016b2351..a45bd3d44b28ab 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -157,6 +157,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [SavedObjectsExportResultDetails](./kibana-plugin-core-server.savedobjectsexportresultdetails.md) | Structure of the export result details entry | | [SavedObjectsFindOptions](./kibana-plugin-core-server.savedobjectsfindoptions.md) | | | [SavedObjectsFindResponse](./kibana-plugin-core-server.savedobjectsfindresponse.md) | Return type of the Saved Objects find() method.\*Note\*: this type is different between the Public and Server Saved Objects clients. | +| [SavedObjectsFindResult](./kibana-plugin-core-server.savedobjectsfindresult.md) | | | [SavedObjectsImportConflictError](./kibana-plugin-core-server.savedobjectsimportconflicterror.md) | Represents a failure to import due to a conflict. | | [SavedObjectsImportError](./kibana-plugin-core-server.savedobjectsimporterror.md) | Represents a failure to import. | | [SavedObjectsImportMissingReferencesError](./kibana-plugin-core-server.savedobjectsimportmissingreferenceserror.md) | Represents a failure to import due to missing references. | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md index a1b1a7a056206d..4ed069d1598fe4 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.md @@ -20,6 +20,6 @@ export interface SavedObjectsFindResponse | --- | --- | --- | | [page](./kibana-plugin-core-server.savedobjectsfindresponse.page.md) | number | | | [per\_page](./kibana-plugin-core-server.savedobjectsfindresponse.per_page.md) | number | | -| [saved\_objects](./kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md) | Array<SavedObject<T>> | | +| [saved\_objects](./kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md) | Array<SavedObjectsFindResult<T>> | | | [total](./kibana-plugin-core-server.savedobjectsfindresponse.total.md) | number | | diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md index adad0dd2b11767..7a91367f6ef0bc 100644 --- a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresponse.saved_objects.md @@ -7,5 +7,5 @@ Signature: ```typescript -saved_objects: Array>; +saved_objects: Array>; ``` diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md new file mode 100644 index 00000000000000..e455074a7d11bb --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.md @@ -0,0 +1,19 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResult](./kibana-plugin-core-server.savedobjectsfindresult.md) + +## SavedObjectsFindResult interface + + +Signature: + +```typescript +export interface SavedObjectsFindResult extends SavedObject +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [score](./kibana-plugin-core-server.savedobjectsfindresult.score.md) | number | The Elasticsearch _score of this result. | + diff --git a/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.score.md b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.score.md new file mode 100644 index 00000000000000..c6646df6ee4700 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.savedobjectsfindresult.score.md @@ -0,0 +1,13 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [SavedObjectsFindResult](./kibana-plugin-core-server.savedobjectsfindresult.md) > [score](./kibana-plugin-core-server.savedobjectsfindresult.score.md) + +## SavedObjectsFindResult.score property + +The Elasticsearch `_score` of this result. + +Signature: + +```typescript +score: number; +``` diff --git a/src/core/public/saved_objects/saved_objects_client.ts b/src/core/public/saved_objects/saved_objects_client.ts index 5c8eca4a33ec57..cb279b2cc4c8f9 100644 --- a/src/core/public/saved_objects/saved_objects_client.ts +++ b/src/core/public/saved_objects/saved_objects_client.ts @@ -303,7 +303,6 @@ export class SavedObjectsClient { query, }); return request.then((resp) => { - resp.saved_objects = resp.saved_objects.map((d) => this.createSavedObject(d)); return renameKeys< PromiseType>, SavedObjectsFindResponsePublic @@ -314,7 +313,10 @@ export class SavedObjectsClient { per_page: 'perPage', page: 'page', }, - resp + { + ...resp, + saved_objects: resp.saved_objects.map((d) => this.createSavedObject(d)), + } ) as SavedObjectsFindResponsePublic; }); }; diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 658c24f835020d..dccd58c24a7d0b 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -217,6 +217,7 @@ export { SavedObjectsErrorHelpers, SavedObjectsExportOptions, SavedObjectsExportResultDetails, + SavedObjectsFindResult, SavedObjectsFindResponse, SavedObjectsImportConflictError, SavedObjectsImportError, diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts index 32485f461f59b9..5da2235828b5c8 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.test.ts @@ -47,6 +47,7 @@ describe('getSortedObjectsForExport()', () => { id: '2', type: 'search', attributes: {}, + score: 1, references: [ { name: 'name', @@ -59,6 +60,7 @@ describe('getSortedObjectsForExport()', () => { id: '1', type: 'index-pattern', attributes: {}, + score: 1, references: [], }, ], @@ -133,6 +135,7 @@ describe('getSortedObjectsForExport()', () => { id: '2', type: 'search', attributes: {}, + score: 1, references: [ { name: 'name', @@ -145,6 +148,7 @@ describe('getSortedObjectsForExport()', () => { id: '1', type: 'index-pattern', attributes: {}, + score: 1, references: [], }, ], @@ -192,6 +196,7 @@ describe('getSortedObjectsForExport()', () => { id: '2', type: 'search', attributes: {}, + score: 1, references: [ { name: 'name', @@ -204,6 +209,7 @@ describe('getSortedObjectsForExport()', () => { id: '1', type: 'index-pattern', attributes: {}, + score: 1, references: [], }, ], @@ -279,6 +285,7 @@ describe('getSortedObjectsForExport()', () => { id: '2', type: 'search', attributes: {}, + score: 1, references: [ { name: 'name', @@ -291,6 +298,7 @@ describe('getSortedObjectsForExport()', () => { id: '1', type: 'index-pattern', attributes: {}, + score: 1, references: [], }, ], @@ -366,6 +374,7 @@ describe('getSortedObjectsForExport()', () => { id: '2', type: 'search', attributes: {}, + score: 1, references: [ { type: 'index-pattern', @@ -378,6 +387,7 @@ describe('getSortedObjectsForExport()', () => { id: '1', type: 'index-pattern', attributes: {}, + score: 1, references: [], }, ], @@ -405,6 +415,7 @@ describe('getSortedObjectsForExport()', () => { attributes: { name: 'baz', }, + score: 1, references: [], }, { @@ -413,6 +424,7 @@ describe('getSortedObjectsForExport()', () => { attributes: { name: 'foo', }, + score: 1, references: [], }, { @@ -421,6 +433,7 @@ describe('getSortedObjectsForExport()', () => { attributes: { name: 'bar', }, + score: 1, references: [], }, ], diff --git a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts index cafaa5a3147db3..6e985c25aeaef9 100644 --- a/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts +++ b/src/core/server/saved_objects/export/get_sorted_objects_for_export.ts @@ -116,8 +116,11 @@ async function fetchObjectsToExport({ } // sorts server-side by _id, since it's only available in fielddata - return findResponse.saved_objects.sort((a: SavedObject, b: SavedObject) => - a.id > b.id ? 1 : -1 + return ( + findResponse.saved_objects + // exclude the find-specific `score` property from the exported objects + .map(({ score, ...obj }) => obj) + .sort((a: SavedObject, b: SavedObject) => (a.id > b.id ? 1 : -1)) ); } else { throw Boom.badRequest('Either `type` or `objects` are required.'); diff --git a/src/core/server/saved_objects/routes/integration_tests/find.test.ts b/src/core/server/saved_objects/routes/integration_tests/find.test.ts index 31bda1d6b9cbd2..33e12dd4e517dd 100644 --- a/src/core/server/saved_objects/routes/integration_tests/find.test.ts +++ b/src/core/server/saved_objects/routes/integration_tests/find.test.ts @@ -79,6 +79,7 @@ describe('GET /api/saved_objects/_find', () => { timeFieldName: '@timestamp', notExpandable: true, attributes: {}, + score: 1, references: [], }, { @@ -88,6 +89,7 @@ describe('GET /api/saved_objects/_find', () => { timeFieldName: '@timestamp', notExpandable: true, attributes: {}, + score: 1, references: [], }, ], diff --git a/src/core/server/saved_objects/service/lib/repository.test.js b/src/core/server/saved_objects/service/lib/repository.test.js index d631ef9cb353cc..ea749235cbb41b 100644 --- a/src/core/server/saved_objects/service/lib/repository.test.js +++ b/src/core/server/saved_objects/service/lib/repository.test.js @@ -1939,7 +1939,7 @@ describe('SavedObjectsRepository', () => { { _index: '.kibana', _id: `${namespace ? `${namespace}:` : ''}config:6.0.0-alpha1`, - _score: 1, + _score: 2, ...mockVersionProps, _source: { namespace, @@ -1954,7 +1954,7 @@ describe('SavedObjectsRepository', () => { { _index: '.kibana', _id: `${namespace ? `${namespace}:` : ''}index-pattern:stocks-*`, - _score: 1, + _score: 3, ...mockVersionProps, _source: { namespace, @@ -1970,7 +1970,7 @@ describe('SavedObjectsRepository', () => { { _index: '.kibana', _id: `${NAMESPACE_AGNOSTIC_TYPE}:something`, - _score: 1, + _score: 4, ...mockVersionProps, _source: { type: NAMESPACE_AGNOSTIC_TYPE, @@ -2131,6 +2131,7 @@ describe('SavedObjectsRepository', () => { type: doc._source.type, ...mockTimestampFields, version: mockVersion, + score: doc._score, attributes: doc._source[doc._source.type], references: [], }); @@ -2153,6 +2154,7 @@ describe('SavedObjectsRepository', () => { type: doc._source.type, ...mockTimestampFields, version: mockVersion, + score: doc._score, attributes: doc._source[doc._source.type], references: [], }); diff --git a/src/core/server/saved_objects/service/lib/repository.ts b/src/core/server/saved_objects/service/lib/repository.ts index 03538f23948459..40c5282a77e499 100644 --- a/src/core/server/saved_objects/service/lib/repository.ts +++ b/src/core/server/saved_objects/service/lib/repository.ts @@ -41,6 +41,7 @@ import { SavedObjectsBulkUpdateResponse, SavedObjectsCreateOptions, SavedObjectsFindResponse, + SavedObjectsFindResult, SavedObjectsUpdateOptions, SavedObjectsUpdateResponse, SavedObjectsBulkUpdateObject, @@ -674,8 +675,11 @@ export class SavedObjectsRepository { page, per_page: perPage, total: response.hits.total, - saved_objects: response.hits.hits.map((hit: SavedObjectsRawDoc) => - this._rawToSavedObject(hit) + saved_objects: response.hits.hits.map( + (hit: SavedObjectsRawDoc): SavedObjectsFindResult => ({ + ...this._rawToSavedObject(hit), + score: (hit as any)._score, + }) ), }; } diff --git a/src/core/server/saved_objects/service/saved_objects_client.ts b/src/core/server/saved_objects/service/saved_objects_client.ts index 8780f07cc3091b..e15a92c92772f3 100644 --- a/src/core/server/saved_objects/service/saved_objects_client.ts +++ b/src/core/server/saved_objects/service/saved_objects_client.ts @@ -79,6 +79,17 @@ export interface SavedObjectsBulkResponse { saved_objects: Array>; } +/** + * + * @public + */ +export interface SavedObjectsFindResult extends SavedObject { + /** + * The Elasticsearch `_score` of this result. + */ + score: number; +} + /** * Return type of the Saved Objects `find()` method. * @@ -88,7 +99,7 @@ export interface SavedObjectsBulkResponse { * @public */ export interface SavedObjectsFindResponse { - saved_objects: Array>; + saved_objects: Array>; total: number; per_page: number; page: number; diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index ecfa09fbd37f39..833c8918a08604 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -2039,11 +2039,16 @@ export interface SavedObjectsFindResponse { // (undocumented) per_page: number; // (undocumented) - saved_objects: Array>; + saved_objects: Array>; // (undocumented) total: number; } +// @public (undocumented) +export interface SavedObjectsFindResult extends SavedObject { + score: number; +} + // @public export interface SavedObjectsImportConflictError { // (undocumented) diff --git a/src/plugins/saved_objects_management/server/lib/find_all.test.ts b/src/plugins/saved_objects_management/server/lib/find_all.test.ts index 2515d11f6d4bbd..823a103d8bab3c 100644 --- a/src/plugins/saved_objects_management/server/lib/find_all.test.ts +++ b/src/plugins/saved_objects_management/server/lib/find_all.test.ts @@ -18,17 +18,18 @@ */ import { times } from 'lodash'; -import { SavedObjectsFindOptions, SavedObject } from 'src/core/server'; +import { SavedObjectsFindOptions, SavedObjectsFindResult } from 'src/core/server'; import { savedObjectsClientMock } from '../../../../core/server/mocks'; import { findAll } from './find_all'; describe('findAll', () => { let savedObjectsClient: ReturnType; - const createObj = (id: number): SavedObject => ({ + const createObj = (id: number): SavedObjectsFindResult => ({ type: 'type', id: `id-${id}`, attributes: {}, + score: 1, references: [], }); diff --git a/src/plugins/saved_objects_management/server/lib/find_relationships.test.ts b/src/plugins/saved_objects_management/server/lib/find_relationships.test.ts index 2c8997c9af21ab..e18a45d9bdf44c 100644 --- a/src/plugins/saved_objects_management/server/lib/find_relationships.test.ts +++ b/src/plugins/saved_objects_management/server/lib/find_relationships.test.ts @@ -77,6 +77,7 @@ describe('findRelationships', () => { type: 'parent-type', id: 'parent-id', attributes: {}, + score: 1, references: [], }, ], diff --git a/test/api_integration/apis/saved_objects/find.js b/test/api_integration/apis/saved_objects/find.js index 7a57d182bc8124..7cb5955e4a43d9 100644 --- a/test/api_integration/apis/saved_objects/find.js +++ b/test/api_integration/apis/saved_objects/find.js @@ -46,6 +46,7 @@ export default function ({ getService }) { attributes: { title: 'Count of requests', }, + score: 0, migrationVersion: resp.body.saved_objects[0].migrationVersion, references: [ { @@ -134,6 +135,7 @@ export default function ({ getService }) { .searchSourceJSON, }, }, + score: 0, references: [ { name: 'kibanaSavedObjectMeta.searchSourceJSON.index', diff --git a/test/api_integration/apis/saved_objects_management/find.ts b/test/api_integration/apis/saved_objects_management/find.ts index e15a9e989d21ff..4d9f1c1658139e 100644 --- a/test/api_integration/apis/saved_objects_management/find.ts +++ b/test/api_integration/apis/saved_objects_management/find.ts @@ -56,6 +56,7 @@ export default function ({ getService }: FtrProviderContext) { type: 'index-pattern', }, ], + score: 0, updated_at: '2017-09-21T18:51:23.794Z', meta: { editUrl: diff --git a/x-pack/plugins/actions/server/actions_client.test.ts b/x-pack/plugins/actions/server/actions_client.test.ts index 2865bbbe1d944a..69fab828e63de4 100644 --- a/x-pack/plugins/actions/server/actions_client.test.ts +++ b/x-pack/plugins/actions/server/actions_client.test.ts @@ -386,6 +386,7 @@ describe('getAll()', () => { foo: 'bar', }, }, + score: 1, references: [], }, ], diff --git a/x-pack/plugins/alerts/server/alerts_client.test.ts b/x-pack/plugins/alerts/server/alerts_client.test.ts index 9685f58b8fb31c..f494f1358980d1 100644 --- a/x-pack/plugins/alerts/server/alerts_client.test.ts +++ b/x-pack/plugins/alerts/server/alerts_client.test.ts @@ -1667,6 +1667,7 @@ describe('find()', () => { }, ], }, + score: 1, references: [ { name: 'action_0', diff --git a/x-pack/plugins/case/server/routes/api/utils.test.ts b/x-pack/plugins/case/server/routes/api/utils.test.ts index 81156b98bab838..2da489e643435c 100644 --- a/x-pack/plugins/case/server/routes/api/utils.test.ts +++ b/x-pack/plugins/case/server/routes/api/utils.test.ts @@ -222,7 +222,12 @@ describe('Utils', () => { ]; const res = transformCases( - { saved_objects: mockCases, total: mockCases.length, per_page: 10, page: 1 }, + { + saved_objects: mockCases.map((obj) => ({ ...obj, score: 1 })), + total: mockCases.length, + per_page: 10, + page: 1, + }, 2, 2, extraCaseData, @@ -232,7 +237,11 @@ describe('Utils', () => { page: 1, per_page: 10, total: mockCases.length, - cases: flattenCaseSavedObjects(mockCases, extraCaseData, '123'), + cases: flattenCaseSavedObjects( + mockCases.map((obj) => ({ ...obj, score: 1 })), + extraCaseData, + '123' + ), count_open_cases: 2, count_closed_cases: 2, }); @@ -500,7 +509,7 @@ describe('Utils', () => { describe('transformComments', () => { it('transforms correctly', () => { const comments = { - saved_objects: mockCaseComments, + saved_objects: mockCaseComments.map((obj) => ({ ...obj, score: 1 })), total: mockCaseComments.length, per_page: 10, page: 1, diff --git a/x-pack/plugins/case/server/routes/api/utils.ts b/x-pack/plugins/case/server/routes/api/utils.ts index b7f3c68d1662fa..ec2881807442fa 100644 --- a/x-pack/plugins/case/server/routes/api/utils.ts +++ b/x-pack/plugins/case/server/routes/api/utils.ts @@ -101,7 +101,7 @@ export const transformCases = ( }); export const flattenCaseSavedObjects = ( - savedObjects: SavedObjectsFindResponse['saved_objects'], + savedObjects: Array>, totalCommentByCase: TotalCommentByCase[], caseConfigureConnectorId: string = 'none' ): CaseResponse[] => @@ -146,7 +146,7 @@ export const transformComments = ( }); export const flattenCommentSavedObjects = ( - savedObjects: SavedObjectsFindResponse['saved_objects'] + savedObjects: Array> ): CommentResponse[] => savedObjects.reduce((acc: CommentResponse[], savedObject: SavedObject) => { return [...acc, flattenCommentSavedObject(savedObject)]; diff --git a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts index 7098f611defa04..ec5d81532e238f 100644 --- a/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts +++ b/x-pack/plugins/encrypted_saved_objects/server/saved_objects/encrypted_saved_objects_client_wrapper.test.ts @@ -676,12 +676,14 @@ describe('#find', () => { id: 'some-id', type: 'unknown-type', attributes: { attrOne: 'one', attrSecret: 'secret', attrThree: 'three' }, + score: 1, references: [], }, { id: 'some-id-2', type: 'unknown-type', attributes: { attrOne: 'one', attrSecret: 'secret', attrThree: 'three' }, + score: 1, references: [], }, ], @@ -722,6 +724,7 @@ describe('#find', () => { attrNotSoSecret: 'not-so-secret', attrThree: 'three', }, + score: 1, references: [], }, { @@ -733,6 +736,7 @@ describe('#find', () => { attrNotSoSecret: '*not-so-secret*', attrThree: 'three', }, + score: 1, references: [], }, ], @@ -793,6 +797,7 @@ describe('#find', () => { attrNotSoSecret: 'not-so-secret', attrThree: 'three', }, + score: 1, references: [], }, { @@ -804,6 +809,7 @@ describe('#find', () => { attrNotSoSecret: '*not-so-secret*', attrThree: 'three', }, + score: 1, references: [], }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index fe66496f70dcdf..9928ce4807da9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -549,6 +549,7 @@ export const getFindResultStatus = (): SavedObjectsFindResponse< searchAfterTimeDurations: ['200.00'], bulkCreateTimeDurations: ['800.43'], }, + score: 1, references: [], updated_at: '2020-02-18T15:26:51.333Z', version: 'WzQ2LDFd', @@ -570,6 +571,7 @@ export const getFindResultStatus = (): SavedObjectsFindResponse< searchAfterTimeDurations: ['200.00'], bulkCreateTimeDurations: ['800.43'], }, + score: 1, references: [], updated_at: '2020-02-18T15:15:58.860Z', version: 'WzMyLDFd', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 6056e692854afe..01ee41e3b877c9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -391,7 +391,7 @@ export const exampleFindRuleStatusResponse: ( total: 1, per_page: 6, page: 1, - saved_objects: mockStatuses, + saved_objects: mockStatuses.map((obj) => ({ ...obj, score: 1 })), }); export const mockLogger: Logger = loggingServiceMock.createLogger(); diff --git a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts index 75cd501a1a9aec..190429d2dacd4d 100644 --- a/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts +++ b/x-pack/plugins/spaces/server/saved_objects/spaces_saved_objects_client.test.ts @@ -138,7 +138,7 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; test(`passes options.type to baseClient if valid singular type specified`, async () => { const { client, baseClient } = await createSpacesSavedObjectsClient(); const expectedReturnValue = { - saved_objects: [createMockResponse()], + saved_objects: [createMockResponse()].map((obj) => ({ ...obj, score: 1 })), total: 1, per_page: 0, page: 0, @@ -158,7 +158,7 @@ const ERROR_NAMESPACE_SPECIFIED = 'Spaces currently determines the namespaces'; test(`supplements options with the current namespace`, async () => { const { client, baseClient } = await createSpacesSavedObjectsClient(); const expectedReturnValue = { - saved_objects: [createMockResponse()], + saved_objects: [createMockResponse()].map((obj) => ({ ...obj, score: 1 })), total: 1, per_page: 0, page: 0, From c8c20e4ca8e768bcce2e471a2f80aef03d4a2a62 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Thu, 18 Jun 2020 13:02:56 +0300 Subject: [PATCH 13/60] Add functional test for Kibana embedded in iframe (#68544) * convert kbn test config into TS * add test for Kibana embedded in iframe * run embedded tests in functional suite * ignore tls errors in functional tests by default * switch test to https * remove env vars mutation * allow to pass ssl config to Kibana * pass ssl config to axios * adopt KbnClient interfaces * adopt KibanaServer * use KbnRequester in security service * set sameSiteCookies:None in test * acceptInsecureCerts in chrome * remove leftovers * fix type error * remove unnecessary field * address comments * refactor plugin * refactor test * make acceptInsecureCerts configurable * run firefox tests on ci * up TS version * fix firefox.sh script * fix path Co-authored-by: Elastic Machine --- .../src/kbn_client/kbn_client.ts | 8 +-- .../src/kbn_client/kbn_client_requester.ts | 38 ++++++++--- .../kbn_client/kbn_client_saved_objects.ts | 16 +++-- .../src/kbn_client/kbn_client_status.ts | 3 +- .../src/kbn_client/kbn_client_ui_settings.ts | 7 +- .../lib/config/schema.ts | 9 +++ .../kbn-test/src/kbn/{index.js => index.ts} | 0 ...{kbn_test_config.js => kbn_test_config.ts} | 24 ++++--- .../kbn-test/src/kbn/{users.js => users.ts} | 0 src/es_archiver/es_archiver.ts | 2 +- .../services/kibana_server/kibana_server.ts | 4 +- test/common/services/security/role.ts | 28 ++++---- .../common/services/security/role_mappings.ts | 32 +++------ test/common/services/security/security.ts | 12 ++-- test/common/services/security/user.ts | 34 ++++------ test/functional/services/common/browser.ts | 5 ++ test/functional/services/remote/remote.ts | 14 ++-- test/functional/services/remote/webdriver.ts | 18 +++-- test/scripts/jenkins_xpack_firefox_smoke.sh | 3 +- x-pack/scripts/functional_tests.js | 1 + .../functional_embedded/config.firefox.ts | 27 ++++++++ x-pack/test/functional_embedded/config.ts | 67 +++++++++++++++++++ .../ftr_provider_context.d.ts | 12 ++++ .../plugins/iframe_embedded/kibana.json | 7 ++ .../plugins/iframe_embedded/package.json | 14 ++++ .../plugins/iframe_embedded/server/index.ts | 11 +++ .../plugins/iframe_embedded/server/plugin.ts | 45 +++++++++++++ x-pack/test/functional_embedded/services.ts | 9 +++ .../tests/iframe_embedded.ts | 42 ++++++++++++ .../test/functional_embedded/tests/index.ts | 14 ++++ 30 files changed, 393 insertions(+), 113 deletions(-) rename packages/kbn-test/src/kbn/{index.js => index.ts} (100%) rename packages/kbn-test/src/kbn/{kbn_test_config.js => kbn_test_config.ts} (76%) rename packages/kbn-test/src/kbn/{users.js => users.ts} (100%) create mode 100644 x-pack/test/functional_embedded/config.firefox.ts create mode 100644 x-pack/test/functional_embedded/config.ts create mode 100644 x-pack/test/functional_embedded/ftr_provider_context.d.ts create mode 100644 x-pack/test/functional_embedded/plugins/iframe_embedded/kibana.json create mode 100644 x-pack/test/functional_embedded/plugins/iframe_embedded/package.json create mode 100644 x-pack/test/functional_embedded/plugins/iframe_embedded/server/index.ts create mode 100644 x-pack/test/functional_embedded/plugins/iframe_embedded/server/plugin.ts create mode 100644 x-pack/test/functional_embedded/services.ts create mode 100644 x-pack/test/functional_embedded/tests/iframe_embedded.ts create mode 100644 x-pack/test/functional_embedded/tests/index.ts diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts index 2eb6c6cc5aac6b..861ea0988692c5 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client.ts @@ -18,7 +18,7 @@ */ import { ToolingLog } from '../tooling_log'; -import { KbnClientRequester, ReqOptions } from './kbn_client_requester'; +import { KibanaConfig, KbnClientRequester, ReqOptions } from './kbn_client_requester'; import { KbnClientStatus } from './kbn_client_status'; import { KbnClientPlugins } from './kbn_client_plugins'; import { KbnClientVersion } from './kbn_client_version'; @@ -26,7 +26,7 @@ import { KbnClientSavedObjects } from './kbn_client_saved_objects'; import { KbnClientUiSettings, UiSettingValues } from './kbn_client_ui_settings'; export class KbnClient { - private readonly requester = new KbnClientRequester(this.log, this.kibanaUrls); + private readonly requester = new KbnClientRequester(this.log, this.kibanaConfig); readonly status = new KbnClientStatus(this.requester); readonly plugins = new KbnClientPlugins(this.status); readonly version = new KbnClientVersion(this.status); @@ -43,10 +43,10 @@ export class KbnClient { */ constructor( private readonly log: ToolingLog, - private readonly kibanaUrls: string[], + private readonly kibanaConfig: KibanaConfig, private readonly uiSettingDefaults?: UiSettingValues ) { - if (!kibanaUrls.length) { + if (!kibanaConfig.url) { throw new Error('missing Kibana urls'); } } diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts index ea4159de557499..2aba2be56f277b 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_requester.ts @@ -16,10 +16,9 @@ * specific language governing permissions and limitations * under the License. */ - import Url from 'url'; - -import Axios from 'axios'; +import Https from 'https'; +import Axios, { AxiosResponse } from 'axios'; import { isAxiosRequestError, isAxiosResponseError } from '../axios'; import { ToolingLog } from '../tooling_log'; @@ -70,20 +69,38 @@ const delay = (ms: number) => setTimeout(resolve, ms); }); +export interface KibanaConfig { + url: string; + ssl?: { + enabled: boolean; + key: string; + certificate: string; + certificateAuthorities: string; + }; +} + export class KbnClientRequester { - constructor(private readonly log: ToolingLog, private readonly kibanaUrls: string[]) {} + private readonly httpsAgent: Https.Agent | null; + constructor(private readonly log: ToolingLog, private readonly kibanaConfig: KibanaConfig) { + this.httpsAgent = + kibanaConfig.ssl && kibanaConfig.ssl.enabled + ? new Https.Agent({ + cert: kibanaConfig.ssl.certificate, + key: kibanaConfig.ssl.key, + ca: kibanaConfig.ssl.certificateAuthorities, + }) + : null; + } private pickUrl() { - const url = this.kibanaUrls.shift()!; - this.kibanaUrls.push(url); - return url; + return this.kibanaConfig.url; } public resolveUrl(relativeUrl: string = '/') { return Url.resolve(this.pickUrl(), relativeUrl); } - async request(options: ReqOptions): Promise { + async request(options: ReqOptions): Promise> { const url = Url.resolve(this.pickUrl(), options.path); const description = options.description || `${options.method} ${url}`; let attempt = 0; @@ -93,7 +110,7 @@ export class KbnClientRequester { attempt += 1; try { - const response = await Axios.request({ + const response = await Axios.request({ method: options.method, url, data: options.body, @@ -101,9 +118,10 @@ export class KbnClientRequester { headers: { 'kbn-xsrf': 'kbn-client', }, + httpsAgent: this.httpsAgent, }); - return response.data; + return response; } catch (error) { const conflictOnGet = isConcliftOnGetError(error); const requestedRetries = options.retries !== undefined; diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts index e671061b343523..7334c6353debfc 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_saved_objects.ts @@ -71,12 +71,13 @@ export class KbnClientSavedObjects { public async migrate() { this.log.debug('Migrating saved objects'); - return await this.requester.request({ + const { data } = await this.requester.request({ description: 'migrate saved objects', path: uriencode`/internal/saved_objects/_migrate`, method: 'POST', body: {}, }); + return data; } /** @@ -85,11 +86,12 @@ export class KbnClientSavedObjects { public async get>(options: GetOptions) { this.log.debug('Gettings saved object: %j', options); - return await this.requester.request>({ + const { data } = await this.requester.request>({ description: 'get saved object', path: uriencode`/api/saved_objects/${options.type}/${options.id}`, method: 'GET', }); + return data; } /** @@ -98,7 +100,7 @@ export class KbnClientSavedObjects { public async create>(options: IndexOptions) { this.log.debug('Creating saved object: %j', options); - return await this.requester.request>({ + const { data } = await this.requester.request>({ description: 'update saved object', path: options.id ? uriencode`/api/saved_objects/${options.type}/${options.id}` @@ -113,6 +115,7 @@ export class KbnClientSavedObjects { references: options.references, }, }); + return data; } /** @@ -121,7 +124,7 @@ export class KbnClientSavedObjects { public async update>(options: UpdateOptions) { this.log.debug('Updating saved object: %j', options); - return await this.requester.request>({ + const { data } = await this.requester.request>({ description: 'update saved object', path: uriencode`/api/saved_objects/${options.type}/${options.id}`, query: { @@ -134,6 +137,7 @@ export class KbnClientSavedObjects { references: options.references, }, }); + return data; } /** @@ -142,10 +146,12 @@ export class KbnClientSavedObjects { public async delete(options: GetOptions) { this.log.debug('Deleting saved object %s/%s', options); - return await this.requester.request({ + const { data } = await this.requester.request({ description: 'delete saved object', path: uriencode`/api/saved_objects/${options.type}/${options.id}`, method: 'DELETE', }); + + return data; } } diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts index 22baf4a3304168..4f203e73620f35 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_status.ts @@ -52,10 +52,11 @@ export class KbnClientStatus { * Get the full server status */ async get() { - return await this.requester.request({ + const { data } = await this.requester.request({ method: 'GET', path: 'api/status', }); + return data; } /** diff --git a/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts b/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts index dbfa87e70032bf..6ee2d3bfe59b0c 100644 --- a/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts +++ b/packages/kbn-dev-utils/src/kbn_client/kbn_client_ui_settings.ts @@ -57,10 +57,11 @@ export class KbnClientUiSettings { * Unset a uiSetting */ async unset(setting: string) { - return await this.requester.request({ + const { data } = await this.requester.request({ path: uriencode`/api/kibana/settings/${setting}`, method: 'DELETE', }); + return data; } /** @@ -105,11 +106,11 @@ export class KbnClientUiSettings { } private async getAll() { - const resp = await this.requester.request({ + const { data } = await this.requester.request({ path: '/api/kibana/settings', method: 'GET', }); - return resp.settings; + return data.settings; } } diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 29ec28175a8519..e9aeee87f1a3b2 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -38,6 +38,14 @@ const urlPartsSchema = () => password: Joi.string(), pathname: Joi.string().regex(/^\//, 'start with a /'), hash: Joi.string().regex(/^\//, 'start with a /'), + ssl: Joi.object() + .keys({ + enabled: Joi.boolean().default(false), + certificate: Joi.string().optional(), + certificateAuthorities: Joi.string().optional(), + key: Joi.string().optional(), + }) + .default(), }) .default(); @@ -122,6 +130,7 @@ export const schema = Joi.object() type: Joi.string().valid('chrome', 'firefox', 'ie', 'msedge').default('chrome'), logPollingMs: Joi.number().default(100), + acceptInsecureCerts: Joi.boolean().default(false), }) .default(), diff --git a/packages/kbn-test/src/kbn/index.js b/packages/kbn-test/src/kbn/index.ts similarity index 100% rename from packages/kbn-test/src/kbn/index.js rename to packages/kbn-test/src/kbn/index.ts diff --git a/packages/kbn-test/src/kbn/kbn_test_config.js b/packages/kbn-test/src/kbn/kbn_test_config.ts similarity index 76% rename from packages/kbn-test/src/kbn/kbn_test_config.js rename to packages/kbn-test/src/kbn/kbn_test_config.ts index c43efabb4b747c..909c94098cf5d6 100644 --- a/packages/kbn-test/src/kbn/kbn_test_config.js +++ b/packages/kbn-test/src/kbn/kbn_test_config.ts @@ -16,26 +16,34 @@ * specific language governing permissions and limitations * under the License. */ - -import { kibanaTestUser } from './users'; import url from 'url'; +import { kibanaTestUser } from './users'; + +interface UrlParts { + protocol?: string; + hostname?: string; + port?: number; + auth?: string; + username?: string; + password?: string; +} export const kbnTestConfig = new (class KbnTestConfig { getPort() { return this.getUrlParts().port; } - getUrlParts() { + getUrlParts(): UrlParts { // allow setting one complete TEST_KIBANA_URL for ES like https://elastic:changeme@example.com:9200 if (process.env.TEST_KIBANA_URL) { const testKibanaUrl = url.parse(process.env.TEST_KIBANA_URL); return { - protocol: testKibanaUrl.protocol.slice(0, -1), + protocol: testKibanaUrl.protocol?.slice(0, -1), hostname: testKibanaUrl.hostname, - port: parseInt(testKibanaUrl.port, 10), + port: testKibanaUrl.port ? parseInt(testKibanaUrl.port, 10) : undefined, auth: testKibanaUrl.auth, - username: testKibanaUrl.auth.split(':')[0], - password: testKibanaUrl.auth.split(':')[1], + username: testKibanaUrl.auth?.split(':')[0], + password: testKibanaUrl.auth?.split(':')[1], }; } @@ -44,7 +52,7 @@ export const kbnTestConfig = new (class KbnTestConfig { return { protocol: process.env.TEST_KIBANA_PROTOCOL || 'http', hostname: process.env.TEST_KIBANA_HOSTNAME || 'localhost', - port: parseInt(process.env.TEST_KIBANA_PORT, 10) || 5620, + port: process.env.TEST_KIBANA_PORT ? parseInt(process.env.TEST_KIBANA_PORT, 10) : 5620, auth: `${username}:${password}`, username, password, diff --git a/packages/kbn-test/src/kbn/users.js b/packages/kbn-test/src/kbn/users.ts similarity index 100% rename from packages/kbn-test/src/kbn/users.js rename to packages/kbn-test/src/kbn/users.ts diff --git a/src/es_archiver/es_archiver.ts b/src/es_archiver/es_archiver.ts index f36cbb3f516b94..e335652195b863 100644 --- a/src/es_archiver/es_archiver.ts +++ b/src/es_archiver/es_archiver.ts @@ -49,7 +49,7 @@ export class EsArchiver { this.client = client; this.dataDir = dataDir; this.log = log; - this.kbnClient = new KbnClient(log, [kibanaUrl]); + this.kbnClient = new KbnClient(log, { url: kibanaUrl }); } /** diff --git a/test/common/services/kibana_server/kibana_server.ts b/test/common/services/kibana_server/kibana_server.ts index 16039d6fee8335..4a251cca044d3c 100644 --- a/test/common/services/kibana_server/kibana_server.ts +++ b/test/common/services/kibana_server/kibana_server.ts @@ -27,9 +27,9 @@ export function KibanaServerProvider({ getService }: FtrProviderContext) { const config = getService('config'); const lifecycle = getService('lifecycle'); const url = Url.format(config.get('servers.kibana')); + const ssl = config.get('servers.kibana').ssl; const defaults = config.get('uiSettings.defaults'); - - const kbn = new KbnClient(log, [url], defaults); + const kbn = new KbnClient(log, { url, ssl }, defaults); if (defaults) { lifecycle.beforeTests.add(async () => { diff --git a/test/common/services/security/role.ts b/test/common/services/security/role.ts index dfc6ff9b164e50..caa5549a70f0c4 100644 --- a/test/common/services/security/role.ts +++ b/test/common/services/security/role.ts @@ -17,27 +17,20 @@ * under the License. */ -import axios, { AxiosInstance } from 'axios'; import util from 'util'; -import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; export class Role { - private log: ToolingLog; - private axios: AxiosInstance; - - constructor(url: string, log: ToolingLog) { - this.log = log; - this.axios = axios.create({ - headers: { 'kbn-xsrf': 'x-pack/ftr/services/security/role' }, - baseURL: url, - maxRedirects: 0, - validateStatus: () => true, // we do our own validation below and throw better error messages - }); - } + constructor(private log: ToolingLog, private kibanaServer: KbnClient) {} public async create(name: string, role: any) { this.log.debug(`creating role ${name}`); - const { data, status, statusText } = await this.axios.put(`/api/security/role/${name}`, role); + const { data, status, statusText } = await this.kibanaServer.request({ + path: `/api/security/role/${name}`, + method: 'PUT', + body: role, + retries: 0, + }); if (status !== 204) { throw new Error( `Expected status code of 204, received ${status} ${statusText}: ${util.inspect(data)}` @@ -47,7 +40,10 @@ export class Role { public async delete(name: string) { this.log.debug(`deleting role ${name}`); - const { data, status, statusText } = await this.axios.delete(`/api/security/role/${name}`); + const { data, status, statusText } = await this.kibanaServer.request({ + path: `/api/security/role/${name}`, + method: 'DELETE', + }); if (status !== 204 && status !== 404) { throw new Error( `Expected status code of 204 or 404, received ${status} ${statusText}: ${util.inspect( diff --git a/test/common/services/security/role_mappings.ts b/test/common/services/security/role_mappings.ts index cc2fa238254987..7951d4b5b47b27 100644 --- a/test/common/services/security/role_mappings.ts +++ b/test/common/services/security/role_mappings.ts @@ -17,30 +17,19 @@ * under the License. */ -import axios, { AxiosInstance } from 'axios'; import util from 'util'; -import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; export class RoleMappings { - private log: ToolingLog; - private axios: AxiosInstance; - - constructor(url: string, log: ToolingLog) { - this.log = log; - this.axios = axios.create({ - headers: { 'kbn-xsrf': 'x-pack/ftr/services/security/role_mappings' }, - baseURL: url, - maxRedirects: 0, - validateStatus: () => true, // we do our own validation below and throw better error messages - }); - } + constructor(private log: ToolingLog, private kbnClient: KbnClient) {} public async create(name: string, roleMapping: Record) { this.log.debug(`creating role mapping ${name}`); - const { data, status, statusText } = await this.axios.post( - `/internal/security/role_mapping/${name}`, - roleMapping - ); + const { data, status, statusText } = await this.kbnClient.request({ + path: `/internal/security/role_mapping/${name}`, + method: 'POST', + body: roleMapping, + }); if (status !== 200) { throw new Error( `Expected status code of 200, received ${status} ${statusText}: ${util.inspect(data)}` @@ -51,9 +40,10 @@ export class RoleMappings { public async delete(name: string) { this.log.debug(`deleting role mapping ${name}`); - const { data, status, statusText } = await this.axios.delete( - `/internal/security/role_mapping/${name}` - ); + const { data, status, statusText } = await this.kbnClient.request({ + path: `/internal/security/role_mapping/${name}`, + method: 'DELETE', + }); if (status !== 200 && status !== 404) { throw new Error( `Expected status code of 200 or 404, received ${status} ${statusText}: ${util.inspect( diff --git a/test/common/services/security/security.ts b/test/common/services/security/security.ts index 6ad0933a2a5a23..fae4c9198cab6d 100644 --- a/test/common/services/security/security.ts +++ b/test/common/services/security/security.ts @@ -17,8 +17,6 @@ * under the License. */ -import { format as formatUrl } from 'url'; - import { Role } from './role'; import { User } from './user'; import { RoleMappings } from './role_mappings'; @@ -28,14 +26,14 @@ import { createTestUserService } from './test_user'; export async function SecurityServiceProvider(context: FtrProviderContext) { const { getService } = context; const log = getService('log'); - const config = getService('config'); - const url = formatUrl(config.get('servers.kibana')); - const role = new Role(url, log); - const user = new User(url, log); + const kibanaServer = getService('kibanaServer'); + + const role = new Role(log, kibanaServer); + const user = new User(log, kibanaServer); const testUser = await createTestUserService(role, user, context); return new (class SecurityService { - roleMappings = new RoleMappings(url, log); + roleMappings = new RoleMappings(log, kibanaServer); testUser = testUser; role = role; user = user; diff --git a/test/common/services/security/user.ts b/test/common/services/security/user.ts index ae02127043234c..58c4d0f1cf34ea 100644 --- a/test/common/services/security/user.ts +++ b/test/common/services/security/user.ts @@ -17,33 +17,22 @@ * under the License. */ -import axios, { AxiosInstance } from 'axios'; import util from 'util'; -import { ToolingLog } from '@kbn/dev-utils'; +import { KbnClient, ToolingLog } from '@kbn/dev-utils'; export class User { - private log: ToolingLog; - private axios: AxiosInstance; - - constructor(url: string, log: ToolingLog) { - this.log = log; - this.axios = axios.create({ - headers: { 'kbn-xsrf': 'x-pack/ftr/services/security/user' }, - baseURL: url, - maxRedirects: 0, - validateStatus: () => true, // we do our own validation below and throw better error messages - }); - } + constructor(private log: ToolingLog, private kbnClient: KbnClient) {} public async create(username: string, user: any) { this.log.debug(`creating user ${username}`); - const { data, status, statusText } = await this.axios.post( - `/internal/security/users/${username}`, - { + const { data, status, statusText } = await this.kbnClient.request({ + path: `/internal/security/users/${username}`, + method: 'POST', + body: { username, ...user, - } - ); + }, + }); if (status !== 200) { throw new Error( `Expected status code of 200, received ${status} ${statusText}: ${util.inspect(data)}` @@ -54,9 +43,10 @@ export class User { public async delete(username: string) { this.log.debug(`deleting user ${username}`); - const { data, status, statusText } = await this.axios.delete( - `/internal/security/users/${username}` - ); + const { data, status, statusText } = await await this.kbnClient.request({ + path: `/internal/security/users/${username}`, + method: 'DELETE', + }); if (status !== 204) { throw new Error( `Expected status code of 204, received ${status} ${statusText}: ${util.inspect(data)}` diff --git a/test/functional/services/common/browser.ts b/test/functional/services/common/browser.ts index 3297f6e094f7c8..d6a4fc91481de2 100644 --- a/test/functional/services/common/browser.ts +++ b/test/functional/services/common/browser.ts @@ -529,5 +529,10 @@ export async function BrowserProvider({ getService }: FtrProviderContext) { await driver.executeScript('document.body.scrollLeft = ' + scrollSize); return this.getScrollLeft(); } + + public async switchToFrame(idOrElement: number | WebElementWrapper) { + const _id = idOrElement instanceof WebElementWrapper ? idOrElement._webElement : idOrElement; + await driver.switchTo().frame(_id); + } })(); } diff --git a/test/functional/services/remote/remote.ts b/test/functional/services/remote/remote.ts index 5a3a775cae0c50..99643929c4682c 100644 --- a/test/functional/services/remote/remote.ts +++ b/test/functional/services/remote/remote.ts @@ -23,7 +23,7 @@ import { resolve } from 'path'; import { mergeMap } from 'rxjs/operators'; import { FtrProviderContext } from '../../ftr_provider_context'; -import { initWebDriver } from './webdriver'; +import { initWebDriver, BrowserConfig } from './webdriver'; import { Browsers } from './browsers'; export async function RemoteProvider({ getService }: FtrProviderContext) { @@ -58,12 +58,12 @@ export async function RemoteProvider({ getService }: FtrProviderContext) { Fs.writeFileSync(path, JSON.stringify(JSON.parse(coverageJson), null, 2)); }; - const { driver, consoleLog$ } = await initWebDriver( - log, - browserType, - lifecycle, - config.get('browser.logPollingMs') - ); + const browserConfig: BrowserConfig = { + logPollingMs: config.get('browser.logPollingMs'), + acceptInsecureCerts: config.get('browser.acceptInsecureCerts'), + }; + + const { driver, consoleLog$ } = await initWebDriver(log, browserType, lifecycle, browserConfig); const isW3CEnabled = (driver as any).executor_.w3c; const caps = await driver.getCapabilities(); diff --git a/test/functional/services/remote/webdriver.ts b/test/functional/services/remote/webdriver.ts index 9fbbf28bbf42cb..27814060e70c1d 100644 --- a/test/functional/services/remote/webdriver.ts +++ b/test/functional/services/remote/webdriver.ts @@ -73,13 +73,18 @@ Executor.prototype.execute = preventParallelCalls( (command: { getName: () => string }) => NO_QUEUE_COMMANDS.includes(command.getName()) ); +export interface BrowserConfig { + logPollingMs: number; + acceptInsecureCerts: boolean; +} + let attemptCounter = 0; let edgePaths: { driverPath: string | undefined; browserPath: string | undefined }; async function attemptToCreateCommand( log: ToolingLog, browserType: Browsers, lifecycle: Lifecycle, - logPollingMs: number + config: BrowserConfig ) { const attemptId = ++attemptCounter; log.debug('[webdriver] Creating session'); @@ -114,6 +119,7 @@ async function attemptToCreateCommand( if (certValidation === '0') { chromeOptions.push('ignore-certificate-errors'); } + if (remoteDebug === '1') { // Visit chrome://inspect in chrome to remotely view/debug chromeOptions.push('headless', 'disable-gpu', 'remote-debugging-port=9222'); @@ -125,6 +131,7 @@ async function attemptToCreateCommand( }); chromeCapabilities.set('unexpectedAlertBehaviour', 'accept'); chromeCapabilities.set('goog:loggingPrefs', { browser: 'ALL' }); + chromeCapabilities.setAcceptInsecureCerts(config.acceptInsecureCerts); const session = await new Builder() .forBrowser(browserType) @@ -137,7 +144,7 @@ async function attemptToCreateCommand( consoleLog$: pollForLogEntry$( session, logging.Type.BROWSER, - logPollingMs, + config.logPollingMs, lifecycle.cleanup.after$ ).pipe( takeUntil(lifecycle.cleanup.after$), @@ -174,7 +181,7 @@ async function attemptToCreateCommand( consoleLog$: pollForLogEntry$( session, logging.Type.BROWSER, - logPollingMs, + config.logPollingMs, lifecycle.cleanup.after$ ).pipe( takeUntil(lifecycle.cleanup.after$), @@ -206,6 +213,7 @@ async function attemptToCreateCommand( 'browser.helperApps.neverAsk.saveToDisk', 'application/comma-separated-values, text/csv, text/plain' ); + firefoxOptions.setAcceptInsecureCerts(config.acceptInsecureCerts); if (headlessBrowser === '1') { // See: https://developer.mozilla.org/en-US/docs/Mozilla/Firefox/Headless_mode @@ -317,7 +325,7 @@ export async function initWebDriver( log: ToolingLog, browserType: Browsers, lifecycle: Lifecycle, - logPollingMs: number + config: BrowserConfig ) { const logger = getLogger('webdriver.http.Executor'); logger.setLevel(logging.Level.FINEST); @@ -348,7 +356,7 @@ export async function initWebDriver( while (true) { const command = await Promise.race([ delay(30 * SECOND), - attemptToCreateCommand(log, browserType, lifecycle, logPollingMs), + attemptToCreateCommand(log, browserType, lifecycle, config), ]); if (!command) { diff --git a/test/scripts/jenkins_xpack_firefox_smoke.sh b/test/scripts/jenkins_xpack_firefox_smoke.sh index fdaee76cafa9de..ae924a5e105527 100755 --- a/test/scripts/jenkins_xpack_firefox_smoke.sh +++ b/test/scripts/jenkins_xpack_firefox_smoke.sh @@ -7,4 +7,5 @@ checks-reporter-with-killswitch "X-Pack firefox smoke test" \ --debug --bail \ --kibana-install-dir "$KIBANA_INSTALL_DIR" \ --include-tag "includeFirefox" \ - --config test/functional/config.firefox.js; + --config test/functional/config.firefox.js \ + --config test/functional_embedded/config.firefox.ts; diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 37b22a687741ed..6cafa3eeef08e3 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -51,6 +51,7 @@ const onlyNotInCoverageTests = [ require.resolve('../test/licensing_plugin/config.legacy.ts'), require.resolve('../test/endpoint_api_integration_no_ingest/config.ts'), require.resolve('../test/reporting_api_integration/config.js'), + require.resolve('../test/functional_embedded/config.ts'), ]; require('@kbn/plugin-helpers').babelRegister(); diff --git a/x-pack/test/functional_embedded/config.firefox.ts b/x-pack/test/functional_embedded/config.firefox.ts new file mode 100644 index 00000000000000..2051d1afd4ab3a --- /dev/null +++ b/x-pack/test/functional_embedded/config.firefox.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const chromeConfig = await readConfigFile(require.resolve('./config')); + + return { + ...chromeConfig.getAll(), + + browser: { + type: 'firefox', + acceptInsecureCerts: true, + }, + + suiteTags: { + exclude: ['skipFirefox'], + }, + + junit: { + reportName: 'Firefox Kibana Embedded in iframe with X-Pack Security', + }, + }; +} diff --git a/x-pack/test/functional_embedded/config.ts b/x-pack/test/functional_embedded/config.ts new file mode 100644 index 00000000000000..95b290ece7db24 --- /dev/null +++ b/x-pack/test/functional_embedded/config.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Fs from 'fs'; +import { resolve } from 'path'; +import { CA_CERT_PATH, KBN_CERT_PATH, KBN_KEY_PATH } from '@kbn/dev-utils'; +import { FtrConfigProviderContext } from '@kbn/test/types/ftr'; +import { pageObjects } from '../functional/page_objects'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const kibanaFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + + const iframeEmbeddedPlugin = resolve(__dirname, './plugins/iframe_embedded'); + + const servers = { + ...kibanaFunctionalConfig.get('servers'), + elasticsearch: { + ...kibanaFunctionalConfig.get('servers.elasticsearch'), + }, + kibana: { + ...kibanaFunctionalConfig.get('servers.kibana'), + protocol: 'https', + ssl: { + enabled: true, + key: Fs.readFileSync(KBN_KEY_PATH).toString('utf8'), + certificate: Fs.readFileSync(KBN_CERT_PATH).toString('utf8'), + certificateAuthorities: Fs.readFileSync(CA_CERT_PATH).toString('utf8'), + }, + }, + }; + + return { + testFiles: [require.resolve('./tests')], + servers, + services: kibanaFunctionalConfig.get('services'), + pageObjects, + browser: { + acceptInsecureCerts: true, + }, + junit: { + reportName: 'Kibana Embedded in iframe with X-Pack Security', + }, + + esTestCluster: kibanaFunctionalConfig.get('esTestCluster'), + apps: { + ...kibanaFunctionalConfig.get('apps'), + }, + + kbnTestServer: { + ...kibanaFunctionalConfig.get('kbnTestServer'), + serverArgs: [ + ...kibanaFunctionalConfig.get('kbnTestServer.serverArgs'), + `--plugin-path=${iframeEmbeddedPlugin}`, + '--server.ssl.enabled=true', + `--server.ssl.key=${KBN_KEY_PATH}`, + `--server.ssl.certificate=${KBN_CERT_PATH}`, + `--server.ssl.certificateAuthorities=${CA_CERT_PATH}`, + + '--xpack.security.sameSiteCookies=None', + '--xpack.security.secureCookies=true', + ], + }, + }; +} diff --git a/x-pack/test/functional_embedded/ftr_provider_context.d.ts b/x-pack/test/functional_embedded/ftr_provider_context.d.ts new file mode 100644 index 00000000000000..5646c06a3cd309 --- /dev/null +++ b/x-pack/test/functional_embedded/ftr_provider_context.d.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GenericFtrProviderContext } from '@kbn/test/types/ftr'; +import { pageObjects } from '../functional/page_objects'; +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; +export { pageObjects }; diff --git a/x-pack/test/functional_embedded/plugins/iframe_embedded/kibana.json b/x-pack/test/functional_embedded/plugins/iframe_embedded/kibana.json new file mode 100644 index 00000000000000..ea9f55bd21c6ea --- /dev/null +++ b/x-pack/test/functional_embedded/plugins/iframe_embedded/kibana.json @@ -0,0 +1,7 @@ +{ + "id": "iframe_embedded", + "version": "1.0.0", + "kibanaVersion": "kibana", + "server": true, + "ui": false +} diff --git a/x-pack/test/functional_embedded/plugins/iframe_embedded/package.json b/x-pack/test/functional_embedded/plugins/iframe_embedded/package.json new file mode 100644 index 00000000000000..9fa1554e5312b5 --- /dev/null +++ b/x-pack/test/functional_embedded/plugins/iframe_embedded/package.json @@ -0,0 +1,14 @@ +{ + "name": "iframe_embedded", + "version": "0.0.0", + "kibana": { + "version": "kibana" + }, + "scripts": { + "kbn": "node ../../../../../scripts/kbn.js", + "build": "rm -rf './target' && tsc" + }, + "devDependencies": { + "typescript": "3.9.5" + } +} diff --git a/x-pack/test/functional_embedded/plugins/iframe_embedded/server/index.ts b/x-pack/test/functional_embedded/plugins/iframe_embedded/server/index.ts new file mode 100644 index 00000000000000..976ef19d4d8a75 --- /dev/null +++ b/x-pack/test/functional_embedded/plugins/iframe_embedded/server/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { PluginInitializerContext } from 'kibana/server'; +import { IframeEmbeddedPlugin } from './plugin'; + +export const plugin = (initContext: PluginInitializerContext) => + new IframeEmbeddedPlugin(initContext); diff --git a/x-pack/test/functional_embedded/plugins/iframe_embedded/server/plugin.ts b/x-pack/test/functional_embedded/plugins/iframe_embedded/server/plugin.ts new file mode 100644 index 00000000000000..890fe14cf03cfa --- /dev/null +++ b/x-pack/test/functional_embedded/plugins/iframe_embedded/server/plugin.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import Url from 'url'; +import { Plugin, CoreSetup, PluginInitializerContext } from 'src/core/server'; + +function renderBody(iframeUrl: string) { + return ` + + + + + Kibana embedded in iframe + + +