From 5b0d679c6044ff9bac1cf413ecd4aeba24faf11e Mon Sep 17 00:00:00 2001 From: "Joey F. Poon" Date: Thu, 22 Jul 2021 10:04:49 -0500 Subject: [PATCH 1/4] [Security Solution] fix metadata api tests (#106340) --- .../endpoint/metadata/api_feature/data.json | 36 +++++++++---------- .../metadata/destination_index/data.json | 12 +++---- .../apps/endpoint/endpoint_list.ts | 10 +++--- .../apis/metadata.ts | 7 ++-- 4 files changed, 32 insertions(+), 33 deletions(-) diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json index 30b4e19dcb1d1c..b3d33f5d45345a 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json @@ -4,7 +4,7 @@ "id": "3KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -26,7 +26,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", "kind": "metric", "category": [ @@ -74,7 +74,7 @@ "id": "3aVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -96,7 +96,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", "kind": "metric", "category": [ @@ -143,7 +143,7 @@ "id": "3qVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -165,7 +165,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", "kind": "metric", "category": [ @@ -210,7 +210,7 @@ "id": "36VN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -232,7 +232,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d18", "kind": "metric", "category": [ @@ -280,7 +280,7 @@ "id": "4KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -302,7 +302,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d19", "kind": "metric", "category": [ @@ -348,7 +348,7 @@ "id": "4aVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -370,7 +370,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d39", "kind": "metric", "category": [ @@ -416,7 +416,7 @@ "id": "4qVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "963b081e-60d1-482c-befd-a5815fa8290f", "version": "6.6.1", @@ -438,7 +438,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d31", "kind": "metric", "category": [ @@ -485,7 +485,7 @@ "id": "46VN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "b3412d6f-b022-4448-8fee-21cc936ea86b", "version": "6.0.0", @@ -507,7 +507,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d23", "kind": "metric", "category": [ @@ -553,7 +553,7 @@ "id": "5KVN2G8BYQH1gtPUuYk7", "index": "metrics-endpoint.metadata-default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "agent": { "id": "3838df35-a095-4af4-8fce-0b6d78793f2e", "version": "6.8.0", @@ -575,7 +575,7 @@ } }, "event": { - "created": 1618841405309, + "created": 1626897841950, "id": "32f5fda2-48e4-4fae-b89e-a18038294d35", "kind": "metric", "category": [ diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json index 22f4afcf99d4d9..b8994a05ea5ccf 100644 --- a/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json +++ b/x-pack/test/functional/es_archives/endpoint/metadata/destination_index/data.json @@ -4,7 +4,7 @@ "id": "M92ScEJT9M9QusfIi3hpEb0AAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "Endpoint": { "policy": { "applied": { @@ -36,7 +36,7 @@ "category": [ "host" ], - "created": 1618841405309, + "created": 1626897841950, "dataset": "endpoint.metadata", "id": "32f5fda2-48e4-4fae-b89e-a18038294d16", "ingested": "2020-09-09T18:25:15.853783Z", @@ -78,7 +78,7 @@ "id": "OU3RgCJaNnR90byeDEHutp8AAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "Endpoint": { "policy": { "applied": { @@ -110,7 +110,7 @@ "category": [ "host" ], - "created": 1618841405309, + "created": 1626897841950, "dataset": "endpoint.metadata", "id": "32f5fda2-48e4-4fae-b89e-a18038294d14", "ingested": "2020-09-09T18:25:14.919526Z", @@ -155,7 +155,7 @@ "id": "YjqDCEuI6JmLeLOSyZx_NhMAAAAAAAAA", "index": "metrics-endpoint.metadata_current_default", "source": { - "@timestamp": 1618841405309, + "@timestamp": 1626897841950, "Endpoint": { "policy": { "applied": { @@ -187,7 +187,7 @@ "category": [ "host" ], - "created": 1618841405309, + "created": 1626897841950, "dataset": "endpoint.metadata", "id": "32f5fda2-48e4-4fae-b89e-a18038294d15", "ingested": "2020-09-09T18:25:15.853404Z", diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts index 1a5158adbd6951..f61edf28957970 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts @@ -38,7 +38,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c', '6.8.0', - 'Apr 19, 2021 @ 14:10:05.309', + 'Jul 21, 2021 @ 20:04:01.950', '', ], [ @@ -49,7 +49,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.192.213.130, 10.70.28.129', '6.6.1', - 'Apr 19, 2021 @ 14:10:05.309', + 'Jul 21, 2021 @ 20:04:01.950', '', ], [ @@ -60,7 +60,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.46.229.234', '6.0.0', - 'Apr 19, 2021 @ 14:10:05.309', + 'Jul 21, 2021 @ 20:04:01.950', '', ], ]; @@ -281,7 +281,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.192.213.130, 10.70.28.129', '6.6.1', - 'Apr 19, 2021 @ 14:10:05.309', + 'Jul 21, 2021 @ 20:04:01.950', '', ], [ @@ -292,7 +292,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { 'windows 10.0', '10.46.229.234', '6.0.0', - 'Apr 19, 2021 @ 14:10:05.309', + 'Jul 21, 2021 @ 20:04:01.950', '', ], ]; diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts index 4656edc7edfb4f..cadb9a420708ad 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts @@ -23,8 +23,7 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); - // Failing: See https://github.com/elastic/kibana/issues/106051 - describe.skip('test metadata api', () => { + describe('test metadata api', () => { describe(`POST ${HOST_METADATA_LIST_ROUTE} when index is empty`, () => { it('metadata api should return empty result when index is empty', async () => { await deleteMetadataStream(getService); @@ -222,7 +221,7 @@ export default function ({ getService }: FtrProviderContext) { (ip: string) => ip === targetEndpointIp ); expect(resultIp).to.eql([targetEndpointIp]); - expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); + expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10); expect(body.request_page_index).to.eql(0); @@ -264,7 +263,7 @@ export default function ({ getService }: FtrProviderContext) { const resultElasticAgentId: string = body.hosts[0].metadata.elastic.agent.id; expect(resultHostId).to.eql(targetEndpointId); expect(resultElasticAgentId).to.eql(targetElasticAgentId); - expect(body.hosts[0].metadata.event.created).to.eql(1618841405309); + expect(body.hosts[0].metadata.event.created).to.eql(1626897841950); expect(body.hosts[0].host_status).to.eql('unhealthy'); expect(body.hosts.length).to.eql(1); expect(body.request_page_size).to.eql(10); From 745db3063a3763f6b704a0bf48d1f3093dba6161 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Thu, 22 Jul 2021 16:11:32 +0100 Subject: [PATCH 2/4] [Security Solution] Flyout overview hover actions (#106362) * flyout-overview * integrate with hover actions * fix types * fix types * move TopN into a popover * fix types * fix up * update field width * fix unit tests * fix agent status field --- .../event_details/alert_summary_view.tsx | 135 ++++++++++-------- .../components/event_details/helpers.tsx | 12 +- .../event_details/table/action_cell.tsx | 8 +- .../event_details/table/field_value_cell.tsx | 8 +- .../table/use_action_cell_data_provider.ts | 2 +- .../hover_actions/actions/show_top_n.tsx | 32 ++--- .../common/components/hover_actions/index.tsx | 8 +- 7 files changed, 114 insertions(+), 91 deletions(-) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx index 841aa6840cc0b7..501ef78d550f9f 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/alert_summary_view.tsx @@ -6,13 +6,11 @@ */ import { EuiBasicTableColumn, EuiSpacer, EuiHorizontalRule, EuiTitle, EuiText } from '@elastic/eui'; -import { get, getOr, find } from 'lodash/fp'; +import { get, getOr, find, isEmpty } from 'lodash/fp'; import React, { useMemo } from 'react'; import styled from 'styled-components'; import * as i18n from './translations'; -import { FormattedFieldValue } from '../../../timelines/components/timeline/body/renderers/formatted_field'; -import { TimelineEventsDetailsItem } from '../../../../common/search_strategy'; import { BrowserFields } from '../../../../common/search_strategy/index_fields'; import { ALERTS_HEADERS_RISK_SCORE, @@ -25,6 +23,7 @@ import { TIMESTAMP, } from '../../../detections/components/alerts_table/translations'; import { + AGENT_STATUS_FIELD_NAME, IP_FIELD_TYPE, SIGNAL_RULE_NAME_FIELD_NAME, } from '../../../timelines/components/timeline/body/renderers/constants'; @@ -35,12 +34,21 @@ import { useRuleWithFallback } from '../../../detections/containers/detection_en import { MarkdownRenderer } from '../markdown_editor'; import { LineClamp } from '../line_clamp'; import { endpointAlertCheck } from '../../utils/endpoint_alert_check'; +import { getEmptyValue } from '../empty_value'; +import { ActionCell } from './table/action_cell'; +import { FieldValueCell } from './table/field_value_cell'; +import { TimelineEventsDetailsItem } from '../../../../common'; +import { EventFieldsData } from './types'; export const Indent = styled.div` padding: 0 8px; word-break: break-word; `; +const StyledEmptyComponent = styled.div` + padding: ${(props) => `${props.theme.eui.paddingSizes.xs} 0`}; +`; + const fields = [ { id: 'signal.status', label: SIGNAL_STATUS }, { id: '@timestamp', label: TIMESTAMP }, @@ -52,7 +60,7 @@ const fields = [ { id: 'signal.rule.severity', label: ALERTS_HEADERS_SEVERITY }, { id: 'signal.rule.risk_score', label: ALERTS_HEADERS_RISK_SCORE }, { id: 'host.name' }, - { id: 'agent.status' }, + { id: 'agent.id', overrideField: AGENT_STATUS_FIELD_NAME, label: i18n.AGENT_STATUS }, { id: 'user.name' }, { id: SOURCE_IP_FIELD_NAME, fieldType: IP_FIELD_TYPE }, { id: DESTINATION_IP_FIELD_NAME, fieldType: IP_FIELD_TYPE }, @@ -76,22 +84,43 @@ const networkFields = [ ]; const getDescription = ({ - contextId, + data, eventId, - fieldName, - value, - fieldType = '', + fieldFromBrowserField, linkValue, -}: AlertSummaryRow['description']) => ( - -); + timelineId, + values, +}: AlertSummaryRow['description']) => { + if (isEmpty(values)) { + return {getEmptyValue()}; + } + + const eventFieldsData = { + ...data, + ...(fieldFromBrowserField ? fieldFromBrowserField : {}), + } as EventFieldsData; + return ( + <> + + + + ); +}; const getSummaryRows = ({ data, @@ -120,25 +149,45 @@ const getSummaryRows = ({ return data != null ? tableFields.reduce((acc, item) => { + const initialDescription = { + contextId: timelineId, + eventId, + value: null, + fieldType: 'string', + linkValue: undefined, + timelineId, + }; const field = data.find((d) => d.field === item.id); if (!field) { - return acc; + return [ + ...acc, + { + title: item.label ?? item.id, + description: initialDescription, + }, + ]; } + const linkValueField = item.linkField != null && data.find((d) => d.field === item.linkField); const linkValue = getOr(null, 'originalValue.0', linkValueField); const value = getOr(null, 'originalValue.0', field); - const category = field.category; - const fieldType = get(`${category}.fields.${field.field}.type`, browserFields) as string; + const category = field.category ?? ''; + const fieldName = field.field ?? ''; + + const browserField = get([category, 'fields', fieldName], browserFields); const description = { - contextId: timelineId, - eventId, - fieldName: item.id, - value, - fieldType: item.fieldType ?? fieldType, + ...initialDescription, + data: { ...field, ...(item.overrideField ? { field: item.overrideField } : {}) }, + values: field.values, linkValue: linkValue ?? undefined, + fieldFromBrowserField: browserField, }; + if (item.id === 'agent.id' && !endpointAlertCheck({ data })) { + return acc; + } + if (item.id === 'signal.threshold_result.terms') { try { const terms = getOr(null, 'originalValue', field); @@ -149,14 +198,14 @@ const getSummaryRows = ({ title: `${entry.field} [threshold]`, description: { ...description, - value: entry.value, + values: [entry.value], }, }; } ); return [...acc, ...thresholdTerms]; } catch (err) { - return acc; + return [...acc]; } } @@ -169,7 +218,7 @@ const getSummaryRows = ({ title: ALERTS_HEADERS_THRESHOLD_CARDINALITY, description: { ...description, - value: `count(${parsedValue.field}) == ${parsedValue.value}`, + values: [`count(${parsedValue.field}) == ${parsedValue.value}`], }, }, ]; @@ -205,28 +254,6 @@ const AlertSummaryViewComponent: React.FC<{ timelineId, ]); - const isEndpointAlert = useMemo(() => { - return endpointAlertCheck({ data }); - }, [data]); - - const endpointId = useMemo(() => { - const findAgentId = find({ category: 'agent', field: 'agent.id' }, data)?.values; - return findAgentId ? findAgentId[0] : ''; - }, [data]); - - const agentStatusRow = { - title: i18n.AGENT_STATUS, - description: { - contextId: timelineId, - eventId, - fieldName: 'agent.status', - value: endpointId, - linkValue: undefined, - }, - }; - - const summaryRowsWithAgentStatus = [...summaryRows, agentStatusRow]; - const ruleId = useMemo(() => { const item = data.find((d) => d.field === 'signal.rule.id'); return Array.isArray(item?.originalValue) @@ -238,11 +265,7 @@ const AlertSummaryViewComponent: React.FC<{ return ( <> - + {maybeRule?.note && ( <> diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx index 2b300789c4d148..ecfa243f89246e 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/helpers.tsx @@ -23,7 +23,7 @@ import { } from '../../../timelines/components/timeline/body/constants'; import * as i18n from './translations'; -import { ColumnHeaderOptions } from '../../../../common'; +import { ColumnHeaderOptions, TimelineEventsDetailsItem } from '../../../../common'; /** * Defines the behavior of the search input that appears above the table of data @@ -55,12 +55,12 @@ export interface Item { export interface AlertSummaryRow { title: string; description: { - contextId: string; + data: TimelineEventsDetailsItem; eventId: string; - fieldName: string; - value: string; - fieldType: string; + fieldFromBrowserField?: Readonly>>; linkValue: string | undefined; + timelineId: string; + values: string[] | null | undefined; }; } @@ -213,7 +213,7 @@ export const getSummaryColumns = ( field: 'title', truncateText: false, render: getTitle, - width: '160px', + width: '33%', name: '', }, { diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/table/action_cell.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/table/action_cell.tsx index 795ecb266b0924..f5cf600e281ad1 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/table/action_cell.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/table/action_cell.tsx @@ -19,8 +19,9 @@ interface Props { data: EventFieldsData; disabled?: boolean; eventId: string; - fieldFromBrowserField: Readonly>>; - getLinkValue: (field: string) => string | null; + fieldFromBrowserField?: Readonly>>; + getLinkValue?: (field: string) => string | null; + linkValue?: string | null | undefined; onFilterAdded?: () => void; timelineId?: string; toggleColumn?: (column: ColumnHeaderOptions) => void; @@ -34,6 +35,7 @@ export const ActionCell: React.FC = React.memo( eventId, fieldFromBrowserField, getLinkValue, + linkValue, onFilterAdded, timelineId, toggleColumn, @@ -47,7 +49,7 @@ export const ActionCell: React.FC = React.memo( fieldFromBrowserField, fieldType: data.type, isObjectArray: data.isObjectArray, - linkValue: getLinkValue(data.field), + linkValue: (getLinkValue && getLinkValue(data.field)) ?? linkValue, values, }); diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/table/field_value_cell.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/table/field_value_cell.tsx index b6524a8c9c895f..2ac0ca23ca8c17 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/table/field_value_cell.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/table/field_value_cell.tsx @@ -17,8 +17,9 @@ export interface FieldValueCellProps { contextId: string; data: EventFieldsData; eventId: string; - fieldFromBrowserField: Readonly>>; - getLinkValue: (field: string) => string | null; + fieldFromBrowserField?: Readonly>>; + getLinkValue?: (field: string) => string | null; + linkValue?: string | null | undefined; values: string[] | null | undefined; } @@ -29,6 +30,7 @@ export const FieldValueCell = React.memo( eventId, fieldFromBrowserField, getLinkValue, + linkValue, values, }: FieldValueCellProps) => { return ( @@ -55,7 +57,7 @@ export const FieldValueCell = React.memo( fieldType={data.type} isObjectArray={data.isObjectArray} value={value} - linkValue={getLinkValue(data.field)} + linkValue={(getLinkValue && getLinkValue(data.field)) ?? linkValue} /> )} diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/table/use_action_cell_data_provider.ts b/x-pack/plugins/security_solution/public/common/components/event_details/table/use_action_cell_data_provider.ts index e580ae6c1fdef9..fbe9767759d281 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/table/use_action_cell_data_provider.ts +++ b/x-pack/plugins/security_solution/public/common/components/event_details/table/use_action_cell_data_provider.ts @@ -31,7 +31,7 @@ export interface UseActionCellDataProvider { eventId?: string; field: string; fieldFormat?: string; - fieldFromBrowserField: Readonly>>; + fieldFromBrowserField?: Readonly>>; fieldType?: string; isObjectArray?: boolean; linkValue?: string | null; diff --git a/x-pack/plugins/security_solution/public/common/components/hover_actions/actions/show_top_n.tsx b/x-pack/plugins/security_solution/public/common/components/hover_actions/actions/show_top_n.tsx index 6e284289243f0f..0fc8a740845213 100644 --- a/x-pack/plugins/security_solution/public/common/components/hover_actions/actions/show_top_n.tsx +++ b/x-pack/plugins/security_solution/public/common/components/hover_actions/actions/show_top_n.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React from 'react'; +import React, { useMemo } from 'react'; import { EuiButtonIcon, EuiPopover, EuiToolTip } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { StatefulTopN } from '../../top_n'; @@ -44,15 +44,18 @@ export const ShowTopNButton: React.FC = React.memo( ? SourcererScopeName.detections : SourcererScopeName.default; const { browserFields, indexPattern } = useSourcererScope(activeScope); - const button = ( - + const button = useMemo( + () => ( + + ), + [field, onClick] ); return showTopN ? ( @@ -80,14 +83,7 @@ export const ShowTopNButton: React.FC = React.memo( /> } > - + {button} ); } diff --git a/x-pack/plugins/security_solution/public/common/components/hover_actions/index.tsx b/x-pack/plugins/security_solution/public/common/components/hover_actions/index.tsx index a7fdb26a525fb7..31bdf78626e7cb 100644 --- a/x-pack/plugins/security_solution/public/common/components/hover_actions/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/hover_actions/index.tsx @@ -39,7 +39,7 @@ export const AdditionalContent = styled.div` AdditionalContent.displayName = 'AdditionalContent'; const StyledHoverActionsContainer = styled.div<{ $showTopN: boolean }>` - padding: ${(props) => (props.$showTopN ? 'none' : `0 ${props.theme.eui.paddingSizes.s}`)}; + padding: ${(props) => `0 ${props.theme.eui.paddingSizes.s}`}; display: flex; &:focus-within { @@ -58,7 +58,7 @@ const StyledHoverActionsContainer = styled.div<{ $showTopN: boolean }>` .timelines__hoverActionButton, .securitySolution__hoverActionButton { - opacity: 0; + opacity: ${(props) => (props.$showTopN ? 1 : 0)}; &:focus { opacity: 1; @@ -268,7 +268,7 @@ export const HoverActions: React.FC = React.memo( ] ); - const showFilters = !showTopN && values != null; + const showFilters = values != null; return ( @@ -342,7 +342,7 @@ export const HoverActions: React.FC = React.memo( value={values} /> )} - {!showTopN && ( + {showFilters && ( Date: Thu, 22 Jul 2021 10:18:19 -0500 Subject: [PATCH 3/4] Revert "[Security solution] [Endpoint] Unify subtitle text in flyout and modal for event filters (#106401)" This reverts commit 7f758731ae3125f93393cad2403d6ea45f3c0b72. --- .../view/components/form/index.tsx | 18 ++++++++++-------- .../view/components/form/translations.ts | 7 +++++++ .../view/event_filters_list_page.tsx | 7 +++++-- .../pages/event_filters/view/translations.ts | 6 ------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/index.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/index.tsx index 29723a5fd3cf80..db5c42241a0cc4 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/index.tsx @@ -31,10 +31,16 @@ import { ExceptionBuilder } from '../../../../../../shared_imports'; import { useEventFiltersSelector } from '../../hooks'; import { getFormEntryStateMutable, getHasNameError, getNewComment } from '../../../store/selector'; -import { NAME_LABEL, NAME_ERROR, NAME_PLACEHOLDER, OS_LABEL, RULE_NAME } from './translations'; +import { + FORM_DESCRIPTION, + NAME_LABEL, + NAME_ERROR, + NAME_PLACEHOLDER, + OS_LABEL, + RULE_NAME, +} from './translations'; import { OS_TITLES } from '../../../../../common/translations'; import { ENDPOINT_EVENT_FILTERS_LIST_ID, EVENT_FILTER_LIST_TYPE } from '../../../constants'; -import { ABOUT_EVENT_FILTERS } from '../../translations'; const OPERATING_SYSTEMS: readonly OperatingSystem[] = [ OperatingSystem.MAC, @@ -199,12 +205,8 @@ export const EventFiltersForm: React.FC = memo( return !isIndexPatternLoading && exception ? ( - {!exception || !exception.item_id ? ( - - {ABOUT_EVENT_FILTERS} - - - ) : null} + {FORM_DESCRIPTION} + {nameInputMemo} {allowSelectOs ? ( diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/translations.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/translations.ts index bfb828699118e4..7391251a936e62 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/translations.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/form/translations.ts @@ -7,6 +7,13 @@ import { i18n } from '@kbn/i18n'; +export const FORM_DESCRIPTION = i18n.translate( + 'xpack.securitySolution.eventFilter.modal.description', + { + defaultMessage: "Events are filtered when the rule's conditions are met:", + } +); + export const NAME_PLACEHOLDER = i18n.translate( 'xpack.securitySolution.eventFilter.form.name.placeholder', { diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx index 95f3e856a6ff6d..2d608bdc6e1575 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx @@ -44,7 +44,6 @@ import { EventFilterDeleteModal } from './components/event_filter_delete_modal'; import { SearchBar } from '../../../components/search_bar'; import { BackToExternalAppButton } from '../../../components/back_to_external_app_button'; -import { ABOUT_EVENT_FILTERS } from './translations'; type EventListPaginatedContent = PaginatedContentProps< Immutable, @@ -196,7 +195,11 @@ export const EventFiltersListPage = memo(() => { defaultMessage="Event Filters" /> } - subtitle={ABOUT_EVENT_FILTERS} + subtitle={i18n.translate('xpack.securitySolution.eventFilters.aboutInfo', { + defaultMessage: + 'Add an event filter to exclude high volume or unwanted events from being written to Elasticsearch. Event ' + + 'filters are processed by the Endpoint Security integration, and are applied to hosts running this integration on their agents.', + })} actions={ doesDataExist && ( { values: { error: getError.message }, }); }; - -export const ABOUT_EVENT_FILTERS = i18n.translate('xpack.securitySolution.eventFilters.aboutInfo', { - defaultMessage: - 'Add an event filter to exclude high volume or unwanted events from being written to Elasticsearch. Event ' + - 'filters are processed by the Endpoint Security integration, and are applied to hosts running this integration on their agents.', -}); From 10ef0e9e3e8bcdd767f4c3c02d21225d11e374cc Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Thu, 22 Jul 2021 17:30:41 +0200 Subject: [PATCH 4/4] [ML] Alerting rule for Anomaly Detection jobs monitoring (#106084) * [ML] init job health alerting rule type * [ML] add health checks selection ui * [ML] define schema * [ML] support all jobs selection * [ML] jobs health service * [ML] add logger * [ML] add context message * [ML] fix default message for i18n * [ML] check response size * [ML] add exclude jobs control * [ML] getResultJobsHealthRuleConfig * [ML] change naming for shared services * [ML] fix excluded jobs filtering * [ML] check for execution results * [ML] update context fields * [ML] unit tests for getResultJobsHealthRuleConfig * [ML] refactor and job ids check * [ML] rename datafeed * [ML] fix translation messages * [ML] hide non-implemented tests * [ML] remove jod ids join from the getJobs call * [ML] add validation for the tests config * [ML] fix excluded jobs udpate * [ML] update jobIdsDescription message * [ML] allow selection all jobs only for include * [ML] better ux for excluded jobs setup * [ML] change rule type name * [ML] fix typo * [ML] change instances names * [ML] fix messages * [ML] hide error callout, show health checks error in EuiFormRow * [ML] add check for job state * [ML] add alertingRules key to the doc links * [ML] update types * [ML] remove redundant type * [ML] fix job and datafeed states check * [ML] fix job and datafeed states check, add comments * [ML] add unit tests --- .../public/doc_links/doc_links_service.ts | 1 + x-pack/plugins/ml/common/constants/alerts.ts | 42 +--- x-pack/plugins/ml/common/types/alerts.ts | 35 ++++ x-pack/plugins/ml/common/util/alerts.test.ts | 52 ++++- x-pack/plugins/ml/common/util/alerts.ts | 25 +++ .../ml/public/alerting/job_selector.tsx | 80 ++++++-- ...aly_detection_jobs_health_rule_trigger.tsx | 148 ++++++++++++++ .../public/alerting/jobs_health_rule/index.ts | 8 + .../register_jobs_health_alerting_rule.ts | 69 +++++++ .../tests_selection_control.tsx | 125 ++++++++++++ .../ml/public/alerting/register_ml_alerts.ts | 5 +- .../ml/server/lib/alerts/alerting_service.ts | 4 +- .../lib/alerts/jobs_health_service.test.ts | 180 +++++++++++++++++ .../server/lib/alerts/jobs_health_service.ts | 185 ++++++++++++++++++ .../register_anomaly_detection_alert_type.ts | 32 +-- .../register_jobs_monitoring_rule_type.ts | 109 +++++++++++ .../server/lib/alerts/register_ml_alerts.ts | 4 + x-pack/plugins/ml/server/plugin.ts | 7 +- .../server/routes/schemas/alerting_schema.ts | 76 +++++-- .../server/shared_services/shared_services.ts | 34 +++- 20 files changed, 1135 insertions(+), 86 deletions(-) create mode 100644 x-pack/plugins/ml/public/alerting/jobs_health_rule/anomaly_detection_jobs_health_rule_trigger.tsx create mode 100644 x-pack/plugins/ml/public/alerting/jobs_health_rule/index.ts create mode 100644 x-pack/plugins/ml/public/alerting/jobs_health_rule/register_jobs_health_alerting_rule.ts create mode 100644 x-pack/plugins/ml/public/alerting/jobs_health_rule/tests_selection_control.tsx create mode 100644 x-pack/plugins/ml/server/lib/alerts/jobs_health_service.test.ts create mode 100644 x-pack/plugins/ml/server/lib/alerts/jobs_health_service.ts create mode 100644 x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index e8453d009e7206..7152c7eb3cb1b7 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -242,6 +242,7 @@ export class DocLinksService { anomalyDetectionJobResource: `${ELASTICSEARCH_DOCS}ml-put-job.html#ml-put-job-path-parms`, anomalyDetectionJobResourceAnalysisConfig: `${ELASTICSEARCH_DOCS}ml-put-job.html#put-analysisconfig`, anomalyDetectionJobTips: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-job-tips`, + alertingRules: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-configuring-alerts.html`, anomalyDetectionModelMemoryLimits: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-model-memory-limits`, calendars: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-ad-finding-anomalies.html#ml-ad-calendars`, classificationEvaluation: `${ELASTIC_WEBSITE_URL}guide/en/machine-learning/${DOC_LINK_VERSION}/ml-dfa-classification.html#ml-dfanalytics-classification-evaluation`, diff --git a/x-pack/plugins/ml/common/constants/alerts.ts b/x-pack/plugins/ml/common/constants/alerts.ts index 30daf0d45c3ac7..604d5ea8c4fc94 100644 --- a/x-pack/plugins/ml/common/constants/alerts.ts +++ b/x-pack/plugins/ml/common/constants/alerts.ts @@ -6,46 +6,22 @@ */ import { i18n } from '@kbn/i18n'; -import { ActionGroup } from '../../../alerting/common'; -import { MINIMUM_FULL_LICENSE } from '../license'; -import { PLUGIN_ID } from './app'; export const ML_ALERT_TYPES = { ANOMALY_DETECTION: 'xpack.ml.anomaly_detection_alert', + AD_JOBS_HEALTH: 'xpack.ml.anomaly_detection_jobs_health', } as const; export type MlAlertType = typeof ML_ALERT_TYPES[keyof typeof ML_ALERT_TYPES]; -export const ANOMALY_SCORE_MATCH_GROUP_ID = 'anomaly_score_match'; -export type AnomalyScoreMatchGroupId = typeof ANOMALY_SCORE_MATCH_GROUP_ID; -export const THRESHOLD_MET_GROUP: ActionGroup = { - id: ANOMALY_SCORE_MATCH_GROUP_ID, - name: i18n.translate('xpack.ml.anomalyDetectionAlert.actionGroupName', { - defaultMessage: 'Anomaly score matched the condition', - }), -}; - -export const ML_ALERT_TYPES_CONFIG: Record< - MlAlertType, - { - name: string; - actionGroups: Array>; - defaultActionGroupId: AnomalyScoreMatchGroupId; - minimumLicenseRequired: string; - producer: string; - } -> = { - [ML_ALERT_TYPES.ANOMALY_DETECTION]: { - name: i18n.translate('xpack.ml.anomalyDetectionAlert.name', { - defaultMessage: 'Anomaly detection alert', - }), - actionGroups: [THRESHOLD_MET_GROUP], - defaultActionGroupId: ANOMALY_SCORE_MATCH_GROUP_ID, - minimumLicenseRequired: MINIMUM_FULL_LICENSE, - producer: PLUGIN_ID, - }, -}; - export const ALERT_PREVIEW_SAMPLE_SIZE = 5; export const TOP_N_BUCKETS_COUNT = 1; + +export const ALL_JOBS_SELECTION = '*'; + +export const HEALTH_CHECK_NAMES = { + datafeed: i18n.translate('xpack.ml.alertTypes.jobsHealthAlertingRule.datafeedCheckName', { + defaultMessage: 'Datafeed is not started', + }), +}; diff --git a/x-pack/plugins/ml/common/types/alerts.ts b/x-pack/plugins/ml/common/types/alerts.ts index 1677a766544a19..877bb2d2933655 100644 --- a/x-pack/plugins/ml/common/types/alerts.ts +++ b/x-pack/plugins/ml/common/types/alerts.ts @@ -108,3 +108,38 @@ export type MlAnomalyDetectionAlertRule = Omit; diff --git a/x-pack/plugins/ml/common/util/alerts.test.ts b/x-pack/plugins/ml/common/util/alerts.test.ts index d9896c967165bc..430e10cc8ffa81 100644 --- a/x-pack/plugins/ml/common/util/alerts.test.ts +++ b/x-pack/plugins/ml/common/util/alerts.test.ts @@ -5,7 +5,11 @@ * 2.0. */ -import { getLookbackInterval, resolveLookbackInterval } from './alerts'; +import { + getLookbackInterval, + getResultJobsHealthRuleConfig, + resolveLookbackInterval, +} from './alerts'; import type { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_jobs'; describe('resolveLookbackInterval', () => { @@ -76,3 +80,49 @@ describe('getLookbackInterval', () => { expect(getLookbackInterval(testJobs)).toBe('32m'); }); }); + +describe('getResultJobsHealthRuleConfig', () => { + test('returns default config for empty configuration', () => { + expect(getResultJobsHealthRuleConfig(null)).toEqual({ + datafeed: { + enabled: true, + }, + mml: { + enabled: true, + }, + delayedData: { + enabled: true, + }, + behindRealtime: { + enabled: true, + }, + errorMessages: { + enabled: true, + }, + }); + }); + test('returns config with overridden values based on provided configuration', () => { + expect( + getResultJobsHealthRuleConfig({ + mml: { enabled: false }, + errorMessages: { enabled: true }, + }) + ).toEqual({ + datafeed: { + enabled: true, + }, + mml: { + enabled: false, + }, + delayedData: { + enabled: true, + }, + behindRealtime: { + enabled: true, + }, + errorMessages: { + enabled: true, + }, + }); + }); +}); diff --git a/x-pack/plugins/ml/common/util/alerts.ts b/x-pack/plugins/ml/common/util/alerts.ts index 5d68677d4fb973..b211423e650623 100644 --- a/x-pack/plugins/ml/common/util/alerts.ts +++ b/x-pack/plugins/ml/common/util/alerts.ts @@ -9,6 +9,7 @@ import { CombinedJobWithStats, Datafeed, Job } from '../types/anomaly_detection_ import { resolveMaxTimeInterval } from './job_utils'; import { isDefined } from '../types/guards'; import { parseInterval } from './parse_interval'; +import { JobsHealthRuleTestsConfig } from '../types/alerts'; const narrowBucketLength = 60; @@ -51,3 +52,27 @@ export function getTopNBuckets(job: Job): number { return Math.ceil(narrowBucketLength / bucketSpan.asSeconds()); } + +/** + * Returns tests configuration combined with default values. + * @param config + */ +export function getResultJobsHealthRuleConfig(config: JobsHealthRuleTestsConfig) { + return { + datafeed: { + enabled: config?.datafeed?.enabled ?? true, + }, + mml: { + enabled: config?.mml?.enabled ?? true, + }, + delayedData: { + enabled: config?.delayedData?.enabled ?? true, + }, + behindRealtime: { + enabled: config?.behindRealtime?.enabled ?? true, + }, + errorMessages: { + enabled: config?.errorMessages?.enabled ?? true, + }, + }; +} diff --git a/x-pack/plugins/ml/public/alerting/job_selector.tsx b/x-pack/plugins/ml/public/alerting/job_selector.tsx index d00d4efc25b8d5..0ef7bba0ddbc58 100644 --- a/x-pack/plugins/ml/public/alerting/job_selector.tsx +++ b/x-pack/plugins/ml/public/alerting/job_selector.tsx @@ -5,12 +5,13 @@ * 2.0. */ -import React, { FC, useCallback, useEffect, useMemo, useState } from 'react'; +import React, { FC, ReactNode, useCallback, useEffect, useMemo, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiComboBox, EuiComboBoxOptionOption, EuiComboBoxProps, EuiFormRow } from '@elastic/eui'; import { JobId } from '../../common/types/anomaly_detection_jobs'; import { MlApiServices } from '../application/services/ml_api_service'; +import { ALL_JOBS_SELECTION } from '../../common/constants/alerts'; interface JobSelection { jobIds?: JobId[]; @@ -25,6 +26,17 @@ export interface JobSelectorControlProps { * Validation is handled by alerting framework */ errors: string[]; + /** Enables multiple selection of jobs and groups */ + multiSelect?: boolean; + label?: ReactNode; + /** + * Allows selecting all jobs, even those created afterward. + */ + allowSelectAll?: boolean; + /** + * Available options to select. By default suggest all existing jobs. + */ + options?: Array>; } export const JobSelectorControl: FC = ({ @@ -32,6 +44,10 @@ export const JobSelectorControl: FC = ({ onChange, adJobsApiService, errors, + multiSelect = false, + label, + allowSelectAll = false, + options: defaultOptions, }) => { const [options, setOptions] = useState>>([]); const jobIds = useMemo(() => new Set(), []); @@ -60,12 +76,39 @@ export const JobSelectorControl: FC = ({ }); setOptions([ + ...(allowSelectAll + ? [ + { + label: i18n.translate('xpack.ml.jobSelector.selectAllGroupLabel', { + defaultMessage: 'Select all', + }), + options: [ + { + label: i18n.translate('xpack.ml.jobSelector.selectAllOptionLabel', { + defaultMessage: '*', + }), + value: ALL_JOBS_SELECTION, + }, + ], + }, + ] + : []), { label: i18n.translate('xpack.ml.jobSelector.jobOptionsLabel', { defaultMessage: 'Jobs', }), options: jobIdOptions.map((v) => ({ label: v })), }, + ...(multiSelect + ? [ + { + label: i18n.translate('xpack.ml.jobSelector.groupOptionsLabel', { + defaultMessage: 'Groups', + }), + options: groupIdOptions.map((v) => ({ label: v })), + }, + ] + : []), ]); } catch (e) { // TODO add error handling @@ -73,25 +116,33 @@ export const JobSelectorControl: FC = ({ }, [adJobsApiService]); const onSelectionChange: EuiComboBoxProps['onChange'] = useCallback( - (selectionUpdate) => { + ((selectionUpdate) => { + if (selectionUpdate.some((selectedOption) => selectedOption.value === ALL_JOBS_SELECTION)) { + onChange({ jobIds: [ALL_JOBS_SELECTION] }); + return; + } + const selectedJobIds: JobId[] = []; const selectedGroupIds: string[] = []; - selectionUpdate.forEach(({ label }: { label: string }) => { - if (jobIds.has(label)) { - selectedJobIds.push(label); - } else if (groupIds.has(label)) { - selectedGroupIds.push(label); + selectionUpdate.forEach(({ label: selectedLabel }: { label: string }) => { + if (jobIds.has(selectedLabel)) { + selectedJobIds.push(selectedLabel); + } else if (groupIds.has(selectedLabel)) { + selectedGroupIds.push(selectedLabel); + } else if (defaultOptions?.some((v) => v.options?.some((o) => o.label === selectedLabel))) { + selectedJobIds.push(selectedLabel); } }); onChange({ ...(selectedJobIds.length > 0 ? { jobIds: selectedJobIds } : {}), ...(selectedGroupIds.length > 0 ? { groupIds: selectedGroupIds } : {}), }); - }, - [jobIds, groupIds] + }) as Exclude['onChange'], undefined>, + [jobIds, groupIds, defaultOptions] ); useEffect(() => { + if (defaultOptions) return; fetchOptions(); }, []); @@ -99,15 +150,20 @@ export const JobSelectorControl: FC = ({ + label ?? ( + + ) } isInvalid={!!errors?.length} error={errors} > - singleSelection + singleSelection={!multiSelect} selectedOptions={selectedOptions} - options={options} + options={defaultOptions ?? options} onChange={onSelectionChange} fullWidth data-test-subj={'mlAnomalyAlertJobSelection'} diff --git a/x-pack/plugins/ml/public/alerting/jobs_health_rule/anomaly_detection_jobs_health_rule_trigger.tsx b/x-pack/plugins/ml/public/alerting/jobs_health_rule/anomaly_detection_jobs_health_rule_trigger.tsx new file mode 100644 index 00000000000000..7c75817e4029f3 --- /dev/null +++ b/x-pack/plugins/ml/public/alerting/jobs_health_rule/anomaly_detection_jobs_health_rule_trigger.tsx @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useCallback, useMemo, useState } from 'react'; +import { EuiComboBoxOptionOption, EuiForm, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import useDebounce from 'react-use/lib/useDebounce'; +import { AlertTypeParamsExpressionProps } from '../../../../triggers_actions_ui/public'; +import { MlAnomalyDetectionJobsHealthRuleParams } from '../../../common/types/alerts'; +import { JobSelectorControl } from '../job_selector'; +import { jobsApiProvider } from '../../application/services/ml_api_service/jobs'; +import { HttpService } from '../../application/services/http_service'; +import { useMlKibana } from '../../application/contexts/kibana'; +import { TestsSelectionControl } from './tests_selection_control'; +import { isPopulatedObject } from '../../../common'; +import { ALL_JOBS_SELECTION } from '../../../common/constants/alerts'; + +export type MlAnomalyAlertTriggerProps = AlertTypeParamsExpressionProps; + +const AnomalyDetectionJobsHealthRuleTrigger: FC = ({ + alertParams, + setAlertParams, + errors, +}) => { + const { + services: { http }, + } = useMlKibana(); + const mlHttpService = useMemo(() => new HttpService(http), [http]); + const adJobsApiService = useMemo(() => jobsApiProvider(mlHttpService), [mlHttpService]); + const [excludeJobsOptions, setExcludeJobsOptions] = useState< + Array> + >([]); + + const includeJobsAndGroupIds: string[] = useMemo( + () => (Object.values(alertParams.includeJobs ?? {}) as string[][]).flat(), + [alertParams.includeJobs] + ); + + const excludeJobsAndGroupIds: string[] = useMemo( + () => (Object.values(alertParams.excludeJobs ?? {}) as string[][]).flat(), + [alertParams.excludeJobs] + ); + + const onAlertParamChange = useCallback( + (param: T) => ( + update: MlAnomalyDetectionJobsHealthRuleParams[T] + ) => { + setAlertParams(param, update); + }, + [] + ); + + const formErrors = Object.values(errors).flat(); + const isFormInvalid = formErrors.length > 0; + + useDebounce( + function updateExcludeJobsOptions() { + const areAllJobsSelected = alertParams.includeJobs?.jobIds?.[0] === ALL_JOBS_SELECTION; + + if (!areAllJobsSelected && !alertParams.includeJobs?.groupIds?.length) { + // It only makes sense to suggest excluded jobs options when at least one group or all jobs are selected + setExcludeJobsOptions([]); + return; + } + + adJobsApiService + .jobs(areAllJobsSelected ? [] : (alertParams.includeJobs.groupIds as string[])) + .then((jobs) => { + setExcludeJobsOptions([ + { + label: i18n.translate('xpack.ml.jobSelector.jobOptionsLabel', { + defaultMessage: 'Jobs', + }), + options: jobs.map((v) => ({ label: v.job_id })), + }, + ]); + }); + }, + 500, + [alertParams.includeJobs] + ); + + return ( + + + } + /> + + + + { + const callback = onAlertParamChange('excludeJobs'); + if (isPopulatedObject(update)) { + callback(update); + } else { + callback(null); + } + }, [])} + errors={Array.isArray(errors.excludeJobs) ? errors.excludeJobs : []} + multiSelect + label={ + + } + options={excludeJobsOptions} + /> + + + + + + ); +}; + +// Default export is required for React.lazy loading + +// eslint-disable-next-line import/no-default-export +export default AnomalyDetectionJobsHealthRuleTrigger; diff --git a/x-pack/plugins/ml/public/alerting/jobs_health_rule/index.ts b/x-pack/plugins/ml/public/alerting/jobs_health_rule/index.ts new file mode 100644 index 00000000000000..f26b38a1370ec3 --- /dev/null +++ b/x-pack/plugins/ml/public/alerting/jobs_health_rule/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { registerJobsHealthAlertingRule } from './register_jobs_health_alerting_rule'; diff --git a/x-pack/plugins/ml/public/alerting/jobs_health_rule/register_jobs_health_alerting_rule.ts b/x-pack/plugins/ml/public/alerting/jobs_health_rule/register_jobs_health_alerting_rule.ts new file mode 100644 index 00000000000000..ef20b51df26006 --- /dev/null +++ b/x-pack/plugins/ml/public/alerting/jobs_health_rule/register_jobs_health_alerting_rule.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { lazy } from 'react'; +import { i18n } from '@kbn/i18n'; +import { TriggersAndActionsUIPublicPluginSetup } from '../../../../triggers_actions_ui/public'; +import { PluginSetupContract as AlertingSetup } from '../../../../alerting/public'; +import { ML_ALERT_TYPES } from '../../../common/constants/alerts'; +import { MlAnomalyDetectionJobsHealthRuleParams } from '../../../common/types/alerts'; + +export function registerJobsHealthAlertingRule( + triggersActionsUi: TriggersAndActionsUIPublicPluginSetup, + alerting?: AlertingSetup +) { + triggersActionsUi.alertTypeRegistry.register({ + id: ML_ALERT_TYPES.AD_JOBS_HEALTH, + description: i18n.translate('xpack.ml.alertTypes.jobsHealthAlertingRule.description', { + defaultMessage: 'Alert when anomaly detection jobs experience operational issues.', + }), + iconClass: 'bell', + documentationUrl(docLinks) { + return docLinks.links.ml.alertingRules; + }, + alertParamsExpression: lazy(() => import('./anomaly_detection_jobs_health_rule_trigger')), + validate: (alertParams: MlAnomalyDetectionJobsHealthRuleParams) => { + const validationResult = { + errors: { + includeJobs: new Array(), + testsConfig: new Array(), + } as Record, + }; + + if (!alertParams.includeJobs?.jobIds?.length && !alertParams.includeJobs?.groupIds?.length) { + validationResult.errors.includeJobs.push( + i18n.translate('xpack.ml.alertTypes.jobsHealthAlertingRule.includeJobs.errorMessage', { + defaultMessage: 'Job selection is required', + }) + ); + } + + if ( + alertParams.testsConfig && + Object.values(alertParams.testsConfig).every((v) => v?.enabled === false) + ) { + validationResult.errors.testsConfig.push( + i18n.translate('xpack.ml.alertTypes.jobsHealthAlertingRule.testsConfig.errorMessage', { + defaultMessage: 'At least one health check must be enabled.', + }) + ); + } + + return validationResult; + }, + requiresAppContext: false, + defaultActionMessage: i18n.translate( + 'xpack.ml.alertTypes.jobsHealthAlertingRule.defaultActionMessage', + { + defaultMessage: `Anomaly detection jobs health check result: +\\{\\{context.message\\}\\} +- Job IDs: \\{\\{context.jobIds\\}\\} +`, + } + ), + }); +} diff --git a/x-pack/plugins/ml/public/alerting/jobs_health_rule/tests_selection_control.tsx b/x-pack/plugins/ml/public/alerting/jobs_health_rule/tests_selection_control.tsx new file mode 100644 index 00000000000000..8c033fe141222a --- /dev/null +++ b/x-pack/plugins/ml/public/alerting/jobs_health_rule/tests_selection_control.tsx @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useCallback } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiFormFieldset, EuiFormRow, EuiSpacer, EuiSwitch } from '@elastic/eui'; +import { JobsHealthRuleTestsConfig } from '../../../common/types/alerts'; +import { getResultJobsHealthRuleConfig } from '../../../common/util/alerts'; +import { HEALTH_CHECK_NAMES } from '../../../common/constants/alerts'; + +interface TestsSelectionControlProps { + config: JobsHealthRuleTestsConfig; + onChange: (update: JobsHealthRuleTestsConfig) => void; + errors?: string[]; +} + +export const TestsSelectionControl: FC = ({ + config, + onChange, + errors, +}) => { + const uiConfig = getResultJobsHealthRuleConfig(config); + + const updateCallback = useCallback( + (update: Partial>) => { + onChange({ + ...(config ?? {}), + ...update, + }); + }, + [onChange, config] + ); + + return ( + + + + + + + + {false && ( + <> + + } + onChange={updateCallback.bind(null, { mml: { enabled: !uiConfig.mml.enabled } })} + checked={uiConfig.mml.enabled} + /> + + + + + } + onChange={updateCallback.bind(null, { + delayedData: { enabled: !uiConfig.delayedData.enabled }, + })} + checked={uiConfig.delayedData.enabled} + /> + + + + + } + onChange={updateCallback.bind(null, { + behindRealtime: { enabled: !uiConfig.behindRealtime.enabled }, + })} + checked={uiConfig.behindRealtime.enabled} + /> + + + + + } + onChange={updateCallback.bind(null, { + errorMessages: { enabled: !uiConfig.errorMessages.enabled }, + })} + checked={uiConfig.errorMessages.enabled} + /> + + + + )} + + ); +}; diff --git a/x-pack/plugins/ml/public/alerting/register_ml_alerts.ts b/x-pack/plugins/ml/public/alerting/register_ml_alerts.ts index b1640ab7aba7d9..99ba61f3d91547 100644 --- a/x-pack/plugins/ml/public/alerting/register_ml_alerts.ts +++ b/x-pack/plugins/ml/public/alerting/register_ml_alerts.ts @@ -14,6 +14,7 @@ import type { PluginSetupContract as AlertingSetup } from '../../../alerting/pub import { PLUGIN_ID } from '../../common/constants/app'; import { formatExplorerUrl } from '../locator/formatters/anomaly_detection'; import { validateLookbackInterval, validateTopNBucket } from './validators'; +import { registerJobsHealthAlertingRule } from './jobs_health_rule'; export function registerMlAlerts( triggersActionsUi: TriggersAndActionsUIPublicPluginSetup, @@ -26,7 +27,7 @@ export function registerMlAlerts( }), iconClass: 'bell', documentationUrl(docLinks) { - return `${docLinks.ELASTIC_WEBSITE_URL}guide/en/machine-learning/${docLinks.DOC_LINK_VERSION}/ml-configuring-alerts.html`; + return docLinks.links.ml.alertingRules; }, alertParamsExpression: lazy(() => import('./ml_anomaly_alert_trigger')), validate: (alertParams: MlAnomalyDetectionAlertParams) => { @@ -137,6 +138,8 @@ export function registerMlAlerts( ), }); + registerJobsHealthAlertingRule(triggersActionsUi, alerting); + if (alerting) { registerNavigation(alerting); } diff --git a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts index e7d3ef97a301b4..e4c1e0fe53f01f 100644 --- a/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts +++ b/x-pack/plugins/ml/server/lib/alerts/alerting_service.ts @@ -436,7 +436,7 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da const jobIds = jobsResponse.map((v) => v.job_id); - const dataFeeds = await datafeedsService.getDatafeedByJobId(jobIds); + const datafeeds = await datafeedsService.getDatafeedByJobId(jobIds); const maxBucketInSeconds = resolveMaxTimeInterval( jobsResponse.map((v) => v.analysis_config.bucket_span) @@ -448,7 +448,7 @@ export function alertingServiceProvider(mlClient: MlClient, datafeedsService: Da } const lookBackTimeInterval: string = - params.lookbackInterval ?? resolveLookbackInterval(jobsResponse, dataFeeds ?? []); + params.lookbackInterval ?? resolveLookbackInterval(jobsResponse, datafeeds ?? []); const topNBuckets: number = params.topNBuckets ?? getTopNBuckets(jobsResponse[0]); diff --git a/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.test.ts b/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.test.ts new file mode 100644 index 00000000000000..59213a7cf6ab1e --- /dev/null +++ b/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.test.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { JobsHealthService, jobsHealthServiceProvider } from './jobs_health_service'; +import type { DatafeedsService } from '../../models/job_service/datafeeds'; +import type { Logger } from 'kibana/server'; +import { MlClient } from '../ml_client'; +import { MlJob, MlJobStats } from '@elastic/elasticsearch/api/types'; + +describe('JobsHealthService', () => { + const mlClient = ({ + getJobs: jest.fn().mockImplementation(({ job_id: jobIds = [] }) => { + let jobs: MlJob[] = []; + + if (jobIds.some((v: string) => v === 'test_group')) { + jobs = [ + ({ + job_id: 'test_job_01', + } as unknown) as MlJob, + ({ + job_id: 'test_job_02', + } as unknown) as MlJob, + ({ + job_id: 'test_job_03', + } as unknown) as MlJob, + ]; + } + + if (jobIds[0]?.startsWith('test_job_')) { + jobs = [ + ({ + job_id: jobIds[0], + } as unknown) as MlJob, + ]; + } + + return Promise.resolve({ + body: { + jobs, + }, + }); + }), + getJobStats: jest.fn().mockImplementation(({ job_id: jobIdsStr }) => { + const jobsIds = jobIdsStr.split(','); + return Promise.resolve({ + body: { + jobs: jobsIds.map((j: string) => { + return { + job_id: j, + state: j === 'test_job_02' ? 'opened' : 'closed', + }; + }) as MlJobStats, + }, + }); + }), + getDatafeedStats: jest.fn().mockImplementation(({ datafeed_id: datafeedIdsStr }) => { + const datafeedIds = datafeedIdsStr.split(','); + return Promise.resolve({ + body: { + datafeeds: datafeedIds.map((d: string) => { + return { + datafeed_id: d, + state: d === 'test_datafeed_02' ? 'stopped' : 'started', + timing_stats: { + job_id: d.replace('datafeed', 'job'), + }, + }; + }) as MlJobStats, + }, + }); + }), + } as unknown) as jest.Mocked; + + const datafeedsService = ({ + getDatafeedByJobId: jest.fn().mockImplementation((jobIds: string[]) => { + return Promise.resolve( + jobIds.map((j) => { + return { + datafeed_id: j.replace('job', 'datafeed'), + }; + }) + ); + }), + } as unknown) as jest.Mocked; + + const logger = ({ + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + } as unknown) as jest.Mocked; + + const jobHealthService: JobsHealthService = jobsHealthServiceProvider( + mlClient, + datafeedsService, + logger + ); + + beforeEach(() => {}); + + afterEach(() => { + jest.clearAllMocks(); + }); + + test('returns empty results when no jobs provided', async () => { + // act + const executionResult = await jobHealthService.getTestsResults('testRule', { + testsConfig: null, + includeJobs: { + jobIds: ['*'], + groupIds: [], + }, + excludeJobs: null, + }); + expect(logger.warn).toHaveBeenCalledWith('Rule "testRule" does not have associated jobs.'); + expect(datafeedsService.getDatafeedByJobId).not.toHaveBeenCalled(); + expect(executionResult).toEqual([]); + }); + + test('returns empty results and does not perform datafeed check when test is disabled', async () => { + const executionResult = await jobHealthService.getTestsResults('testRule', { + testsConfig: { + datafeed: { + enabled: false, + }, + behindRealtime: null, + delayedData: null, + errorMessages: null, + mml: null, + }, + includeJobs: { + jobIds: ['test_job_01'], + groupIds: [], + }, + excludeJobs: null, + }); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith(`Performing health checks for job IDs: test_job_01`); + expect(datafeedsService.getDatafeedByJobId).not.toHaveBeenCalled(); + expect(executionResult).toEqual([]); + }); + + test('returns results based on provided selection', async () => { + const executionResult = await jobHealthService.getTestsResults('testRule_03', { + testsConfig: null, + includeJobs: { + jobIds: [], + groupIds: ['test_group'], + }, + excludeJobs: { + jobIds: ['test_job_03'], + groupIds: [], + }, + }); + expect(logger.warn).not.toHaveBeenCalled(); + expect(logger.debug).toHaveBeenCalledWith( + `Performing health checks for job IDs: test_job_01, test_job_02` + ); + expect(datafeedsService.getDatafeedByJobId).toHaveBeenCalledWith([ + 'test_job_01', + 'test_job_02', + ]); + expect(mlClient.getJobStats).toHaveBeenCalledWith({ job_id: 'test_job_01,test_job_02' }); + expect(mlClient.getDatafeedStats).toHaveBeenCalledWith({ + datafeed_id: 'test_datafeed_01,test_datafeed_02', + }); + expect(executionResult).toEqual([ + { + name: 'Datafeed is not started', + context: { + jobIds: ['test_job_02'], + message: 'Datafeed is not started for the following jobs:', + }, + }, + ]); + }); +}); diff --git a/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.ts b/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.ts new file mode 100644 index 00000000000000..db4907decc3f0c --- /dev/null +++ b/x-pack/plugins/ml/server/lib/alerts/jobs_health_service.ts @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest, SavedObjectsClientContract } from 'kibana/server'; +import { i18n } from '@kbn/i18n'; +import { Logger } from 'kibana/server'; +import { MlJobState } from '@elastic/elasticsearch/api/types'; +import { MlClient } from '../ml_client'; +import { + AnomalyDetectionJobsHealthRuleParams, + JobSelection, +} from '../../routes/schemas/alerting_schema'; +import { datafeedsProvider, DatafeedsService } from '../../models/job_service/datafeeds'; +import { ALL_JOBS_SELECTION, HEALTH_CHECK_NAMES } from '../../../common/constants/alerts'; +import { DatafeedStats } from '../../../common/types/anomaly_detection_jobs'; +import { GetGuards } from '../../shared_services/shared_services'; +import { AnomalyDetectionJobsHealthAlertContext } from './register_jobs_monitoring_rule_type'; +import { getResultJobsHealthRuleConfig } from '../../../common/util/alerts'; + +interface TestResult { + name: string; + context: AnomalyDetectionJobsHealthAlertContext; +} + +type TestsResults = TestResult[]; + +type NotStartedDatafeedResponse = Array; + +export function jobsHealthServiceProvider( + mlClient: MlClient, + datafeedsService: DatafeedsService, + logger: Logger +) { + /** + * Extracts result list of job ids based on included and excluded selection of jobs and groups. + * @param includeJobs + * @param excludeJobs + */ + const getResultJobIds = async (includeJobs: JobSelection, excludeJobs?: JobSelection | null) => { + const jobAndGroupIds = [...(includeJobs.jobIds ?? []), ...(includeJobs.groupIds ?? [])]; + + const includeAllJobs = jobAndGroupIds.some((id) => id === ALL_JOBS_SELECTION); + + // Extract jobs from group ids and make sure provided jobs assigned to a current space + const jobsResponse = ( + await mlClient.getJobs({ + ...(includeAllJobs ? {} : { job_id: jobAndGroupIds }), + }) + ).body.jobs; + + let resultJobIds = jobsResponse.map((v) => v.job_id); + + if (excludeJobs && (!!excludeJobs.jobIds.length || !!excludeJobs?.groupIds.length)) { + const excludedJobAndGroupIds = [ + ...(excludeJobs?.jobIds ?? []), + ...(excludeJobs?.groupIds ?? []), + ]; + const excludedJobsResponse = ( + await mlClient.getJobs({ + job_id: excludedJobAndGroupIds, + }) + ).body.jobs; + + const excludedJobsIds: Set = new Set(excludedJobsResponse.map((v) => v.job_id)); + + resultJobIds = resultJobIds.filter((v) => !excludedJobsIds.has(v)); + } + + return resultJobIds; + }; + + return { + /** + * Gets not started datafeeds for opened jobs. + * @param jobIds + */ + async getNotStartedDatafeeds(jobIds: string[]): Promise { + const datafeeds = await datafeedsService.getDatafeedByJobId(jobIds); + + if (datafeeds) { + const { + body: { jobs: jobsStats }, + } = await mlClient.getJobStats({ job_id: jobIds.join(',') }); + + const { + body: { datafeeds: datafeedsStats }, + } = await mlClient.getDatafeedStats({ + datafeed_id: datafeeds.map((d) => d.datafeed_id).join(','), + }); + + // match datafeed stats with the job ids + return (datafeedsStats as DatafeedStats[]) + .map((datafeedStats) => { + const jobId = datafeedStats.timing_stats.job_id; + const jobState = + jobsStats.find((jobStats) => jobStats.job_id === jobId)?.state ?? 'failed'; + return { + ...datafeedStats, + job_id: jobId, + job_state: jobState, + }; + }) + .filter((datafeedStat) => { + // Find opened jobs with not started datafeeds + return datafeedStat.job_state === 'opened' && datafeedStat.state !== 'started'; + }); + } + }, + /** + * Retrieves report grouped by test. + */ + async getTestsResults( + ruleInstanceName: string, + { testsConfig, includeJobs, excludeJobs }: AnomalyDetectionJobsHealthRuleParams + ): Promise { + const config = getResultJobsHealthRuleConfig(testsConfig); + + const results: TestsResults = []; + + const jobIds = await getResultJobIds(includeJobs, excludeJobs); + + if (jobIds.length === 0) { + logger.warn(`Rule "${ruleInstanceName}" does not have associated jobs.`); + return results; + } + + logger.debug(`Performing health checks for job IDs: ${jobIds.join(', ')}`); + + if (config.datafeed.enabled) { + const response = await this.getNotStartedDatafeeds(jobIds); + if (response && response.length > 0) { + results.push({ + name: HEALTH_CHECK_NAMES.datafeed, + context: { + jobIds: [...new Set(response.map((v) => v.job_id))], + message: i18n.translate( + 'xpack.ml.alertTypes.jobsHealthAlertingRule.datafeedStateMessage', + { + defaultMessage: 'Datafeed is not started for the following jobs:', + } + ), + }, + }); + } + } + + return results; + }, + }; +} + +export type JobsHealthService = ReturnType; + +export function getJobsHealthServiceProvider(getGuards: GetGuards) { + return { + jobsHealthServiceProvider( + savedObjectsClient: SavedObjectsClientContract, + request: KibanaRequest, + logger: Logger + ) { + return { + getTestsResults: async ( + ...args: Parameters + ): ReturnType => { + return await getGuards(request, savedObjectsClient) + .isFullLicense() + .hasMlCapabilities(['canGetJobs']) + .ok(({ mlClient, scopedClient }) => + jobsHealthServiceProvider( + mlClient, + datafeedsProvider(scopedClient, mlClient), + logger + ).getTestsResults(...args) + ); + }, + }; + }, + }; +} + +export type JobsHealthServiceProvider = ReturnType; diff --git a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts index 07bca8f3aae745..e30ea01b27cb53 100644 --- a/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts +++ b/x-pack/plugins/ml/server/lib/alerts/register_anomaly_detection_alert_type.ts @@ -7,11 +7,7 @@ import { i18n } from '@kbn/i18n'; import { KibanaRequest } from 'kibana/server'; -import { - ML_ALERT_TYPES, - ML_ALERT_TYPES_CONFIG, - AnomalyScoreMatchGroupId, -} from '../../../common/constants/alerts'; +import { ML_ALERT_TYPES } from '../../../common/constants/alerts'; import { PLUGIN_ID } from '../../../common/constants/app'; import { MINIMUM_FULL_LICENSE } from '../../../common/license'; import { @@ -21,13 +17,12 @@ import { import { RegisterAlertParams } from './register_ml_alerts'; import { InfluencerAnomalyAlertDoc, RecordAnomalyAlertDoc } from '../../../common/types/alerts'; import { + ActionGroup, AlertInstanceContext, AlertInstanceState, AlertTypeState, } from '../../../../alerting/common'; -const alertTypeConfig = ML_ALERT_TYPES_CONFIG[ML_ALERT_TYPES.ANOMALY_DETECTION]; - export type AnomalyDetectionAlertContext = { name: string; jobIds: string[]; @@ -40,6 +35,17 @@ export type AnomalyDetectionAlertContext = { anomalyExplorerUrl: string; } & AlertInstanceContext; +export const ANOMALY_SCORE_MATCH_GROUP_ID = 'anomaly_score_match'; + +export type AnomalyScoreMatchGroupId = typeof ANOMALY_SCORE_MATCH_GROUP_ID; + +export const THRESHOLD_MET_GROUP: ActionGroup = { + id: ANOMALY_SCORE_MATCH_GROUP_ID, + name: i18n.translate('xpack.ml.anomalyDetectionAlert.actionGroupName', { + defaultMessage: 'Anomaly score matched the condition', + }), +}; + export function registerAnomalyDetectionAlertType({ alerting, mlSharedServices, @@ -53,9 +59,11 @@ export function registerAnomalyDetectionAlertType({ AnomalyScoreMatchGroupId >({ id: ML_ALERT_TYPES.ANOMALY_DETECTION, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + name: i18n.translate('xpack.ml.anomalyDetectionAlert.name', { + defaultMessage: 'Anomaly detection alert', + }), + actionGroups: [THRESHOLD_MET_GROUP], + defaultActionGroupId: ANOMALY_SCORE_MATCH_GROUP_ID, validate: { params: mlAnomalyDetectionAlertParams, }, @@ -76,7 +84,7 @@ export function registerAnomalyDetectionAlertType({ { name: 'jobIds', description: i18n.translate('xpack.ml.alertContext.jobIdsDescription', { - defaultMessage: 'List of job IDs that triggered the alert instance', + defaultMessage: 'List of job IDs that triggered the alert', }), }, { @@ -132,7 +140,7 @@ export function registerAnomalyDetectionAlertType({ if (executionResult) { const alertInstanceName = executionResult.name; const alertInstance = services.alertInstanceFactory(alertInstanceName); - alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, executionResult); + alertInstance.scheduleActions(ANOMALY_SCORE_MATCH_GROUP_ID, executionResult); } }, }); diff --git a/x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts b/x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts new file mode 100644 index 00000000000000..3547b44cc73e4c --- /dev/null +++ b/x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { KibanaRequest } from 'kibana/server'; +import { ML_ALERT_TYPES } from '../../../common/constants/alerts'; +import { PLUGIN_ID } from '../../../common/constants/app'; +import { MINIMUM_FULL_LICENSE } from '../../../common/license'; +import { + anomalyDetectionJobsHealthRuleParams, + AnomalyDetectionJobsHealthRuleParams, +} from '../../routes/schemas/alerting_schema'; +import { RegisterAlertParams } from './register_ml_alerts'; +import { + ActionGroup, + AlertInstanceContext, + AlertInstanceState, + AlertTypeState, +} from '../../../../alerting/common'; + +export type AnomalyDetectionJobsHealthAlertContext = { + jobIds: string[]; + message: string; +} & AlertInstanceContext; + +export const ANOMALY_DETECTION_JOB_REALTIME_ISSUE = 'anomaly_detection_realtime_issue'; + +export type AnomalyDetectionJobRealtimeIssue = typeof ANOMALY_DETECTION_JOB_REALTIME_ISSUE; + +export const REALTIME_ISSUE_DETECTED: ActionGroup = { + id: ANOMALY_DETECTION_JOB_REALTIME_ISSUE, + name: i18n.translate('xpack.ml.jobsHealthAlertingRule.actionGroupName', { + defaultMessage: 'Real-time issue detected', + }), +}; + +export function registerJobsMonitoringRuleType({ + alerting, + mlServicesProviders, + logger, +}: RegisterAlertParams) { + alerting.registerType< + AnomalyDetectionJobsHealthRuleParams, + never, // Only use if defining useSavedObjectReferences hook + AlertTypeState, + AlertInstanceState, + AnomalyDetectionJobsHealthAlertContext, + AnomalyDetectionJobRealtimeIssue + >({ + id: ML_ALERT_TYPES.AD_JOBS_HEALTH, + name: i18n.translate('xpack.ml.jobsHealthAlertingRule.name', { + defaultMessage: 'Anomaly detection jobs health', + }), + actionGroups: [REALTIME_ISSUE_DETECTED], + defaultActionGroupId: ANOMALY_DETECTION_JOB_REALTIME_ISSUE, + validate: { + params: anomalyDetectionJobsHealthRuleParams, + }, + actionVariables: { + context: [ + { + name: 'jobIds', + description: i18n.translate( + 'xpack.ml.alertTypes.jobsHealthAlertingRule.alertContext.jobIdsDescription', + { + defaultMessage: 'List of job IDs that triggered the alert', + } + ), + }, + { + name: 'message', + description: i18n.translate( + 'xpack.ml.alertTypes.jobsHealthAlertingRule.alertContext.messageDescription', + { + defaultMessage: 'Alert info message', + } + ), + }, + ], + }, + producer: PLUGIN_ID, + minimumLicenseRequired: MINIMUM_FULL_LICENSE, + isExportable: true, + async executor({ services, params, alertId, state, previousStartedAt, startedAt, name }) { + const fakeRequest = {} as KibanaRequest; + const { getTestsResults } = mlServicesProviders.jobsHealthServiceProvider( + services.savedObjectsClient, + fakeRequest, + logger + ); + const executionResult = await getTestsResults(name, params); + + if (executionResult.length > 0) { + logger.info( + `Scheduling actions for tests: ${executionResult.map((v) => v.name).join(', ')}` + ); + + executionResult.forEach(({ name: alertInstanceName, context }) => { + const alertInstance = services.alertInstanceFactory(alertInstanceName); + alertInstance.scheduleActions(ANOMALY_DETECTION_JOB_REALTIME_ISSUE, context); + }); + } + }, + }); +} diff --git a/x-pack/plugins/ml/server/lib/alerts/register_ml_alerts.ts b/x-pack/plugins/ml/server/lib/alerts/register_ml_alerts.ts index 8368c606598f0e..6f1e000c9a4303 100644 --- a/x-pack/plugins/ml/server/lib/alerts/register_ml_alerts.ts +++ b/x-pack/plugins/ml/server/lib/alerts/register_ml_alerts.ts @@ -9,13 +9,17 @@ import { Logger } from 'kibana/server'; import { AlertingPlugin } from '../../../../alerting/server'; import { registerAnomalyDetectionAlertType } from './register_anomaly_detection_alert_type'; import { SharedServices } from '../../shared_services'; +import { registerJobsMonitoringRuleType } from './register_jobs_monitoring_rule_type'; +import { MlServicesProviders } from '../../shared_services/shared_services'; export interface RegisterAlertParams { alerting: AlertingPlugin['setup']; logger: Logger; mlSharedServices: SharedServices; + mlServicesProviders: MlServicesProviders; } export function registerMlAlerts(params: RegisterAlertParams) { registerAnomalyDetectionAlertType(params); + registerJobsMonitoringRuleType(params); } diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 213be9421c41df..35f66e86b955a1 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -196,7 +196,7 @@ export class MlServerPlugin initMlServerLog({ log: this.log }); - const sharedServices = createSharedServices( + const { internalServicesProviders, sharedServicesProviders } = createSharedServices( this.mlLicense, getSpaces, plugins.cloud, @@ -211,7 +211,8 @@ export class MlServerPlugin registerMlAlerts({ alerting: plugins.alerting, logger: this.log, - mlSharedServices: sharedServices, + mlSharedServices: sharedServicesProviders, + mlServicesProviders: internalServicesProviders, }); } @@ -219,7 +220,7 @@ export class MlServerPlugin registerCollector(plugins.usageCollection, this.kibanaIndexConfig.kibana.index); } - return { ...sharedServices }; + return sharedServicesProviders; } public start(coreStart: CoreStart): MlPluginStart { diff --git a/x-pack/plugins/ml/server/routes/schemas/alerting_schema.ts b/x-pack/plugins/ml/server/routes/schemas/alerting_schema.ts index df22ccfe208217..4e0f9a9aa7c928 100644 --- a/x-pack/plugins/ml/server/routes/schemas/alerting_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/alerting_schema.ts @@ -10,22 +10,24 @@ import { i18n } from '@kbn/i18n'; import { ALERT_PREVIEW_SAMPLE_SIZE } from '../../../common/constants/alerts'; import { ANOMALY_RESULT_TYPE } from '../../../common/constants/anomalies'; -export const mlAnomalyDetectionAlertParams = schema.object({ - jobSelection: schema.object( - { - jobIds: schema.arrayOf(schema.string(), { defaultValue: [] }), - groupIds: schema.arrayOf(schema.string(), { defaultValue: [] }), +const jobsSelectionSchema = schema.object( + { + jobIds: schema.arrayOf(schema.string(), { defaultValue: [] }), + groupIds: schema.arrayOf(schema.string(), { defaultValue: [] }), + }, + { + validate: (v) => { + if (!v.jobIds?.length && !v.groupIds?.length) { + return i18n.translate('xpack.ml.alertTypes.anomalyDetection.jobSelection.errorMessage', { + defaultMessage: 'Job selection is required', + }); + } }, - { - validate: (v) => { - if (!v.jobIds?.length && !v.groupIds?.length) { - return i18n.translate('xpack.ml.alertTypes.anomalyDetection.jobSelection.errorMessage', { - defaultMessage: 'Job selection is required', - }); - } - }, - } - ), + } +); + +export const mlAnomalyDetectionAlertParams = schema.object({ + jobSelection: jobsSelectionSchema, /** Anomaly score threshold */ severity: schema.number({ min: 0, max: 100 }), /** Result type to alert upon */ @@ -58,3 +60,47 @@ export type MlAnomalyDetectionAlertParams = TypeOf; + +export const anomalyDetectionJobsHealthRuleParams = schema.object({ + includeJobs: jobsSelectionSchema, + excludeJobs: schema.nullable(jobsSelectionSchema), + testsConfig: schema.nullable( + schema.object({ + datafeed: schema.nullable( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + }) + ), + mml: schema.nullable( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + }) + ), + delayedData: schema.nullable( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + docsCount: schema.nullable(schema.number()), + timeInterval: schema.nullable(schema.string()), + }) + ), + behindRealtime: schema.nullable( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + timeInterval: schema.nullable(schema.string()), + }) + ), + errorMessages: schema.nullable( + schema.object({ + enabled: schema.boolean({ defaultValue: true }), + }) + ), + }) + ), +}); + +export type AnomalyDetectionJobsHealthRuleParams = TypeOf< + typeof anomalyDetectionJobsHealthRuleParams +>; + +export type TestsConfig = AnomalyDetectionJobsHealthRuleParams['testsConfig']; +export type JobSelection = AnomalyDetectionJobsHealthRuleParams['includeJobs']; diff --git a/x-pack/plugins/ml/server/shared_services/shared_services.ts b/x-pack/plugins/ml/server/shared_services/shared_services.ts index caed3fd9332983..3766a48b0537d0 100644 --- a/x-pack/plugins/ml/server/shared_services/shared_services.ts +++ b/x-pack/plugins/ml/server/shared_services/shared_services.ts @@ -30,6 +30,10 @@ import { getAlertingServiceProvider, MlAlertingServiceProvider, } from './providers/alerting_service'; +import { + getJobsHealthServiceProvider, + JobsHealthServiceProvider, +} from '../lib/alerts/jobs_health_service'; export type SharedServices = JobServiceProvider & AnomalyDetectorsProvider & @@ -38,6 +42,8 @@ export type SharedServices = JobServiceProvider & ResultsServiceProvider & MlAlertingServiceProvider; +export type MlServicesProviders = JobsHealthServiceProvider; + interface Guards { isMinimumLicense(): Guards; isFullLicense(): Guards; @@ -71,7 +77,10 @@ export function createSharedServices( getClusterClient: () => IClusterClient | null, getInternalSavedObjectsClient: () => SavedObjectsClientContract | null, isMlReady: () => Promise -): SharedServices { +): { + sharedServicesProviders: SharedServices; + internalServicesProviders: MlServicesProviders; +} { const { isFullLicense, isMinimumLicense } = licenseChecks(mlLicense); function getGuards( request: KibanaRequest, @@ -118,12 +127,23 @@ export function createSharedServices( } return { - ...getJobServiceProvider(getGuards), - ...getAnomalyDetectorsProvider(getGuards), - ...getModulesProvider(getGuards), - ...getResultsServiceProvider(getGuards), - ...getMlSystemProvider(getGuards, mlLicense, getSpaces, cloud, resolveMlCapabilities), - ...getAlertingServiceProvider(getGuards), + /** + * Exposed providers for shared services used by other plugins + */ + sharedServicesProviders: { + ...getJobServiceProvider(getGuards), + ...getAnomalyDetectorsProvider(getGuards), + ...getModulesProvider(getGuards), + ...getResultsServiceProvider(getGuards), + ...getMlSystemProvider(getGuards, mlLicense, getSpaces, cloud, resolveMlCapabilities), + ...getAlertingServiceProvider(getGuards), + }, + /** + * Services providers for ML internal usage + */ + internalServicesProviders: { + ...getJobsHealthServiceProvider(getGuards), + }, }; }