[SIEM] Show all SIEM ML Jobs in Anomaly Detection UI by default (elas…

…tic#48067) (elastic#48361)

## Summary

Resolves elastic#46874 by showing all SIEM ML Jobs in the Anomaly Detection UI by default -- no installation necessary! These changes also check for job compatibility using the ML API's `recognize` endpoint, and will display an alert message for those jobs that are not compatible. This also introduces the ability to filter by groups, and refactors/cleans up functions within this feature that were being used elsewhere in the app. Finally, there is improved error support in that errors are filtered to only the job the user is trying to install/enable instead of showing errors for the entire module.

##### Compatibility Callout
![image](https://user-images.githubusercontent.com/2946766/66734388-247c4980-ee20-11e9-8d5d-fa685fa3daf4.png)

##### Filter by Groups
![image](https://user-images.githubusercontent.com/2946766/66734663-36aab780-ee21-11e9-9685-9305e53f2263.png)



### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

- [x] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)
- [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)
- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials -- will work with @benskelker on creating updated documentation.
- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios
- [ ] ~This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~

### For maintainers

- [ ] ~This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~
- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/public/components/embeddables/embedded_map.tsx

@@ -15,16 +15,16 @@ import { start } from '../../../../../../../src/legacy/core_plugins/embeddable_a
 import { EmbeddablePanel } from '../../../../../../../src/legacy/core_plugins/embeddable_api/public/np_ready/public';
 
 import { Loader } from '../loader';
-import { useIndexPatterns } from '../ml_popover/hooks/use_index_patterns';
+import { useIndexPatterns } from '../../hooks/use_index_patterns';
 import { useKibanaUiSetting } from '../../lib/settings/use_kibana_ui_setting';
 import { DEFAULT_INDEX_KEY } from '../../../common/constants';
-import { getIndexPatternTitleIdMapping } from '../ml_popover/helpers';
 import { IndexPatternsMissingPrompt } from './index_patterns_missing_prompt';
 import { MapEmbeddable, SetQuery } from './types';
 import * as i18n from './translations';
 import { useStateToaster } from '../toasters';
 import { createEmbeddable, displayErrorToast, setupEmbeddablesAPI } from './embedded_map_helpers';
 import { MapToolTip } from './map_tool_tip/map_tool_tip';
+import { getIndexPatternTitleIdMapping } from '../../hooks/api/helpers';
 
 const EmbeddableWrapper = styled(EuiFlexGroup)`
   position: relative;

x-pack/legacy/plugins/siem/public/components/ml/anomaly/use_anomalies_table_data.ts

@@ -71,6 +71,8 @@ export const useAnomaliesTableData = ({
   const [anomalyScore] = useKibanaUiSetting(DEFAULT_ANOMALY_SCORE);
   const [kbnVersion] = useKibanaUiSetting(DEFAULT_KBN_VERSION);
 
+  const siemJobIds = siemJobs.filter(job => job.isInstalled).map(job => job.id);
+
   useEffect(() => {
     let isSubscribed = true;
     const abortCtrl = new AbortController();
@@ -82,11 +84,11 @@ export const useAnomaliesTableData = ({
       earliestMs: number,
       latestMs: number
     ) {
-      if (userPermissions && !skip && siemJobs.length > 0) {
+      if (userPermissions && !skip && siemJobIds.length > 0) {
         try {
           const data = await anomaliesTableData(
             {
-              jobIds: siemJobs,
+              jobIds: siemJobIds,
               criteriaFields: criteriaFieldsInput,
               aggregationInterval: 'auto',
               threshold: getThreshold(anomalyScore, threshold),
@@ -114,7 +116,7 @@ export const useAnomaliesTableData = ({
         }
       } else if (!userPermissions && isSubscribed) {
         setLoading(false);
-      } else if (siemJobs.length === 0 && isSubscribed) {
+      } else if (siemJobIds.length === 0 && isSubscribed) {
         setLoading(false);
       } else if (isSubscribed) {
         setTableData(null);
@@ -134,7 +136,7 @@ export const useAnomaliesTableData = ({
     endDate,
     skip,
     userPermissions,
-    siemJobs.join(),
+    siemJobIds.sort().join(),
   ]);
 
   return [loading, tableData];

x-pack/legacy/plugins/siem/public/components/ml/api/anomalies_table_data.ts

@@ -9,7 +9,7 @@ import chrome from 'ui/chrome';
 import { useKibanaUiSetting } from '../../../lib/settings/use_kibana_ui_setting';
 import { DEFAULT_KBN_VERSION } from '../../../../common/constants';
 import { Anomalies, InfluencerInput, CriteriaFields } from '../types';
-import { throwIfNotOk } from './throw_if_not_ok';
+import { throwIfNotOk } from '../../../hooks/api/api';
 export interface Body {
   jobIds: string[];
   criteriaFields: CriteriaFields[];
@@ -25,7 +25,7 @@ export interface Body {
 
 export const anomaliesTableData = async (
   body: Body,
-  headers: Record<string, string | undefined>,
+  headers: Record<string, string>,
   signal: AbortSignal
 ): Promise<Anomalies> => {
   const [kbnVersion] = useKibanaUiSetting(DEFAULT_KBN_VERSION);

x-pack/legacy/plugins/siem/public/components/ml/api/get_ml_capabilities.ts

@@ -9,7 +9,7 @@ import chrome from 'ui/chrome';
 import { DEFAULT_KBN_VERSION } from '../../../../common/constants';
 import { useKibanaUiSetting } from '../../../lib/settings/use_kibana_ui_setting';
 import { InfluencerInput, MlCapabilities } from '../types';
-import { throwIfNotOk } from './throw_if_not_ok';
+import { throwIfNotOk } from '../../../hooks/api/api';
 
 export interface Body {
   jobIds: string[];
@@ -25,7 +25,7 @@ export interface Body {
 }
 
 export const getMlCapabilities = async (
-  headers: Record<string, string | undefined>,
+  headers: Record<string, string>,
   signal: AbortSignal
 ): Promise<MlCapabilities> => {
   const [kbnVersion] = useKibanaUiSetting(DEFAULT_KBN_VERSION);

x-pack/legacy/plugins/siem/public/components/ml/api/throw_if_not_ok.test.ts

@@ -6,14 +6,13 @@
 
 import fetchMock from 'fetch-mock';
 import {
-  throwIfNotOk,
-  parseJsonFromBody,
+  isMlStartJobError,
   MessageBody,
-  tryParseResponse,
+  parseJsonFromBody,
   throwIfErrorAttached,
-  isMlStartJobError,
-  ToasterErrors,
   throwIfErrorAttachedToSetup,
+  ToasterErrors,
+  tryParseResponse,
 } from './throw_if_not_ok';
 import { SetupMlResponse } from '../../ml_popover/types';
 
@@ -22,32 +21,6 @@ describe('throw_if_not_ok', () => {
     fetchMock.reset();
   });
 
-  describe('#throwIfNotOk', () => {
-    test('does a throw if it is given response that is not ok and the body is not parsable', async () => {
-      fetchMock.mock('http://example.com', 500);
-      const response = await fetch('http://example.com');
-      await expect(throwIfNotOk(response)).rejects.toThrow('Network Error: Internal Server Error');
-    });
-
-    test('does a throw and returns a body if it is parsable', async () => {
-      fetchMock.mock('http://example.com', {
-        status: 500,
-        body: {
-          statusCode: 500,
-          message: 'I am a custom message',
-        },
-      });
-      const response = await fetch('http://example.com');
-      await expect(throwIfNotOk(response)).rejects.toThrow('I am a custom message');
-    });
-
-    test('does NOT do a throw if it is given response is not ok', async () => {
-      fetchMock.mock('http://example.com', 200);
-      const response = await fetch('http://example.com');
-      await expect(throwIfNotOk(response)).resolves.toEqual(undefined);
-    });
-  });
-
   describe('#parseJsonFromBody', () => {
     test('parses a json from the body correctly', async () => {
       fetchMock.mock('http://example.com', {

x-pack/legacy/plugins/siem/public/components/ml/api/throw_if_not_ok.ts

@@ -35,21 +35,6 @@ export class ToasterErrors extends Error implements ToasterErrorsType {
   }
 }
 
-export const throwIfNotOk = async (response: Response): Promise<void> => {
-  if (!response.ok) {
-    const body = await parseJsonFromBody(response);
-    if (body != null && body.message) {
-      if (body.statusCode != null) {
-        throw new ToasterErrors([body.message, `${i18n.STATUS_CODE} ${body.statusCode}`]);
-      } else {
-        throw new ToasterErrors([body.message]);
-      }
-    } else {
-      throw new ToasterErrors([`${i18n.NETWORK_ERROR} ${response.statusText}`]);
-    }
-  }
-};
-
 export const parseJsonFromBody = async (response: Response): Promise<MessageBody | null> => {
   try {
     const text = await response.text();
@@ -67,10 +52,13 @@ export const tryParseResponse = (response: string): string => {
   }
 };
 
-export const throwIfErrorAttachedToSetup = (setupResponse: SetupMlResponse): void => {
+export const throwIfErrorAttachedToSetup = (
+  setupResponse: SetupMlResponse,
+  jobIdErrorFilter: string[] = []
+): void => {
   const jobErrors = setupResponse.jobs.reduce<string[]>(
     (accum, job) =>
-      job.error != null
+      job.error != null && jobIdErrorFilter.includes(job.id)
         ? [
             ...accum,
             job.error.msg,
@@ -83,7 +71,7 @@ export const throwIfErrorAttachedToSetup = (setupResponse: SetupMlResponse): voi
 
   const dataFeedErrors = setupResponse.datafeeds.reduce<string[]>(
     (accum, dataFeed) =>
-      dataFeed.error != null
+      dataFeed.error != null && jobIdErrorFilter.includes(dataFeed.id.substr('datafeed-'.length))
         ? [
             ...accum,
             dataFeed.error.msg,

x-pack/legacy/plugins/siem/public/components/ml/api/translations.ts

@@ -12,10 +12,3 @@ export const STATUS_CODE = i18n.translate(
     defaultMessage: 'Status Code:',
   }
 );
-
-export const NETWORK_ERROR = i18n.translate(
-  'xpack.siem.components.ml.api.errors.networkErrorFailureTitle',
-  {
-    defaultMessage: 'Network Error:',
-  }
-);