WIP

x-pack/legacy/plugins/infra/common/log_analysis/log_analysis.ts

@@ -26,8 +26,29 @@ export type SetupStatus =
   | 'initializing' // acquiring job statuses to determine setup status
   | 'unknown' // job status could not be acquired (failed request etc)
   | 'required' // jobs are missing
+  | 'requiredForReconfiguration' // the configurations don't match the source configurations
+  | 'requiredForUpdate' // the definitions don't match the module definitions
   | 'pending' // In the process of setting up the module for the first time or retrying, waiting for response
   | 'succeeded' // setup succeeded, notifying user
   | 'failed' // setup failed, notifying user
   | 'hiddenAfterSuccess' // hide the setup screen and we show the results for the first time
   | 'skipped'; // setup hidden because the module is in a correct state already
+
+/**
+ * Maps a job status to the possibility that results have already been produced
+ * before this state was reached.
+ */
+export const isJobStatusWithResults = (jobStatus: JobStatus) =>
+  ['started', 'finished', 'stopped', 'failed'].includes(jobStatus);
+
+export const isHealthyJobStatus = (jobStatus: JobStatus) =>
+  ['started', 'finished'].includes(jobStatus);
+
+/**
+ * Maps a setup status to the possibility that results have already been
+ * produced before this state was reached.
+ */
+export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
+  ['skipped', 'hiddenAfterSuccess', 'requiredForReconfiguration', 'requiredForUpdate'].includes(
+    setupStatus
+  );

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './log_analysis_job_problem_indicator';

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/log_analysis_job_problem_indicator.tsx

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { isHealthyJobStatus, JobStatus } from '../../../../common/log_analysis';
+
+export const LogAnalysisJobProblemIndicator: React.FC<{
+  jobStatus: JobStatus;
+}> = ({ jobStatus }) => {
+  if (isHealthyJobStatus(jobStatus)) {
+    return null;
+  } else {
+    return <div>misconfigured</div>;
+  }
+};

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts

@@ -47,6 +47,14 @@ const jobStateRT = rt.keyof({
   opening: null,
 });
 
+export const jobCustomSettingsRT = rt.type({
+  logs_source_config: rt.type({
+    indexPattern: rt.string,
+    timestampField: rt.string,
+    bucketSpan: rt.number,
+  }),
+});
+
 export const jobSummaryRT = rt.intersection([
   rt.type({
     id: rt.string,
@@ -56,6 +64,7 @@ export const jobSummaryRT = rt.intersection([
     datafeedIndices: rt.array(rt.string),
     datafeedState: datafeedStateRT,
     fullJob: rt.partial({
+      custom_settings: jobCustomSettingsRT,
       finished_time: rt.number,
     }),
   }),

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -4,12 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import * as rt from 'io-ts';
-import { kfetch } from 'ui/kfetch';
-
 import { fold } from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { identity } from 'fp-ts/lib/function';
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { jobCustomSettingsRT } from './ml_get_jobs_summary_api';
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 import { getJobIdPrefix } from '../../../../../common/log_analysis';
 
@@ -33,25 +34,33 @@ export const callSetupMlModuleAPI = async (
   timeField: string,
   bucketSpan: number
 ) => {
+  const filteredIndexPattern = removeSampleDataIndex(indexPattern);
   const response = await kfetch({
     method: 'POST',
     pathname: `/api/ml/modules/setup/${MODULE_ID}`,
     body: JSON.stringify(
       setupMlModuleRequestPayloadRT.encode({
         start,
         end,
-        indexPatternName: removeSampleDataIndex(indexPattern),
+        indexPatternName: filteredIndexPattern,
         prefix: getJobIdPrefix(spaceId, sourceId),
         startDatafeed: true,
         jobOverrides: [
           {
-            job_id: 'log-entry-rate',
+            job_id: 'log-entry-rate' as const,
             analysis_config: {
               bucket_span: `${bucketSpan}ms`,
             },
             data_description: {
               time_field: timeField,
             },
+            custom_settings: {
+              logs_source_config: {
+                indexPattern: filteredIndexPattern,
+                timestampField: timeField,
+                bucketSpan,
+              },
+            },
           },
         ],
         datafeedOverrides: [],
@@ -70,11 +79,22 @@ const setupMlModuleTimeParamsRT = rt.partial({
   end: rt.number,
 });
 
+const setupMlModuleLogEntryRateJobOverridesRT = rt.type({
+  job_id: rt.literal('log-entry-rate'),
+  analysis_config: rt.type({
+    bucket_span: rt.string,
+  }),
+  data_description: rt.type({
+    time_field: rt.string,
+  }),
+  custom_settings: jobCustomSettingsRT,
+});
+
 const setupMlModuleRequestParamsRT = rt.type({
   indexPatternName: rt.string,
   prefix: rt.string,
   startDatafeed: rt.boolean,
-  jobOverrides: rt.array(rt.object),
+  jobOverrides: rt.array(setupMlModuleLogEntryRateJobOverridesRT),
   datafeedOverrides: rt.array(rt.object),
 });
 

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -100,7 +100,6 @@ export const useLogAnalysisJobs = ({
     retry,
     setupStatus: statusState.setupStatus,
     viewResults,
-    fetchJobStatus,
   };
 };
 

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_status_state.tsx

@@ -9,9 +9,11 @@ import { useReducer } from 'react';
 import {
   getDatafeedId,
   getJobId,
+  isHealthyJobStatus,
   JobStatus,
   JobType,
   SetupStatus,
+  bucketSpan,
 } from '../../../../common/log_analysis';
 import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api';
 import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
@@ -101,9 +103,7 @@ function statusReducer(state: StatusReducerState, action: StatusReducerAction):
         ...state.jobStatus,
         'log-entry-rate': getJobStatus(getJobId(spaceId, sourceId, 'log-entry-rate'))(payload),
       };
-      const nextSetupStatus = Object.values(nextJobStatus).every(jobState =>
-        ['started', 'finished'].includes(jobState)
-      )
+      const nextSetupStatus = Object.values(nextJobStatus).every(isHealthyJobStatus)
         ? 'skipped'
         : 'required';
       return {
@@ -181,6 +181,30 @@ const getJobStatus = (jobId: string) => (jobSummaries: FetchJobStatusResponsePay
       }
     )[0] || 'missing';
 
+const isConfigurationConsistent = (
+  jobId: string,
+  sourceConfiguration: {
+    bucketSpan: number;
+    indexPattern: string;
+    timestampField: string;
+  }
+) => (jobSummaries: FetchJobStatusResponsePayload): boolean =>
+  jobSummaries
+    .filter(jobSummary => jobSummary.id === jobId)
+    .every(jobSummary => {
+      if (!jobSummary.fullJob || !jobSummary.fullJob.custom_settings) {
+        return false;
+      }
+
+      const jobConfiguration = jobSummary.fullJob.custom_settings.logs_source_config;
+
+      return (
+        jobConfiguration.bucketSpan === sourceConfiguration.bucketSpan &&
+        jobConfiguration.indexPattern === sourceConfiguration.indexPattern &&
+        jobConfiguration.timestampField === sourceConfiguration.timestampField
+      );
+    });
+
 export const useStatusState = () => {
   return useReducer(statusReducer, initialState);
 };

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_content.tsx

@@ -7,6 +7,7 @@
 import { i18n } from '@kbn/i18n';
 import React, { useContext, useEffect } from 'react';
 
+import { isSetupStatusWithResults } from '../../../../common/log_analysis';
 import { LoadingPage } from '../../../components/loading_page';
 import { LogAnalysisCapabilities, LogAnalysisJobs } from '../../../containers/logs/log_analysis';
 import { Source } from '../../../containers/source';
@@ -39,7 +40,7 @@ export const AnalysisPageContent = () => {
     );
   } else if (setupStatus === 'unknown') {
     return <AnalysisSetupStatusUnknownContent retry={fetchJobStatus} />;
-  } else if (setupStatus === 'skipped' || setupStatus === 'hiddenAfterSuccess') {
+  } else if (isSetupStatusWithResults(setupStatus)) {
     return (
       <AnalysisResultsContent
         sourceId={sourceId}

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_results_content.tsx

@@ -33,8 +33,9 @@ import { useInterval } from '../../../hooks/use_interval';
 import { useTrackPageview } from '../../../hooks/use_track_metric';
 import { FirstUseCallout } from './first_use';
 import { LogRateResults } from './sections/log_rate';
+import { LogAnalysisJobProblemIndicator } from '../../../components/logging/log_analysis_job_status';
 
-const JOB_STATUS_POLLING_INTERVAL = 10000;
+const JOB_STATUS_POLLING_INTERVAL = 30000;
 
 export const AnalysisResultsContent = ({
   sourceId,
@@ -163,6 +164,7 @@ export const AnalysisResultsContent = ({
               <EuiPageContent>
                 <EuiPageContentBody>
                   {isFirstUse && !hasResults ? <FirstUseCallout /> : null}
+                  <LogAnalysisJobProblemIndicator jobStatus={jobStatus['log-entry-rate']} />
                   <LogRateResults
                     isLoading={isLoading}
                     results={logEntryRate}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/logs_ui_analysis/ml/log_entry_rate.json

@@ -26,6 +26,6 @@
   },
   "custom_settings": {
     "created_by": "ml-module-logs-ui-analysis",
-    "job_revision": 2
+    "job_revision": 3
   }
 }