[Infrastructure UI][Rules] Refactor Metric Threshold rule to push evaluations to Elasticsearch

x-pack/plugins/infra/public/alerting/metric_threshold/components/expression.tsx

@@ -252,11 +252,6 @@ export const Expressions: React.FC<Props> = (props) => {
     [onFilterChange]
   );
 
-  const areAllAggsRate = useMemo(
-    () => ruleParams.criteria?.every((c) => c.aggType === Aggregators.RATE),
-    [ruleParams.criteria]
-  );
-
   const hasGroupBy = useMemo(
     () => ruleParams.groupBy && ruleParams.groupBy.length > 0,
     [ruleParams.groupBy]
@@ -386,31 +381,6 @@ export const Expressions: React.FC<Props> = (props) => {
             checked={ruleParams.alertOnNoData}
             onChange={(e) => setRuleParams('alertOnNoData', e.target.checked)}
           />
-          <EuiCheckbox
-            id="metrics-alert-partial-buckets-toggle"
-            label={
-              <>
-                {i18n.translate('xpack.infra.metrics.alertFlyout.shouldDropPartialBuckets', {
-                  defaultMessage: 'Drop partial buckets when evaluating data',
-                })}{' '}
-                <EuiToolTip
-                  content={i18n.translate(
-                    'xpack.infra.metrics.alertFlyout.dropPartialBucketsHelpText',
-                    {
-                      defaultMessage:
-                        "Enable this to drop the most recent bucket of evaluation data if it's less than {timeSize}{timeUnit}.",
-                      values: { timeSize, timeUnit },
-                    }
-                  )}
-                >
-                  <EuiIcon type="questionInCircle" color="subdued" />
-                </EuiToolTip>
-              </>
-            }
-            checked={areAllAggsRate || ruleParams.shouldDropPartialBuckets}
-            disabled={areAllAggsRate}
-            onChange={(e) => setRuleParams('shouldDropPartialBuckets', e.target.checked)}
-          />
         </EuiPanel>
       </EuiAccordion>
       <EuiSpacer size={'m'} />

x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/lib/calculate_rate_timeranges.ts

@@ -4,15 +4,14 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import { InfraTimerangeInput } from '../../../../../common/http_api';
 
-export const calculateRateTimeranges = (timerange: InfraTimerangeInput) => {
+export const calculateRateTimeranges = (timerange: { to: number; from: number }) => {
   // This is the total number of milliseconds for the entire timerange
   const totalTime = timerange.to - timerange.from;
   // Halfway is the to minus half the total time;
-  const halfway = timerange.to - totalTime / 2;
+  const halfway = Math.round(timerange.to - totalTime / 2);
   // The interval is half the total time (divided by 1000 to convert to seconds)
-  const intervalInSeconds = totalTime / 2000;
+  const intervalInSeconds = Math.round(totalTime / (2 * 1000));
 
   // The first bucket is from the beginning of the time range to the halfway point
   const firstBucketRange = {

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_bucket_selector.ts

@@ -0,0 +1,118 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  Aggregators,
+  Comparator,
+  MetricExpressionParams,
+} from '../../../../../common/alerting/metrics';
+import { createConditionScript } from './create_condition_script';
+import { createLastPeriod } from './wrap_in_period';
+
+const EMPTY_SHOULD_WARN = {
+  bucket_script: {
+    buckets_path: {},
+    script: '0',
+  },
+};
+
+export const createBucketSelector = (
+  condition: MetricExpressionParams,
+  alertOnGroupDisappear: boolean = false,
+  groupBy?: string | string[],
+  lastPeriodEnd?: number
+) => {
+  const hasGroupBy = groupBy != null;
+  const hasWarn = condition.warningThreshold != null && condition.warningComparator != null;
+  const isPercentile = [Aggregators.P95, Aggregators.P99].includes(condition.aggType);
+  const isCount = condition.aggType === Aggregators.COUNT;
+  const isRate = condition.aggType === Aggregators.RATE;
+  const bucketPath = isCount
+    ? 'currentPeriod>_count'
+    : isRate
+    ? `aggregatedValue`
+    : isPercentile
+    ? `currentPeriod>aggregatedValue[${condition.aggType === Aggregators.P95 ? '95' : '99'}]`
+    : 'currentPeriod>aggregatedValue';
+
+  const shouldWarn = hasWarn
+    ? {
+        bucket_script: {
+          buckets_path: {
+            value: bucketPath,
+          },
+          script: createConditionScript(
+            condition.warningThreshold as number[],
+            condition.warningComparator as Comparator
+          ),
+        },
+      }
+    : EMPTY_SHOULD_WARN;
+
+  const shouldTrigger = {
+    bucket_script: {
+      buckets_path: {
+        value: bucketPath,
+      },
+      script: createConditionScript(condition.threshold, condition.comparator),
+    },
+  };
+
+  const aggs: any = {
+    shouldWarn,
+    shouldTrigger,
+  };
+
+  if (hasGroupBy && alertOnGroupDisappear && lastPeriodEnd) {
+    const wrappedPeriod = createLastPeriod(lastPeriodEnd, condition);
+    aggs.lastPeriod = wrappedPeriod.lastPeriod;
+    aggs.missingGroup = {
+      bucket_script: {
+        buckets_path: {
+          lastPeriod: 'lastPeriod>_count',
+          currentPeriod: 'currentPeriod>_count',
+        },
+        script: 'params.lastPeriod > 0 && params.currentPeriod < 1 ? 1 : 0',
+      },
+    };
+    aggs.newOrRecoveredGroup = {
+      bucket_script: {
+        buckets_path: {
+          lastPeriod: 'lastPeriod>_count',
+          currentPeriod: 'currentPeriod>_count',
+        },
+        script: 'params.lastPeriod < 1 && params.currentPeriod > 0 ? 1 : 0',
+      },
+    };
+  }
+
+  if (hasGroupBy) {
+    const evalutionBucketPath =
+      alertOnGroupDisappear && lastPeriodEnd
+        ? {
+            shouldWarn: 'shouldWarn',
+            shouldTrigger: 'shouldTrigger',
+            missingGroup: 'missingGroup',
+            newOrRecoveredGroup: 'newOrRecoveredGroup',
+          }
+        : { shouldWarn: 'shouldWarn', shouldTrigger: 'shouldTrigger' };
+
+    const evaluationScript =
+      alertOnGroupDisappear && lastPeriodEnd
+        ? '(params.missingGroup != null && params.missingGroup > 0) || (params.shouldWarn != null && params.shouldWarn > 0) || (params.shouldTrigger != null && params.shouldTrigger > 0) || (params.newOrRecoveredGroup != null && params.newOrRecoveredGroup > 0)'
+        : '(params.shouldWarn != null && params.shouldWarn > 0) || (params.shouldTrigger != null && params.shouldTrigger > 0)';
+
+    aggs.evaluation = {
+      bucket_selector: {
+        buckets_path: evalutionBucketPath,
+        script: evaluationScript,
+      },
+    };
+  }
+
+  return aggs;
+};

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_condition_script.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { Comparator } from '../../../../../common/alerting/metrics';
+
+export const createConditionScript = (threshold: number[], comparator: Comparator) => {
+  if (comparator === Comparator.BETWEEN && threshold.length === 2) {
+    return `params.value > ${threshold[0]} && params.value < ${threshold[1]} ? 1 : 0`;
+  }
+  if (comparator === Comparator.OUTSIDE_RANGE && threshold.length === 2) {
+    return `params.value < ${threshold[0]} && params.value > ${threshold[1]} ? 1 : 0`;
+  }
+  return `params.value ${comparator} ${threshold[0]} ? 1 : 0`;
+};

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_percentile_aggregation.ts

@@ -17,7 +17,7 @@ export const createPercentileAggregation = (
       percentiles: {
         field,
         percents: [value],
-        keyed: false,
+        keyed: true,
       },
     },
   };

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_rate_aggregation.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import moment from 'moment';
+import { calculateRateTimeranges } from '../../inventory_metric_threshold/lib/calculate_rate_timeranges';
+import { TIMESTAMP_FIELD } from '../../../../../common/constants';
+
+export const createRateAggsBucketScript = (
+  timeframe: { start: number; end: number },
+  id: string
+) => {
+  const { intervalInSeconds } = calculateRateTimeranges({
+    to: timeframe.end,
+    from: timeframe.start,
+  });
+  return {
+    [id]: {
+      bucket_script: {
+        buckets_path: {
+          first: `currentPeriod>${id}_first_bucket.maxValue`,
+          second: `currentPeriod>${id}_second_bucket.maxValue`,
+        },
+        script: `params.second > 0.0 && params.first > 0.0 && params.second > params.first ? (params.second - params.first) / ${intervalInSeconds}: null`,
+      },
+    },
+  };
+};
+
+export const createRateAggsBuckets = (
+  timeframe: { start: number; end: number },
+  id: string,
+  field: string
+) => {
+  const { firstBucketRange, secondBucketRange } = calculateRateTimeranges({
+    to: timeframe.end,
+    from: timeframe.start,
+  });
+
+  return {
+    [`${id}_first_bucket`]: {
+      filter: {
+        range: {
+          [TIMESTAMP_FIELD]: {
+            gte: moment(firstBucketRange.from).toISOString(),
+            lt: moment(firstBucketRange.to).toISOString(),
+          },
+        },
+      },
+      aggs: { maxValue: { max: { field } } },
+    },
+    [`${id}_second_bucket`]: {
+      filter: {
+        range: {
+          [TIMESTAMP_FIELD]: {
+            gte: moment(secondBucketRange.from).toISOString(),
+            lt: moment(secondBucketRange.to).toISOString(),
+          },
+        },
+      },
+      aggs: { maxValue: { max: { field } } },
+    },
+  };
+};

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_timerange.test.ts

@@ -36,23 +36,23 @@ describe('createTimerange(interval, aggType, timeframe)', () => {
     describe('Rate Aggs', () => {
       it('should return a 20 second range for last 1 second', () => {
         const subject = createTimerange(1000, Aggregators.RATE);
-        expect(subject.end - subject.start).toEqual(1000 * 5);
+        expect(subject.end - subject.start).toEqual(1000 * 2);
       });
       it('should return a 5 minute range for last 1 minute', () => {
         const subject = createTimerange(60000, Aggregators.RATE);
-        expect(subject.end - subject.start).toEqual(60000 * 5);
+        expect(subject.end - subject.start).toEqual(60000 * 2);
       });
       it('should return 25 minute range for last 5 minutes', () => {
         const subject = createTimerange(300000, Aggregators.RATE);
-        expect(subject.end - subject.start).toEqual(300000 * 5);
+        expect(subject.end - subject.start).toEqual(300000 * 2);
       });
       it('should return 5 hour range for last hour', () => {
         const subject = createTimerange(3600000, Aggregators.RATE);
-        expect(subject.end - subject.start).toEqual(3600000 * 5);
+        expect(subject.end - subject.start).toEqual(3600000 * 2);
       });
       it('should return a 5 day range for last day', () => {
         const subject = createTimerange(86400000, Aggregators.RATE);
-        expect(subject.end - subject.start).toEqual(86400000 * 5);
+        expect(subject.end - subject.start).toEqual(86400000 * 2);
       });
     });
   });
@@ -78,23 +78,23 @@ describe('createTimerange(interval, aggType, timeframe)', () => {
       });
     });
     describe('Rate Aggs', () => {
-      it('should return 25 minute range when given 4 minute timeframe', () => {
+      it('should return 8 minute range when given 4 minute timeframe', () => {
         const end = moment();
         const timeframe = {
           start: end.clone().subtract(4, 'minutes').valueOf(),
           end: end.valueOf(),
         };
         const subject = createTimerange(300000, Aggregators.RATE, timeframe);
-        expect(subject.end - subject.start).toEqual(300000 * 5);
+        expect(subject.end - subject.start).toEqual(300000 * 2);
       });
-      it('should return 25 minute range when given 6 minute timeframe', () => {
+      it('should return 12 minute range when given 6 minute timeframe', () => {
         const end = moment();
         const timeframe = {
           start: end.clone().subtract(6, 'minutes').valueOf(),
           end: end.valueOf(),
         };
         const subject = createTimerange(300000, Aggregators.RATE, timeframe);
-        expect(subject.end - subject.start).toEqual(300000 * 5);
+        expect(subject.end - subject.start).toEqual(300000 * 2);
       });
     });
   });
@@ -113,7 +113,7 @@ describe('createTimerange(interval, aggType, timeframe)', () => {
       });
     });
     describe('Rate Aggs', () => {
-      it('should return 25 minute range for last 5 minutes', () => {
+      it('should return 10 minute range for last 5 minutes', () => {
         const end = moment();
         const timeframe = {
           end: end.valueOf(),
@@ -122,7 +122,7 @@ describe('createTimerange(interval, aggType, timeframe)', () => {
         expect(subject).toEqual({
           start: end
             .clone()
-            .subtract(300 * 5, 'seconds')
+            .subtract(300 * 2, 'seconds')
             .valueOf(),
           end: end.valueOf(),
         });

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/create_timerange.ts

@@ -11,14 +11,14 @@ import { Aggregators } from '../../../../../common/alerting/metrics';
 export const createTimerange = (
   interval: number,
   aggType: Aggregators,
-  timeframe?: { end: number; start?: number }
+  timeframe?: { end: number; start?: number },
+  lastPeriodEnd?: number
 ) => {
   const to = moment(timeframe ? timeframe.end : Date.now()).valueOf();
 
   // Rate aggregations need 5 buckets worth of data
-  const minimumBuckets = aggType === Aggregators.RATE ? 5 : 1;
-
-  const calculatedFrom = to - interval * minimumBuckets;
+  const minimumBuckets = aggType === Aggregators.RATE ? 2 : 1;
+  const calculatedFrom = lastPeriodEnd ? lastPeriodEnd - interval : to - interval * minimumBuckets;
 
   // Use either the timeframe.start when the start is less then calculatedFrom
   // OR use the calculatedFrom