Merge branch 'master' into 89404

x-pack/plugins/alerts/README.md

@@ -15,6 +15,7 @@ Table of Contents
 	- [Usage](#usage)
 	- [Alerts API keys](#alerts-api-keys)
 	- [Limitations](#limitations)
+	- [Plugin status](#plugin-status)
 	- [Alert types](#alert-types)
 		- [Methods](#methods)
 		- [Executor](#executor)
@@ -79,6 +80,27 @@ Note that the `manage_own_api_key` cluster privilege is not enough - it can be u
     is unauthorized for user [user-name-here]
 ```
 
+## Plugin status
+
+The plugin status of an alert is customized by including information about checking failures for the framework decryption:
+```
+core.status.set(
+        combineLatest([
+          core.status.derivedStatus$,
+          getHealthStatusStream(startPlugins.taskManager),
+        ]).pipe(
+          map(([derivedStatus, healthStatus]) => {
+            if (healthStatus.level > derivedStatus.level) {
+              return healthStatus as ServiceStatus;
+            } else {
+              return derivedStatus;
+            }
+          })
+        )
+      );
+```
+To check for framework decryption failures, we use the task `alerting_health_check`, which runs every 60 minutes by default. To change the default schedule, use the kibana.yml configuration option `xpack.alerts.healthCheck.interval`.
+
 ## Alert types
 
 ### Methods

x-pack/plugins/apm/common/utils/as_mutable_array.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// Sometimes we use `as const` to have a more specific type,
+// because TypeScript by default will widen the value type of an
+// array literal. Consider the following example:
+//
+// const filter = [
+//   { term: { 'agent.name': 'nodejs' } },
+//   { range: { '@timestamp': { gte: 'now-15m ' }}
+// ];
+
+// The result value type will be:
+
+// const filter: ({
+//   term: {
+//     'agent.name'?: string
+//   };
+//   range?: undefined
+// } | {
+//   term?: undefined;
+//   range: {
+//     '@timestamp': {
+//       gte: string
+//     }
+//   }
+// })[];
+
+// This can sometimes leads to issues. In those cases, we can
+// use `as const`. However, the Readonly<any> type is not compatible
+// with Array<any>. This function returns a mutable version of a type.
+
+export function asMutableArray<T extends Readonly<any>>(
+  arr: T
+): T extends Readonly<[...infer U]> ? U : unknown[] {
+  return arr as any;
+}

x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts

@@ -40,15 +40,15 @@ interface AggregationParams {
   kuery?: string;
   setup: ServicesItemsSetup;
   searchAggregatedTransactions: boolean;
+  maxNumServices: number;
 }
 
-const MAX_NUMBER_OF_SERVICES = 500;
-
 export async function getServiceTransactionStats({
   environment,
   kuery,
   setup,
   searchAggregatedTransactions,
+  maxNumServices,
 }: AggregationParams) {
   return withApmSpan('get_service_transaction_stats', async () => {
     const { apmEventClient, start, end } = setup;
@@ -92,7 +92,7 @@ export async function getServiceTransactionStats({
           services: {
             terms: {
               field: SERVICE_NAME,
-              size: MAX_NUMBER_OF_SERVICES,
+              size: maxNumServices,
             },
             aggs: {
               transactionType: {
@@ -104,7 +104,6 @@ export async function getServiceTransactionStats({
                   environments: {
                     terms: {
                       field: SERVICE_ENVIRONMENT,
-                      missing: '',
                     },
                   },
                   sample: {
@@ -147,9 +146,9 @@ export async function getServiceTransactionStats({
         return {
           serviceName: bucket.key as string,
           transactionType: topTransactionTypeBucket.key as string,
-          environments: topTransactionTypeBucket.environments.buckets
-            .map((environmentBucket) => environmentBucket.key as string)
-            .filter(Boolean),
+          environments: topTransactionTypeBucket.environments.buckets.map(
+            (environmentBucket) => environmentBucket.key as string
+          ),
           agentName: topTransactionTypeBucket.sample.top[0].metrics[
             AGENT_NAME
           ] as AgentName,

x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent';
+import {
+  AGENT_NAME,
+  SERVICE_ENVIRONMENT,
+  SERVICE_NAME,
+} from '../../../../common/elasticsearch_fieldnames';
+import { environmentQuery, kqlQuery, rangeQuery } from '../../../utils/queries';
+import { ProcessorEvent } from '../../../../common/processor_event';
+import { Setup, SetupTimeRange } from '../../helpers/setup_request';
+import { withApmSpan } from '../../../utils/with_apm_span';
+
+export function getServicesFromMetricDocuments({
+  environment,
+  setup,
+  maxNumServices,
+  kuery,
+}: {
+  setup: Setup & SetupTimeRange;
+  environment?: string;
+  maxNumServices: number;
+  kuery?: string;
+}) {
+  return withApmSpan('get_services_from_metric_documents', async () => {
+    const { apmEventClient, start, end } = setup;
+
+    const response = await apmEventClient.search({
+      apm: {
+        events: [ProcessorEvent.metric],
+      },
+      body: {
+        size: 0,
+        query: {
+          bool: {
+            filter: [
+              ...rangeQuery(start, end),
+              ...environmentQuery(environment),
+              ...kqlQuery(kuery),
+            ],
+          },
+        },
+        aggs: {
+          services: {
+            terms: {
+              field: SERVICE_NAME,
+              size: maxNumServices,
+            },
+            aggs: {
+              environments: {
+                terms: {
+                  field: SERVICE_ENVIRONMENT,
+                },
+              },
+              latest: {
+                top_metrics: {
+                  metrics: { field: AGENT_NAME } as const,
+                  sort: { '@timestamp': 'desc' },
+                },
+              },
+            },
+          },
+        },
+      },
+    });
+
+    return (
+      response.aggregations?.services.buckets.map((bucket) => {
+        return {
+          serviceName: bucket.key as string,
+          environments: bucket.environments.buckets.map(
+            (envBucket) => envBucket.key as string
+          ),
+          agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName,
+        };
+      }) ?? []
+    );
+  });
+}

x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts

@@ -6,15 +6,18 @@
  */
 
 import { Logger } from '@kbn/logging';
+import { asMutableArray } from '../../../../common/utils/as_mutable_array';
 import { joinByKey } from '../../../../common/utils/join_by_key';
-import { getServicesProjection } from '../../../projections/services';
 import { withApmSpan } from '../../../utils/with_apm_span';
 import { Setup, SetupTimeRange } from '../../helpers/setup_request';
 import { getHealthStatuses } from './get_health_statuses';
+import { getServicesFromMetricDocuments } from './get_services_from_metric_documents';
 import { getServiceTransactionStats } from './get_service_transaction_stats';
 
 export type ServicesItemsSetup = Setup & SetupTimeRange;
 
+const MAX_NUMBER_OF_SERVICES = 500;
+
 export async function getServicesItems({
   environment,
   kuery,
@@ -32,33 +35,49 @@ export async function getServicesItems({
     const params = {
       environment,
       kuery,
-      projection: getServicesProjection({
-        kuery,
-        setup,
-        searchAggregatedTransactions,
-      }),
       setup,
       searchAggregatedTransactions,
+      maxNumServices: MAX_NUMBER_OF_SERVICES,
     };
 
-    const [transactionStats, healthStatuses] = await Promise.all([
+    const [
+      transactionStats,
+      servicesFromMetricDocuments,
+      healthStatuses,
+    ] = await Promise.all([
       getServiceTransactionStats(params),
+      getServicesFromMetricDocuments(params),
       getHealthStatuses(params).catch((err) => {
         logger.error(err);
         return [];
       }),
     ]);
 
-    const apmServices = transactionStats.map(({ serviceName }) => serviceName);
+    const foundServiceNames = transactionStats.map(
+      ({ serviceName }) => serviceName
+    );
+
+    const servicesWithOnlyMetricDocuments = servicesFromMetricDocuments.filter(
+      ({ serviceName }) => !foundServiceNames.includes(serviceName)
+    );
+
+    const allServiceNames = foundServiceNames.concat(
+      servicesWithOnlyMetricDocuments.map(({ serviceName }) => serviceName)
+    );
 
     // make sure to exclude health statuses from services
     // that are not found in APM data
     const matchedHealthStatuses = healthStatuses.filter(({ serviceName }) =>
-      apmServices.includes(serviceName)
+      allServiceNames.includes(serviceName)
     );
 
-    const allMetrics = [...transactionStats, ...matchedHealthStatuses];
-
-    return joinByKey(allMetrics, 'serviceName');
+    return joinByKey(
+      asMutableArray([
+        ...transactionStats,
+        ...servicesWithOnlyMetricDocuments,
+        ...matchedHealthStatuses,
+      ] as const),
+      'serviceName'
+    );
   });
 }

x-pack/plugins/case/common/api/cases/index.ts

@@ -11,4 +11,3 @@ export * from './comment';
 export * from './status';
 export * from './user_actions';
 export * from './sub_case';
-export * from './commentable_case';