From 9be5d2cc7fac065311fa15dbfe7671a78eaaa0e9 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Fri, 4 Jun 2021 11:36:29 -0400 Subject: [PATCH 1/5] [Fleet] Add fleet server telemetry --- .../server/collectors/agent_collectors.ts | 3 +- .../collectors/fleet_server_collector.ts | 73 +++++++++++++++++++ .../fleet/server/collectors/register.ts | 25 +++++++ 3 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 0eb392e7843345..8487557075e756 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -5,8 +5,7 @@ * 2.0. */ -import type { SavedObjectsClient } from 'kibana/server'; -import type { ElasticsearchClient } from 'kibana/server'; +import type { SavedObjectsClient, ElasticsearchClient } from 'kibana/server'; import type { FleetConfigType } from '../../common/types'; import * as AgentService from '../services/agents'; diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts new file mode 100644 index 00000000000000..a0f7882d04be04 --- /dev/null +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { SavedObjectsClient, ElasticsearchClient } from 'kibana/server'; + +import { packagePolicyService, settingsService } from '../services'; +import { getAgentStatusForAgentPolicy } from '../services/agents'; +import { isFleetServerSetup } from '../services/fleet_server'; + +const DEFAULT_USAGE = { + total_all_statuses: 0, + healthy: 0, + num_host_urls: 0, +}; + +export interface FleetServerUsage { + healthy: number; + total_all_statuses: number; + num_host_urls: number; +} + +export const getFleetServerUsage = async ( + soClient?: SavedObjectsClient, + esClient?: ElasticsearchClient +): Promise => { + if (!soClient || !esClient || !(await isFleetServerSetup())) { + return DEFAULT_USAGE; + } + + const numHostsUrls = + (await settingsService.getSettings(soClient)).fleet_server_hosts?.length ?? 0; + + // Find all policies with Fleet server than query agent status + + let hasMore = true; + const policyIds = new Set(); + let page = 1; + while (hasMore) { + const res = await packagePolicyService.list(soClient, { + page: page++, + perPage: 20, + kuery: 'ingest-package-policies.package.name:fleet-server', + }); + + for (const item of res.items) { + policyIds.add(item.policy_id); + } + + if (res.items.length === 0) { + hasMore = false; + } + } + + if (policyIds.size === 0) { + return DEFAULT_USAGE; + } + + const { total, inactive, online } = await getAgentStatusForAgentPolicy( + soClient, + esClient, + undefined, + [...policyIds].map((policyId) => `(policy_id:"${policyId}")`).join(' or ') + ); + + return { + total_all_statuses: total + inactive, + healthy: online, + num_host_urls: numHostsUrls, + }; +}; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index 842bb95fe813f1..ad153af7ef2f82 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -16,11 +16,14 @@ import type { AgentUsage } from './agent_collectors'; import { getInternalClients } from './helpers'; import { getPackageUsage } from './package_collectors'; import type { PackageUsage } from './package_collectors'; +import { getFleetServerUsage } from './fleet_server_collector'; +import type { FleetServerUsage } from './fleet_server_collector'; interface Usage { agents_enabled: boolean; agents: AgentUsage; packages: PackageUsage[]; + fleet_server: FleetServerUsage; } export function registerFleetUsageCollector( @@ -43,6 +46,7 @@ export function registerFleetUsageCollector( return { agents_enabled: getIsAgentsEnabled(config), agents: await getAgentUsage(config, soClient, esClient), + fleet_server: await getFleetServerUsage(soClient, esClient), packages: await getPackageUsage(soClient), }; }, @@ -80,6 +84,27 @@ export function registerFleetUsageCollector( }, }, }, + fleet_server: { + total_all_statuses: { + type: 'long', + _meta: { + description: + 'The total number of Fleet Server agents in any state, both enrolled and inactive.', + }, + }, + healthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in a healthy state.', + }, + }, + num_host_urls: { + type: 'long', + _meta: { + description: 'The number of Fleet Server hosts configured in Fleet settings.', + }, + }, + }, packages: { type: 'array', items: { From aa73aa6c344f60d9dc194f1e58b939885f641550 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 7 Jun 2021 09:16:33 -0400 Subject: [PATCH 2/5] Update mappings --- .../schema/xpack_plugins.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 12e83008b2e5a5..46927e1180550f 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -2055,6 +2055,28 @@ } } }, + "fleet_server": { + "properties": { + "total_all_statuses": { + "type": "long", + "_meta": { + "description": "The total number of Fleet Server agents in any state, both enrolled and inactive." + } + }, + "healthy": { + "type": "long", + "_meta": { + "description": "The total number of enrolled Fleet Server agents in a healthy state." + } + }, + "num_host_urls": { + "type": "long", + "_meta": { + "description": "The number of Fleet Server hosts configured in Fleet settings." + } + } + } + }, "packages": { "type": "array", "items": { From a0dacdcb0cfee9d8b2ac2823236c9f723652cd59 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 7 Jun 2021 15:22:36 -0400 Subject: [PATCH 3/5] Add integration tests for Fleet telemetry --- .../collectors/fleet_server_collector.ts | 2 +- .../apis/fleet_telemetry.ts | 122 ++++++++++++++++++ .../test/fleet_api_integration/apis/index.js | 3 + 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index a0f7882d04be04..76ad344993bcd7 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -42,7 +42,7 @@ export const getFleetServerUsage = async ( const res = await packagePolicyService.list(soClient, { page: page++, perPage: 20, - kuery: 'ingest-package-policies.package.name:fleet-server', + kuery: 'ingest-package-policies.package.name:fleet_server', }); for (const item of res.items) { diff --git a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts new file mode 100644 index 00000000000000..5716fa1d305e5c --- /dev/null +++ b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { skipIfNoDockerRegistry } from '../helpers'; +import { setupFleetAndAgents } from './agents/services'; + +export default function (providerContext: FtrProviderContext) { + const { getService } = providerContext; + const supertest = getService('supertest'); + const es = getService('es'); + const esArchiver = getService('esArchiver'); + + let agentCount = 0; + async function generateAgent(status: string, policyId: string) { + let data: any = {}; + + switch (status) { + case 'unhealthy': + data = { last_checkin_status: 'error' }; + break; + case 'offline': + data = { last_checkin: '2017-06-07T18:59:04.498Z' }; + break; + default: + data = { last_checkin: new Date().toISOString() }; + } + + await es.index({ + index: '.fleet-agents', + body: { + id: `agent-${++agentCount}`, + active: true, + last_checkin: new Date().toISOString(), + policy_id: policyId, + policy_revision: 1, + ...data, + }, + refresh: 'wait_for', + }); + } + + describe('fleet_telemetry', () => { + skipIfNoDockerRegistry(providerContext); + before(async () => { + await esArchiver.load('empty_kibana'); + await esArchiver.load('fleet/empty_fleet_server'); + }); + + setupFleetAndAgents(providerContext); + + after(async () => { + await esArchiver.unload('empty_kibana'); + await esArchiver.unload('fleet/empty_fleet_server'); + }); + + before(async () => { + // Get FleetServer policy id + const { body: apiResponse } = await supertest.get(`/api/fleet/agent_policies`).expect(200); + const defaultFleetServerPolicy = apiResponse.items.find( + (item: any) => item.is_default_fleet_server + ); + + const defaultServerPolicy = apiResponse.items.find((item: any) => item.is_default); + + if (!defaultFleetServerPolicy) { + throw new Error('No default Fleet server policy'); + } + + if (!defaultServerPolicy) { + throw new Error('No default policy'); + } + + await supertest + .put(`/api/fleet/settings`) + .set('kbn-xsrf', 'xxxx') + .send({ fleet_server_hosts: ['https://test1.fr', 'https://test2.fr'] }) + .expect(200); + + // Default Fleet Server + await generateAgent('healthy', defaultFleetServerPolicy.id); + await generateAgent('healthy', defaultFleetServerPolicy.id); + await generateAgent('unhealthy', defaultFleetServerPolicy.id); + + // Default policy + await generateAgent('healthy', defaultServerPolicy.id); + await generateAgent('offline', defaultServerPolicy.id); + await generateAgent('unhealthy', defaultServerPolicy.id); + }); + + it('should return the correct telemetry values for fleet', async () => { + const { + body: [apiResponse], + } = await supertest + .post(`/api/telemetry/v2/clusters/_stats`) + .set('kbn-xsrf', 'xxxx') + .send({ + unencrypted: true, + }) + .expect(200); + + expect(apiResponse.stack_stats.kibana.plugins.fleet.agents).eql({ + total_enrolled: 6, + healthy: 3, + unhealthy: 2, + offline: 1, + total_all_statuses: 6, + }); + + expect(apiResponse.stack_stats.kibana.plugins.fleet.fleet_server).eql({ + total_all_statuses: 3, + healthy: 2, + num_host_urls: 2, + }); + }); + }); +} diff --git a/x-pack/test/fleet_api_integration/apis/index.js b/x-pack/test/fleet_api_integration/apis/index.js index 2357f549c101ca..ca6315e1934ab5 100644 --- a/x-pack/test/fleet_api_integration/apis/index.js +++ b/x-pack/test/fleet_api_integration/apis/index.js @@ -48,5 +48,8 @@ export default function ({ loadTestFile }) { // Outputs loadTestFile(require.resolve('./outputs')); + + // Telemetry + loadTestFile(require.resolve('./fleet_telemetry')); }); } From 1b0c8387df5070f9b1b685f5d0665a230de649fe Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 8 Jun 2021 15:50:50 -0400 Subject: [PATCH 4/5] Breakdown all agent statuses in agent and fleet collector --- .../server/collectors/agent_collectors.ts | 4 +++ .../collectors/fleet_server_collector.ts | 16 ++++++++-- .../fleet/server/collectors/register.ts | 30 +++++++++++++++++++ .../schema/xpack_plugins.json | 30 +++++++++++++++++++ .../apis/fleet_telemetry.ts | 5 ++++ 5 files changed, 83 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts index 8487557075e756..6a9a4cd9ba83ce 100644 --- a/x-pack/plugins/fleet/server/collectors/agent_collectors.ts +++ b/x-pack/plugins/fleet/server/collectors/agent_collectors.ts @@ -17,6 +17,7 @@ export interface AgentUsage { unhealthy: number; offline: number; total_all_statuses: number; + updating: number; } export const getAgentUsage = async ( @@ -32,6 +33,7 @@ export const getAgentUsage = async ( unhealthy: 0, offline: 0, total_all_statuses: 0, + updating: 0, }; } @@ -41,6 +43,7 @@ export const getAgentUsage = async ( online, error, offline, + updating, } = await AgentService.getAgentStatusForAgentPolicy(soClient, esClient); return { total_enrolled: total, @@ -48,5 +51,6 @@ export const getAgentUsage = async ( unhealthy: error, offline, total_all_statuses: total + inactive, + updating, }; }; diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index 76ad344993bcd7..ab17b67655d04d 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -12,12 +12,20 @@ import { isFleetServerSetup } from '../services/fleet_server'; const DEFAULT_USAGE = { total_all_statuses: 0, + total_enrolled: 0, healthy: 0, + unhealthy: 0, + offline: 0, + updating: 0, num_host_urls: 0, }; export interface FleetServerUsage { + total_enrolled: number; healthy: number; + unhealthy: number; + offline: number; + updating: number; total_all_statuses: number; num_host_urls: number; } @@ -58,7 +66,7 @@ export const getFleetServerUsage = async ( return DEFAULT_USAGE; } - const { total, inactive, online } = await getAgentStatusForAgentPolicy( + const { total, inactive, online, error, updating, offline } = await getAgentStatusForAgentPolicy( soClient, esClient, undefined, @@ -66,8 +74,12 @@ export const getFleetServerUsage = async ( ); return { - total_all_statuses: total + inactive, + total_enrolled: total, healthy: online, + unhealthy: error, + offline, + updating, + total_all_statuses: total + inactive, num_host_urls: numHostsUrls, }; }; diff --git a/x-pack/plugins/fleet/server/collectors/register.ts b/x-pack/plugins/fleet/server/collectors/register.ts index ad153af7ef2f82..a097d423e7dd28 100644 --- a/x-pack/plugins/fleet/server/collectors/register.ts +++ b/x-pack/plugins/fleet/server/collectors/register.ts @@ -71,6 +71,12 @@ export function registerFleetUsageCollector( description: 'The total number of enrolled agents in an unhealthy state', }, }, + updating: { + type: 'long', + _meta: { + description: 'The total number of enrolled agents in an updating state', + }, + }, offline: { type: 'long', _meta: { @@ -85,6 +91,12 @@ export function registerFleetUsageCollector( }, }, fleet_server: { + total_enrolled: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents, in any state', + }, + }, total_all_statuses: { type: 'long', _meta: { @@ -98,6 +110,24 @@ export function registerFleetUsageCollector( description: 'The total number of enrolled Fleet Server agents in a healthy state.', }, }, + unhealthy: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in an unhealthy state', + }, + }, + updating: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents in an updating state', + }, + }, + offline: { + type: 'long', + _meta: { + description: 'The total number of enrolled Fleet Server agents currently offline', + }, + }, num_host_urls: { type: 'long', _meta: { diff --git a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json index 46927e1180550f..7c96dce3fac7f5 100644 --- a/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json +++ b/x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @@ -2041,6 +2041,12 @@ "description": "The total number of enrolled agents in an unhealthy state" } }, + "updating": { + "type": "long", + "_meta": { + "description": "The total number of enrolled agents in an updating state" + } + }, "offline": { "type": "long", "_meta": { @@ -2057,6 +2063,12 @@ }, "fleet_server": { "properties": { + "total_enrolled": { + "type": "long", + "_meta": { + "description": "The total number of enrolled Fleet Server agents, in any state" + } + }, "total_all_statuses": { "type": "long", "_meta": { @@ -2069,6 +2081,24 @@ "description": "The total number of enrolled Fleet Server agents in a healthy state." } }, + "unhealthy": { + "type": "long", + "_meta": { + "description": "The total number of enrolled Fleet Server agents in an unhealthy state" + } + }, + "updating": { + "type": "long", + "_meta": { + "description": "The total number of enrolled Fleet Server agents in an updating state" + } + }, + "offline": { + "type": "long", + "_meta": { + "description": "The total number of enrolled Fleet Server agents currently offline" + } + }, "num_host_urls": { "type": "long", "_meta": { diff --git a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts index 5716fa1d305e5c..5cf0db2a6d9171 100644 --- a/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts +++ b/x-pack/test/fleet_api_integration/apis/fleet_telemetry.ts @@ -109,12 +109,17 @@ export default function (providerContext: FtrProviderContext) { healthy: 3, unhealthy: 2, offline: 1, + updating: 0, total_all_statuses: 6, }); expect(apiResponse.stack_stats.kibana.plugins.fleet.fleet_server).eql({ total_all_statuses: 3, + total_enrolled: 3, healthy: 2, + unhealthy: 1, + offline: 0, + updating: 0, num_host_urls: 2, }); }); From 6cbd41e29426b44510df947c0ed7dde28d706b97 Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Tue, 8 Jun 2021 16:54:21 -0400 Subject: [PATCH 5/5] use Array.from instead of spread --- .../plugins/fleet/server/collectors/fleet_server_collector.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts index ab17b67655d04d..d861b211b88484 100644 --- a/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts +++ b/x-pack/plugins/fleet/server/collectors/fleet_server_collector.ts @@ -70,7 +70,9 @@ export const getFleetServerUsage = async ( soClient, esClient, undefined, - [...policyIds].map((policyId) => `(policy_id:"${policyId}")`).join(' or ') + Array.from(policyIds) + .map((policyId) => `(policy_id:"${policyId}")`) + .join(' or ') ); return {