From bfe2c0005fe304bea78f8bd00ae1aef877c61bdc Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 7 Jul 2020 16:57:23 -0500 Subject: [PATCH] ssl fixes and tests --- .../elasticsearch_fieldnames.test.ts.snap | 18 +++++ .../common/elasticsearch_fieldnames.test.ts | 15 ++++ x-pack/plugins/apm/dev_docs/telemetry.md | 3 + .../scripts/upload-telemetry-data/index.ts | 3 +- .../collect_data_telemetry/tasks.test.ts | 68 +++++++++++++++++++ .../collect_data_telemetry/tasks.ts | 2 +- 6 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index f3dc7abcf82395..06ca3145bfce9a 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -8,6 +8,12 @@ exports[`Error CLIENT_GEO 1`] = `undefined`; exports[`Error CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Error CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Error CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Error CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Error CONTAINER_ID 1`] = `undefined`; exports[`Error DESTINATION_ADDRESS 1`] = `undefined`; @@ -146,6 +152,12 @@ exports[`Span CLIENT_GEO 1`] = `undefined`; exports[`Span CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Span CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Span CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Span CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Span CONTAINER_ID 1`] = `undefined`; exports[`Span DESTINATION_ADDRESS 1`] = `undefined`; @@ -284,6 +296,12 @@ exports[`Transaction CLIENT_GEO 1`] = `undefined`; exports[`Transaction CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; +exports[`Transaction CLOUD_AVAILABILITY_ZONE 1`] = `"europe-west1-c"`; + +exports[`Transaction CLOUD_PROVIDER 1`] = `"gcp"`; + +exports[`Transaction CLOUD_REGION 1`] = `"europe-west1"`; + exports[`Transaction CONTAINER_ID 1`] = `"container1234567890abcdef"`; exports[`Transaction DESTINATION_ADDRESS 1`] = `undefined`; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts index e081f27e28b7b1..a9eb95cf37d4da 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -23,6 +23,11 @@ describe('Transaction', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, http: { request: { method: 'GET' }, response: { status_code: 200 }, @@ -74,6 +79,11 @@ describe('Span', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, processor: { name: 'transaction', event: 'span', @@ -121,6 +131,11 @@ describe('Error', () => { name: 'java', version: 'agent version', }, + cloud: { + availability_zone: 'europe-west1-c', + provider: 'gcp', + region: 'europe-west1', + }, error: { exception: [ { diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md index e655394c57052f..fa8e057a595954 100644 --- a/x-pack/plugins/apm/dev_docs/telemetry.md +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -40,6 +40,9 @@ and/or config/kibana.dev.yml files. Running the script with `--clear` will delete the index first. +If you're using an Elasticsearch instance without TLS verification (if you have `elasticsearch.ssl.verificationMode: none` set in your kibana.yml) +you can run the script with `env NODE_TLS_REJECT_UNAUTHORIZED=0` to avoid TLS connection errors. + After running the script you should see sample telemetry data in the "xpack-phone-home" index. ### Updating Data Telemetry Mappings diff --git a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts index c830303ac0d8f2..a44fad82f20e64 100644 --- a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts +++ b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -49,9 +49,8 @@ async function uploadData() { ...(httpAuth ? { auth: { ...httpAuth, username: 'elastic' }, - ssl: { rejectUnauthorized: false }, } - : { ssl: { rejectUnauthorized: false } }), + : {}), }); // The new template is the template downloaded from the telemetry repo, with diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts new file mode 100644 index 00000000000000..c648cf4cc116a6 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { tasks } from './tasks'; +import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; + +describe('data telemetry collection tasks', () => { + const indices = { + 'apm_oss.errorIndices': 'apm-8.0.0-error', + 'apm_oss.metricsIndices': 'apm-8.0.0-metric', + 'apm_oss.spanIndices': 'apm-8.0.0-span', + 'apm_oss.transactionIndices': 'apm-8.0.0-transaction', + } as ApmIndicesConfig; + + describe('cloud', () => { + const cloudTask = tasks.find((task) => task.name === 'cloud'); + + it('returns a map of cloud provider data', async () => { + const search = jest.fn().mockResolvedValueOnce({ + aggregations: { + availability_zone: { + buckets: [ + { doc_count: 1, key: 'us-west-1' }, + { doc_count: 1, key: 'europe-west1-c' }, + ], + }, + provider: { + buckets: [ + { doc_count: 1, key: 'aws' }, + { doc_count: 1, key: 'gcp' }, + ], + }, + region: { + buckets: [ + { doc_count: 1, key: 'us-west' }, + { doc_count: 1, key: 'europe-west1' }, + ], + }, + }, + }); + + expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + cloud: { + availability_zone: ['us-west-1', 'europe-west1-c'], + provider: ['aws', 'gcp'], + region: ['us-west', 'europe-west1'], + }, + }); + }); + + describe('with no results', () => { + it('returns an empty map', async () => { + const search = jest.fn().mockResolvedValueOnce({}); + + expect(await cloudTask?.executor({ indices, search } as any)).toEqual({ + cloud: { + availability_zone: [], + provider: [], + region: [], + }, + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 79d0338d09aa50..f27af9a2cc5165 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -85,7 +85,7 @@ export const tasks: TelemetryTask[] = [ const { aggregations } = response; if (!aggregations) { - return { [az]: [], [provider]: [], [region]: [] }; + return { cloud: { [az]: [], [provider]: [], [region]: [] } }; } const cloud = { [az]: getBucketKeys(aggregations[az]),