From 4b21a01932dea6002f9e83913aa8af8fe4983bf5 Mon Sep 17 00:00:00 2001 From: CJ Cenizal Date: Thu, 2 Aug 2018 11:11:19 -0700 Subject: [PATCH 01/11] remove legacy method --- ...bulk_uploader.combine_stats_legacy.test.js | 187 ------------------ .../server/kibana_monitoring/bulk_uploader.js | 42 +--- 2 files changed, 2 insertions(+), 227 deletions(-) delete mode 100644 x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js deleted file mode 100644 index 2ecd9a8329eab4..00000000000000 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.combine_stats_legacy.test.js +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { KIBANA_STATS_TYPE_MONITORING, KIBANA_USAGE_TYPE, KIBANA_SETTINGS_TYPE } from '../../common/constants'; -import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants'; -import { BulkUploader } from './bulk_uploader'; - -const getInitial = () => { - return [ - [ - { 'index': { '_type': KIBANA_STATS_TYPE_MONITORING } }, - { - 'host': 'tsullivan.local', - 'concurrent_connections': 0, - 'os': { - 'load': { '1m': 2.28857421875, '5m': 2.45068359375, '15m': 2.29248046875 }, - 'memory': { 'total_in_bytes': 17179869184, 'free_in_bytes': 527749120, 'used_in_bytes': 16652120064 }, - 'uptime_in_millis': 1211027000 - }, - 'process': { - 'event_loop_delay': 4.222616970539093, - 'memory': { - 'heap': { 'total_in_bytes': 219455488, 'used_in_bytes': 152622064, 'size_limit': 1501560832 }, - 'resident_set_size_in_bytes': 245923840 - }, - 'uptime_in_millis': 18467 - }, - 'requests': { - 'disconnects': 0, - 'total': 2, - }, - 'response_times': { 'average': 47, 'max': 47 }, - 'timestamp': '2017-07-26T00:14:20.771Z', - } - ], - [ - { 'index': { '_type': KIBANA_USAGE_TYPE } }, - { - 'dashboard': { 'total': 0 }, - 'visualization': { 'total': 0 }, - 'search': { 'total': 0 }, - 'index_pattern': { 'total': 2 }, - 'index': '.kibana' - } - ], - [ - { 'index': { '_type': KIBANA_REPORTING_TYPE } }, - { - 'available': true, - 'enabled': false, - '_all': 55, - 'csv': { - 'available': true, - 'count': 25 - }, - 'printable_pdf': { - 'available': true, - 'count': 30 - } - } - ], - [ - { 'index': { '_type': KIBANA_SETTINGS_TYPE } }, - { 'xpack': { 'defaultAdminEmail': 'tim@elastic.co' } } - ] - ]; -}; - -// TODO use jest snapshotting -const getResult = () => { - return [ - [ - { 'index': { '_type': 'kibana_stats' } }, - { - 'host': 'tsullivan.local', - 'concurrent_connections': 0, - 'os': { - 'load': { '1m': 2.28857421875, '5m': 2.45068359375, '15m': 2.29248046875 }, - 'memory': { 'total_in_bytes': 17179869184, 'free_in_bytes': 527749120, 'used_in_bytes': 16652120064 }, - 'uptime_in_millis': 1211027000 - }, - 'process': { - 'event_loop_delay': 4.222616970539093, - 'memory': { - 'heap': { 'total_in_bytes': 219455488, 'used_in_bytes': 152622064, 'size_limit': 1501560832 }, - 'resident_set_size_in_bytes': 245923840 - }, - 'uptime_in_millis': 18467 - }, - 'requests': { - 'disconnects': 0, - 'total': 2, - }, - 'response_times': { 'average': 47, 'max': 47 }, - 'timestamp': '2017-07-26T00:14:20.771Z', - 'usage': { - 'dashboard': { 'total': 0 }, - 'visualization': { 'total': 0 }, - 'search': { 'total': 0 }, - 'index_pattern': { 'total': 2 }, - 'index': '.kibana', - 'xpack': { - 'reporting': { - '_all': 55, - 'available': true, - 'csv': { - 'available': true, - 'count': 25, - }, - 'enabled': false, - 'printable_pdf': { - 'available': true, - 'count': 30, - } - } - } - } - } - ], - [ - { 'index': { '_type': 'kibana_settings' } }, - { - 'xpack': { 'defaultAdminEmail': 'tim@elastic.co' }, - } - ] - ]; -}; - -describe('Collector Types Combiner', () => { - describe('with all the data types present', () => { - it('provides settings, and combined stats/usage data', () => { - // default gives all the data types - const initial = getInitial(); - const result = BulkUploader.combineStatsLegacy(initial); - expect(result).toEqual(getResult()); - }); - }); - describe('with settings data missing', () => { - it('provides combined stats/usage data', () => { - // default gives all the data types - const initial = getInitial(); - const trimmedInitial = [ initial[0], initial[1], initial[2] ]; // just stats, usage and reporting, no settings - const result = BulkUploader.combineStatsLegacy(trimmedInitial); - const expectedResult = getResult(); - const trimmedExpectedResult = [ expectedResult[0] ]; // single combined item - expect(result).toEqual(trimmedExpectedResult); - }); - }); - describe('with usage data missing', () => { - it('provides settings, and stats data', () => { - // default gives all the data types - const initial = getInitial(); - const trimmedInitial = [ initial[0], initial[3] ]; // just stats and settings, no usage or reporting - const result = BulkUploader.combineStatsLegacy(trimmedInitial); - const expectedResult = getResult(); - delete expectedResult[0][1].usage; // usage stats should not be present in the result - const trimmedExpectedResult = [ expectedResult[0], expectedResult[1] ]; - expect(result).toEqual(trimmedExpectedResult); - }); - }); - describe('with stats data missing', () => { - it('provides settings data', () => { - // default gives all the data types - const initial = getInitial(); - const trimmedInitial = [ initial[3] ]; // just settings - const result = BulkUploader.combineStatsLegacy(trimmedInitial); - const expectedResult = getResult(); - const trimmedExpectedResult = [ expectedResult[1] ]; // just settings - expect(result).toEqual(trimmedExpectedResult); - }); - }); - - it('throws an error if duplicate types are registered', () => { - const combineWithDuplicate = () => { - const initial = getInitial(); - const withDuplicate = [ initial[0] ].concat(initial); - return BulkUploader.combineStatsLegacy(withDuplicate); - }; - expect(combineWithDuplicate).toThrow( - 'Duplicate collector type identifiers found in payload! ' + - 'kibana_stats_monitoring,kibana_stats_monitoring,kibana,reporting,kibana_settings' - ); - }); -}); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 31fe51e5163ef4..d5645e62fe23e4 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -162,45 +162,7 @@ export class BulkUploader { } } - static combineStatsLegacy(payload) { - throw 'This is deprecated'; - BulkUploader.checkPayloadTypesUnique(payload); - - // default the item to [] to allow destructuring - const findItem = type => payload.find(item => get(item, '[0].index._type') === type) || []; - - // kibana usage and stats - let statsResult; - const [ statsHeader, statsPayload ] = findItem(KIBANA_STATS_TYPE_MONITORING); - const [ reportingHeader, reportingPayload ] = findItem(KIBANA_REPORTING_TYPE); - - if (statsHeader && statsPayload) { - statsHeader.index._type = 'kibana_stats'; // HACK to convert kibana_stats_monitoring to just kibana_stats for bwc - const [ usageHeader, usagePayload ] = findItem(KIBANA_USAGE_TYPE); - const kibanaUsage = (usageHeader && usagePayload) ? usagePayload : null; - const reportingUsage = (reportingHeader && reportingPayload) ? reportingPayload : null; - statsResult = [ statsHeader, statsPayload ]; - if (kibanaUsage) { - set(statsResult, '[1].usage', kibanaUsage); - } - if (reportingUsage) { - set(statsResult, '[1].usage.xpack.reporting', reportingUsage); - } - } - - // kibana settings - let settingsResult; - const [ settingsHeader, settingsPayload ] = findItem(KIBANA_SETTINGS_TYPE); - if (settingsHeader && settingsPayload) { - settingsResult = [ settingsHeader, settingsPayload ]; - } - - // return new payload with the combined data - // adds usage data to stats data - // strips usage out as a top-level type - const result = [ statsResult, settingsResult ]; - - // remove result items that are undefined - return result.filter(Boolean); + static combineStatsLegacy() { + throw new Error('This is deprecated'); } } From 601feb96fdd09fe86325cb5aa5b3022deb23d209 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:03:52 -0700 Subject: [PATCH 02/11] remove duplicate method --- src/server/usage/classes/collector_set.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index 7b42d95b4e853a..f6c2634b17e610 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -98,10 +98,6 @@ export class CollectorSet { }); } - getCollectorByType(type, collectors = this._collectors) { - return collectors.find(collector => collector.type === type); - } - bulkFormat(data, collectors = this._collectors) { if (!Array.isArray(collectors)) { throw new Error(`bulkFormat method given bad collectors parameter: ` + typeof collectors); From 6dd6e09f98a200b15c6c5badc6613e4912b67c0f Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:06:18 -0700 Subject: [PATCH 03/11] pick apart formatForBulkUpload --- src/server/usage/classes/collector.js | 17 +++++++++++++++-- src/server/usage/classes/collector_set.js | 16 ++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/server/usage/classes/collector.js b/src/server/usage/classes/collector.js index 8b56a3bc8accff..fdc716df96ec5e 100644 --- a/src/server/usage/classes/collector.js +++ b/src/server/usage/classes/collector.js @@ -25,19 +25,28 @@ export class Collector { * @param {String} options.type - property name as the key for the data * @param {Function} options.init (optional) - initialization function * @param {Function} options.fetch - function to query data + * @param {Function} options.formatForBulkUpload - optional + * @param {Function} options.rest - optional other properties */ - constructor(server, { type, init, fetch, formatForBulkUpload = null } = {}) { + constructor(server, { type, init, fetch, formatForBulkUpload = null, ...options } = {}) { if (type === undefined) { throw new Error('Collector must be instantiated with a options.type string property'); } + if (typeof init !== 'undefined' && typeof init !== 'function') { + throw new Error('If init property is passed, Collector must be instantiated with a options.init as a function property'); + } if (typeof fetch !== 'function') { throw new Error('Collector must be instantiated with a options.fetch function property'); } + Object.assign(this, options); // spread in other properties and mutate "this" + this.type = type; this.init = init; this.fetch = fetch; - this.formatForBulkUpload = formatForBulkUpload; + + const defaultFormatterForBulkUpload = result => [ { type, payload: result } ]; + this._formatForBulkUpload = formatForBulkUpload || defaultFormatterForBulkUpload; this.log = getCollectorLogger(server); } @@ -48,4 +57,8 @@ export class Collector { } return this.fetch(callCluster); } + + formatForBulkUpload(result) { + return this._formatForBulkUpload(result); + } } diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index f6c2634b17e610..1095dd5f667a0e 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -23,12 +23,6 @@ import { getCollectorLogger } from '../lib'; import { Collector } from './collector'; import { UsageCollector } from './usage_collector'; -function defaultFormatterForBulkUpload(collector, result) { - return [ - { type: collector.type, payload: result } - ]; -} - /* * A collector object has types registered into it with the register(type) * function. Each type that gets registered defines how to fetch its own data @@ -108,10 +102,12 @@ export class CollectorSet { return accum; } - const collector = this.getCollectorByType(collectedData.type); - const formatter = collector.formatForBulkUpload || defaultFormatterForBulkUpload.bind(null, collector); - accum.push(formatter(collectedData.result)); - return accum; + const payload = this.getCollectorByType(type).formatForBulkUpload(result); + console.log(JSON.stringify({ payload })); + return [ + ...accum, + payload // TODO flatten it here + ]; }, []); } From b8aab579e993bf930a117bc9b1459b6b7199d94a Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:07:45 -0700 Subject: [PATCH 04/11] let a collector be filtered out of a set --- .../status/collectors/get_ops_stats_collector.js | 3 ++- src/server/usage/classes/collector_set.js | 15 ++++++++++++--- .../server/kibana_monitoring/bulk_uploader.js | 7 +++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/server/status/collectors/get_ops_stats_collector.js b/src/server/status/collectors/get_ops_stats_collector.js index ba4a3d92699ae1..a11c5bf9b1dc9f 100644 --- a/src/server/status/collectors/get_ops_stats_collector.js +++ b/src/server/status/collectors/get_ops_stats_collector.js @@ -44,6 +44,7 @@ export function getOpsStatsCollector(server, kbnServer) { kibana: getKibanaInfoForStats(server, kbnServer), ...kbnServer.metrics // latest metrics captured from the ops event listener in src/server/status/index }; - } + }, + internalIgnore: true, // Ignore this one from internal uploader. A different stats collector is used there. }); } diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index 1095dd5f667a0e..329fc1cd15ee3c 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -46,6 +46,7 @@ export class CollectorSet { */ this.makeStatsCollector = options => new Collector(server, options); this.makeUsageCollector = options => new UsageCollector(server, options); + this.makeCollectorSetFromArray = collectorsArray => new CollectorSet(server, collectorsArray); } /* @@ -111,9 +112,17 @@ export class CollectorSet { }, []); } - async bulkFetchUsage(callCluster) { - const usageCollectors = this._collectors.filter(c => c instanceof UsageCollector); - return this.bulkFetch(callCluster, usageCollectors); + /* + * @return {new CollectorSet} + */ + getFilteredCollectorSet(filter) { + const filtered = this._collectors.filter(filter); + return this.makeCollectorSetFromArray(filtered); + } + + async bulkFetchUsage(fetchMechanisms) { + const usageCollectors = this.getFilteredCollectorSet(c => c instanceof UsageCollector); + return this.bulkFetch(fetchMechanisms, usageCollectors); } // convert an array of fetched stats results into key/object diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index d5645e62fe23e4..7355e8397be843 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -66,9 +66,12 @@ export class BulkUploader { */ start(collectorSet) { this._log.info('Starting monitoring stats collection'); - this._fetchAndUpload(collectorSet); // initial fetch + + // filter out API-only collectors + const filterThem = _collectorSet => _collectorSet.getFilteredCollectorSet(c => c.internalIgnore !== true); + this._fetchAndUpload(filterThem(collectorSet)); // initial fetch this._timer = setInterval(() => { - this._fetchAndUpload(collectorSet); + this._fetchAndUpload(filterThem(collectorSet)); }, this._interval); } From 0ea6a87b8baabd786963ab8fb5e435022fd5a122 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:09:21 -0700 Subject: [PATCH 05/11] toBulkUploadFormat => getCollectedData --- .../server/kibana_monitoring/bulk_uploader.js | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 7355e8397be843..974ccd7af6cd7a 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -4,16 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get, set, flatten, uniq, compact } from 'lodash'; +import { flatten, uniq, compact } from 'lodash'; import { callClusterFactory } from '../../../xpack_main'; import { LOGGING_TAG, KIBANA_MONITORING_LOGGING_TAG, - KIBANA_STATS_TYPE_MONITORING, - KIBANA_SETTINGS_TYPE, - KIBANA_USAGE_TYPE, } from '../../common/constants'; -import { KIBANA_REPORTING_TYPE } from '../../../reporting/common/constants'; import { sendBulkPayload, monitoringBulk, @@ -100,8 +96,11 @@ export class BulkUploader { * @return {Promise} - resolves to undefined */ async _fetchAndUpload(collectorSet) { - const data = await collectorSet.bulkFetch(this._callClusterWithInternalUser); - const payload = BulkUploader.toBulkUploadFormat(data, collectorSet); + const data = await collectorSet.bulkFetch({ + callCluster: this._callClusterWithInternalUser, + savedObjectsClient: this._savedObjectsClient, + }); + const payload = BulkUploader.getCollectedData(data, collectorSet); if (payload) { try { @@ -151,7 +150,7 @@ export class BulkUploader { * Bulk stats are transformed into a bulk upload format * Non-legacy transformation is done in CollectorSet.toApiStats */ - static toBulkUploadFormat(uploadData, collectorSet) { + static getCollectedData(uploadData, collectorSet) { if (compact(uploadData).length > 0) { return flatten(BulkUploader.deepMergeUploadData(uploadData, collectorSet)); } From d11f46917eb559510232d08f82cf507fe31d7e6d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:09:38 -0700 Subject: [PATCH 06/11] integrated saved objects client with summarize method --- .../service/create_saved_objects_service.js | 9 ++++ .../saved_objects/service/lib/repository.js | 49 +++++++++++++++++ .../service/lib/repository.test.js | 33 ++++++++++++ .../service/saved_objects_client.js | 7 +++ .../status/routes/api/register_stats.js | 7 +-- .../usage/classes/__tests__/collector_set.js | 8 ++- src/server/usage/classes/collector.js | 15 ++++-- src/server/usage/classes/collector_set.js | 17 +++--- .../__tests__/bulk_uploader.js | 3 ++ .../server/kibana_monitoring/bulk_uploader.js | 4 +- .../collectors/get_kibana_usage_collector.js | 53 ++----------------- .../collectors/get_ops_stats_collector.js | 4 +- .../collectors/get_settings_collector.js | 6 +-- .../server/kibana_monitoring/init.js | 2 +- .../usage/get_reporting_usage_collector.js | 4 +- .../get_reporting_usage_collector.test.js | 8 +-- 16 files changed, 148 insertions(+), 81 deletions(-) diff --git a/src/server/saved_objects/service/create_saved_objects_service.js b/src/server/saved_objects/service/create_saved_objects_service.js index fd471cad2dd23d..aa7e51617e6722 100644 --- a/src/server/saved_objects/service/create_saved_objects_service.js +++ b/src/server/saved_objects/service/create_saved_objects_service.js @@ -82,6 +82,13 @@ export function createSavedObjectsService(server) { } }); + const unscopedClientProvider = { + getClient(callCluster) { + const repository = repositoryProvider.getRepository(callCluster); + return new SavedObjectsClient(repository); + } + }; + return { types: Object.keys(getRootPropertiesObjects(mappings)), SavedObjectsClient, @@ -94,5 +101,7 @@ export function createSavedObjectsService(server) { scopedClientProvider.setClientFactory(...args), addScopedSavedObjectsClientWrapperFactory: (...args) => scopedClientProvider.addClientWrapperFactory(...args), + getUnscopedSavedObjectsClient: (...args) => + unscopedClientProvider.getClient(...args), }; } diff --git a/src/server/saved_objects/service/lib/repository.js b/src/server/saved_objects/service/lib/repository.js index 4ff8eaa8d6b6e9..a0548a1c5b4163 100644 --- a/src/server/saved_objects/service/lib/repository.js +++ b/src/server/saved_objects/service/lib/repository.js @@ -17,6 +17,7 @@ * under the License. */ +import { get, snakeCase } from 'lodash'; import uuid from 'uuid'; import { getRootType } from '../../../mappings'; @@ -210,6 +211,54 @@ export class SavedObjectsRepository { ); } + /** + * Summarize + */ + async summarize() { + const TYPES = [ + 'dashboard', + 'graph-workspace', + 'index-pattern', + 'search', + 'timelion-sheet', + 'visualization', + ]; + + const esOptions = { + index: this._index, + size: 0, + ignore: [404], + filterPath: 'aggregations.types.buckets', + body: { + query: { + terms: { type: TYPES }, + }, + aggs: { + types: { + terms: { field: 'type', size: TYPES.length } + } + } + } + }; + const response = await this._callCluster('search', esOptions); + const buckets = get(response, 'aggregations.types.buckets', []); + const bucketCounts = buckets.reduce((acc, bucket) => ({ + ...acc, + [bucket.key]: bucket.doc_count, + }), {}); + + return { + index: this._index, + ...TYPES.reduce((acc, type) => ({ // combine the bucketCounts and 0s for types that don't have documents + ...acc, + [snakeCase(type)]: { + total: bucketCounts[type] || 0 + } + }), {}) + }; + + } + /** * @param {object} [options={}] * @property {(string|Array)} [options.type] diff --git a/src/server/saved_objects/service/lib/repository.test.js b/src/server/saved_objects/service/lib/repository.test.js index 39174d204a1465..b4f191188ba218 100644 --- a/src/server/saved_objects/service/lib/repository.test.js +++ b/src/server/saved_objects/service/lib/repository.test.js @@ -666,4 +666,37 @@ describe('SavedObjectsRepository', () => { } }); }); + + describe('summarize', () => { + beforeEach(() => { + const aggResults = { + aggregations: { + types: { + buckets: [ + { key: 'dashboard', doc_count: 13 }, + { key: 'graph-workspace', doc_count: 2 }, + { key: 'index-pattern', doc_count: 4 }, + { key: 'search', doc_count: 13 }, + { key: 'timelion-sheet', doc_count: 4 }, + { key: 'visualization', doc_count: 65 }, + ] + } + } + }; + callAdminCluster.returns(aggResults); + }); + + it('summarizes saved objects', async () => { + const response = await savedObjectsRepository.summarize(); + expect(response).toEqual({ + index: '.kibana-test', + index_pattern: { total: 4 }, + dashboard: { total: 13 }, + graph_workspace: { total: 2 }, + search: { total: 13 }, + timelion_sheet: { total: 4 }, + visualization: { total: 65 } + }); + }); + }); }); diff --git a/src/server/saved_objects/service/saved_objects_client.js b/src/server/saved_objects/service/saved_objects_client.js index 7741800fc2e272..402690799e1176 100644 --- a/src/server/saved_objects/service/saved_objects_client.js +++ b/src/server/saved_objects/service/saved_objects_client.js @@ -187,4 +187,11 @@ export class SavedObjectsClient { async update(type, id, attributes, options = {}) { return this._repository.update(type, id, attributes, options); } + + /* + * Summarize the objects + */ + async summarize() { + return this._repository.summarize(); + } } diff --git a/src/server/status/routes/api/register_stats.js b/src/server/status/routes/api/register_stats.js index d7497d8f58640c..1f71b94a7add86 100644 --- a/src/server/status/routes/api/register_stats.js +++ b/src/server/status/routes/api/register_stats.js @@ -40,8 +40,8 @@ export function registerStatsApi(kbnServer, server, config) { return uuid; }; - const getUsage = async callCluster => { - const usage = await collectorSet.bulkFetchUsage(callCluster); + const getUsage = async (callCluster, savedObjectsClient) => { + const usage = await collectorSet.bulkFetchUsage({ callCluster, savedObjectsClient }); return collectorSet.toObject(usage); }; @@ -66,9 +66,10 @@ export function registerStatsApi(kbnServer, server, config) { if (isExtended) { const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); const callCluster = (...args) => callWithRequest(req, ...args); + const savedObjectsClient = req.getSavedObjectsClient(); try { const [ usage, clusterUuid ] = await Promise.all([ - getUsage(callCluster), + getUsage(callCluster, savedObjectsClient), getClusterUuid(callCluster), ]); diff --git a/src/server/usage/classes/__tests__/collector_set.js b/src/server/usage/classes/__tests__/collector_set.js index 3d1a23b8dc5d60..65d6ed2d6f7e49 100644 --- a/src/server/usage/classes/__tests__/collector_set.js +++ b/src/server/usage/classes/__tests__/collector_set.js @@ -52,13 +52,17 @@ describe('CollectorSet', () => { it('should log debug status of fetching from the collector', async () => { const mockCallCluster = () => Promise.resolve({ passTest: 1000 }); + const mockSavedObjectsClient = { summarize: () => ({}) }; const collectors = new CollectorSet(server); collectors.register(new Collector(server, { type: 'MY_TEST_COLLECTOR', - fetch: caller => caller() + fetch: ({ callCluster: caller }) => caller() })); - const result = await collectors.bulkFetch(mockCallCluster); + const result = await collectors.bulkFetch({ + callCluster: mockCallCluster, + savedObjectsClient: mockSavedObjectsClient + }); const calls = server.log.getCalls(); expect(calls.length).to.be(1); expect(calls[0].args).to.eql([ diff --git a/src/server/usage/classes/collector.js b/src/server/usage/classes/collector.js index fdc716df96ec5e..d1e6d6690e5a09 100644 --- a/src/server/usage/classes/collector.js +++ b/src/server/usage/classes/collector.js @@ -51,11 +51,18 @@ export class Collector { this.log = getCollectorLogger(server); } - fetchInternal(callCluster) { - if (typeof callCluster !== 'function') { - throw new Error('A `callCluster` function must be passed to the fetch methods of collectors'); + /* + * @param {Object} fetchMechanisms - an object with a callCluster function and a savedObjectsClient object + */ + fetchInternal(fetchMechanisms) { + const { callCluster, savedObjectsClient } = fetchMechanisms; + if (typeof callCluster !== 'function' || typeof savedObjectsClient !== 'object') { + throw new Error( + 'An object must be passed to the fetch methods of collectors having ' + + 'properties of a callCluster function and savedObjectsClient object' + ); } - return this.fetch(callCluster); + return this.fetch(fetchMechanisms); } formatForBulkUpload(result) { diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index 329fc1cd15ee3c..0126894ffd4103 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -32,13 +32,11 @@ export class CollectorSet { /* * @param {Object} server - server object - * @param {Number} options.interval - in milliseconds - * @param {Function} options.combineTypes - * @param {Function} options.onPayload + * @param {Array} collectors to initialize, usually as a result of filtering another CollectorSet instance */ - constructor(server) { + constructor(server, collectors = []) { this._log = getCollectorLogger(server); - this._collectors = []; + this._collectors = collectors; /* * Helper Factory methods @@ -72,9 +70,10 @@ export class CollectorSet { /* * Call a bunch of fetch methods and then do them in bulk + * @param {Object} fetchMechanisms - an object with a callCluster function and a savedObjectsClient object * @param {Array} collectors - an array of collectors, default to all registered collectors */ - bulkFetch(callCluster, collectors = this._collectors) { + bulkFetch(fetchMechanisms, collectors = this._collectors) { if (!Array.isArray(collectors)) { throw new Error(`bulkFetch method given bad collectors parameter: ` + typeof collectors); } @@ -84,7 +83,7 @@ export class CollectorSet { this._log.debug(`Fetching data from ${collectorType} collector`); return Promise.props({ type: collectorType, - result: collector.fetchInternal(callCluster) // use the wrapper for fetch, kicks in error checking + result: collector.fetchInternal(fetchMechanisms) // use the wrapper for fetch, kicks in error checking }) .catch(err => { this._log.warn(err); @@ -98,8 +97,8 @@ export class CollectorSet { throw new Error(`bulkFormat method given bad collectors parameter: ` + typeof collectors); } - return data.reduce((accum, collectedData) => { - if (isEmpty(collectedData)) { + return data.reduce((accum, { type, result }) => { + if (isEmpty(result)) { return accum; } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js index 3c5065a782c951..6594ede794e714 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/__tests__/bulk_uploader.js @@ -52,6 +52,9 @@ describe('BulkUploader', () => { }, }, usage: {}, + savedObjects: { + getUnscopedSavedObjectsClient: sinon.stub() + } }; }); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index 974ccd7af6cd7a..e727f7414e6707 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { flatten, uniq, compact } from 'lodash'; +import { compact, flatten, uniq } from 'lodash'; import { callClusterFactory } from '../../../xpack_main'; import { LOGGING_TAG, @@ -52,7 +52,7 @@ export class BulkUploader { }); this._callClusterWithInternalUser = callClusterFactory(server).getCallClusterInternal(); - + this._savedObjectsClient = server.savedObjects.getUnscopedSavedObjectsClient(this._callClusterWithInternalUser); } /* diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js index 8b4060d27d36e7..17fedea91ec5b2 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js @@ -4,66 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get, snakeCase } from 'lodash'; import { KIBANA_USAGE_TYPE } from '../../../common/constants'; -const TYPES = [ - 'dashboard', - 'visualization', - 'search', - 'index-pattern', - 'graph-workspace', - 'timelion-sheet', -]; - /** - * Fetches saved object client counts by querying the saved object index + * Fetches saved object counts by querying the .kibana index */ export function getKibanaUsageCollector(server) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_USAGE_TYPE, - async fetch(callCluster) { - const index = server.config().get('kibana.index'); - const savedObjectCountSearchParams = { - index, - ignoreUnavailable: true, - filterPath: 'aggregations.types.buckets', - body: { - size: 0, - query: { - terms: { type: TYPES }, - }, - aggs: { - types: { - terms: { field: 'type', size: TYPES.length } - } - } - } - }; - - const resp = await callCluster('search', savedObjectCountSearchParams); - const buckets = get(resp, 'aggregations.types.buckets', []); - - // get the doc_count from each bucket - const bucketCounts = buckets.reduce((acc, bucket) => ({ - ...acc, - [bucket.key]: bucket.doc_count, - }), {}); - - return { - index, - ...TYPES.reduce((acc, type) => ({ // combine the bucketCounts and 0s for types that don't have documents - ...acc, - [snakeCase(type)]: { - total: bucketCounts[type] || 0 - } - }), {}) - }; + async fetch({ savedObjectsClient }) { + return savedObjectsClient.summarize(); }, formatForBulkUpload: result => { return [{ - type: 'kibana_stats', + type: KIBANA_USAGE_TYPE, payload: { usage: result } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js index 6214b8eb81998b..60f2e7ebc9660e 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js @@ -57,14 +57,14 @@ export function getOpsStatsCollector(server, kbnServer) { const { kibana, ...rest } = result; return [ { - type: 'kibana_stats', + type: KIBANA_STATS_TYPE_MONITORING, payload: { kibana, ...rest, } }, { - type: 'kibana_settings', + type: KIBANA_STATS_TYPE_MONITORING + '_settings', payload: { kibana, } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js index f552f69af18465..49faa0032088f2 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js @@ -56,11 +56,11 @@ export async function checkForEmailValue( export function getSettingsCollector(server, kbnServer) { const config = server.config(); - const { collectorSet } = server.usage; + return collectorSet.makeStatsCollector({ - type: KIBANA_SETTINGS_TYPE, - async fetch(callCluster) { + type: KIBANA_SETTINGS_TYPE + '_settings_2', + async fetch({ callCluster }) { let kibanaSettingsData; const defaultAdminEmail = await checkForEmailValue(config, callCluster); diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js index 6dcb0ae1fc076c..5df1686234a4f1 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/init.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/init.js @@ -16,9 +16,9 @@ import { BulkUploader } from './bulk_uploader'; * @param {Object} server HapiJS server instance */ export function initBulkUploader(_kbnServer, server) { - const config = server.config(); const interval = config.get('xpack.monitoring.kibana.collection.interval'); + return new BulkUploader(server, { interval }); diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js index 196e14abc52ba8..e3e64a80959f46 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js @@ -117,7 +117,7 @@ export function getReportingUsageCollector(server) { const { collectorSet } = server.usage; return collectorSet.makeUsageCollector({ type: KIBANA_REPORTING_TYPE, - fetch: async callCluster => { + fetch: async ({ callCluster }) => { const xpackInfo = server.plugins.xpack_main.info; const config = server.config(); const available = xpackInfo && xpackInfo.isAvailable(); // some form of reporting (csv at least) is available for all valid licenses @@ -150,7 +150,7 @@ export function getReportingUsageCollector(server) { }, formatForBulkUpload: result => { return [{ - type: 'kibana_stats', + type: KIBANA_REPORTING_TYPE, payload: { usage: { xpack: { diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.test.js b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.test.js index 078fe1349c3a34..d1839d03578652 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.test.js +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.test.js @@ -66,7 +66,7 @@ test('sets enabled to false when reporting is turned off', async () => { const serverMock = getServerMock({ config: () => ({ get: mockConfigGet }) }); const callClusterMock = jest.fn(); const { fetch: getReportingUsage } = getReportingUsageCollector(serverMock); - const usageStats = await getReportingUsage(callClusterMock); + const usageStats = await getReportingUsage({ callCluster: callClusterMock }); expect(usageStats.enabled).toBe(false); }); @@ -77,7 +77,7 @@ describe('with a basic license', async () => { serverWithBasicLicenseMock.plugins.xpack_main.info.license.getType = sinon.stub().returns('basic'); const callClusterMock = jest.fn(() => Promise.resolve({})); const { fetch: getReportingUsage } = getReportingUsageCollector(serverWithBasicLicenseMock); - usageStats = await getReportingUsage(callClusterMock); + usageStats = await getReportingUsage({ callCluster: callClusterMock }); }); test('sets enables to true', async () => { @@ -100,7 +100,7 @@ describe('with no license', async () => { serverWithNoLicenseMock.plugins.xpack_main.info.license.getType = sinon.stub().returns('none'); const callClusterMock = jest.fn(() => Promise.resolve({})); const { fetch: getReportingUsage } = getReportingUsageCollector(serverWithNoLicenseMock); - usageStats = await getReportingUsage(callClusterMock); + usageStats = await getReportingUsage({ callCluster: callClusterMock }); }); test('sets enables to true', async () => { @@ -123,7 +123,7 @@ describe('with platinum license', async () => { serverWithPlatinumLicenseMock.plugins.xpack_main.info.license.getType = sinon.stub().returns('platinum'); const callClusterMock = jest.fn(() => Promise.resolve({})); const { fetch: getReportingUsage } = getReportingUsageCollector(serverWithPlatinumLicenseMock); - usageStats = await getReportingUsage(callClusterMock); + usageStats = await getReportingUsage({ callCluster: callClusterMock }); }); test('sets enables to true', async () => { From 72ffae91a9414fc45065819ca51d45baa531f940 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:09:57 -0700 Subject: [PATCH 07/11] remove console.log --- src/server/usage/classes/collector_set.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index 0126894ffd4103..cae5492f5e8b12 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -103,7 +103,6 @@ export class CollectorSet { } const payload = this.getCollectorByType(type).formatForBulkUpload(result); - console.log(JSON.stringify({ payload })); return [ ...accum, payload // TODO flatten it here From a853b09a3575385d63d5caf282e864c1966f5a67 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:51:11 -0700 Subject: [PATCH 08/11] fix the fact that bulkFetch receives a CollectorSet instance, not an array --- src/server/usage/classes/collector_set.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/server/usage/classes/collector_set.js b/src/server/usage/classes/collector_set.js index cae5492f5e8b12..bf79092aa61791 100644 --- a/src/server/usage/classes/collector_set.js +++ b/src/server/usage/classes/collector_set.js @@ -44,7 +44,8 @@ export class CollectorSet { */ this.makeStatsCollector = options => new Collector(server, options); this.makeUsageCollector = options => new UsageCollector(server, options); - this.makeCollectorSetFromArray = collectorsArray => new CollectorSet(server, collectorsArray); + + this._makeCollectorSetFromArray = collectorsArray => new CollectorSet(server, collectorsArray); } /* @@ -73,12 +74,12 @@ export class CollectorSet { * @param {Object} fetchMechanisms - an object with a callCluster function and a savedObjectsClient object * @param {Array} collectors - an array of collectors, default to all registered collectors */ - bulkFetch(fetchMechanisms, collectors = this._collectors) { - if (!Array.isArray(collectors)) { + bulkFetch(fetchMechanisms, collectors = this) { + if (!(collectors instanceof CollectorSet)) { throw new Error(`bulkFetch method given bad collectors parameter: ` + typeof collectors); } - return Promise.map(collectors, collector => { + const fetchPromises = collectors.map(collector => { const collectorType = collector.type; this._log.debug(`Fetching data from ${collectorType} collector`); return Promise.props({ @@ -90,13 +91,10 @@ export class CollectorSet { this._log.warn(`Unable to fetch data from ${collectorType} collector`); }); }); + return Promise.all(fetchPromises); } - bulkFormat(data, collectors = this._collectors) { - if (!Array.isArray(collectors)) { - throw new Error(`bulkFormat method given bad collectors parameter: ` + typeof collectors); - } - + bulkFormat(data) { return data.reduce((accum, { type, result }) => { if (isEmpty(result)) { return accum; @@ -115,7 +113,7 @@ export class CollectorSet { */ getFilteredCollectorSet(filter) { const filtered = this._collectors.filter(filter); - return this.makeCollectorSetFromArray(filtered); + return this._makeCollectorSetFromArray(filtered); } async bulkFetchUsage(fetchMechanisms) { @@ -163,4 +161,8 @@ export class CollectorSet { }; }, {}); } + + map(mapFn) { + return this._collectors.map(mapFn); + } } From c76d82b230e2bfb07b9f97e728c664d526b4b83c Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 13:57:41 -0700 Subject: [PATCH 09/11] roll back some changes to type strings --- .../kibana_monitoring/collectors/get_ops_stats_collector.js | 4 ++-- .../kibana_monitoring/collectors/get_settings_collector.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js index 60f2e7ebc9660e..6214b8eb81998b 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_ops_stats_collector.js @@ -57,14 +57,14 @@ export function getOpsStatsCollector(server, kbnServer) { const { kibana, ...rest } = result; return [ { - type: KIBANA_STATS_TYPE_MONITORING, + type: 'kibana_stats', payload: { kibana, ...rest, } }, { - type: KIBANA_STATS_TYPE_MONITORING + '_settings', + type: 'kibana_settings', payload: { kibana, } diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js index 49faa0032088f2..9ac95e69d054fc 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.js @@ -59,7 +59,7 @@ export function getSettingsCollector(server, kbnServer) { const { collectorSet } = server.usage; return collectorSet.makeStatsCollector({ - type: KIBANA_SETTINGS_TYPE + '_settings_2', + type: KIBANA_SETTINGS_TYPE, async fetch({ callCluster }) { let kibanaSettingsData; const defaultAdminEmail = await checkForEmailValue(config, callCluster); From 878b2cc7c2b9ef8deeb1309aff43c82602815150 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 14:11:16 -0700 Subject: [PATCH 10/11] roll back more type string changes --- .../kibana_monitoring/collectors/get_kibana_usage_collector.js | 2 +- .../reporting/server/usage/get_reporting_usage_collector.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js index 17fedea91ec5b2..2a479bbf3e49b3 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_kibana_usage_collector.js @@ -18,7 +18,7 @@ export function getKibanaUsageCollector(server) { }, formatForBulkUpload: result => { return [{ - type: KIBANA_USAGE_TYPE, + type: 'kibana_stats', payload: { usage: result } diff --git a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js index e3e64a80959f46..ec4468a0f65c48 100644 --- a/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js +++ b/x-pack/plugins/reporting/server/usage/get_reporting_usage_collector.js @@ -150,7 +150,7 @@ export function getReportingUsageCollector(server) { }, formatForBulkUpload: result => { return [{ - type: KIBANA_REPORTING_TYPE, + type: 'kibana_stats', payload: { usage: { xpack: { From b1cc66ca69cc3391164c7873e68aefd4feecfa3d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Fri, 3 Aug 2018 14:13:20 -0700 Subject: [PATCH 11/11] roll back a name change --- .../monitoring/server/kibana_monitoring/bulk_uploader.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js index e727f7414e6707..66589289c81661 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/bulk_uploader.js @@ -100,7 +100,7 @@ export class BulkUploader { callCluster: this._callClusterWithInternalUser, savedObjectsClient: this._savedObjectsClient, }); - const payload = BulkUploader.getCollectedData(data, collectorSet); + const payload = BulkUploader.toBulkUploadFormat(data, collectorSet); if (payload) { try { @@ -150,7 +150,7 @@ export class BulkUploader { * Bulk stats are transformed into a bulk upload format * Non-legacy transformation is done in CollectorSet.toApiStats */ - static getCollectedData(uploadData, collectorSet) { + static toBulkUploadFormat(uploadData, collectorSet) { if (compact(uploadData).length > 0) { return flatten(BulkUploader.deepMergeUploadData(uploadData, collectorSet)); }