From 59c682d7550d0097794f558483be4234fc618969 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Mon, 11 Mar 2019 10:54:51 -0600 Subject: [PATCH 01/18] Add file upload x-pack plugin --- x-pack/index.js | 2 + .../common/constants/file_datavisualizer.ts | 11 + x-pack/plugins/file_upload/index.js | 22 + .../__tests__/elasticsearch_fileupload.js | 58 ++ .../client/call_with_request_factory.js | 23 + .../server/client/elasticsearch_fileupload.js | 624 ++++++++++++++++++ .../file_upload/server/client/errors.js | 13 + .../file_data_visualizer/import_data.js | 171 +++++ .../models/file_data_visualizer/index.js | 8 + .../server/routes/file_data_visualizer.js | 35 + 10 files changed, 967 insertions(+) create mode 100644 x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts create mode 100644 x-pack/plugins/file_upload/index.js create mode 100644 x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js create mode 100644 x-pack/plugins/file_upload/server/client/call_with_request_factory.js create mode 100644 x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js create mode 100644 x-pack/plugins/file_upload/server/client/errors.js create mode 100644 x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js create mode 100644 x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js create mode 100644 x-pack/plugins/file_upload/server/routes/file_data_visualizer.js diff --git a/x-pack/index.js b/x-pack/index.js index 3c9f683ed0796a..23c3190078bcd1 100644 --- a/x-pack/index.js +++ b/x-pack/index.js @@ -37,6 +37,7 @@ import { translations } from './plugins/translations'; import { upgradeAssistant } from './plugins/upgrade_assistant'; import { uptime } from './plugins/uptime'; import { ossTelemetry } from './plugins/oss_telemetry'; +import { fileUpload } from './plugins/file_upload'; module.exports = function (kibana) { return [ @@ -73,5 +74,6 @@ module.exports = function (kibana) { upgradeAssistant(kibana), uptime(kibana), ossTelemetry(kibana), + fileUpload(kibana), ]; }; diff --git a/x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts b/x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts new file mode 100644 index 00000000000000..0d6338e4c3e25d --- /dev/null +++ b/x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const MAX_BYTES = 104857600; + +// Value to use in the Elasticsearch index mapping meta data to identify the +// index as having been created by the File Upload Plugin. +export const INDEX_META_DATA_CREATED_BY = 'file-upload-plugin'; diff --git a/x-pack/plugins/file_upload/index.js b/x-pack/plugins/file_upload/index.js new file mode 100644 index 00000000000000..457f073cf1bb06 --- /dev/null +++ b/x-pack/plugins/file_upload/index.js @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { mirrorPluginStatus } from '../../server/lib/mirror_plugin_status'; +import { fileDataVisualizerRoutes } from './server/routes/file_data_visualizer'; + +export const fileUpload = kibana => { + return new kibana.Plugin({ + require: ['elasticsearch', 'xpack_main'], + name: 'file_upload', + id: 'file_upload', + + init(server) { + const { xpack_main: xpackMainPlugin } = server.plugins; + + mirrorPluginStatus(xpackMainPlugin, this); + fileDataVisualizerRoutes(server); + } + }); +}; diff --git a/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js b/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js new file mode 100644 index 00000000000000..a7eeb8dcf69420 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import expect from 'expect.js'; +import { + elasticsearchJsPlugin +} from '../elasticsearch_fileupload'; + +describe('ML - Endpoints', () => { + + // Check all paths in the ML elasticsearchJsPlugin start with a leading forward slash + // so they work if Kibana is run behind a reverse proxy + const PATH_START = '/'; + const urls = []; + + // Stub objects + const Client = { + prototype: {} + }; + + const components = { + clientAction: { + factory: function (obj) { + // add each endpoint URL to a list + if (obj.urls) { + obj.urls.forEach((url) => { + urls.push(url.fmt); + }); + } + if (obj.url) { + urls.push(obj.url.fmt); + } + }, + namespaceFactory() { + return { + prototype: {} + }; + } + } + }; + + // Stub elasticsearchJsPlugin + elasticsearchJsPlugin(Client, null, components); + + describe('paths', () => { + it(`should start with ${PATH_START}`, () => { + urls.forEach((url) => { + expect(url[0]).to.eql(PATH_START); + }); + }); + }); + +}); diff --git a/x-pack/plugins/file_upload/server/client/call_with_request_factory.js b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js new file mode 100644 index 00000000000000..f984910042d678 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import { once } from 'lodash'; +import { elasticsearchJsPlugin } from './elasticsearch_fileupload'; + +const callWithRequest = once((server) => { + const config = { plugins: [ elasticsearchJsPlugin ] }; + const cluster = server.plugins.elasticsearch.createCluster('fileupload', config); + + return cluster.callWithRequest; +}); + +export const callWithRequestFactory = (server, request) => { + return (...args) => { + return callWithRequest(server)(request, ...args); + }; +}; diff --git a/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js b/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js new file mode 100644 index 00000000000000..8e0b613099a259 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js @@ -0,0 +1,624 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +export const elasticsearchJsPlugin = (Client, config, components) => { + const ca = components.clientAction.factory; + + Client.prototype.fileupload = components.clientAction.namespaceFactory(); + const fileupload = Client.prototype.fileupload.prototype; + + /** + * Perform a [fileupload.authenticate](Retrieve details about the currently authenticated user) request + * + * @param {Object} params - An object with parameters used to carry out this action + */ + fileupload.jobs = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>', + req: { + jobId: { + type: 'list' + } + } + }, + { + fmt: '/_fileupload/anomaly_detectors/', + } + ], + method: 'GET' + }); + + fileupload.jobStats = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_stats', + req: { + jobId: { + type: 'list' + } + } + }, + { + fmt: '/_fileupload/anomaly_detectors/_stats', + } + ], + method: 'GET' + }); + + fileupload.addJob = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>', + req: { + jobId: { + type: 'string' + } + } + } + ], + needBody: true, + method: 'PUT' + }); + + fileupload.openJob = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_open', + req: { + jobId: { + type: 'string' + } + } + } + ], + method: 'POST' + }); + + fileupload.closeJob = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_close?force=<%=force%>', + req: { + jobId: { + type: 'string' + }, + force: { + type: 'boolean' + } + } + }, + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_close', + req: { + jobId: { + type: 'string' + } + } + } + ], + method: 'POST' + }); + + fileupload.deleteJob = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>?&force=<%=force%>&wait_for_completion=false', + req: { + jobId: { + type: 'string' + }, + force: { + type: 'boolean' + } + } + }, + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>?&wait_for_completion=false', + req: { + jobId: { + type: 'string' + } + } + } + ], + method: 'DELETE' + }); + + fileupload.updateJob = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_update', + req: { + jobId: { + type: 'string' + } + } + } + ], + needBody: true, + method: 'POST' + }); + + fileupload.datafeeds = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>', + req: { + datafeedId: { + type: 'list' + } + } + }, + { + fmt: '/_fileupload/datafeeds/', + } + ], + method: 'GET' + }); + + fileupload.datafeedStats = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_stats', + req: { + datafeedId: { + type: 'list' + } + } + }, + { + fmt: '/_fileupload/datafeeds/_stats', + } + ], + method: 'GET' + }); + + fileupload.addDatafeed = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>', + req: { + datafeedId: { + type: 'string' + } + } + } + ], + needBody: true, + method: 'PUT' + }); + + fileupload.updateDatafeed = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_update', + req: { + datafeedId: { + type: 'string' + } + } + } + ], + needBody: true, + method: 'POST' + }); + + fileupload.deleteDatafeed = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>?force=<%=force%>', + req: { + datafeedId: { + type: 'string' + }, + force: { + type: 'boolean' + } + } + }, + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>', + req: { + datafeedId: { + type: 'string' + } + } + } + ], + method: 'DELETE' + }); + + fileupload.startDatafeed = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start?&start=<%=start%>&end=<%=end%>', + req: { + datafeedId: { + type: 'string' + }, + start: { + type: 'string' + }, + end: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start?&start=<%=start%>', + req: { + datafeedId: { + type: 'string' + }, + start: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start', + req: { + datafeedId: { + type: 'string' + } + } + } + ], + method: 'POST' + }); + + fileupload.stopDatafeed = ca({ + urls: [ + { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_stop', + req: { + datafeedId: { + type: 'string' + } + } + } + ], + method: 'POST' + }); + + fileupload.validateDetector = ca({ + url: { + fmt: '/_fileupload/anomaly_detectors/_validate/detector' + }, + needBody: true, + method: 'POST' + }); + + fileupload.datafeedPreview = ca({ + url: { + fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_preview', + req: { + datafeedId: { + type: 'string' + } + } + }, + method: 'GET' + }); + + fileupload.forecast = ca({ + urls: [ + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_forecast?&duration=<%=duration%>', + req: { + jobId: { + type: 'string' + }, + duration: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_forecast', + req: { + jobId: { + type: 'string' + } + } + } + ], + method: 'POST' + }); + + fileupload.overallBuckets = ca({ + url: { + fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/results/overall_buckets', + req: { + jobId: { + type: 'string' + } + } + }, + method: 'POST' + }); + + fileupload.privilegeCheck = ca({ + url: { + fmt: '/_security/user/_has_privileges' + }, + needBody: true, + method: 'POST' + }); + + fileupload.calendars = ca({ + urls: [ + { + fmt: '/_fileupload/calendars/<%=calendarId%>', + req: { + calendarId: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/calendars/', + } + ], + method: 'GET' + }); + + fileupload.deleteCalendar = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>', + req: { + calendarId: { + type: 'string' + } + } + }, + method: 'DELETE' + }); + + fileupload.addCalendar = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>', + req: { + calendarId: { + type: 'string' + } + } + }, + needBody: true, + method: 'PUT' + }); + + fileupload.addJobToCalendar = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>/jobs/<%=jobId%>', + req: { + calendarId: { + type: 'string' + }, + jobId: { + type: 'string' + } + } + }, + method: 'PUT' + }); + + fileupload.removeJobFromCalendar = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>/jobs/<%=jobId%>', + req: { + calendarId: { + type: 'string' + }, + jobId: { + type: 'string' + } + } + }, + method: 'DELETE' + }); + + fileupload.events = ca({ + urls: [ + { + fmt: '/_fileupload/calendars/<%=calendarId%>/events', + req: { + calendarId: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/calendars/<%=calendarId%>/events?&job_id=<%=jobId%>', + req: { + calendarId: { + type: 'string' + }, + jobId: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/calendars/<%=calendarId%>/events?&after=<%=start%>&before=<%=end%>', + req: { + calendarId: { + type: 'string' + }, + start: { + type: 'string' + }, + end: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/calendars/<%=calendarId%>/events?&after=<%=start%>&before=<%=end%>&job_id=<%=jobId%>', + req: { + calendarId: { + type: 'string' + }, + start: { + type: 'string' + }, + end: { + type: 'string' + }, + jobId: { + type: 'string' + } + } + } + ], + method: 'GET' + }); + + fileupload.addEvent = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>/events', + req: { + calendarId: { + type: 'string' + } + } + }, + needBody: true, + method: 'POST' + }); + + fileupload.deleteEvent = ca({ + url: { + fmt: '/_fileupload/calendars/<%=calendarId%>/events/<%=eventId%>', + req: { + calendarId: { + type: 'string' + }, + eventId: { + type: 'string' + } + } + }, + method: 'DELETE' + }); + + fileupload.filters = ca({ + urls: [ + { + fmt: '/_fileupload/filters/<%=filterId%>', + req: { + filterId: { + type: 'string' + } + } + }, + { + fmt: '/_fileupload/filters/', + } + ], + method: 'GET' + }); + + fileupload.addFilter = ca({ + url: { + fmt: '/_fileupload/filters/<%=filterId%>', + req: { + filterId: { + type: 'string' + } + } + }, + needBody: true, + method: 'PUT' + }); + + fileupload.updateFilter = ca({ + urls: [ + { + fmt: '/_fileupload/filters/<%=filterId%>/_update', + req: { + filterId: { + type: 'string' + } + } + } + ], + needBody: true, + method: 'POST' + }); + + fileupload.deleteFilter = ca({ + url: { + fmt: '/_fileupload/filters/<%=filterId%>', + req: { + filterId: { + type: 'string' + } + } + }, + method: 'DELETE' + }); + + fileupload.info = ca({ + url: { + fmt: '/_fileupload/info' + }, + method: 'GET' + }); + + fileupload.fileStructure = ca({ + urls: [ + { + // eslint-disable-next-line max-len + fmt: '/_fileupload/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', + req: { + charset: { + type: 'string' + }, + format: { + type: 'string' + }, + has_header_row: { + type: 'string' + }, + column_names: { + type: 'string' + }, + delimiter: { + type: 'string' + }, + quote: { + type: 'string' + }, + should_trim_fields: { + type: 'string' + }, + grok_pattern: { + type: 'string' + }, + timestamp_field: { + type: 'string' + }, + timestamp_format: { + type: 'string' + }, + lines_to_sample: { + type: 'string' + }, + } + }, + { + fmt: '/_fileupload/find_file_structure' + } + ], + needBody: true, + method: 'POST' + }); + +}; diff --git a/x-pack/plugins/file_upload/server/client/errors.js b/x-pack/plugins/file_upload/server/client/errors.js new file mode 100644 index 00000000000000..98da148192a898 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/errors.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import { boomify } from 'boom'; + +export function wrapError(error) { + return boomify(error, { statusCode: error.status }); +} diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js new file mode 100644 index 00000000000000..b3a800d3382014 --- /dev/null +++ b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; + +export function importDataProvider(callWithRequest) { + async function importData(id, index, settings, mappings, ingestPipeline, data) { + let createdIndex; + let createdPipelineId; + const docCount = data.length; + + try { + + const { + id: pipelineId, + pipeline, + } = ingestPipeline; + + if (id === undefined) { + // first chunk of data, create the index and id to return + id = generateId(); + + await createIndex(index, settings, mappings); + createdIndex = index; + + // create the pipeline if one has been supplied + if (pipelineId !== undefined) { + const success = await createPipeline(pipelineId, pipeline); + if (success.acknowledged !== true) { + throw success; + } + } + createdPipelineId = pipelineId; + + } else { + createdIndex = index; + createdPipelineId = pipelineId; + } + + let failures = []; + if (data.length) { + const resp = await indexData(index, createdPipelineId, data); + if (resp.success === false) { + if (resp.ingestError) { + // all docs failed, abort + throw resp; + } else { + // some docs failed. + // still report success but with a list of failures + failures = (resp.failures || []); + } + } + } + + return { + success: true, + id, + index: createdIndex, + pipelineId: createdPipelineId, + docCount, + failures, + }; + } catch (error) { + return { + success: false, + id, + index: createdIndex, + pipelineId: createdPipelineId, + error: (error.error !== undefined) ? error.error : error, + docCount, + ingestError: error.ingestError, + failures: (error.failures || []) + }; + } + } + + async function createIndex(index, settings, mappings) { + const body = { + mappings: { + _meta: { + created_by: INDEX_META_DATA_CREATED_BY + }, + properties: mappings + } + }; + + if (settings && Object.keys(settings).length) { + body.settings = settings; + } + + await callWithRequest('indices.create', { index, body }); + } + + async function indexData(index, pipelineId, data) { + try { + const body = []; + for (let i = 0; i < data.length; i++) { + body.push({ index: {} }); + body.push(data[i]); + } + + const settings = { index, body }; + if (pipelineId !== undefined) { + settings.pipeline = pipelineId; + } + + const resp = await callWithRequest('bulk', settings); + if (resp.errors) { + throw resp; + } else { + return { + success: true, + docs: data.length, + failures: [], + }; + } + } catch (error) { + + let failures = []; + let ingestError = false; + if (error.errors !== undefined && Array.isArray(error.items)) { + // an expected error where some or all of the bulk request + // docs have failed to be ingested. + failures = getFailures(error.items, data); + } else { + // some other error has happened. + ingestError = true; + } + + return { + success: false, + error, + docCount: data.length, + failures, + ingestError, + }; + } + + } + + async function createPipeline(id, pipeline) { + return await callWithRequest('ingest.putPipeline', { id, body: pipeline }); + } + + function getFailures(items, data) { + const failures = []; + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (item.index && item.index.error) { + failures.push({ + item: i, + reason: item.index.error.reason, + doc: data[i], + }); + } + } + return failures; + } + + return { + importData, + }; +} + + +function generateId() { + return Math.random().toString(36).substr(2, 9); +} diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js new file mode 100644 index 00000000000000..e91b658c4358bf --- /dev/null +++ b/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js b/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js new file mode 100644 index 00000000000000..12c5c386b39ae1 --- /dev/null +++ b/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { callWithRequestFactory } from '../client/call_with_request_factory'; +import { wrapError } from '../client/errors'; +import { importDataProvider } from '../models/file_data_visualizer'; +import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; + + +function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { + const { importData: importDataFunc } = importDataProvider(callWithRequest); + return importDataFunc(id, index, settings, mappings, ingestPipeline, data); +} + +export function fileDataVisualizerRoutes(server, commonRouteConfig) { + server.route({ + method: 'POST', + path: '/api/fileupload/import', + handler(request) { + const callWithRequest = callWithRequestFactory(server, request); + const { id } = request.query; + const { index, data, settings, mappings, ingestPipeline } = request.payload; + + return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) + .catch(wrapError); + }, + config: { + ...commonRouteConfig, + payload: { maxBytes: MAX_BYTES }, + } + }); +} From 9abbc67eaee6f493063978a266260f833b055e08 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Mon, 11 Mar 2019 11:23:47 -0600 Subject: [PATCH 02/18] Clean up --- .../common/constants/{file_datavisualizer.ts => file_import.ts} | 0 .../server/client/__tests__/elasticsearch_fileupload.js | 2 +- .../server/models/file_data_visualizer/import_data.js | 2 +- .../plugins/file_upload/server/routes/file_data_visualizer.js | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename x-pack/plugins/file_upload/common/constants/{file_datavisualizer.ts => file_import.ts} (100%) diff --git a/x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts b/x-pack/plugins/file_upload/common/constants/file_import.ts similarity index 100% rename from x-pack/plugins/file_upload/common/constants/file_datavisualizer.ts rename to x-pack/plugins/file_upload/common/constants/file_import.ts diff --git a/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js b/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js index a7eeb8dcf69420..7dafe7c396b913 100644 --- a/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js +++ b/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js @@ -11,7 +11,7 @@ import { elasticsearchJsPlugin } from '../elasticsearch_fileupload'; -describe('ML - Endpoints', () => { +describe('File upload - Endpoints', () => { // Check all paths in the ML elasticsearchJsPlugin start with a leading forward slash // so they work if Kibana is run behind a reverse proxy diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js index b3a800d3382014..659e36f2513d5a 100644 --- a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js +++ b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_import'; export function importDataProvider(callWithRequest) { async function importData(id, index, settings, mappings, ingestPipeline, data) { diff --git a/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js b/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js index 12c5c386b39ae1..4312664c03eb78 100644 --- a/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js +++ b/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js @@ -7,7 +7,7 @@ import { callWithRequestFactory } from '../client/call_with_request_factory'; import { wrapError } from '../client/errors'; import { importDataProvider } from '../models/file_data_visualizer'; -import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; +import { MAX_BYTES } from '../../common/constants/file_import'; function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { From 13c187aa75ccafe3a1921c14aed1eb05d3ceaafc Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Tue, 12 Mar 2019 08:16:20 -0600 Subject: [PATCH 03/18] Remove unneeded cluster config --- .../client/call_with_request_factory.js | 5 +- .../server/client/elasticsearch_fileupload.js | 624 ------------------ 2 files changed, 1 insertion(+), 628 deletions(-) delete mode 100644 x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js diff --git a/x-pack/plugins/file_upload/server/client/call_with_request_factory.js b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js index f984910042d678..a42d3c4d661494 100644 --- a/x-pack/plugins/file_upload/server/client/call_with_request_factory.js +++ b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js @@ -7,12 +7,9 @@ import { once } from 'lodash'; -import { elasticsearchJsPlugin } from './elasticsearch_fileupload'; const callWithRequest = once((server) => { - const config = { plugins: [ elasticsearchJsPlugin ] }; - const cluster = server.plugins.elasticsearch.createCluster('fileupload', config); - + const cluster = server.plugins.elasticsearch.createCluster('fileupload'); return cluster.callWithRequest; }); diff --git a/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js b/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js deleted file mode 100644 index 8e0b613099a259..00000000000000 --- a/x-pack/plugins/file_upload/server/client/elasticsearch_fileupload.js +++ /dev/null @@ -1,624 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - - -export const elasticsearchJsPlugin = (Client, config, components) => { - const ca = components.clientAction.factory; - - Client.prototype.fileupload = components.clientAction.namespaceFactory(); - const fileupload = Client.prototype.fileupload.prototype; - - /** - * Perform a [fileupload.authenticate](Retrieve details about the currently authenticated user) request - * - * @param {Object} params - An object with parameters used to carry out this action - */ - fileupload.jobs = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>', - req: { - jobId: { - type: 'list' - } - } - }, - { - fmt: '/_fileupload/anomaly_detectors/', - } - ], - method: 'GET' - }); - - fileupload.jobStats = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_stats', - req: { - jobId: { - type: 'list' - } - } - }, - { - fmt: '/_fileupload/anomaly_detectors/_stats', - } - ], - method: 'GET' - }); - - fileupload.addJob = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>', - req: { - jobId: { - type: 'string' - } - } - } - ], - needBody: true, - method: 'PUT' - }); - - fileupload.openJob = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_open', - req: { - jobId: { - type: 'string' - } - } - } - ], - method: 'POST' - }); - - fileupload.closeJob = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_close?force=<%=force%>', - req: { - jobId: { - type: 'string' - }, - force: { - type: 'boolean' - } - } - }, - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_close', - req: { - jobId: { - type: 'string' - } - } - } - ], - method: 'POST' - }); - - fileupload.deleteJob = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>?&force=<%=force%>&wait_for_completion=false', - req: { - jobId: { - type: 'string' - }, - force: { - type: 'boolean' - } - } - }, - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>?&wait_for_completion=false', - req: { - jobId: { - type: 'string' - } - } - } - ], - method: 'DELETE' - }); - - fileupload.updateJob = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_update', - req: { - jobId: { - type: 'string' - } - } - } - ], - needBody: true, - method: 'POST' - }); - - fileupload.datafeeds = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>', - req: { - datafeedId: { - type: 'list' - } - } - }, - { - fmt: '/_fileupload/datafeeds/', - } - ], - method: 'GET' - }); - - fileupload.datafeedStats = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_stats', - req: { - datafeedId: { - type: 'list' - } - } - }, - { - fmt: '/_fileupload/datafeeds/_stats', - } - ], - method: 'GET' - }); - - fileupload.addDatafeed = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>', - req: { - datafeedId: { - type: 'string' - } - } - } - ], - needBody: true, - method: 'PUT' - }); - - fileupload.updateDatafeed = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_update', - req: { - datafeedId: { - type: 'string' - } - } - } - ], - needBody: true, - method: 'POST' - }); - - fileupload.deleteDatafeed = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>?force=<%=force%>', - req: { - datafeedId: { - type: 'string' - }, - force: { - type: 'boolean' - } - } - }, - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>', - req: { - datafeedId: { - type: 'string' - } - } - } - ], - method: 'DELETE' - }); - - fileupload.startDatafeed = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start?&start=<%=start%>&end=<%=end%>', - req: { - datafeedId: { - type: 'string' - }, - start: { - type: 'string' - }, - end: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start?&start=<%=start%>', - req: { - datafeedId: { - type: 'string' - }, - start: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_start', - req: { - datafeedId: { - type: 'string' - } - } - } - ], - method: 'POST' - }); - - fileupload.stopDatafeed = ca({ - urls: [ - { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_stop', - req: { - datafeedId: { - type: 'string' - } - } - } - ], - method: 'POST' - }); - - fileupload.validateDetector = ca({ - url: { - fmt: '/_fileupload/anomaly_detectors/_validate/detector' - }, - needBody: true, - method: 'POST' - }); - - fileupload.datafeedPreview = ca({ - url: { - fmt: '/_fileupload/datafeeds/<%=datafeedId%>/_preview', - req: { - datafeedId: { - type: 'string' - } - } - }, - method: 'GET' - }); - - fileupload.forecast = ca({ - urls: [ - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_forecast?&duration=<%=duration%>', - req: { - jobId: { - type: 'string' - }, - duration: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/_forecast', - req: { - jobId: { - type: 'string' - } - } - } - ], - method: 'POST' - }); - - fileupload.overallBuckets = ca({ - url: { - fmt: '/_fileupload/anomaly_detectors/<%=jobId%>/results/overall_buckets', - req: { - jobId: { - type: 'string' - } - } - }, - method: 'POST' - }); - - fileupload.privilegeCheck = ca({ - url: { - fmt: '/_security/user/_has_privileges' - }, - needBody: true, - method: 'POST' - }); - - fileupload.calendars = ca({ - urls: [ - { - fmt: '/_fileupload/calendars/<%=calendarId%>', - req: { - calendarId: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/calendars/', - } - ], - method: 'GET' - }); - - fileupload.deleteCalendar = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>', - req: { - calendarId: { - type: 'string' - } - } - }, - method: 'DELETE' - }); - - fileupload.addCalendar = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>', - req: { - calendarId: { - type: 'string' - } - } - }, - needBody: true, - method: 'PUT' - }); - - fileupload.addJobToCalendar = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>/jobs/<%=jobId%>', - req: { - calendarId: { - type: 'string' - }, - jobId: { - type: 'string' - } - } - }, - method: 'PUT' - }); - - fileupload.removeJobFromCalendar = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>/jobs/<%=jobId%>', - req: { - calendarId: { - type: 'string' - }, - jobId: { - type: 'string' - } - } - }, - method: 'DELETE' - }); - - fileupload.events = ca({ - urls: [ - { - fmt: '/_fileupload/calendars/<%=calendarId%>/events', - req: { - calendarId: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/calendars/<%=calendarId%>/events?&job_id=<%=jobId%>', - req: { - calendarId: { - type: 'string' - }, - jobId: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/calendars/<%=calendarId%>/events?&after=<%=start%>&before=<%=end%>', - req: { - calendarId: { - type: 'string' - }, - start: { - type: 'string' - }, - end: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/calendars/<%=calendarId%>/events?&after=<%=start%>&before=<%=end%>&job_id=<%=jobId%>', - req: { - calendarId: { - type: 'string' - }, - start: { - type: 'string' - }, - end: { - type: 'string' - }, - jobId: { - type: 'string' - } - } - } - ], - method: 'GET' - }); - - fileupload.addEvent = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>/events', - req: { - calendarId: { - type: 'string' - } - } - }, - needBody: true, - method: 'POST' - }); - - fileupload.deleteEvent = ca({ - url: { - fmt: '/_fileupload/calendars/<%=calendarId%>/events/<%=eventId%>', - req: { - calendarId: { - type: 'string' - }, - eventId: { - type: 'string' - } - } - }, - method: 'DELETE' - }); - - fileupload.filters = ca({ - urls: [ - { - fmt: '/_fileupload/filters/<%=filterId%>', - req: { - filterId: { - type: 'string' - } - } - }, - { - fmt: '/_fileupload/filters/', - } - ], - method: 'GET' - }); - - fileupload.addFilter = ca({ - url: { - fmt: '/_fileupload/filters/<%=filterId%>', - req: { - filterId: { - type: 'string' - } - } - }, - needBody: true, - method: 'PUT' - }); - - fileupload.updateFilter = ca({ - urls: [ - { - fmt: '/_fileupload/filters/<%=filterId%>/_update', - req: { - filterId: { - type: 'string' - } - } - } - ], - needBody: true, - method: 'POST' - }); - - fileupload.deleteFilter = ca({ - url: { - fmt: '/_fileupload/filters/<%=filterId%>', - req: { - filterId: { - type: 'string' - } - } - }, - method: 'DELETE' - }); - - fileupload.info = ca({ - url: { - fmt: '/_fileupload/info' - }, - method: 'GET' - }); - - fileupload.fileStructure = ca({ - urls: [ - { - // eslint-disable-next-line max-len - fmt: '/_fileupload/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', - req: { - charset: { - type: 'string' - }, - format: { - type: 'string' - }, - has_header_row: { - type: 'string' - }, - column_names: { - type: 'string' - }, - delimiter: { - type: 'string' - }, - quote: { - type: 'string' - }, - should_trim_fields: { - type: 'string' - }, - grok_pattern: { - type: 'string' - }, - timestamp_field: { - type: 'string' - }, - timestamp_format: { - type: 'string' - }, - lines_to_sample: { - type: 'string' - }, - } - }, - { - fmt: '/_fileupload/find_file_structure' - } - ], - needBody: true, - method: 'POST' - }); - -}; From bd921a764347fb80e64c05ed777917ab0881586e Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Tue, 12 Mar 2019 16:43:02 -0600 Subject: [PATCH 04/18] Remove unneeded test --- .../__tests__/elasticsearch_fileupload.js | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js diff --git a/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js b/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js deleted file mode 100644 index 7dafe7c396b913..00000000000000 --- a/x-pack/plugins/file_upload/server/client/__tests__/elasticsearch_fileupload.js +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - - -import expect from 'expect.js'; -import { - elasticsearchJsPlugin -} from '../elasticsearch_fileupload'; - -describe('File upload - Endpoints', () => { - - // Check all paths in the ML elasticsearchJsPlugin start with a leading forward slash - // so they work if Kibana is run behind a reverse proxy - const PATH_START = '/'; - const urls = []; - - // Stub objects - const Client = { - prototype: {} - }; - - const components = { - clientAction: { - factory: function (obj) { - // add each endpoint URL to a list - if (obj.urls) { - obj.urls.forEach((url) => { - urls.push(url.fmt); - }); - } - if (obj.url) { - urls.push(obj.url.fmt); - } - }, - namespaceFactory() { - return { - prototype: {} - }; - } - } - }; - - // Stub elasticsearchJsPlugin - elasticsearchJsPlugin(Client, null, components); - - describe('paths', () => { - it(`should start with ${PATH_START}`, () => { - urls.forEach((url) => { - expect(url[0]).to.eql(PATH_START); - }); - }); - }); - -}); From 444565879dc02caa72f3b7783c957168ca2bffe2 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Tue, 12 Mar 2019 16:50:07 -0600 Subject: [PATCH 05/18] First pass basic telemetry (not connected). --- .../call_with_internal_user_factory.d.ts | 9 + .../client/call_with_internal_user_factory.js | 20 ++ .../call_with_internal_user_factory.test.ts | 32 +++ .../file_upload/server/telemetry/index.ts | 8 + .../server/telemetry/make_usage_collector.ts | 44 ++++ .../server/telemetry/telemetry.test.ts | 197 ++++++++++++++++++ .../file_upload/server/telemetry/telemetry.ts | 73 +++++++ 7 files changed, 383 insertions(+) create mode 100644 x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.d.ts create mode 100644 x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.js create mode 100644 x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.test.ts create mode 100644 x-pack/plugins/file_upload/server/telemetry/index.ts create mode 100644 x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts create mode 100644 x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts create mode 100644 x-pack/plugins/file_upload/server/telemetry/telemetry.ts diff --git a/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.d.ts b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.d.ts new file mode 100644 index 00000000000000..0b39c81cee6ff9 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.d.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; + +export function callWithInternalUserFactory(server: Server): any; diff --git a/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.js b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.js new file mode 100644 index 00000000000000..dc3131484e75fe --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.js @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + + +import { once } from 'lodash'; + +const _callWithInternalUser = once((server) => { + const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin'); + return callWithInternalUser; +}); + +export const callWithInternalUserFactory = (server) => { + return (...args) => { + return _callWithInternalUser(server)(...args); + }; +}; diff --git a/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.test.ts b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.test.ts new file mode 100644 index 00000000000000..d77541e7d3d6c1 --- /dev/null +++ b/x-pack/plugins/file_upload/server/client/call_with_internal_user_factory.test.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { callWithInternalUserFactory } from './call_with_internal_user_factory'; + +describe('call_with_internal_user_factory', () => { + describe('callWithInternalUserFactory', () => { + let server: any; + let callWithInternalUser: any; + + beforeEach(() => { + callWithInternalUser = jest.fn(); + server = { + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, + }; + }); + + it('should use internal user "admin"', () => { + const callWithInternalUserInstance = callWithInternalUserFactory(server); + callWithInternalUserInstance(); + + expect(server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith('admin'); + }); + }); +}); diff --git a/x-pack/plugins/file_upload/server/telemetry/index.ts b/x-pack/plugins/file_upload/server/telemetry/index.ts new file mode 100644 index 00000000000000..d05f7cc63c8967 --- /dev/null +++ b/x-pack/plugins/file_upload/server/telemetry/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './telemetry'; +export { makeUsageCollector } from './make_usage_collector'; diff --git a/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts new file mode 100644 index 00000000000000..67aa9a66120fe0 --- /dev/null +++ b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; + +import { + createTelemetry, + getSavedObjectsClient, + Telemetry, + TELEMETRY_DOC_ID, + TelemetrySavedObject, +} from './telemetry'; + +// TODO this type should be defined by the platform +interface KibanaHapiServer extends Server { + usage: { + collectorSet: { + makeUsageCollector: any; + register: any; + }; + }; +} + +export function makeUsageCollector(server: KibanaHapiServer): void { + const fileUploadUsageCollector = server.usage.collectorSet.makeUsageCollector({ + type: 'fileUpload', + fetch: async (): Promise => { + try { + const savedObjectsClient = getSavedObjectsClient(server); + const telemetrySavedObject = (await savedObjectsClient.get( + 'file-upload-telemetry', + TELEMETRY_DOC_ID + )) as TelemetrySavedObject; + return telemetrySavedObject.attributes; + } catch (err) { + return createTelemetry(); + } + }, + }); + server.usage.collectorSet.register(fileUploadUsageCollector); +} diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts new file mode 100644 index 00000000000000..1f19edaeb229ba --- /dev/null +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -0,0 +1,197 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + createTelemetry, + getSavedObjectsClient, + incrementFileDataVisualizerIndexCreationCount, + storeTelemetry, + Telemetry, + TELEMETRY_DOC_ID, +} from './telemetry'; + +describe('file_upload_telemetry', () => { + describe('createTelemetry', () => { + it('should create a file upload telemetry object', () => { + const fileUploadTelemetry = createTelemetry(1); + expect(fileUploadTelemetry.file_upload.index_creation_count).toBe(1); + }); + it('should ignore undefined or unknown values', () => { + const fileUploadTelemetry = createTelemetry(undefined); + expect(fileUploadTelemetry.file_upload.index_creation_count).toBe(0); + }); + }); + + describe('storeTelemetry', () => { + let server: any; + let fileUploadTelemetry: Telemetry; + let savedObjectsClientInstance: any; + + beforeEach(() => { + savedObjectsClientInstance = { create: jest.fn() }; + const callWithInternalUser = jest.fn(); + const internalRepository = jest.fn(); + server = { + savedObjects: { + SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), + getSavedObjectsRepository: jest.fn(() => internalRepository), + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, + }; + fileUploadTelemetry = { + file_upload: { + index_creation_count: 1, + }, + }; + }); + + it('should call savedObjectsClient create with the given Telemetry object', () => { + storeTelemetry(server, fileUploadTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][1]).toBe(fileUploadTelemetry); + }); + + it('should call savedObjectsClient create with the file-upload-telemetry document type and ID', () => { + storeTelemetry(server, fileUploadTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); + expect(savedObjectsClientInstance.create.mock.calls[0][2].id).toBe(TELEMETRY_DOC_ID); + }); + + it('should call savedObjectsClient create with overwrite: true', () => { + storeTelemetry(server, fileUploadTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][2].overwrite).toBe(true); + }); + }); + + describe('getSavedObjectsClient', () => { + let server: any; + let savedObjectsClientInstance: any; + let callWithInternalUser: any; + let internalRepository: any; + + beforeEach(() => { + savedObjectsClientInstance = { create: jest.fn() }; + callWithInternalUser = jest.fn(); + internalRepository = jest.fn(); + server = { + savedObjects: { + SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), + getSavedObjectsRepository: jest.fn(() => internalRepository), + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, + }; + }); + + it('should return a SavedObjectsClient initialized with the saved objects internal repository', () => { + const result = getSavedObjectsClient(server); + + expect(result).toBe(savedObjectsClientInstance); + expect(server.savedObjects.SavedObjectsClient).toHaveBeenCalledWith(internalRepository); + }); + }); + + describe('incrementFileImportCount', () => { + let server: any; + let savedObjectsClientInstance: any; + let callWithInternalUser: any; + let internalRepository: any; + + function createSavedObjectsClientInstance( + telemetryEnabled?: boolean, + indexCreationCount?: number + ) { + return { + create: jest.fn(), + get: jest.fn(obj => { + switch (obj) { + case 'telemetry': + if (telemetryEnabled === undefined) { + throw Error; + } + return { + attributes: { + import_telemetry: { + enabled: telemetryEnabled, + }, + }, + }; + case 'file-upload-telemetry': + // emulate that a non-existing saved object will throw an error + if (indexCreationCount === undefined) { + throw Error; + } + return { + file_upload: { + index_creation_count: indexCreationCount, + }, + }; + } + }), + }; + } + + function mockInit(telemetryEnabled?: boolean, indexCreationCount?: number): void { + savedObjectsClientInstance = createSavedObjectsClientInstance( + telemetryEnabled, + indexCreationCount + ); + callWithInternalUser = jest.fn(); + internalRepository = jest.fn(); + server = { + savedObjects: { + SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), + getSavedObjectsRepository: jest.fn(() => internalRepository), + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, + }; + } + + it('should not increment if telemetry status cannot be determined', async () => { + mockInit(); + await incrementFileDataVisualizerIndexCreationCount(server); + + expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0); + }); + + it('should not increment if telemetry status is disabled', async () => { + mockInit(false); + await incrementFileDataVisualizerIndexCreationCount(server); + + expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0); + }); + + it('should initialize index_creation_count with 1', async () => { + mockInit(true); + await incrementFileDataVisualizerIndexCreationCount(server); + + expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); + expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({ + file_upload: { index_creation_count: 1 }, + }); + }); + + it('should increment index_creation_count to 2', async () => { + mockInit(true, 1); + await incrementFileDataVisualizerIndexCreationCount(server); + + expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); + expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({ + file_upload: { index_creation_count: 2 }, + }); + }); + }); +}); diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts new file mode 100644 index 00000000000000..d727e10f515cb1 --- /dev/null +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; +import { callWithInternalUserFactory } from '../client/call_with_internal_user_factory'; + +export const TELEMETRY_DOC_ID = 'file-upload-telemetry'; + +export interface Telemetry { + file_upload: { + index_creation_count: number; + }; +} + +export interface TelemetrySavedObject { + attributes: Telemetry; +} + +export function createTelemetry(count: number = 0): Telemetry { + return { + file_upload: { + index_creation_count: count, + }, + }; +} + +export function storeTelemetry(server: Server, fileUploadTelemetry: Telemetry): void { + const savedObjectsClient = getSavedObjectsClient(server); + savedObjectsClient.create('telemetry', fileUploadTelemetry, { + id: TELEMETRY_DOC_ID, + overwrite: true, + }); +} + +export function getSavedObjectsClient(server: Server): any { + const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects; + const callWithInternalUser = callWithInternalUserFactory(server); + const internalRepository = getSavedObjectsRepository(callWithInternalUser); + return new SavedObjectsClient(internalRepository); +} + +export async function incrementFileDataVisualizerIndexCreationCount(server: Server): Promise { + const savedObjectsClient = getSavedObjectsClient(server); + + try { + const { attributes } = await savedObjectsClient.get('telemetry', 'telemetry'); + if (attributes.telemetry.enabled === false) { + return; + } + } catch (error) { + // if we aren't allowed to get the telemetry document, + // we assume we couldn't opt in to telemetry and won't increment the index count. + return; + } + + let indicesCount = 1; + + try { + const { attributes } = (await savedObjectsClient.get( + 'telemetry', + TELEMETRY_DOC_ID + )) as TelemetrySavedObject; + indicesCount = attributes.file_upload.index_creation_count + 1; + } catch (e) { + /* silently fail, this will happen if the saved object doesn't exist yet. */ + } + + const fileUploadTelemetry = createTelemetry(indicesCount); + storeTelemetry(server, fileUploadTelemetry); +} From be3ab95b1465878c7b1059e3e8686e7a3446038e Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 13 Mar 2019 15:33:21 -0600 Subject: [PATCH 06/18] Basic telemetry connected --- x-pack/plugins/file_upload/index.js | 10 +++++-- x-pack/plugins/file_upload/mappings.json | 9 ++++++ ...file_data_visualizer.js => file_upload.js} | 10 ++++++- .../file_upload/server/telemetry/telemetry.ts | 28 ++++--------------- 4 files changed, 32 insertions(+), 25 deletions(-) create mode 100644 x-pack/plugins/file_upload/mappings.json rename x-pack/plugins/file_upload/server/routes/{file_data_visualizer.js => file_upload.js} (72%) diff --git a/x-pack/plugins/file_upload/index.js b/x-pack/plugins/file_upload/index.js index 457f073cf1bb06..adcc5544fc5a53 100644 --- a/x-pack/plugins/file_upload/index.js +++ b/x-pack/plugins/file_upload/index.js @@ -4,19 +4,25 @@ * you may not use this file except in compliance with the Elastic License. */ import { mirrorPluginStatus } from '../../server/lib/mirror_plugin_status'; -import { fileDataVisualizerRoutes } from './server/routes/file_data_visualizer'; +import { fileUploadRoutes } from './server/routes/file_upload'; +import { makeUsageCollector } from './server/telemetry/'; +import mappings from './mappings'; export const fileUpload = kibana => { return new kibana.Plugin({ require: ['elasticsearch', 'xpack_main'], name: 'file_upload', id: 'file_upload', + uiExports: { + mappings, + }, init(server) { const { xpack_main: xpackMainPlugin } = server.plugins; mirrorPluginStatus(xpackMainPlugin, this); - fileDataVisualizerRoutes(server); + fileUploadRoutes(server); + makeUsageCollector(server); } }); }; diff --git a/x-pack/plugins/file_upload/mappings.json b/x-pack/plugins/file_upload/mappings.json new file mode 100644 index 00000000000000..f3a33937cb2bc9 --- /dev/null +++ b/x-pack/plugins/file_upload/mappings.json @@ -0,0 +1,9 @@ +{ + "file-upload-telemetry": { + "properties": { + "index_creation_count": { + "type" : "long" + } + } + } + } diff --git a/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js b/x-pack/plugins/file_upload/server/routes/file_upload.js similarity index 72% rename from x-pack/plugins/file_upload/server/routes/file_data_visualizer.js rename to x-pack/plugins/file_upload/server/routes/file_upload.js index 4312664c03eb78..d924aea91c8556 100644 --- a/x-pack/plugins/file_upload/server/routes/file_data_visualizer.js +++ b/x-pack/plugins/file_upload/server/routes/file_upload.js @@ -8,6 +8,7 @@ import { callWithRequestFactory } from '../client/call_with_request_factory'; import { wrapError } from '../client/errors'; import { importDataProvider } from '../models/file_data_visualizer'; import { MAX_BYTES } from '../../common/constants/file_import'; +import { incrementIndexCreationCount } from '../telemetry/telemetry'; function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { @@ -15,7 +16,7 @@ function importData(callWithRequest, id, index, settings, mappings, ingestPipeli return importDataFunc(id, index, settings, mappings, ingestPipeline, data); } -export function fileDataVisualizerRoutes(server, commonRouteConfig) { +export function fileUploadRoutes(server, commonRouteConfig) { server.route({ method: 'POST', path: '/api/fileupload/import', @@ -24,6 +25,13 @@ export function fileDataVisualizerRoutes(server, commonRouteConfig) { const { id } = request.query; const { index, data, settings, mappings, ingestPipeline } = request.payload; + // `id` being `undefined` tells us that this is a new import due to create a new index. + // follow-up import calls to just add additional data will include the `id` of the created + // index, we'll ignore those and don't increment the counter. + if (id === undefined) { + incrementIndexCreationCount(server); + } + return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) .catch(wrapError); }, diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index d727e10f515cb1..a4bb10dca1b15f 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -10,9 +10,7 @@ import { callWithInternalUserFactory } from '../client/call_with_internal_user_f export const TELEMETRY_DOC_ID = 'file-upload-telemetry'; export interface Telemetry { - file_upload: { - index_creation_count: number; - }; + index_creation_count: number; } export interface TelemetrySavedObject { @@ -21,15 +19,13 @@ export interface TelemetrySavedObject { export function createTelemetry(count: number = 0): Telemetry { return { - file_upload: { - index_creation_count: count, - }, + index_creation_count: count, }; } export function storeTelemetry(server: Server, fileUploadTelemetry: Telemetry): void { const savedObjectsClient = getSavedObjectsClient(server); - savedObjectsClient.create('telemetry', fileUploadTelemetry, { + savedObjectsClient.create('file-upload-telemetry', fileUploadTelemetry, { id: TELEMETRY_DOC_ID, overwrite: true, }); @@ -42,28 +38,16 @@ export function getSavedObjectsClient(server: Server): any { return new SavedObjectsClient(internalRepository); } -export async function incrementFileDataVisualizerIndexCreationCount(server: Server): Promise { +export async function incrementIndexCreationCount(server: Server): Promise { const savedObjectsClient = getSavedObjectsClient(server); - - try { - const { attributes } = await savedObjectsClient.get('telemetry', 'telemetry'); - if (attributes.telemetry.enabled === false) { - return; - } - } catch (error) { - // if we aren't allowed to get the telemetry document, - // we assume we couldn't opt in to telemetry and won't increment the index count. - return; - } - let indicesCount = 1; try { const { attributes } = (await savedObjectsClient.get( - 'telemetry', + 'file-upload-telemetry', TELEMETRY_DOC_ID )) as TelemetrySavedObject; - indicesCount = attributes.file_upload.index_creation_count + 1; + indicesCount = attributes.index_creation_count + 1; } catch (e) { /* silently fail, this will happen if the saved object doesn't exist yet. */ } From 85c74bc836a4cb8398538f7e623bb9791bfb74d8 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 13 Mar 2019 18:20:58 -0600 Subject: [PATCH 07/18] Review feedback --- .../plugins/file_upload/common/constants/file_import.ts | 2 +- .../server/client/call_with_request_factory.js | 2 +- .../server/models/file_data_visualizer/import_data.js | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/file_upload/common/constants/file_import.ts b/x-pack/plugins/file_upload/common/constants/file_import.ts index 0d6338e4c3e25d..1f9dbd154032dc 100644 --- a/x-pack/plugins/file_upload/common/constants/file_import.ts +++ b/x-pack/plugins/file_upload/common/constants/file_import.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -export const MAX_BYTES = 104857600; +export const MAX_BYTES = 10485760; // Value to use in the Elasticsearch index mapping meta data to identify the // index as having been created by the File Upload Plugin. diff --git a/x-pack/plugins/file_upload/server/client/call_with_request_factory.js b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js index a42d3c4d661494..0040fcb6c802af 100644 --- a/x-pack/plugins/file_upload/server/client/call_with_request_factory.js +++ b/x-pack/plugins/file_upload/server/client/call_with_request_factory.js @@ -9,7 +9,7 @@ import { once } from 'lodash'; const callWithRequest = once((server) => { - const cluster = server.plugins.elasticsearch.createCluster('fileupload'); + const cluster = server.plugins.elasticsearch.getCluster('data'); return cluster.callWithRequest; }); diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js index 659e36f2513d5a..4311ac7f2a3884 100644 --- a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js +++ b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js @@ -5,6 +5,7 @@ */ import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_import'; +import uuid from 'uuid'; export function importDataProvider(callWithRequest) { async function importData(id, index, settings, mappings, ingestPipeline, data) { @@ -21,7 +22,7 @@ export function importDataProvider(callWithRequest) { if (id === undefined) { // first chunk of data, create the index and id to return - id = generateId(); + id = uuid.v1(); await createIndex(index, settings, mappings); createdIndex = index; @@ -164,8 +165,3 @@ export function importDataProvider(callWithRequest) { importData, }; } - - -function generateId() { - return Math.random().toString(36).substr(2, 9); -} From 60156044aa52e8ed3128ac92f4308ba1f973b9e4 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 20 Mar 2019 16:19:58 -0600 Subject: [PATCH 08/18] Revise telemetry to use savedObjectRepository. Capture metrics on app and file types --- x-pack/plugins/file_upload/index.js | 5 ++ x-pack/plugins/file_upload/mappings.json | 16 +++- .../file_upload/server/routes/file_upload.js | 19 +++-- .../server/telemetry/make_usage_collector.ts | 2 +- .../file_upload/server/telemetry/telemetry.ts | 77 ++++++++++++++----- 5 files changed, 87 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/file_upload/index.js b/x-pack/plugins/file_upload/index.js index adcc5544fc5a53..24907082adb2c8 100644 --- a/x-pack/plugins/file_upload/index.js +++ b/x-pack/plugins/file_upload/index.js @@ -16,6 +16,11 @@ export const fileUpload = kibana => { uiExports: { mappings, }, + savedObjectSchemas: { + 'file-upload-telemetry': { + isNamespaceAgnostic: true + } + }, init(server) { const { xpack_main: xpackMainPlugin } = server.plugins; diff --git a/x-pack/plugins/file_upload/mappings.json b/x-pack/plugins/file_upload/mappings.json index f3a33937cb2bc9..197be209125263 100644 --- a/x-pack/plugins/file_upload/mappings.json +++ b/x-pack/plugins/file_upload/mappings.json @@ -1,9 +1,17 @@ { "file-upload-telemetry": { - "properties": { - "index_creation_count": { - "type" : "long" - } + "properties": { + "filesUploadedTotalCount": { + "type": "long" + }, + "filesUploadedTypesTotalCount": { + "dynamic": "true", + "properties": {} + }, + "filesUploadedByApp": { + "dynamic": "true", + "properties": {} } } } +} \ No newline at end of file diff --git a/x-pack/plugins/file_upload/server/routes/file_upload.js b/x-pack/plugins/file_upload/server/routes/file_upload.js index d924aea91c8556..3c08f4c8d01c91 100644 --- a/x-pack/plugins/file_upload/server/routes/file_upload.js +++ b/x-pack/plugins/file_upload/server/routes/file_upload.js @@ -8,7 +8,7 @@ import { callWithRequestFactory } from '../client/call_with_request_factory'; import { wrapError } from '../client/errors'; import { importDataProvider } from '../models/file_data_visualizer'; import { MAX_BYTES } from '../../common/constants/file_import'; -import { incrementIndexCreationCount } from '../telemetry/telemetry'; +import { updateTelemetry } from '../telemetry/telemetry'; function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { @@ -17,21 +17,28 @@ function importData(callWithRequest, id, index, settings, mappings, ingestPipeli } export function fileUploadRoutes(server, commonRouteConfig) { + server.route({ method: 'POST', path: '/api/fileupload/import', - handler(request) { - const callWithRequest = callWithRequestFactory(server, request); - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.payload; + async handler(request) { + + const { savedObjects: { getSavedObjectsRepository } } = server; + const { callWithInternalUser } = + server.plugins.elasticsearch.getCluster('data'); + const internalRepository = getSavedObjectsRepository(callWithInternalUser); + const { index, data, settings, mappings, ingestPipeline, app, fileType } + = request.payload; // `id` being `undefined` tells us that this is a new import due to create a new index. // follow-up import calls to just add additional data will include the `id` of the created // index, we'll ignore those and don't increment the counter. + const { id } = request.query; if (id === undefined) { - incrementIndexCreationCount(server); + await updateTelemetry(internalRepository, app, fileType); } + const callWithRequest = callWithRequestFactory(server, request); return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) .catch(wrapError); }, diff --git a/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts index 67aa9a66120fe0..404b630f4e3ca5 100644 --- a/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts +++ b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts @@ -26,7 +26,7 @@ interface KibanaHapiServer extends Server { export function makeUsageCollector(server: KibanaHapiServer): void { const fileUploadUsageCollector = server.usage.collectorSet.makeUsageCollector({ - type: 'fileUpload', + type: 'fileUploadTelemetry', fetch: async (): Promise => { try { const savedObjectsClient = getSavedObjectsClient(server); diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index a4bb10dca1b15f..704384df6b3865 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -5,12 +5,15 @@ */ import { Server } from 'hapi'; +import _ from 'lodash'; import { callWithInternalUserFactory } from '../client/call_with_internal_user_factory'; export const TELEMETRY_DOC_ID = 'file-upload-telemetry'; export interface Telemetry { - index_creation_count: number; + filesUploadedTotalCount: number; + filesUploadedTypesTotalCount: object; + filesUploadedByApp: object; } export interface TelemetrySavedObject { @@ -19,18 +22,12 @@ export interface TelemetrySavedObject { export function createTelemetry(count: number = 0): Telemetry { return { - index_creation_count: count, + filesUploadedTotalCount: count, + filesUploadedTypesTotalCount: {}, + filesUploadedByApp: {}, }; } -export function storeTelemetry(server: Server, fileUploadTelemetry: Telemetry): void { - const savedObjectsClient = getSavedObjectsClient(server); - savedObjectsClient.create('file-upload-telemetry', fileUploadTelemetry, { - id: TELEMETRY_DOC_ID, - overwrite: true, - }); -} - export function getSavedObjectsClient(server: Server): any { const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects; const callWithInternalUser = callWithInternalUserFactory(server); @@ -38,20 +35,58 @@ export function getSavedObjectsClient(server: Server): any { return new SavedObjectsClient(internalRepository); } -export async function incrementIndexCreationCount(server: Server): Promise { - const savedObjectsClient = getSavedObjectsClient(server); - let indicesCount = 1; +export async function updateTelemetry( + internalRepository, + app = 'unspecified-app', + fileType = 'unspecified-file-type' +) { + const nameAndId = 'file-upload-telemetry'; + let telemetrySavedObject; try { - const { attributes } = (await savedObjectsClient.get( - 'file-upload-telemetry', - TELEMETRY_DOC_ID - )) as TelemetrySavedObject; - indicesCount = attributes.index_creation_count + 1; + telemetrySavedObject = await internalRepository.get(nameAndId, nameAndId); } catch (e) { - /* silently fail, this will happen if the saved object doesn't exist yet. */ + // Fail silently } - const fileUploadTelemetry = createTelemetry(indicesCount); - storeTelemetry(server, fileUploadTelemetry); + if (telemetrySavedObject && telemetrySavedObject.attributes) { + const { + filesUploadedTotalCount, + filesUploadedTypesTotalCount, + filesUploadedByApp, + } = telemetrySavedObject.attributes; + + await internalRepository.update(nameAndId, nameAndId, { + filesUploadedTotalCount: (filesUploadedTotalCount || 0) + 1, + filesUploadedTypesTotalCount: { + ...filesUploadedTypesTotalCount, + [fileType]: _.get(filesUploadedTypesTotalCount, fileType, 0) + 1, + }, + filesUploadedByApp: { + ...filesUploadedByApp, + [app]: { + ..._.get(filesUploadedByApp, app, {}), + [fileType]: _.get(filesUploadedByApp, `${app}.${fileType}`, 0) + 1, + }, + }, + }); + } else { + await internalRepository.create( + 'file-upload-telemetry', + { + filesUploadedTotalCount: 1, + filesUploadedTypesTotalCount: { + [fileType]: 1, + }, + filesUploadedByApp: { + [app]: { + [fileType]: 1, + }, + }, + }, + { + id: 'file-upload-telemetry', + } + ); + } } From 6b222f6f086d6a724181767553c68fee021152a7 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Fri, 22 Mar 2019 13:01:26 -0600 Subject: [PATCH 09/18] Lots of cleanup, consolidation of logic --- x-pack/plugins/file_upload/mappings.json | 2 +- .../file_upload/server/routes/file_upload.js | 15 +-- .../server/telemetry/make_usage_collector.ts | 22 +---- .../file_upload/server/telemetry/telemetry.ts | 98 +++++++++---------- 4 files changed, 52 insertions(+), 85 deletions(-) diff --git a/x-pack/plugins/file_upload/mappings.json b/x-pack/plugins/file_upload/mappings.json index 197be209125263..b62a244b2bfa1d 100644 --- a/x-pack/plugins/file_upload/mappings.json +++ b/x-pack/plugins/file_upload/mappings.json @@ -4,7 +4,7 @@ "filesUploadedTotalCount": { "type": "long" }, - "filesUploadedTypesTotalCount": { + "filesUploadedTypesTotalCounts": { "dynamic": "true", "properties": {} }, diff --git a/x-pack/plugins/file_upload/server/routes/file_upload.js b/x-pack/plugins/file_upload/server/routes/file_upload.js index 3c08f4c8d01c91..46620151905a3d 100644 --- a/x-pack/plugins/file_upload/server/routes/file_upload.js +++ b/x-pack/plugins/file_upload/server/routes/file_upload.js @@ -11,7 +11,9 @@ import { MAX_BYTES } from '../../common/constants/file_import'; import { updateTelemetry } from '../telemetry/telemetry'; -function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { +function importData({ + callWithRequest, id, index, settings, mappings, ingestPipeline, data +}) { const { importData: importDataFunc } = importDataProvider(callWithRequest); return importDataFunc(id, index, settings, mappings, ingestPipeline, data); } @@ -23,23 +25,16 @@ export function fileUploadRoutes(server, commonRouteConfig) { path: '/api/fileupload/import', async handler(request) { - const { savedObjects: { getSavedObjectsRepository } } = server; - const { callWithInternalUser } = - server.plugins.elasticsearch.getCluster('data'); - const internalRepository = getSavedObjectsRepository(callWithInternalUser); - const { index, data, settings, mappings, ingestPipeline, app, fileType } - = request.payload; - // `id` being `undefined` tells us that this is a new import due to create a new index. // follow-up import calls to just add additional data will include the `id` of the created // index, we'll ignore those and don't increment the counter. const { id } = request.query; if (id === undefined) { - await updateTelemetry(internalRepository, app, fileType); + await updateTelemetry({ server, ...request.payload }); } const callWithRequest = callWithRequestFactory(server, request); - return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) + return importData({ callWithRequest, id, ...request.payload }) .catch(wrapError); }, config: { diff --git a/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts index 404b630f4e3ca5..67830f2004d43f 100644 --- a/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts +++ b/x-pack/plugins/file_upload/server/telemetry/make_usage_collector.ts @@ -5,14 +5,7 @@ */ import { Server } from 'hapi'; - -import { - createTelemetry, - getSavedObjectsClient, - Telemetry, - TELEMETRY_DOC_ID, - TelemetrySavedObject, -} from './telemetry'; +import { getTelemetry, Telemetry } from './telemetry'; // TODO this type should be defined by the platform interface KibanaHapiServer extends Server { @@ -27,18 +20,7 @@ interface KibanaHapiServer extends Server { export function makeUsageCollector(server: KibanaHapiServer): void { const fileUploadUsageCollector = server.usage.collectorSet.makeUsageCollector({ type: 'fileUploadTelemetry', - fetch: async (): Promise => { - try { - const savedObjectsClient = getSavedObjectsClient(server); - const telemetrySavedObject = (await savedObjectsClient.get( - 'file-upload-telemetry', - TELEMETRY_DOC_ID - )) as TelemetrySavedObject; - return telemetrySavedObject.attributes; - } catch (err) { - return createTelemetry(); - } - }, + fetch: async (): Promise => await getTelemetry(server), }); server.usage.collectorSet.register(fileUploadUsageCollector); } diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 704384df6b3865..2b3a33cec6edb8 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -12,7 +12,7 @@ export const TELEMETRY_DOC_ID = 'file-upload-telemetry'; export interface Telemetry { filesUploadedTotalCount: number; - filesUploadedTypesTotalCount: object; + filesUploadedTypesTotalCounts: object; filesUploadedByApp: object; } @@ -20,73 +20,63 @@ export interface TelemetrySavedObject { attributes: Telemetry; } -export function createTelemetry(count: number = 0): Telemetry { +export function initTelemetry(): Telemetry { return { - filesUploadedTotalCount: count, - filesUploadedTypesTotalCount: {}, + filesUploadedTotalCount: 0, + filesUploadedTypesTotalCounts: {}, filesUploadedByApp: {}, }; } -export function getSavedObjectsClient(server: Server): any { - const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects; +function getInternalRepository(server: Server): any { + const { getSavedObjectsRepository } = server.savedObjects; const callWithInternalUser = callWithInternalUserFactory(server); - const internalRepository = getSavedObjectsRepository(callWithInternalUser); - return new SavedObjectsClient(internalRepository); + return getSavedObjectsRepository(callWithInternalUser); } -export async function updateTelemetry( - internalRepository, - app = 'unspecified-app', - fileType = 'unspecified-file-type' -) { - const nameAndId = 'file-upload-telemetry'; - +export async function getTelemetry(server: Server): Promise { + const internalRepository = getInternalRepository(server); let telemetrySavedObject; + try { - telemetrySavedObject = await internalRepository.get(nameAndId, nameAndId); + telemetrySavedObject = await internalRepository.get(TELEMETRY_DOC_ID, TELEMETRY_DOC_ID); } catch (e) { // Fail silently } - if (telemetrySavedObject && telemetrySavedObject.attributes) { - const { - filesUploadedTotalCount, - filesUploadedTypesTotalCount, - filesUploadedByApp, - } = telemetrySavedObject.attributes; - - await internalRepository.update(nameAndId, nameAndId, { - filesUploadedTotalCount: (filesUploadedTotalCount || 0) + 1, - filesUploadedTypesTotalCount: { - ...filesUploadedTypesTotalCount, - [fileType]: _.get(filesUploadedTypesTotalCount, fileType, 0) + 1, - }, - filesUploadedByApp: { - ...filesUploadedByApp, - [app]: { - ..._.get(filesUploadedByApp, app, {}), - [fileType]: _.get(filesUploadedByApp, `${app}.${fileType}`, 0) + 1, - }, - }, + if (!telemetrySavedObject) { + telemetrySavedObject = await internalRepository.create(TELEMETRY_DOC_ID, initTelemetry(), { + id: TELEMETRY_DOC_ID, }); - } else { - await internalRepository.create( - 'file-upload-telemetry', - { - filesUploadedTotalCount: 1, - filesUploadedTypesTotalCount: { - [fileType]: 1, - }, - filesUploadedByApp: { - [app]: { - [fileType]: 1, - }, - }, - }, - { - id: 'file-upload-telemetry', - } - ); } + return telemetrySavedObject.attributes; +} + +export async function updateTelemetry({ + server, + app = 'unspecified-app', + fileType = 'unspecified-file-type', +}: { + server: Server; + app: string; + fileType: string; +}) { + const telemetry = await getTelemetry(server); + const internalRepository = getInternalRepository(server); + const { filesUploadedTotalCount, filesUploadedTypesTotalCounts, filesUploadedByApp } = telemetry; + + await internalRepository.update(TELEMETRY_DOC_ID, TELEMETRY_DOC_ID, { + filesUploadedTotalCount: filesUploadedTotalCount + 1, + filesUploadedTypesTotalCounts: { + ...filesUploadedTypesTotalCounts, + [fileType]: _.get(filesUploadedTypesTotalCounts, fileType, 0) + 1, + }, + filesUploadedByApp: { + ...filesUploadedByApp, + [app]: { + ..._.get(filesUploadedByApp, app, {}), + [fileType]: _.get(filesUploadedByApp, `${app}.${fileType}`, 0) + 1, + }, + }, + }); } From 2aae63e59edbb6418c20c5acd5839d36506116c2 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Mon, 25 Mar 2019 18:50:13 -0600 Subject: [PATCH 10/18] Clean up, reorg --- .../file_data_visualizer/import_data.js | 167 ------------------ .../models/file_data_visualizer/index.js | 8 - .../file_upload/server/routes/file_upload.js | 2 +- .../file_upload/server/telemetry/telemetry.ts | 6 + 4 files changed, 7 insertions(+), 176 deletions(-) delete mode 100644 x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js delete mode 100644 x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js deleted file mode 100644 index 4311ac7f2a3884..00000000000000 --- a/x-pack/plugins/file_upload/server/models/file_data_visualizer/import_data.js +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_import'; -import uuid from 'uuid'; - -export function importDataProvider(callWithRequest) { - async function importData(id, index, settings, mappings, ingestPipeline, data) { - let createdIndex; - let createdPipelineId; - const docCount = data.length; - - try { - - const { - id: pipelineId, - pipeline, - } = ingestPipeline; - - if (id === undefined) { - // first chunk of data, create the index and id to return - id = uuid.v1(); - - await createIndex(index, settings, mappings); - createdIndex = index; - - // create the pipeline if one has been supplied - if (pipelineId !== undefined) { - const success = await createPipeline(pipelineId, pipeline); - if (success.acknowledged !== true) { - throw success; - } - } - createdPipelineId = pipelineId; - - } else { - createdIndex = index; - createdPipelineId = pipelineId; - } - - let failures = []; - if (data.length) { - const resp = await indexData(index, createdPipelineId, data); - if (resp.success === false) { - if (resp.ingestError) { - // all docs failed, abort - throw resp; - } else { - // some docs failed. - // still report success but with a list of failures - failures = (resp.failures || []); - } - } - } - - return { - success: true, - id, - index: createdIndex, - pipelineId: createdPipelineId, - docCount, - failures, - }; - } catch (error) { - return { - success: false, - id, - index: createdIndex, - pipelineId: createdPipelineId, - error: (error.error !== undefined) ? error.error : error, - docCount, - ingestError: error.ingestError, - failures: (error.failures || []) - }; - } - } - - async function createIndex(index, settings, mappings) { - const body = { - mappings: { - _meta: { - created_by: INDEX_META_DATA_CREATED_BY - }, - properties: mappings - } - }; - - if (settings && Object.keys(settings).length) { - body.settings = settings; - } - - await callWithRequest('indices.create', { index, body }); - } - - async function indexData(index, pipelineId, data) { - try { - const body = []; - for (let i = 0; i < data.length; i++) { - body.push({ index: {} }); - body.push(data[i]); - } - - const settings = { index, body }; - if (pipelineId !== undefined) { - settings.pipeline = pipelineId; - } - - const resp = await callWithRequest('bulk', settings); - if (resp.errors) { - throw resp; - } else { - return { - success: true, - docs: data.length, - failures: [], - }; - } - } catch (error) { - - let failures = []; - let ingestError = false; - if (error.errors !== undefined && Array.isArray(error.items)) { - // an expected error where some or all of the bulk request - // docs have failed to be ingested. - failures = getFailures(error.items, data); - } else { - // some other error has happened. - ingestError = true; - } - - return { - success: false, - error, - docCount: data.length, - failures, - ingestError, - }; - } - - } - - async function createPipeline(id, pipeline) { - return await callWithRequest('ingest.putPipeline', { id, body: pipeline }); - } - - function getFailures(items, data) { - const failures = []; - for (let i = 0; i < items.length; i++) { - const item = items[i]; - if (item.index && item.index.error) { - failures.push({ - item: i, - reason: item.index.error.reason, - doc: data[i], - }); - } - } - return failures; - } - - return { - importData, - }; -} diff --git a/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js b/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js deleted file mode 100644 index e91b658c4358bf..00000000000000 --- a/x-pack/plugins/file_upload/server/models/file_data_visualizer/index.js +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - -export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/file_upload/server/routes/file_upload.js b/x-pack/plugins/file_upload/server/routes/file_upload.js index 46620151905a3d..ac07d80962bdce 100644 --- a/x-pack/plugins/file_upload/server/routes/file_upload.js +++ b/x-pack/plugins/file_upload/server/routes/file_upload.js @@ -6,7 +6,7 @@ import { callWithRequestFactory } from '../client/call_with_request_factory'; import { wrapError } from '../client/errors'; -import { importDataProvider } from '../models/file_data_visualizer'; +import { importDataProvider } from '../models/import_data'; import { MAX_BYTES } from '../../common/constants/file_import'; import { updateTelemetry } from '../telemetry/telemetry'; diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 2b3a33cec6edb8..8bdf5892f2b014 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -20,6 +20,12 @@ export interface TelemetrySavedObject { attributes: Telemetry; } +export function getInternalRepository(server: Server): any { + const { getSavedObjectsRepository } = server.savedObjects; + const callWithInternalUser = callWithInternalUserFactory(server); + return getSavedObjectsRepository(callWithInternalUser); +} + export function initTelemetry(): Telemetry { return { filesUploadedTotalCount: 0, From 8de9a9733e9fa007ab3c33f46054c1540331dbf4 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Mon, 25 Mar 2019 19:01:37 -0600 Subject: [PATCH 11/18] Update telem tests and telem functions --- .../server/telemetry/telemetry.test.ts | 261 ++++++------------ .../file_upload/server/telemetry/telemetry.ts | 41 ++- 2 files changed, 116 insertions(+), 186 deletions(-) diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index 1f19edaeb229ba..50e41c9928585b 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -4,194 +4,109 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - createTelemetry, - getSavedObjectsClient, - incrementFileDataVisualizerIndexCreationCount, - storeTelemetry, - Telemetry, - TELEMETRY_DOC_ID, -} from './telemetry'; - -describe('file_upload_telemetry', () => { - describe('createTelemetry', () => { - it('should create a file upload telemetry object', () => { - const fileUploadTelemetry = createTelemetry(1); - expect(fileUploadTelemetry.file_upload.index_creation_count).toBe(1); - }); - it('should ignore undefined or unknown values', () => { - const fileUploadTelemetry = createTelemetry(undefined); - expect(fileUploadTelemetry.file_upload.index_creation_count).toBe(0); - }); - }); - - describe('storeTelemetry', () => { - let server: any; - let fileUploadTelemetry: Telemetry; - let savedObjectsClientInstance: any; - - beforeEach(() => { - savedObjectsClientInstance = { create: jest.fn() }; - const callWithInternalUser = jest.fn(); - const internalRepository = jest.fn(); - server = { - savedObjects: { - SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), - getSavedObjectsRepository: jest.fn(() => internalRepository), - }, - plugins: { - elasticsearch: { - getCluster: jest.fn(() => ({ callWithInternalUser })), - }, - }, - }; - fileUploadTelemetry = { - file_upload: { - index_creation_count: 1, - }, - }; - }); - - it('should call savedObjectsClient create with the given Telemetry object', () => { - storeTelemetry(server, fileUploadTelemetry); - expect(savedObjectsClientInstance.create.mock.calls[0][1]).toBe(fileUploadTelemetry); - }); - - it('should call savedObjectsClient create with the file-upload-telemetry document type and ID', () => { - storeTelemetry(server, fileUploadTelemetry); - expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); - expect(savedObjectsClientInstance.create.mock.calls[0][2].id).toBe(TELEMETRY_DOC_ID); +import { getTelemetry, incrementCounts, updateTelemetry } from './telemetry'; + +let server: any; +let callWithInternalUser: any; +let internalRepository: any; + +function mockInit(getVal: any = null): void { + internalRepository = { + get: jest.fn(() => getVal), + create: jest.fn(() => ({ attributes: 'test' })), + update: jest.fn(() => ({ attributes: 'test' })), + }; + callWithInternalUser = jest.fn(); + server = { + savedObjects: { + getSavedObjectsRepository: jest.fn(() => internalRepository), + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, + }; +} + +describe('file upload plugin telemetry', () => { + describe('getTelemetry', () => { + it('should create new telemetry if no telemetry exists', async () => { + mockInit(); + await getTelemetry(server, internalRepository); + // Expect internalRepository.get to get called + expect(internalRepository.get.mock.calls.length).toBe(1); + // Expect internalRepository.create to get called + expect(internalRepository.create.mock.calls.length).toBe(1); }); - it('should call savedObjectsClient create with overwrite: true', () => { - storeTelemetry(server, fileUploadTelemetry); - expect(savedObjectsClientInstance.create.mock.calls[0][2].overwrite).toBe(true); + it('should get existing telemetry', async () => { + mockInit({}); + await getTelemetry(server, internalRepository); + // Expect internalRepository.get to get called + expect(internalRepository.get.mock.calls.length).toBe(1); + // Expect internalRepository.create NOT to get called + expect(internalRepository.create.mock.calls.length).toBe(0); }); }); - describe('getSavedObjectsClient', () => { - let server: any; - let savedObjectsClientInstance: any; - let callWithInternalUser: any; - let internalRepository: any; - - beforeEach(() => { - savedObjectsClientInstance = { create: jest.fn() }; - callWithInternalUser = jest.fn(); - internalRepository = jest.fn(); - server = { - savedObjects: { - SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), - getSavedObjectsRepository: jest.fn(() => internalRepository), - }, - plugins: { - elasticsearch: { - getCluster: jest.fn(() => ({ callWithInternalUser })), - }, + describe('updateTelemetry', () => { + it('total count should equal sum of all file counts', async () => { + mockInit({ + attributes: { + filesUploadedTotalCount: 2, }, - }; - }); - - it('should return a SavedObjectsClient initialized with the saved objects internal repository', () => { - const result = getSavedObjectsClient(server); - - expect(result).toBe(savedObjectsClientInstance); - expect(server.savedObjects.SavedObjectsClient).toHaveBeenCalledWith(internalRepository); + }); + await updateTelemetry({ server, internalRepo: internalRepository }); + expect(internalRepository.update.mock.calls.length).toBe(1); + // Expect internalRepository.get to get called + expect(internalRepository.get.mock.calls.length).toBe(2); + // Expect internalRepository.create to get called + expect(internalRepository.create.mock.calls.length).toBe(0); }); }); - describe('incrementFileImportCount', () => { - let server: any; - let savedObjectsClientInstance: any; - let callWithInternalUser: any; - let internalRepository: any; - - function createSavedObjectsClientInstance( - telemetryEnabled?: boolean, - indexCreationCount?: number - ) { - return { - create: jest.fn(), - get: jest.fn(obj => { - switch (obj) { - case 'telemetry': - if (telemetryEnabled === undefined) { - throw Error; - } - return { - attributes: { - import_telemetry: { - enabled: telemetryEnabled, - }, - }, - }; - case 'file-upload-telemetry': - // emulate that a non-existing saved object will throw an error - if (indexCreationCount === undefined) { - throw Error; - } - return { - file_upload: { - index_creation_count: indexCreationCount, - }, - }; - } - }), - }; - } - - function mockInit(telemetryEnabled?: boolean, indexCreationCount?: number): void { - savedObjectsClientInstance = createSavedObjectsClientInstance( - telemetryEnabled, - indexCreationCount - ); - callWithInternalUser = jest.fn(); - internalRepository = jest.fn(); - server = { - savedObjects: { - SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), - getSavedObjectsRepository: jest.fn(() => internalRepository), + describe('incrementCounts', () => { + const oldCounts = { + filesUploadedTotalCount: 3, + filesUploadedTypesTotalCounts: { + json: 1, + csv: 2, + }, + filesUploadedByApp: { + maps: { + json: 1, + csv: 1, }, - plugins: { - elasticsearch: { - getCluster: jest.fn(() => ({ callWithInternalUser })), - }, + ml: { + csv: 1, }, - }; - } - - it('should not increment if telemetry status cannot be determined', async () => { - mockInit(); - await incrementFileDataVisualizerIndexCreationCount(server); - - expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0); - }); - - it('should not increment if telemetry status is disabled', async () => { - mockInit(false); - await incrementFileDataVisualizerIndexCreationCount(server); - - expect(savedObjectsClientInstance.create.mock.calls).toHaveLength(0); + }, + }; + const app = 'maps'; + const fileType = 'json'; + + it('app, file and total count should increment by 1', async () => { + const newCounts = incrementCounts({ app, fileType, ...oldCounts }); + expect(newCounts.filesUploadedTotalCount).toEqual(4); + expect(newCounts.filesUploadedTypesTotalCounts[fileType]).toEqual(2); + expect(newCounts.filesUploadedByApp[app][fileType]).toEqual(2); }); - it('should initialize index_creation_count with 1', async () => { - mockInit(true); - await incrementFileDataVisualizerIndexCreationCount(server); - - expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); - expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({ - file_upload: { index_creation_count: 1 }, - }); + it('total count should equal sum of all file type counts', async () => { + const newCounts = incrementCounts({ app, fileType, ...oldCounts }); + const fileTypeCounts = + newCounts.filesUploadedTypesTotalCounts.json + newCounts.filesUploadedTypesTotalCounts.csv; + expect(newCounts.filesUploadedTotalCount).toEqual(fileTypeCounts); }); - it('should increment index_creation_count to 2', async () => { - mockInit(true, 1); - await incrementFileDataVisualizerIndexCreationCount(server); - - expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe('file-upload-telemetry'); - expect(savedObjectsClientInstance.create.mock.calls[0][1]).toEqual({ - file_upload: { index_creation_count: 2 }, - }); + it('total count should equal sum of all app counts', async () => { + const newCounts = incrementCounts({ app, fileType, ...oldCounts }); + const fileAppCounts = + newCounts.filesUploadedByApp.maps.json + + newCounts.filesUploadedByApp.maps.csv + + newCounts.filesUploadedByApp.ml.csv; + expect(newCounts.filesUploadedTotalCount).toEqual(fileAppCounts); }); }); }); diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 8bdf5892f2b014..6b0f937e8673bc 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -34,14 +34,8 @@ export function initTelemetry(): Telemetry { }; } -function getInternalRepository(server: Server): any { - const { getSavedObjectsRepository } = server.savedObjects; - const callWithInternalUser = callWithInternalUserFactory(server); - return getSavedObjectsRepository(callWithInternalUser); -} - -export async function getTelemetry(server: Server): Promise { - const internalRepository = getInternalRepository(server); +export async function getTelemetry(server: Server, internalRepo?: object): Promise { + const internalRepository = internalRepo || getInternalRepository(server); let telemetrySavedObject; try { @@ -60,18 +54,39 @@ export async function getTelemetry(server: Server): Promise { export async function updateTelemetry({ server, + internalRepo, app = 'unspecified-app', fileType = 'unspecified-file-type', }: { server: Server; + internalRepo: object; app: string; fileType: string; }) { - const telemetry = await getTelemetry(server); - const internalRepository = getInternalRepository(server); - const { filesUploadedTotalCount, filesUploadedTypesTotalCounts, filesUploadedByApp } = telemetry; + const telemetry = await getTelemetry(server, internalRepo); + const internalRepository = internalRepo || getInternalRepository(server); + + await internalRepository.update( + TELEMETRY_DOC_ID, + TELEMETRY_DOC_ID, + incrementCounts({ app, fileType, ...telemetry }) + ); +} - await internalRepository.update(TELEMETRY_DOC_ID, TELEMETRY_DOC_ID, { +export function incrementCounts({ + filesUploadedTotalCount, + filesUploadedTypesTotalCounts, + filesUploadedByApp, + fileType, + app, +}: { + filesUploadedTotalCount: number; + filesUploadedTypesTotalCounts: object; + filesUploadedByApp: object; + fileType: string; + app: string; +}) { + return { filesUploadedTotalCount: filesUploadedTotalCount + 1, filesUploadedTypesTotalCounts: { ...filesUploadedTypesTotalCounts, @@ -84,5 +99,5 @@ export async function updateTelemetry({ [fileType]: _.get(filesUploadedByApp, `${app}.${fileType}`, 0) + 1, }, }, - }); + }; } From 63ec74e08f4017d20938d89f8a32207d58ffecbf Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Tue, 26 Mar 2019 07:35:24 -0600 Subject: [PATCH 12/18] Add back import data model --- .../server/models/import_data/import_data.js | 167 ++++++++++++++++++ .../server/models/import_data/index.js | 8 + 2 files changed, 175 insertions(+) create mode 100644 x-pack/plugins/file_upload/server/models/import_data/import_data.js create mode 100644 x-pack/plugins/file_upload/server/models/import_data/index.js diff --git a/x-pack/plugins/file_upload/server/models/import_data/import_data.js b/x-pack/plugins/file_upload/server/models/import_data/import_data.js new file mode 100644 index 00000000000000..4311ac7f2a3884 --- /dev/null +++ b/x-pack/plugins/file_upload/server/models/import_data/import_data.js @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_import'; +import uuid from 'uuid'; + +export function importDataProvider(callWithRequest) { + async function importData(id, index, settings, mappings, ingestPipeline, data) { + let createdIndex; + let createdPipelineId; + const docCount = data.length; + + try { + + const { + id: pipelineId, + pipeline, + } = ingestPipeline; + + if (id === undefined) { + // first chunk of data, create the index and id to return + id = uuid.v1(); + + await createIndex(index, settings, mappings); + createdIndex = index; + + // create the pipeline if one has been supplied + if (pipelineId !== undefined) { + const success = await createPipeline(pipelineId, pipeline); + if (success.acknowledged !== true) { + throw success; + } + } + createdPipelineId = pipelineId; + + } else { + createdIndex = index; + createdPipelineId = pipelineId; + } + + let failures = []; + if (data.length) { + const resp = await indexData(index, createdPipelineId, data); + if (resp.success === false) { + if (resp.ingestError) { + // all docs failed, abort + throw resp; + } else { + // some docs failed. + // still report success but with a list of failures + failures = (resp.failures || []); + } + } + } + + return { + success: true, + id, + index: createdIndex, + pipelineId: createdPipelineId, + docCount, + failures, + }; + } catch (error) { + return { + success: false, + id, + index: createdIndex, + pipelineId: createdPipelineId, + error: (error.error !== undefined) ? error.error : error, + docCount, + ingestError: error.ingestError, + failures: (error.failures || []) + }; + } + } + + async function createIndex(index, settings, mappings) { + const body = { + mappings: { + _meta: { + created_by: INDEX_META_DATA_CREATED_BY + }, + properties: mappings + } + }; + + if (settings && Object.keys(settings).length) { + body.settings = settings; + } + + await callWithRequest('indices.create', { index, body }); + } + + async function indexData(index, pipelineId, data) { + try { + const body = []; + for (let i = 0; i < data.length; i++) { + body.push({ index: {} }); + body.push(data[i]); + } + + const settings = { index, body }; + if (pipelineId !== undefined) { + settings.pipeline = pipelineId; + } + + const resp = await callWithRequest('bulk', settings); + if (resp.errors) { + throw resp; + } else { + return { + success: true, + docs: data.length, + failures: [], + }; + } + } catch (error) { + + let failures = []; + let ingestError = false; + if (error.errors !== undefined && Array.isArray(error.items)) { + // an expected error where some or all of the bulk request + // docs have failed to be ingested. + failures = getFailures(error.items, data); + } else { + // some other error has happened. + ingestError = true; + } + + return { + success: false, + error, + docCount: data.length, + failures, + ingestError, + }; + } + + } + + async function createPipeline(id, pipeline) { + return await callWithRequest('ingest.putPipeline', { id, body: pipeline }); + } + + function getFailures(items, data) { + const failures = []; + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (item.index && item.index.error) { + failures.push({ + item: i, + reason: item.index.error.reason, + doc: data[i], + }); + } + } + return failures; + } + + return { + importData, + }; +} diff --git a/x-pack/plugins/file_upload/server/models/import_data/index.js b/x-pack/plugins/file_upload/server/models/import_data/index.js new file mode 100644 index 00000000000000..e91b658c4358bf --- /dev/null +++ b/x-pack/plugins/file_upload/server/models/import_data/index.js @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +export { importDataProvider } from './import_data'; From 60a8a7cdb944a8e2fe6d9dd804aab7dc8d600a0d Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Tue, 30 Apr 2019 15:47:20 -0600 Subject: [PATCH 13/18] Clean up and update telemetry tests --- .../file_upload/server/telemetry/telemetry.test.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index 50e41c9928585b..b051c5205eb588 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -34,24 +34,21 @@ describe('file upload plugin telemetry', () => { it('should create new telemetry if no telemetry exists', async () => { mockInit(); await getTelemetry(server, internalRepository); - // Expect internalRepository.get to get called expect(internalRepository.get.mock.calls.length).toBe(1); - // Expect internalRepository.create to get called expect(internalRepository.create.mock.calls.length).toBe(1); }); it('should get existing telemetry', async () => { mockInit({}); await getTelemetry(server, internalRepository); - // Expect internalRepository.get to get called + expect(internalRepository.update.mock.calls.length).toBe(0); expect(internalRepository.get.mock.calls.length).toBe(1); - // Expect internalRepository.create NOT to get called expect(internalRepository.create.mock.calls.length).toBe(0); }); }); describe('updateTelemetry', () => { - it('total count should equal sum of all file counts', async () => { + it('should update existing telemetry', async () => { mockInit({ attributes: { filesUploadedTotalCount: 2, @@ -59,9 +56,7 @@ describe('file upload plugin telemetry', () => { }); await updateTelemetry({ server, internalRepo: internalRepository }); expect(internalRepository.update.mock.calls.length).toBe(1); - // Expect internalRepository.get to get called - expect(internalRepository.get.mock.calls.length).toBe(2); - // Expect internalRepository.create to get called + expect(internalRepository.get.mock.calls.length).toBe(1); expect(internalRepository.create.mock.calls.length).toBe(0); }); }); From e264ec1ca39732edfdc894da37ccee1d0f865042 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 1 May 2019 14:48:00 -0600 Subject: [PATCH 14/18] Fix telemetry test issues and update corresponding code --- .../server/telemetry/telemetry.test.ts | 69 ++++++++++--------- .../file_upload/server/telemetry/telemetry.ts | 10 +-- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index b051c5205eb588..12a13a0e346904 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -6,58 +6,59 @@ import { getTelemetry, incrementCounts, updateTelemetry } from './telemetry'; -let server: any; -let callWithInternalUser: any; -let internalRepository: any; +const internalRepository = () => ({ + get: jest.fn(() => null), + create: jest.fn(() => ({ attributes: 'test' })), + update: jest.fn(() => ({ attributes: 'test' })), +}); +const server: any = { + savedObjects: { + getSavedObjectsRepository: jest.fn(() => internalRepository()), + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })), + }, + }, +}; +const callWithInternalUser = jest.fn(); -function mockInit(getVal: any = null): void { - internalRepository = { +function mockInit(getVal: any = { attributes: {}}): any { + return { + ...internalRepository(), get: jest.fn(() => getVal), - create: jest.fn(() => ({ attributes: 'test' })), - update: jest.fn(() => ({ attributes: 'test' })), - }; - callWithInternalUser = jest.fn(); - server = { - savedObjects: { - getSavedObjectsRepository: jest.fn(() => internalRepository), - }, - plugins: { - elasticsearch: { - getCluster: jest.fn(() => ({ callWithInternalUser })), - }, - }, }; } describe('file upload plugin telemetry', () => { describe('getTelemetry', () => { it('should create new telemetry if no telemetry exists', async () => { - mockInit(); - await getTelemetry(server, internalRepository); - expect(internalRepository.get.mock.calls.length).toBe(1); - expect(internalRepository.create.mock.calls.length).toBe(1); + const internalRepo = mockInit({}); + await getTelemetry(server, internalRepo); + expect(internalRepo.get.mock.calls.length).toBe(1); + expect(internalRepo.create.mock.calls.length).toBe(1); }); it('should get existing telemetry', async () => { - mockInit({}); - await getTelemetry(server, internalRepository); - expect(internalRepository.update.mock.calls.length).toBe(0); - expect(internalRepository.get.mock.calls.length).toBe(1); - expect(internalRepository.create.mock.calls.length).toBe(0); + const internalRepo = mockInit(); + await getTelemetry(server, internalRepo); + expect(internalRepo.update.mock.calls.length).toBe(0); + expect(internalRepo.get.mock.calls.length).toBe(1); + expect(internalRepo.create.mock.calls.length).toBe(0); }); }); describe('updateTelemetry', () => { it('should update existing telemetry', async () => { - mockInit({ + const internalRepo = mockInit({ attributes: { filesUploadedTotalCount: 2, }, }); - await updateTelemetry({ server, internalRepo: internalRepository }); - expect(internalRepository.update.mock.calls.length).toBe(1); - expect(internalRepository.get.mock.calls.length).toBe(1); - expect(internalRepository.create.mock.calls.length).toBe(0); + await updateTelemetry({ server, internalRepo }); + expect(internalRepo.update.mock.calls.length).toBe(1); + expect(internalRepo.get.mock.calls.length).toBe(1); + expect(internalRepo.create.mock.calls.length).toBe(0); }); }); @@ -82,7 +83,7 @@ describe('file upload plugin telemetry', () => { const fileType = 'json'; it('app, file and total count should increment by 1', async () => { - const newCounts = incrementCounts({ app, fileType, ...oldCounts }); + const newCounts: any = incrementCounts({ app, fileType, ...oldCounts }); expect(newCounts.filesUploadedTotalCount).toEqual(4); expect(newCounts.filesUploadedTypesTotalCounts[fileType]).toEqual(2); expect(newCounts.filesUploadedByApp[app][fileType]).toEqual(2); @@ -96,7 +97,7 @@ describe('file upload plugin telemetry', () => { }); it('total count should equal sum of all app counts', async () => { - const newCounts = incrementCounts({ app, fileType, ...oldCounts }); + const newCounts: any = incrementCounts({ app, fileType, ...oldCounts }); const fileAppCounts = newCounts.filesUploadedByApp.maps.json + newCounts.filesUploadedByApp.maps.csv + diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 6b0f937e8673bc..35f046da50c1fd 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -44,7 +44,7 @@ export async function getTelemetry(server: Server, internalRepo?: object): Promi // Fail silently } - if (!telemetrySavedObject) { + if (!telemetrySavedObject || _.isEmpty(telemetrySavedObject)) { telemetrySavedObject = await internalRepository.create(TELEMETRY_DOC_ID, initTelemetry(), { id: TELEMETRY_DOC_ID, }); @@ -58,10 +58,10 @@ export async function updateTelemetry({ app = 'unspecified-app', fileType = 'unspecified-file-type', }: { - server: Server; - internalRepo: object; - app: string; - fileType: string; + server: any; + internalRepo?: object; + app?: string; + fileType?: string; }) { const telemetry = await getTelemetry(server, internalRepo); const internalRepository = internalRepo || getInternalRepository(server); From 377a3a1be4a3c3ee95cbadd83b886e4be9b7deec Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 1 May 2019 14:54:09 -0600 Subject: [PATCH 15/18] Up chunk limit to 30 MB --- x-pack/plugins/file_upload/common/constants/file_import.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/file_upload/common/constants/file_import.ts b/x-pack/plugins/file_upload/common/constants/file_import.ts index 1f9dbd154032dc..1bfc3595f83b29 100644 --- a/x-pack/plugins/file_upload/common/constants/file_import.ts +++ b/x-pack/plugins/file_upload/common/constants/file_import.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -export const MAX_BYTES = 10485760; +export const MAX_BYTES = 31457280; -// Value to use in the Elasticsearch index mapping meta data to identify the +// Value to use in the Elasticsearch index mapping metadata to identify the // index as having been created by the File Upload Plugin. export const INDEX_META_DATA_CREATED_BY = 'file-upload-plugin'; From 3e4897774a59133279c79398a6895a4a4c0ad2cf Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 1 May 2019 17:40:37 -0600 Subject: [PATCH 16/18] Add file upload telemetry to saved objects management builder --- .../ui_capabilities/common/saved_objects_management_builder.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/test/ui_capabilities/common/saved_objects_management_builder.ts b/x-pack/test/ui_capabilities/common/saved_objects_management_builder.ts index c522d68180fd36..c1626af777b4d1 100644 --- a/x-pack/test/ui_capabilities/common/saved_objects_management_builder.ts +++ b/x-pack/test/ui_capabilities/common/saved_objects_management_builder.ts @@ -41,6 +41,7 @@ export class SavedObjectsManagementBuilder { 'infrastructure-ui-source', 'upgrade-assistant-reindex-operation', 'upgrade-assistant-telemetry', + 'file-upload-telemetry', 'index-pattern', 'visualization', 'search', From 23c33ff803dd844bb61cefe37d162b4445bf061d Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Wed, 1 May 2019 18:50:14 -0600 Subject: [PATCH 17/18] Missing space --- x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts index 12a13a0e346904..647c4bb77187f9 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.test.ts @@ -23,7 +23,7 @@ const server: any = { }; const callWithInternalUser = jest.fn(); -function mockInit(getVal: any = { attributes: {}}): any { +function mockInit(getVal: any = { attributes: {} }): any { return { ...internalRepository(), get: jest.fn(() => getVal), From 4361f885a4766b2216220ff93bf1a3910c65cc45 Mon Sep 17 00:00:00 2001 From: Aaron Caldwell Date: Thu, 2 May 2019 08:08:16 -0600 Subject: [PATCH 18/18] Add descriptive comments to dynamic keys in telemetry fields --- x-pack/plugins/file_upload/server/telemetry/telemetry.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts index 35f046da50c1fd..6597c7d6c8a4bc 100644 --- a/x-pack/plugins/file_upload/server/telemetry/telemetry.ts +++ b/x-pack/plugins/file_upload/server/telemetry/telemetry.ts @@ -90,10 +90,12 @@ export function incrementCounts({ filesUploadedTotalCount: filesUploadedTotalCount + 1, filesUploadedTypesTotalCounts: { ...filesUploadedTypesTotalCounts, + // Example: 'json', 'txt', 'csv', etc. [fileType]: _.get(filesUploadedTypesTotalCounts, fileType, 0) + 1, }, filesUploadedByApp: { ...filesUploadedByApp, + // Example: 'maps', 'ml', etc. [app]: { ..._.get(filesUploadedByApp, app, {}), [fileType]: _.get(filesUploadedByApp, `${app}.${fileType}`, 0) + 1,