From a4fe581653337cfbe6ba7bc6cbf4b084fbb3b25f Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Thu, 6 Feb 2020 13:30:10 +0100 Subject: [PATCH] [ML] change import endpoint call to fileupload plugin, update file analyzer endpoint --- .../services/ml_api_service/datavisualizer.js | 6 +- ..._visualizer.js => file_data_visualizer.ts} | 70 ++++++++++++------- .../{index.js => index.ts} | 3 +- .../ml/server/routes/file_data_visualizer.js | 69 ------------------ .../ml/server/routes/file_data_visualizer.ts | 41 +++++++++++ 5 files changed, 90 insertions(+), 99 deletions(-) rename x-pack/legacy/plugins/ml/server/models/file_data_visualizer/{file_data_visualizer.js => file_data_visualizer.ts} (53%) rename x-pack/legacy/plugins/ml/server/models/file_data_visualizer/{index.js => index.ts} (66%) delete mode 100644 x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js create mode 100644 x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts diff --git a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/datavisualizer.js b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/datavisualizer.js index c9f6bc08e75ec9..584641068405c1 100644 --- a/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/datavisualizer.js +++ b/x-pack/legacy/plugins/ml/public/application/services/ml_api_service/datavisualizer.js @@ -8,7 +8,7 @@ import chrome from 'ui/chrome'; import { http } from '../http_service'; -const basePath = chrome.addBasePath('/api/ml'); +const basePath = chrome.addBasePath('/api'); export const fileDatavisualizer = { analyzeFile(obj, params = {}) { @@ -22,7 +22,7 @@ export const fileDatavisualizer = { } } return http({ - url: `${basePath}/file_data_visualizer/analyze_file${paramString}`, + url: `${basePath}/ml/file_data_visualizer/analyze_file${paramString}`, method: 'POST', data: obj, }); @@ -33,7 +33,7 @@ export const fileDatavisualizer = { const { index, data, settings, mappings, ingestPipeline } = obj; return http({ - url: `${basePath}/file_data_visualizer/import${paramString}`, + url: `${basePath}/fileupload/import${paramString}`, method: 'POST', data: { index, diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts similarity index 53% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts index 28bb7c24cf12ec..9c76d886b623ef 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -6,19 +6,37 @@ import Boom from 'boom'; import fs from 'fs'; +import { RequestHandlerContext } from 'kibana/server'; import os from 'os'; -const util = require('util'); -// const readFile = util.promisify(fs.readFile); +import util from 'util'; + const readdir = util.promisify(fs.readdir); const writeFile = util.promisify(fs.writeFile); -export function fileDataVisualizerProvider(callWithRequest) { - async function analyzeFile(data, overrides) { +export interface InputData { + [key: string]: any; +} + +export interface InputOverrides { + [key: string]: string; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export function fileDataVisualizerProvider(context: RequestHandlerContext) { + async function analyzeFile(data: any, overrides: any) { let cached = false; let results = []; try { - results = await callWithRequest('ml.fileStructure', { body: data, ...overrides }); + results = await context.ml!.mlClient.callAsCurrentUser('ml.fileStructure', { + body: data, + ...overrides, + }); if (false) { // disabling caching for now cached = await cacheData(data); @@ -37,7 +55,7 @@ export function fileDataVisualizerProvider(callWithRequest) { }; } - async function cacheData(data) { + async function cacheData(data: InputData) { const outputPath = `${os.tmpdir()}/kibana-ml`; const tempFile = 'es-ml-tempFile'; const tempFilePath = `${outputPath}/${tempFile}`; @@ -52,13 +70,13 @@ export function fileDataVisualizerProvider(callWithRequest) { } } - function createOutputDir(dir) { + function createOutputDir(dir: string) { if (fs.existsSync(dir) === false) { fs.mkdirSync(dir); } } - async function deleteOutputFiles(outputPath) { + async function deleteOutputFiles(outputPath: string) { const files = await readdir(outputPath); files.forEach(f => { fs.unlinkSync(`${outputPath}/${f}`); @@ -70,28 +88,30 @@ export function fileDataVisualizerProvider(callWithRequest) { }; } -function formatOverrides(overrides) { +function formatOverrides(overrides: InputOverrides) { let hasOverrides = false; - const reducedOverrides = Object.keys(overrides).reduce((p, c) => { - if (overrides[c] !== '') { - p[c] = overrides[c]; - hasOverrides = true; - } - return p; - }, {}); + const reducedOverrides: FormattedOverrides = Object.keys(overrides).reduce((acc, overrideKey) => { + const overrideValue: string = overrides[overrideKey]; + if (overrideValue !== '') { + acc[overrideKey] = overrideValue; - if (reducedOverrides.column_names !== undefined) { - reducedOverrides.column_names = reducedOverrides.column_names.split(','); - } + if (overrideKey === 'column_names') { + acc.column_names = overrideValue.split(','); + } - if (reducedOverrides.has_header_row !== undefined) { - reducedOverrides.has_header_row = reducedOverrides.has_header_row === 'true'; - } + if (overrideKey === 'has_header_row') { + acc.has_header_row = overrideValue === 'true'; + } - if (reducedOverrides.should_trim_fields !== undefined) { - reducedOverrides.should_trim_fields = reducedOverrides.should_trim_fields === 'true'; - } + if (overrideKey === 'should_trim_fields') { + acc.should_trim_fields = overrideValue === 'true'; + } + + hasOverrides = true; + } + return acc; + }, {} as FormattedOverrides); return { reducedOverrides, diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts similarity index 66% rename from x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js rename to x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts index 3bda5599e71816..ed2b3d62bc0e17 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.js +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/index.ts @@ -4,5 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { fileDataVisualizerProvider } from './file_data_visualizer'; -export { importDataProvider } from './import_data'; +export { fileDataVisualizerProvider, InputOverrides, InputData } from './file_data_visualizer'; diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js deleted file mode 100644 index fc6a0ff7569282..00000000000000 --- a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.js +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { callWithRequestFactory } from '../client/call_with_request_factory'; -import { wrapError } from '../client/errors'; -import { fileDataVisualizerProvider, importDataProvider } from '../models/file_data_visualizer'; -import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; - -import { incrementFileDataVisualizerIndexCreationCount } from '../lib/ml_telemetry/ml_telemetry'; - -function analyzeFiles(callWithRequest, data, overrides) { - const { analyzeFile } = fileDataVisualizerProvider(callWithRequest); - return analyzeFile(data, overrides); -} - -function importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data) { - const { importData: importDataFunc } = importDataProvider(callWithRequest); - return importDataFunc(id, index, settings, mappings, ingestPipeline, data); -} - -export function fileDataVisualizerRoutes({ - commonRouteConfig, - elasticsearchPlugin, - route, - savedObjects, -}) { - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/analyze_file', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const data = request.payload; - - return analyzeFiles(callWithRequest, data, request.query).catch(wrapError); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); - - route({ - method: 'POST', - path: '/api/ml/file_data_visualizer/import', - handler(request) { - const callWithRequest = callWithRequestFactory(elasticsearchPlugin, request); - const { id } = request.query; - const { index, data, settings, mappings, ingestPipeline } = request.payload; - - // `id` being `undefined` tells us that this is a new import due to create a new index. - // follow-up import calls to just add additional data will include the `id` of the created - // index, we'll ignore those and don't increment the counter. - if (id === undefined) { - incrementFileDataVisualizerIndexCreationCount(elasticsearchPlugin, savedObjects); - } - - return importData(callWithRequest, id, index, settings, mappings, ingestPipeline, data).catch( - wrapError - ); - }, - config: { - ...commonRouteConfig, - payload: { maxBytes: MAX_BYTES }, - }, - }); -} diff --git a/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts new file mode 100644 index 00000000000000..bed79c9501ce77 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/routes/file_data_visualizer.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { schema } from '@kbn/config-schema'; +import { RequestHandlerContext } from 'kibana/server'; +import { wrapError } from '../client/error_wrapper'; +import { + InputOverrides, + InputData, + fileDataVisualizerProvider, +} from '../models/file_data_visualizer'; + +import { licensePreRoutingFactory } from '../new_platform/licence_check_pre_routing_factory'; +import { RouteInitialization } from '../new_platform/plugin'; + +function analyzeFiles(context: RequestHandlerContext, data: InputData, overrides: InputOverrides) { + const { analyzeFile } = fileDataVisualizerProvider(context); + return analyzeFile(data, overrides); +} + +export function fileDataVisualizerRoutes({ router, xpackMainPlugin }: RouteInitialization) { + router.post( + { + path: '/api/ml/file_data_visualizer/analyze_file', + validate: { + body: schema.any(), + }, + }, + licensePreRoutingFactory(xpackMainPlugin, async (context, request, response) => { + try { + const result = await analyzeFiles(context, request.body, request.query); + return response.ok({ body: result }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); +}