diff --git a/src/core/lib/kbn_internal_native_observable/index.js b/src/core/lib/kbn_internal_native_observable/index.js index 504ddc1e342972..7e8b4077990962 100644 --- a/src/core/lib/kbn_internal_native_observable/index.js +++ b/src/core/lib/kbn_internal_native_observable/index.js @@ -4,7 +4,7 @@ import { observable as SymbolObservable } from 'rxjs/internal/symbol/observable' // see https://github.com/tc39/proposal-observable. // // One change has been applied to work with current libraries: using the -// Symbol.observable ponyfill instead of relying on the implementation in the +// Symbol_observable ponyfill instead of relying on the implementation in the // spec. // === Abstract Operations === diff --git a/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_config.test.ts.snap b/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_config.test.ts.snap new file mode 100644 index 00000000000000..497396b46cff82 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_config.test.ts.snap @@ -0,0 +1,40 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`creates elasticsearch client config when shouldAuth is false (for admin clients) 1`] = ` +Object { + "apiVersion": undefined, + "host": Object { + "headers": undefined, + "host": null, + "path": null, + "port": null, + "protocol": null, + "query": null, + }, + "keepAlive": true, + "logQueries": undefined, + "pingTimeout": undefined, + "requestTimeout": undefined, + "username": undefined, +} +`; + +exports[`creates elasticsearch client config when shouldAuth is true by default (for data clients) 1`] = ` +Object { + "apiVersion": undefined, + "host": Object { + "auth": "foo:bar", + "headers": undefined, + "host": null, + "path": null, + "port": null, + "protocol": null, + "query": null, + }, + "keepAlive": true, + "logQueries": undefined, + "pingTimeout": undefined, + "requestTimeout": undefined, + "username": "foo", +} +`; diff --git a/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_service.test.ts.snap b/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_service.test.ts.snap new file mode 100644 index 00000000000000..7d46d26692b924 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/__snapshots__/elasticsearch_service.test.ts.snap @@ -0,0 +1,20 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`should not create multiple clients while service is running 1`] = ` +Array [ + Array [ + Object { + "options": Object { + "shouldAuth": false, + }, + "type": "admin", + }, + ], + Array [ + Object { + "options": undefined, + "type": "data", + }, + ], +] +`; diff --git a/src/core/server/elasticsearch/__tests__/admin_client.test.ts b/src/core/server/elasticsearch/__tests__/admin_client.test.ts new file mode 100644 index 00000000000000..a40ca693ec89b2 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/admin_client.test.ts @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const mockCallAPI = jest.fn(); + +jest.mock('../call_api', () => ({ + callAPI: mockCallAPI, +})); + +import { Client } from 'elasticsearch'; +import { AdminClient } from '../admin_client'; + +let client: AdminClient; +let esClient: Client; + +beforeEach(() => { + esClient = new Client({}); + client = new AdminClient(esClient); +}); + +describe('call passes correct arguments to callAPI', () => { + test('when only endpoint is specified', () => { + client.call('foo'); + expect(mockCallAPI).toHaveBeenCalledWith(esClient, 'foo', {}, { wrap401Errors: true }); + }); + + test('when endpoint and clientParams are specified', () => { + client.call('foo', { bar: 'baz' }); + expect(mockCallAPI).toHaveBeenCalledWith( + esClient, + 'foo', + { bar: 'baz' }, + { wrap401Errors: true } + ); + }); + + test('when endpoint, clientParams, and options are specified', () => { + client.call('foo', {}, { wrap401Errors: true }); + expect(mockCallAPI).toHaveBeenCalledWith(esClient, 'foo', {}, { wrap401Errors: true }); + }); + + test('when endpoint contains periods', () => { + client.call('foo.bar.baz'); + expect(mockCallAPI).toHaveBeenCalledWith(esClient, 'foo.bar.baz', {}, { wrap401Errors: true }); + }); +}); diff --git a/src/core/server/elasticsearch/__tests__/call_api.test.ts b/src/core/server/elasticsearch/__tests__/call_api.test.ts new file mode 100644 index 00000000000000..26715b6e5fb27c --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/call_api.test.ts @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { callAPI } from '../call_api'; + +describe('things', () => { + test('should call the api when it exists, with the right context and params', async () => { + let apiContext; + const baz = jest.fn(function(this: any) { + apiContext = this; + }); + const clientParams = {}; + const client: any = { foo: { bar: { baz } } }; + await callAPI(client, 'foo.bar.baz', clientParams); + + expect(baz).toHaveBeenCalledWith(clientParams); + expect(apiContext).toBe(client.foo.bar); + }); + + test('should fail when endpoint does not exist on client', async () => { + expect.assertions(1); + + const client: any = {}; + + try { + await callAPI(client, 'foo.bar.baz', {}); + } catch (error) { + expect(error.message).toEqual('called with an invalid endpoint: foo.bar.baz'); + } + }); + + test('should handle top-level endpoint', async () => { + let apiContext; + const fooFn = jest.fn(function(this: any) { + apiContext = this; + }); + const client: any = { foo: fooFn }; + await callAPI(client, 'foo', {}); + + expect(apiContext).toBe(client); + }); + + test('should handle failing api call', async () => { + expect.assertions(2); + + const fooFn = () => { + throw new Error('api call failed'); + }; + + const client: any = { foo: fooFn }; + + try { + await callAPI(client, 'foo', {}); + } catch (error) { + expect(error.message).toEqual('api call failed'); + expect(error.wrap401Errors).toBeUndefined(); + } + }); +}); + +// TODO: change this test after implementing +// homegrown error lib or boom +// https://github.com/elastic/kibana/issues/12464 +describe('should wrap 401 errors', () => { + test('when wrap401Errors is undefined', async () => { + expect.assertions(2); + + const fooFn = () => { + const err: any = new Error('api call failed'); + err.statusCode = 401; + throw err; + }; + + const client: any = { foo: fooFn }; + + try { + await callAPI(client, 'foo', {}); + } catch (error) { + expect(error.message).toEqual('api call failed'); + expect(error.wrap401Errors).toBe(true); + } + }); +}); + +describe('should not wrap 401 errors', () => { + test('when wrap401Errors is false', async () => { + expect.assertions(2); + + const fooFn = () => { + const err: any = new Error('api call failed'); + err.statusCode = 401; + throw err; + }; + + const client: any = { foo: fooFn }; + + try { + await callAPI(client, 'foo', {}, { wrap401Errors: false }); + } catch (error) { + expect(error.message).toEqual('api call failed'); + expect(error.wrap401Errors).toBeUndefined(); + } + }); + + test('when statusCode is not 401', async () => { + expect.assertions(2); + + const fooFn = () => { + const err: any = new Error('api call failed'); + err.statusCode = 400; + throw err; + }; + + const client: any = { foo: fooFn }; + + try { + await callAPI(client, 'foo', {}, { wrap401Errors: true }); + } catch (error) { + expect(error.message).toEqual('api call failed'); + expect(error.wrap401Errors).toBeUndefined(); + } + }); +}); diff --git a/src/core/server/elasticsearch/__tests__/elasticsearch_config.test.ts b/src/core/server/elasticsearch/__tests__/elasticsearch_config.test.ts new file mode 100644 index 00000000000000..75761fb0797697 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/elasticsearch_config.test.ts @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* tslint:disable:no-empty */ +import { ElasticsearchConfig } from '../elasticsearch_config'; +import { ClusterSchema } from '../schema'; + +test('filters headers', () => { + const clusterSchema = {} as ClusterSchema; + const headers = { foo: 'bar', baz: 'qux' }; + const config = new ElasticsearchConfig('data', clusterSchema); + config.requestHeadersWhitelist = ['foo']; + + const expectedHeaders = { foo: 'bar' }; + expect(config.filterHeaders(headers)).toEqual(expectedHeaders); +}); + +describe('creates elasticsearch client config', () => { + test('when shouldAuth is true by default (for data clients)', () => { + const clusterSchema = { + password: 'bar', + pingTimeout: { asMilliseconds: () => {} }, + requestTimeout: { asMilliseconds: () => {} }, + url: '', + username: 'foo', + } as ClusterSchema; + const config = new ElasticsearchConfig('data', clusterSchema); + + expect(config.toElasticsearchClientConfig()).toMatchSnapshot(); + }); + + test('when shouldAuth is false (for admin clients)', () => { + const clusterSchema = { + pingTimeout: { asMilliseconds: () => {} }, + requestTimeout: { asMilliseconds: () => {} }, + url: '', + } as ClusterSchema; + const config = new ElasticsearchConfig('data', clusterSchema); + + expect(config.toElasticsearchClientConfig({ shouldAuth: false })).toMatchSnapshot(); + }); +}); diff --git a/src/core/server/elasticsearch/__tests__/elasticsearch_service.test.ts b/src/core/server/elasticsearch/__tests__/elasticsearch_service.test.ts new file mode 100644 index 00000000000000..12a2d1dbf32185 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/elasticsearch_service.test.ts @@ -0,0 +1,132 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* tslint:disable:no-empty */ +// @ts-ignore +import es from 'elasticsearch'; +jest.mock('elasticsearch', () => ({ + Client: mockFn, +})); + +const mockFn = jest.fn(() => { + return { + close: jest.fn(), + }; +}); + +import { ConnectableObservable, of } from 'rxjs'; +import { first } from 'rxjs/operators'; +import { logger } from '../../logging/__mocks__'; +import { AdminClient } from '../admin_client'; +import { ElasticsearchConfigs } from '../elasticsearch_configs'; +import { ElasticsearchService } from '../elasticsearch_service'; +import { ScopedDataClient } from '../scoped_data_client'; + +it('should not create multiple clients while service is running', async () => { + const createElasticsearchConfig = (type: string) => ({ + filterHeaders: () => {}, + requestHeadersWhitelist: [], + toElasticsearchClientConfig: (options: any) => ({ type, options }), + }); + const elasticsearchConfigs = { + forType: (type: string) => createElasticsearchConfig(type), + }; + const configs$: ConnectableObservable = of(elasticsearchConfigs); + const service = new ElasticsearchService(configs$, logger); + + // Start subscribing to this.clients$, + // which means new elasticsearch data and admin clients are created, + // calling mockCreateClient once for each client (twice) + await service.start(); + + // Get the latest elasticsearch data client + // and create a ScopedDataClient around it + await service.getScopedDataClient({ foo: 'bar' }); + // Calling it again does not create any new elasticsearch clients + await service.getScopedDataClient({ foo: 'bar' }); + + await service.stop(); + + // We expect it to be called only twice: once for the data client + // and once for the admin client. + expect(mockFn).toHaveBeenCalledTimes(2); + + // Check that specifically the admin client and the data client + // were created + expect(mockFn.mock.calls).toMatchSnapshot(); +}); + +it('should get an AdminClient', async () => { + const elasticsearchConfigs = { + forType: () => ({ toElasticsearchClientConfig: () => {} }), + }; + + const configs$: ConnectableObservable = of(elasticsearchConfigs); + + const service = new ElasticsearchService(configs$, logger); + const adminClient = await service + .getAdminClient$() + .pipe(first()) + .toPromise(Promise); + + expect(adminClient).toBeInstanceOf(AdminClient); +}); + +it('should get a ScopedDataClient', async () => { + const elasticsearchConfig = { + filterHeaders: () => {}, + requestHeadersWhitelist: [], + toElasticsearchClientConfig: () => {}, + }; + + const elasticsearchConfigs = { + forType: () => elasticsearchConfig, + }; + + const configs$: ConnectableObservable = of(elasticsearchConfigs); + + const service = new ElasticsearchService(configs$, logger); + const dataClient = await service.getScopedDataClient({ foo: 'bar' }); + + expect(dataClient).toBeInstanceOf(ScopedDataClient); +}); + +it('should get a ScopedDataClient observable', async () => { + const elasticsearchConfig = { + filterHeaders: jest.fn(), + requestHeadersWhitelist: [], + toElasticsearchClientConfig: () => {}, + }; + + const elasticsearchConfigs = { + forType: () => elasticsearchConfig, + }; + + const configs$ = of(elasticsearchConfigs); + + const service = new ElasticsearchService(configs$, logger); + const dataClient$ = service.getScopedDataClient$({ foo: 'bar' }); + + const dataClient = await dataClient$.pipe(first()).toPromise(Promise); + + expect(dataClient).toBeInstanceOf(ScopedDataClient); + expect(elasticsearchConfig.filterHeaders).toHaveBeenCalledWith({ + foo: 'bar', + }); +}); diff --git a/src/core/server/elasticsearch/__tests__/scoped_data_client.test.ts b/src/core/server/elasticsearch/__tests__/scoped_data_client.test.ts new file mode 100644 index 00000000000000..e0c88b3a9c1e19 --- /dev/null +++ b/src/core/server/elasticsearch/__tests__/scoped_data_client.test.ts @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const mockCallAPI = jest.fn(); + +jest.mock('../call_api', () => ({ + callAPI: mockCallAPI, +})); + +import { Client } from 'elasticsearch'; +import { ScopedDataClient } from '../scoped_data_client'; + +let client: ScopedDataClient; +let esClient: Client; + +beforeEach(() => { + esClient = new Client({}); + client = new ScopedDataClient(esClient, { foo: 'bar' }); +}); + +describe('call passes correct arguments to callAPI', () => { + test('when only endpoint is specified', () => { + client.call('foo'); + expect(mockCallAPI).toHaveBeenCalledWith( + esClient, + 'foo', + { headers: { foo: 'bar' } }, + { wrap401Errors: true } + ); + }); + + test('when endpoint and clientParams are specified', () => { + client.call('foo', { bar: 'baz' }); + expect(mockCallAPI).toHaveBeenCalledWith( + esClient, + 'foo', + { headers: { foo: 'bar' }, bar: 'baz' }, + { wrap401Errors: true } + ); + }); + + test('when endpoint, clientParams, and options are specified', () => { + client.call('foo', {}, { wrap401Errors: true }); + expect(mockCallAPI).toHaveBeenCalledWith( + esClient, + 'foo', + { headers: { foo: 'bar' } }, + { wrap401Errors: true } + ); + }); + + test('when endpoint contains periods', () => { + client.call('foo.bar.baz'); + expect(mockCallAPI).toHaveBeenCalledWith( + esClient, + 'foo.bar.baz', + { headers: { foo: 'bar' } }, + { wrap401Errors: true } + ); + }); +}); diff --git a/src/core/server/elasticsearch/admin_client.ts b/src/core/server/elasticsearch/admin_client.ts new file mode 100644 index 00000000000000..15b58030734885 --- /dev/null +++ b/src/core/server/elasticsearch/admin_client.ts @@ -0,0 +1,52 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore +import { Client } from 'elasticsearch'; +import { callAPI } from './call_api'; + +export interface CallAPIOptions { + wrap401Errors: boolean; +} +export interface CallAPIClientParams { + [key: string]: any; +} + +export class AdminClient { + constructor(private readonly client: Client) {} + + /** + * Call the elasticsearch API via the given client + * which is bound to the admin cluster. + * + * @param endpoint Dot-delimited string that corresponds + * to the endpoint path. + * @param clientParams Params that get passed directly to the + * API for the endpoint. + * @param options Object that can specify whether to wrap + * 401 errors. + */ + public call( + endpoint: string, + clientParams: CallAPIClientParams = {}, + options: CallAPIOptions = { wrap401Errors: true } + ): any { + return callAPI(this.client, endpoint, clientParams, options); + } +} diff --git a/src/core/server/elasticsearch/api.ts b/src/core/server/elasticsearch/api.ts new file mode 100644 index 00000000000000..3c5c58ac5eef2e --- /dev/null +++ b/src/core/server/elasticsearch/api.ts @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Router } from '../http'; +import { LoggerFactory } from '../logging'; +import { ElasticsearchService } from './elasticsearch_service'; + +export function registerElasticsearchRoutes( + router: Router, + logger: LoggerFactory, + service: ElasticsearchService +) { + const log = logger.get('elasticsearch', 'routes'); + + log.info('creating elasticsearch api'); + + router.get( + { + path: '/:field', + validate: schema => ({ + params: schema.object({ + field: schema.string(), + }), + query: schema.object({ + key: schema.maybe(schema.string()), + }), + }), + }, + async (req, res) => { + // WUHU! Both of these are typed! + log.info(`field param: ${req.params.field}`); + log.info(`query param: ${req.query.key}`); + + log.info('request received on [data] cluster'); + + const cluster = await service.getScopedDataClient(req.headers); + + log.info('got scoped [data] cluster, now calling it'); + + const response = cluster.call('search', {}); + + return res.ok({ + params: req.params, + query: req.query, + total_count: response.hits.total, + }); + } + ); + + return router; +} diff --git a/src/core/server/elasticsearch/call_api.ts b/src/core/server/elasticsearch/call_api.ts new file mode 100644 index 00000000000000..2251dc70e17cb6 --- /dev/null +++ b/src/core/server/elasticsearch/call_api.ts @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore +import { Client } from 'elasticsearch'; +import { get } from 'lodash'; + +export interface CallAPIOptions { + wrap401Errors: boolean; +} +export interface CallAPIClientParams { + [key: string]: any; +} + +export async function callAPI( + client: Client, + endpoint: string, + clientParams: CallAPIClientParams, + options: CallAPIOptions = { wrap401Errors: true } +) { + const clientPath = endpoint.split('.'); + const api: any = get(client, clientPath); + + if (api === undefined) { + throw new Error(`called with an invalid endpoint: ${endpoint}`); + } + + const apiContext = clientPath.length === 1 ? client : get(client, clientPath.slice(0, -1)); + + try { + return await api.call(apiContext, clientParams); + } catch (err) { + if (options.wrap401Errors && err.statusCode === 401) { + // TODO: decide on using homegrown error lib or boom + // https://github.com/elastic/kibana/issues/12464 + + err.wrap401Errors = true; + throw err; + } + + throw err; + } +} diff --git a/src/core/server/elasticsearch/elasticsearch_config.ts b/src/core/server/elasticsearch/elasticsearch_config.ts new file mode 100644 index 00000000000000..b03c9ab43a05fe --- /dev/null +++ b/src/core/server/elasticsearch/elasticsearch_config.ts @@ -0,0 +1,139 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore avoid converting elasticsearch lib for now +import { ConfigOptions } from 'elasticsearch'; +import { readFileSync } from 'fs'; +import { noop } from 'lodash'; +import url from 'url'; + +import { assertNever, pick } from '../../utils'; +import { filterHeaders, Headers } from '../http/router/headers'; +import { ClusterSchema } from './schema'; + +export enum ElasticsearchClusterType { + admin = 'admin', + data = 'data', +} + +export class ElasticsearchConfig { + public requestHeadersWhitelist: string[]; + + /** + * @internal + */ + constructor( + readonly clusterType: ElasticsearchClusterType, + private readonly config: ClusterSchema + ) { + this.requestHeadersWhitelist = config.requestHeadersWhitelist; + } + + /** + * Filter given headers by requestHeadersWhitelist + * + * e.g. + * + * ``` + * elasticsearchConfigs.forType(ElasticsearchClusterType.data).filterHeaders(request.headers); + * ``` + * + * @param headers Full headers (for a request) + * + */ + public filterHeaders(headers: Headers): Headers { + return filterHeaders(headers, this.requestHeadersWhitelist); + } + + /** + * Config for Elasticsearch client, e.g. + * + * ``` + * new elasticsearch.Client(config.toElasticsearchClientConfig()) + * ``` + * + * @param shouldAuth Whether or not to the config should include the username + * and password. Used to create a client that is not + * authenticated using the config, but from each request. + */ + public toElasticsearchClientConfig({ shouldAuth = true } = {}): ConfigOptions { + const config: ConfigOptions = pick(this.config, ['apiVersion', 'username', 'logQueries']); + + config.pingTimeout = this.config.pingTimeout.asMilliseconds(); + config.requestTimeout = this.config.requestTimeout.asMilliseconds(); + + config.keepAlive = true; + + const uri = url.parse(this.config.url); + + config.host = { + headers: this.config.customHeaders, + host: uri.hostname, + path: uri.pathname, + port: uri.port, + protocol: uri.protocol, + query: uri.query, + }; + + if (shouldAuth && this.config.username !== undefined && this.config.password !== undefined) { + config.host.auth = `${this.config.username}:${this.config.password}`; + } + + if (this.config.ssl === undefined) { + return config; + } + + let ssl: { [key: string]: any } = {}; + if (this.config.ssl.certificate && this.config.ssl.key) { + ssl = { + cert: readFileSync(this.config.ssl.certificate), + key: readFileSync(this.config.ssl.key), + passphrase: this.config.ssl.keyPassphrase, + }; + } + + const verificationMode = this.config.ssl.verificationMode; + + switch (verificationMode) { + case 'none': + ssl.rejectUnauthorized = false; + break; + case 'certificate': + ssl.rejectUnauthorized = true; + + // by default NodeJS checks the server identity + ssl.checkServerIdentity = noop; + break; + case 'full': + ssl.rejectUnauthorized = true; + break; + default: + assertNever(verificationMode as never); + } + + const ca = this.config.ssl.certificateAuthorities; + if (ca && ca.length > 0) { + ssl.ca = ca.map((authority: string) => readFileSync(authority)); + } + + config.ssl = ssl; + + return config; + } +} diff --git a/src/core/server/elasticsearch/elasticsearch_configs.ts b/src/core/server/elasticsearch/elasticsearch_configs.ts new file mode 100644 index 00000000000000..2f7909120a8753 --- /dev/null +++ b/src/core/server/elasticsearch/elasticsearch_configs.ts @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Env } from '../config'; +import { ElasticsearchClusterType, ElasticsearchConfig } from './elasticsearch_config'; +import { ElasticsearchConfigsSchema, elasticsearchSchema } from './schema'; + +export class ElasticsearchConfigs { + /** + * @internal + */ + public static schema = elasticsearchSchema; + + private readonly configs: { [type in ElasticsearchClusterType]: ElasticsearchConfig }; + + /** + * @internal + */ + constructor(config: ElasticsearchConfigsSchema, env: Env) { + this.configs = { + [ElasticsearchClusterType.admin]: new ElasticsearchConfig( + ElasticsearchClusterType.admin, + config + ), + [ElasticsearchClusterType.data]: new ElasticsearchConfig( + ElasticsearchClusterType.data, + config + ), + }; + } + + public forType(type: ElasticsearchClusterType) { + return this.configs[type]; + } +} diff --git a/src/core/server/elasticsearch/elasticsearch_service.ts b/src/core/server/elasticsearch/elasticsearch_service.ts new file mode 100644 index 00000000000000..1696e562c35f19 --- /dev/null +++ b/src/core/server/elasticsearch/elasticsearch_service.ts @@ -0,0 +1,120 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore don't type elasticsearch now +import { Client } from 'elasticsearch'; +import { combineLatest, ConnectableObservable, from, Observable, Subscription } from 'rxjs'; +import { filter, first, map, publishReplay, refCount, switchMap } from 'rxjs/operators'; +import { CoreService } from '../../types/core_service'; +import { Headers } from '../http/router/headers'; +import { LoggerFactory } from '../logging'; +import { AdminClient } from './admin_client'; +import { ElasticsearchClusterType } from './elasticsearch_config'; +import { ElasticsearchConfigs } from './elasticsearch_configs'; +import { ScopedDataClient } from './scoped_data_client'; + +interface Clients { + [ElasticsearchClusterType.admin]: Client; + [ElasticsearchClusterType.data]: Client; +} + +export class ElasticsearchService implements CoreService { + private clients$: ConnectableObservable; + private subscription?: Subscription; + + constructor( + // private readonly configs$: ConnectableObservable, + private readonly configs$: Observable, + logger: LoggerFactory + ) { + const log = logger.get('elasticsearch'); + + this.clients$ = from(configs$).pipe( + filter(() => { + if (this.subscription !== undefined) { + log.error('clusters cannot be changed after they are created'); + return false; + } + + return true; + }), + switchMap(configs => { + return new Observable(observer => { + log.info('creating Elasticsearch clients'); + + const clients = { + admin: new Client( + configs.forType(ElasticsearchClusterType.admin).toElasticsearchClientConfig({ + shouldAuth: false, + }) + ), + data: new Client( + configs.forType(ElasticsearchClusterType.data).toElasticsearchClientConfig() + ), + }; + + observer.next(clients); + + return () => { + log.info('closing Elasticsearch clients'); + + clients.data.close(); + clients.admin.close(); + }; + }); + }), + publishReplay(1), + refCount() + ) as ConnectableObservable; + } + + public async start() { + // ensure that we don't unnecessarily re-create clients by always having + // at least one current connection + this.subscription = this.clients$.subscribe(); + } + + public async stop() { + if (this.subscription !== undefined) { + this.subscription.unsubscribe(); + } + } + + public getAdminClient$() { + return this.clients$.pipe(map(clients => new AdminClient(clients.admin))); + } + + public getScopedDataClient$(headers: Headers) { + return combineLatest(this.clients$, this.configs$).pipe( + map( + ([clients, configs]) => + new ScopedDataClient( + clients.data, + configs.forType(ElasticsearchClusterType.data).filterHeaders(headers) + ) + ) + ); + } + + public async getScopedDataClient(headers: Headers) { + return this.getScopedDataClient$(headers) + .pipe(first()) + .toPromise(); + } +} diff --git a/src/core/server/elasticsearch/index.ts b/src/core/server/elasticsearch/index.ts new file mode 100644 index 00000000000000..dfed33d0bbf6a6 --- /dev/null +++ b/src/core/server/elasticsearch/index.ts @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Observable } from 'rxjs'; + +import { Router } from '../http'; +import { LoggerFactory } from '../logging'; +import { registerElasticsearchRoutes } from './api'; +import { ElasticsearchConfigs } from './elasticsearch_configs'; +import { ElasticsearchService } from './elasticsearch_service'; + +export { ElasticsearchClusterType } from './elasticsearch_config'; +export { AdminClient } from './admin_client'; +export { ScopedDataClient } from './scoped_data_client'; +export { ElasticsearchService, ElasticsearchConfigs }; + +export class ElasticsearchModule { + public readonly service: ElasticsearchService; + + constructor( + readonly config$: Observable, + private readonly logger: LoggerFactory + ) { + this.service = new ElasticsearchService(this.config$, logger); + } + + public createRoutes() { + const router = new Router('/elasticsearch'); + + return registerElasticsearchRoutes(router, this.logger, this.service); + } +} diff --git a/src/core/server/elasticsearch/schema.ts b/src/core/server/elasticsearch/schema.ts new file mode 100644 index 00000000000000..8d4368c81c5b58 --- /dev/null +++ b/src/core/server/elasticsearch/schema.ts @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { schema, TypeOf } from '../config/schema'; + +/** + * @internal + */ +export const sslSchema = schema.object({ + certificate: schema.maybe(schema.string()), + certificateAuthorities: schema.maybe(schema.arrayOf(schema.string(), { minSize: 1 })), + key: schema.maybe(schema.string()), + keyPassphrase: schema.maybe(schema.string()), + verificationMode: schema.oneOf([ + schema.literal('none'), + schema.literal('certificate'), + schema.literal('full'), + ]), +}); + +const DEFAULT_REQUEST_HEADERS = ['authorization']; + +/** + * @internal + */ +const sharedElasticsearchFields = { + apiVersion: schema.string({ defaultValue: 'master' }), + customHeaders: schema.maybe(schema.object({})), + logQueries: schema.boolean({ defaultValue: false }), + password: schema.maybe(schema.string()), + pingTimeout: schema.duration({ defaultValue: '30s' }), + preserveHost: schema.boolean({ defaultValue: true }), + requestHeadersWhitelist: schema.arrayOf(schema.string(), { + defaultValue: DEFAULT_REQUEST_HEADERS, + }), + requestTimeout: schema.duration({ defaultValue: '30s' }), + shardTimeout: schema.duration({ defaultValue: '30s' }), + ssl: schema.maybe(sslSchema), + startupTimeout: schema.duration({ defaultValue: '5s' }), + url: schema.string({ defaultValue: 'http://localhost:9200' }), + username: schema.maybe(schema.string()), +}; + +/** + * @internal + */ +const clusterSchema = schema.object({ + ...sharedElasticsearchFields, +}); + +/** + * @internal + */ +export const elasticsearchSchema = schema.object({ + enabled: schema.boolean({ defaultValue: true }), + ...sharedElasticsearchFields, + healthCheck: schema.object({ + delay: schema.duration({ defaultValue: '2500ms' }), + }), +}); + +/** + * @internal + */ +export type ElasticsearchConfigsSchema = TypeOf; + +/** + * @internal + */ +export type ClusterSchema = TypeOf; diff --git a/src/core/server/elasticsearch/scoped_data_client.ts b/src/core/server/elasticsearch/scoped_data_client.ts new file mode 100644 index 00000000000000..40b5639dd80281 --- /dev/null +++ b/src/core/server/elasticsearch/scoped_data_client.ts @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// @ts-ignore don't type elasticsearch now +import { Client } from 'elasticsearch'; +import { Headers } from '../http/router/headers'; +import { callAPI } from './call_api'; + +export interface CallAPIOptions { + wrap401Errors: boolean; +} +export interface CallAPIClientParams { + [key: string]: any; +} + +export class ScopedDataClient { + constructor(private readonly client: Client, private readonly headers: Headers) {} + + /** + * Call the elasticsearch API via the given client + * which is bound to the data cluster. + * + * @param endpoint Dot-delimited string that corresponds + * to the endpoint path. + * @param clientParams Params that get passed directly to the + * API for the endpoint. + * @param options Object that can specify whether to wrap + * 401 errors. + */ + public call( + endpoint: string, + clientParams: CallAPIClientParams = {}, + options: CallAPIOptions = { wrap401Errors: true } + ): any { + clientParams = { ...clientParams, headers: this.headers }; + + return callAPI(this.client, endpoint, clientParams, options); + } +} diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 27d9a12081897b..5b8f199c909741 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -17,18 +17,24 @@ * under the License. */ +import { from } from 'rxjs'; import { ConfigService, Env } from './config'; +import { ElasticsearchConfigs, ElasticsearchModule } from './elasticsearch'; import { HttpConfig, HttpModule, Router } from './http'; import { Logger, LoggerFactory } from './logging'; export class Server { + private readonly elasticsearch: ElasticsearchModule; private readonly http: HttpModule; private readonly log: Logger; constructor(private readonly configService: ConfigService, logger: LoggerFactory, env: Env) { this.log = logger.get('server'); + const esConfigs$ = configService.atPath('elasticsearch', ElasticsearchConfigs); const httpConfig$ = configService.atPath('server', HttpConfig); + + this.elasticsearch = new ElasticsearchModule(from(esConfigs$), logger); this.http = new HttpModule(httpConfig$, logger, env); } @@ -39,6 +45,7 @@ export class Server { router.get({ path: '/', validate: false }, async (req, res) => res.ok({ version: '0.0.1' })); this.http.service.registerRouter(router); + await this.elasticsearch.service.start(); await this.http.service.start(); const unhandledConfigPaths = await this.configService.getUnusedPaths(); @@ -51,5 +58,6 @@ export class Server { this.log.debug('stopping server'); await this.http.service.stop(); + await this.elasticsearch.service.stop(); } } diff --git a/src/server/config/config.js b/src/server/config/config.js index 5bdeda1fa8ac28..73187d7839a2b9 100644 --- a/src/server/config/config.js +++ b/src/server/config/config.js @@ -163,7 +163,7 @@ export class Config { if (_.size(schema._inner.children)) { for (let i = 0; i < schema._inner.children.length; i++) { const child = schema._inner.children[i]; - // If the child is an object recurse through it's children and return + // If the child is an object recurse through its children and return // true if there's a match if (child.schema._type === 'object') { if (has(key, child.schema, path.concat([child.key]))) return true; diff --git a/src/ui/__tests__/__fixtures__/elasticsearch_plugin/index.js b/src/ui/__tests__/__fixtures__/elasticsearch_plugin/index.js new file mode 100644 index 00000000000000..1ddb2bdbee227e --- /dev/null +++ b/src/ui/__tests__/__fixtures__/elasticsearch_plugin/index.js @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export default function (kibana) { + return new kibana.Plugin({ + config(Joi) { + return Joi.object({ + enabled: Joi.boolean().default(true), + ssl: Joi.object({ + verificationMode: Joi.string().default('full'), + }), + }).default(); + }, + init() {}, + uiExports: {}, + }); +} diff --git a/src/ui/__tests__/__fixtures__/elasticsearch_plugin/package.json b/src/ui/__tests__/__fixtures__/elasticsearch_plugin/package.json new file mode 100644 index 00000000000000..75f95f485f2bda --- /dev/null +++ b/src/ui/__tests__/__fixtures__/elasticsearch_plugin/package.json @@ -0,0 +1,4 @@ +{ + "name": "elasticsearch", + "version": "kibana" +} diff --git a/src/ui/__tests__/ui_exports_replace_injected_vars.js b/src/ui/__tests__/ui_exports_replace_injected_vars.js index b7762ef104b901..a0a1ddc1f87685 100644 --- a/src/ui/__tests__/ui_exports_replace_injected_vars.js +++ b/src/ui/__tests__/ui_exports_replace_injected_vars.js @@ -56,7 +56,15 @@ describe('UiExports', function () { logging: { silent: true }, // no logs optimize: { enabled: false }, plugins: { - paths: [resolve(__dirname, './fixtures/test_app')] // inject an app so we can hit /app/{id} + paths: [ + resolve(__dirname, './__fixtures__/elasticsearch_plugin'), + resolve(__dirname, './fixtures/test_app'), + ] // inject an app so we can hit /app/{id} + }, + elasticsearch: { + ssl: { + verificationMode: 'full' + } }, });