Skip to content

Commit

Permalink
[Visualizations] Expensive queries are causing unnecessary load and d…
Browse files Browse the repository at this point in the history 
…elays on Elasticsearch (#99031) (#110447)

* [Visualizations] Expensive queries are causing unnecessary load and delays on Elasticsearch

Part of: #93770

* fix CI

* fix typo

* fix namespaces issue

* fix tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@alexwizp @kibanamachine
alexwizp and kibanamachine authored Aug 30, 2021
1 parent 949e123 commit acc8bc6
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 177 deletions.
24 changes: 10 additions & 14 deletions src/plugins/visualizations/server/plugin.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,29 @@

import { i18n } from '@kbn/i18n';
import { schema } from '@kbn/config-schema';
import { Observable } from 'rxjs';
import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
import {

import { VISUALIZE_ENABLE_LABS_SETTING } from '../common/constants';
import { visualizationSavedObjectType } from './saved_objects';
import { registerVisualizationsCollector } from './usage_collector';

import type { VisualizationsPluginSetup, VisualizationsPluginStart } from './types';
import type {
PluginInitializerContext,
CoreSetup,
CoreStart,
Plugin,
Logger,
} from '../../../core/server';

import { VISUALIZE_ENABLE_LABS_SETTING } from '../common/constants';

import { visualizationSavedObjectType } from './saved_objects';

import { VisualizationsPluginSetup, VisualizationsPluginStart } from './types';
import { registerVisualizationsCollector } from './usage_collector';
import { EmbeddableSetup } from '../../embeddable/server';
import type { UsageCollectionSetup } from '../../usage_collection/server';
import type { EmbeddableSetup } from '../../embeddable/server';
import { visualizeEmbeddableFactory } from './embeddable/visualize_embeddable_factory';

export class VisualizationsPlugin
implements Plugin<VisualizationsPluginSetup, VisualizationsPluginStart> {
private readonly logger: Logger;
private readonly config: Observable<{ kibana: { index: string } }>;

constructor(initializerContext: PluginInitializerContext) {
this.logger = initializerContext.logger.get();
this.config = initializerContext.config.legacy.globalConfig$;
}

public setup(
Expand All @@ -61,7 +57,7 @@ export class VisualizationsPlugin
});

if (plugins.usageCollection) {
registerVisualizationsCollector(plugins.usageCollection, this.config);
registerVisualizationsCollector(plugins.usageCollection);
}

plugins.embeddable.registerEmbeddableFactory(visualizeEmbeddableFactory());
Expand Down
134 changes: 57 additions & 77 deletions src/plugins/visualizations/server/usage_collector/get_usage_collector.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,126 +7,106 @@
*/

import moment from 'moment';
import { ElasticsearchClient } from 'src/core/server';
import { getStats } from './get_usage_collector';
import type { SavedObjectsClientContract } from '../../../../core/server';

const defaultMockSavedObjects = [
{
_id: 'visualization:coolviz-123',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "shell_beads"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
id: 'visualization:coolviz-123',
attributes: { visState: '{"type": "shell_beads"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
];

const enlargedMockSavedObjects = [
// default space
{
_id: 'visualization:coolviz-123',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
id: 'visualization:coolviz-123',
namespaces: ['default'],
attributes: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
{
_id: 'visualization:coolviz-456',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "printing_press"}' },
updated_at: moment().subtract(20, 'days').startOf('day').toString(),
},
id: 'visualization:coolviz-456',
namespaces: ['default'],
attributes: { visState: '{"type": "printing_press"}' },
updated_at: moment().subtract(20, 'days').startOf('day').toString(),
},
{
_id: 'meat:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(2, 'months').startOf('day').toString(),
},
id: 'meat:visualization:coolviz-789',
namespaces: ['default'],
attributes: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(2, 'months').startOf('day').toString(),
},
// meat space
{
_id: 'meat:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(89, 'days').startOf('day').toString(),
},
id: 'meat:visualization:coolviz-789',
namespaces: ['meat'],
attributes: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(89, 'days').startOf('day').toString(),
},
{
_id: 'meat:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "cuneiform"}' },
updated_at: moment().subtract(5, 'months').startOf('day').toString(),
},
id: 'meat:visualization:coolviz-789',
namespaces: ['meat'],
attributes: { visState: '{"type": "cuneiform"}' },
updated_at: moment().subtract(5, 'months').startOf('day').toString(),
},
{
_id: 'meat:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "cuneiform"}' },
updated_at: moment().subtract(2, 'days').startOf('day').toString(),
},
id: 'meat:visualization:coolviz-789',
namespaces: ['meat'],
attributes: { visState: '{"type": "cuneiform"}' },
updated_at: moment().subtract(2, 'days').startOf('day').toString(),
},
{
_id: 'meat:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
id: 'meat:visualization:coolviz-789',
attributes: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(7, 'days').startOf('day').toString(),
},
// cyber space
{
_id: 'cyber:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(7, 'months').startOf('day').toString(),
},
id: 'cyber:visualization:coolviz-789',
namespaces: ['cyber'],
attributes: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(7, 'months').startOf('day').toString(),
},
{
_id: 'cyber:visualization:coolviz-789',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(3, 'days').startOf('day').toString(),
},
id: 'cyber:visualization:coolviz-789',
namespaces: ['cyber'],
attributes: { visState: '{"type": "floppy_disk"}' },
updated_at: moment().subtract(3, 'days').startOf('day').toString(),
},
{
_id: 'cyber:visualization:coolviz-123',
_source: {
type: 'visualization',
visualization: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(15, 'days').startOf('day').toString(),
},
id: 'cyber:visualization:coolviz-123',
namespaces: ['cyber'],
attributes: { visState: '{"type": "cave_painting"}' },
updated_at: moment().subtract(15, 'days').startOf('day').toString(),
},
];

describe('Visualizations usage collector', () => {
const mockIndex = '';

const getMockCallCluster = (hits: unknown[]) =>
({
search: () => Promise.resolve({ body: { hits: { hits } } }) as unknown,
} as ElasticsearchClient);
const getMockCallCluster = (savedObjects: unknown[]) =>
(({
createPointInTimeFinder: jest.fn().mockResolvedValue({
close: jest.fn(),
find: function* asyncGenerator() {
yield { saved_objects: savedObjects };
},
}),
} as unknown) as SavedObjectsClientContract);

test('Returns undefined when no results found (undefined)', async () => {
const result = await getStats(getMockCallCluster(undefined as any), mockIndex);
const result = await getStats(getMockCallCluster(undefined as any));

expect(result).toBeUndefined();
});

test('Returns undefined when no results found (0 results)', async () => {
const result = await getStats(getMockCallCluster([]), mockIndex);
const result = await getStats(getMockCallCluster([]));
expect(result).toBeUndefined();
});

test('Summarizes visualizations response data', async () => {
const result = await getStats(getMockCallCluster(defaultMockSavedObjects), mockIndex);
const result = await getStats(getMockCallCluster(defaultMockSavedObjects));

expect(result).toMatchObject({
shell_beads: {
Expand Down Expand Up @@ -181,7 +161,7 @@ describe('Visualizations usage collector', () => {
},
};

const result = await getStats(getMockCallCluster(enlargedMockSavedObjects), mockIndex);
const result = await getStats(getMockCallCluster(enlargedMockSavedObjects));

expect(result).toMatchObject(expectedStats);
});
Expand Down
102 changes: 45 additions & 57 deletions src/plugins/visualizations/server/usage_collector/get_usage_collector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@
* Side Public License, v 1.
*/

import { countBy, get, groupBy, mapValues, max, min, values } from 'lodash';
import { ElasticsearchClient } from 'kibana/server';
import type { estypes } from '@elastic/elasticsearch';

import { countBy, groupBy, mapValues, max, min, values } from 'lodash';
import { getPastDays } from './get_past_days';
type ESResponse = estypes.SearchResponse<{ visualization: { visState: string } }>;

import type { SavedObjectsClientContract, SavedObjectsFindResult } from '../../../../core/server';
import type { SavedVisState } from '../../../visualizations/common';

interface VisSummary {
type: string;
Expand All @@ -35,61 +34,50 @@ export interface VisualizationUsage {
* Parse the response data into telemetry payload
*/
export async function getStats(
esClient: ElasticsearchClient,
index: string
soClient: SavedObjectsClientContract
): Promise<VisualizationUsage | undefined> {
const searchParams = {
size: 10000, // elasticsearch index.max_result_window default value
index,
ignoreUnavailable: true,
filterPath: [
'hits.hits._id',
'hits.hits._source.visualization',
'hits.hits._source.updated_at',
],
body: {
query: {
bool: { filter: { term: { type: 'visualization' } } },
},
},
};
const { body: esResponse } = await esClient.search<ESResponse>(searchParams);
const size = get(esResponse, 'hits.hits.length', 0);
if (size < 1) {
return;
}

// `map` to get the raw types
const visSummaries: VisSummary[] = esResponse.hits.hits.map((hit) => {
const spacePhrases = hit._id.toString().split(':');
const lastUpdated: string = get(hit, '_source.updated_at');
const space = spacePhrases.length === 3 ? spacePhrases[0] : 'default'; // if in a custom space, the format of a saved object ID is space:type:id
const visualization = get(hit, '_source.visualization', { visState: '{}' });
const visState: { type?: string } = JSON.parse(visualization.visState);
return {
type: visState.type || '_na_',
space,
past_days: getPastDays(lastUpdated),
};
const finder = await soClient.createPointInTimeFinder({
type: 'visualization',
perPage: 1000,
namespaces: ['*'],
});

// organize stats per type
const visTypes = groupBy(visSummaries, 'type');
const visSummaries: VisSummary[] = [];

// get the final result
return mapValues(visTypes, (curr) => {
const total = curr.length;
const spacesBreakdown = countBy(curr, 'space');
const spaceCounts: number[] = values(spacesBreakdown);
for await (const response of finder.find()) {
(response.saved_objects || []).forEach((so: SavedObjectsFindResult<any>) => {
if (so.attributes?.visState) {
const visState: SavedVisState = JSON.parse(so.attributes.visState);

return {
total,
spaces_min: min(spaceCounts),
spaces_max: max(spaceCounts),
spaces_avg: total / spaceCounts.length,
saved_7_days_total: curr.filter((c) => c.past_days <= 7).length,
saved_30_days_total: curr.filter((c) => c.past_days <= 30).length,
saved_90_days_total: curr.filter((c) => c.past_days <= 90).length,
};
});
visSummaries.push({
type: visState.type ?? '_na_',
space: so.namespaces?.[0] ?? 'default',
past_days: getPastDays(so.updated_at!),
});
}
});
}
await finder.close();

if (visSummaries.length) {
// organize stats per type
const visTypes = groupBy(visSummaries, 'type');

// get the final result
return mapValues(visTypes, (curr) => {
const total = curr.length;
const spacesBreakdown = countBy(curr, 'space');
const spaceCounts: number[] = values(spacesBreakdown);

return {
total,
spaces_min: min(spaceCounts),
spaces_max: max(spaceCounts),
spaces_avg: total / spaceCounts.length,
saved_7_days_total: curr.filter((c) => c.past_days <= 7).length,
saved_30_days_total: curr.filter((c) => c.past_days <= 30).length,
saved_90_days_total: curr.filter((c) => c.past_days <= 90).length,
};
});
}
}
Loading

0 comments on commit acc8bc6

Please sign in to comment.