From 3f3969dedf38d125d35d0d3051bdcc653fc5e7f6 Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Thu, 13 Feb 2020 09:22:37 -0700 Subject: [PATCH] [Maps] Fix document source top hits split by scripted field (#57481) * [Maps] Fix document source top hits split by scripted field * fix i18n message * review feedback * remove unneeded scss file --- .../es_search_source/es_search_source.js | 45 +++++++-- .../apps/maps/documents_source/top_hits.js | 95 +++++++++++-------- .../es_archives/maps/kibana/data.json | 56 +++++++++++ 3 files changed, 145 insertions(+), 51 deletions(-) diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js index b8644adddcf7e2..93ef40162a584e 100644 --- a/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js +++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_search_source/es_search_source.js @@ -29,6 +29,31 @@ import { loadIndexSettings } from './load_index_settings'; import { DEFAULT_FILTER_BY_MAP_BOUNDS } from './constants'; import { ESDocField } from '../../fields/es_doc_field'; +function getField(indexPattern, fieldName) { + const field = indexPattern.fields.getByName(fieldName); + if (!field) { + throw new Error( + i18n.translate('xpack.maps.source.esSearch.fieldNotFoundMsg', { + defaultMessage: `Unable to find '{fieldName}' in index-pattern '{indexPatternTitle}'.`, + values: { fieldName, indexPatternTitle: indexPattern.title }, + }) + ); + } + return field; +} + +function addFieldToDSL(dsl, field) { + return !field.scripted + ? { ...dsl, field: field.name } + : { + ...dsl, + script: { + source: field.script, + lang: field.lang, + }, + }; +} + export class ESSearchSource extends AbstractESSource { static type = ES_SEARCH; static title = i18n.translate('xpack.maps.source.esSearchTitle', { @@ -242,7 +267,7 @@ export class ESSearchSource extends AbstractESSource { } async _getTopHits(layerName, searchFilters, registerCancelCallback) { - const { topHitsSplitField, topHitsSize } = this._descriptor; + const { topHitsSplitField: topHitsSplitFieldName, topHitsSize } = this._descriptor; const indexPattern = await this.getIndexPattern(); const geoField = await this._getGeoField(); @@ -279,20 +304,20 @@ export class ESSearchSource extends AbstractESSource { }; } + const topHitsSplitField = getField(indexPattern, topHitsSplitFieldName); + const cardinalityAgg = { precision_threshold: 1 }; + const termsAgg = { + size: DEFAULT_MAX_BUCKETS_LIMIT, + shard_size: DEFAULT_MAX_BUCKETS_LIMIT, + }; + const searchSource = await this._makeSearchSource(searchFilters, 0); searchSource.setField('aggs', { totalEntities: { - cardinality: { - field: topHitsSplitField, - precision_threshold: 1, - }, + cardinality: addFieldToDSL(cardinalityAgg, topHitsSplitField), }, entitySplit: { - terms: { - field: topHitsSplitField, - size: DEFAULT_MAX_BUCKETS_LIMIT, - shard_size: DEFAULT_MAX_BUCKETS_LIMIT, - }, + terms: addFieldToDSL(termsAgg, topHitsSplitField), aggs: { entityHits: { top_hits: topHits, diff --git a/x-pack/test/functional/apps/maps/documents_source/top_hits.js b/x-pack/test/functional/apps/maps/documents_source/top_hits.js index 65b008e3f07c96..59a4bea27ebabf 100644 --- a/x-pack/test/functional/apps/maps/documents_source/top_hits.js +++ b/x-pack/test/functional/apps/maps/documents_source/top_hits.js @@ -13,60 +13,73 @@ export default function({ getPageObjects, getService }) { const inspector = getService('inspector'); const find = getService('find'); - describe('top hits', () => { - before(async () => { - await PageObjects.maps.loadSavedMap('document example top hits'); - }); - - it('should not fetch any search hits', async () => { - await inspector.open(); - await inspector.openInspectorRequestsView(); - const requestStats = await inspector.getTableData(); - const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits'); - expect(hits).to.equal('0'); // aggregation requests do not return any documents - }); - - it('should display top hits per entity', async () => { - const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(10); - }); - - describe('configuration', () => { + describe('geo top hits', () => { + describe('split on string field', () => { before(async () => { - await PageObjects.maps.openLayerPanel('logstash'); - // Can not use testSubjects because data-test-subj is placed range input and number input - const sizeInput = await find.byCssSelector( - `input[data-test-subj="layerPanelTopHitsSize"][type='number']` - ); - await sizeInput.click(); - await sizeInput.clearValue(); - await sizeInput.type('3'); - await PageObjects.maps.waitForLayersToLoad(); + await PageObjects.maps.loadSavedMap('document example top hits'); }); - after(async () => { - await PageObjects.maps.closeLayerPanel(); + it('should not fetch any search hits', async () => { + await inspector.open(); + await inspector.openInspectorRequestsView(); + const requestStats = await inspector.getTableData(); + const hits = PageObjects.maps.getInspectorStatRowHit(requestStats, 'Hits'); + expect(hits).to.equal('0'); // aggregation requests do not return any documents }); - it('should update top hits when configation changes', async () => { + it('should display top hits per entity', async () => { const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(15); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(10); }); - }); - describe('query', () => { - before(async () => { - await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + describe('configuration', () => { + before(async () => { + await PageObjects.maps.openLayerPanel('logstash'); + // Can not use testSubjects because data-test-subj is placed range input and number input + const sizeInput = await find.byCssSelector( + `input[data-test-subj="layerPanelTopHitsSize"][type='number']` + ); + await sizeInput.click(); + await sizeInput.clearValue(); + await sizeInput.type('3'); + await PageObjects.maps.waitForLayersToLoad(); + }); + + after(async () => { + await PageObjects.maps.closeLayerPanel(); + }); + + it('should update top hits when configation changes', async () => { + const mapboxStyle = await PageObjects.maps.getMapboxStyle(); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(15); + }); + }); + + describe('query', () => { + before(async () => { + await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + }); + + after(async () => { + await PageObjects.maps.setAndSubmitQuery(''); + }); + + it('should apply query to top hits request', async () => { + await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + const mapboxStyle = await PageObjects.maps.getMapboxStyle(); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(2); + }); }); + }); - after(async () => { - await PageObjects.maps.setAndSubmitQuery(''); + describe('split on scripted field', () => { + before(async () => { + await PageObjects.maps.loadSavedMap('document example top hits split with scripted field'); }); - it('should apply query to top hits request', async () => { - await PageObjects.maps.setAndSubmitQuery('machine.os.raw : "win 8"'); + it('should display top hits per entity', async () => { const mapboxStyle = await PageObjects.maps.getMapboxStyle(); - expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(2); + expect(mapboxStyle.sources[VECTOR_SOURCE_ID].data.features.length).to.equal(24); }); }); }); diff --git a/x-pack/test/functional/es_archives/maps/kibana/data.json b/x-pack/test/functional/es_archives/maps/kibana/data.json index e24734841bf55c..e50ec593cc9902 100644 --- a/x-pack/test/functional/es_archives/maps/kibana/data.json +++ b/x-pack/test/functional/es_archives/maps/kibana/data.json @@ -278,6 +278,62 @@ } } +{ + "type": "doc", + "value": { + "id": "map:4ea1e4f0-4dba-11ea-b554-4ba0def79f86", + "index": ".kibana", + "source": { + "map": { + "title" : "document example top hits split with scripted field", + "description" : "", + "mapStateJSON" : "{\"zoom\":4.1,\"center\":{\"lon\":-100.61091,\"lat\":33.23887},\"timeFilters\":{\"from\":\"2015-09-20T00:00:00.000Z\",\"to\":\"2015-09-24T01:00:00.000Z\"},\"refreshConfig\":{\"isPaused\":true,\"interval\":1000},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON" : "[{\"id\":\"0hmz5\",\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"id\":\"road_map\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"TILE\",\"properties\":{}},\"type\":\"VECTOR_TILE\",\"minZoom\":0,\"maxZoom\":24},{\"id\":\"z52lq\",\"label\":\"logstash\",\"minZoom\":0,\"maxZoom\":24,\"sourceDescriptor\":{\"id\":\"e1a5e1a6-676c-4a89-8ea9-0d91d64b73c6\",\"type\":\"ES_SEARCH\",\"geoField\":\"geo.coordinates\",\"limit\":2048,\"filterByMapBounds\":true,\"showTooltip\":true,\"tooltipProperties\":[],\"useTopHits\":true,\"topHitsSplitField\":\"hour_of_day\",\"topHitsSize\":1,\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"visible\":true,\"temporary\":false,\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#e6194b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":10}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}},\"previousStyle\":null},\"type\":\"VECTOR\"}]", + "uiStateJSON" : "{\"isLayerTOCOpen\":true,\"openTOCDetails\":[]}", + "bounds" : { + "type" : "Polygon", + "coordinates" : [ + [ + [ + -141.61334, + 47.30762 + ], + [ + -141.61334, + 16.49119 + ], + [ + -59.60848, + 16.49119 + ], + [ + -59.60848, + 47.30762 + ], + [ + -141.61334, + 47.30762 + ] + ] + ] + } + }, + "type" : "map", + "references" : [ + { + "name" : "layer_1_source_index_pattern", + "type" : "index-pattern", + "id" : "c698b940-e149-11e8-a35a-370a8516603a" + } + ], + "migrationVersion" : { + "map" : "7.7.0" + }, + "updated_at" : "2020-02-12T17:08:36.671Z" + } + } +} + { "type": "doc", "value": {