From d9e315dafcec6f148f1f804cba290f2047beeb25 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 20 Apr 2020 12:29:05 +0200 Subject: [PATCH 1/3] Decode fields: layerListJSON, mapStateJSON --- dev/import-beats/kibana.go | 35 +++- .../0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json | 147 +++++++++++++- .../513a3d70-4482-11ea-ad63-791a5dc86f10.json | 182 +++++++++++++++++- .../dae24080-739a-11ea-a345-f985c61fe654.json | 141 +++++++++++++- .../a97de660-73a5-11ea-a345-f985c61fe654.json | 141 +++++++++++++- .../dbae13c0-685c-11ea-8d6a-292ef5d68366.json | 155 ++++++++++++++- .../281ca660-67b1-11ea-a76f-bf44814e437d.json | 162 +++++++++++++++- 7 files changed, 943 insertions(+), 20 deletions(-) diff --git a/dev/import-beats/kibana.go b/dev/import-beats/kibana.go index c7764d553..81f1610e4 100644 --- a/dev/import-beats/kibana.go +++ b/dev/import-beats/kibana.go @@ -19,12 +19,22 @@ import ( ) var ( - encodedFields = []string{ + fieldsToEncode = []string{ + "attributes.kibanaSavedObjectMeta.searchSourceJSON", + "attributes.optionsJSON", + "attributes.panelsJSON", "attributes.uiStateJSON", "attributes.visState", + } + + fieldsToDecode = []string{ + "attributes.kibanaSavedObjectMeta.searchSourceJSON", + "attributes.layerListJSON", + "attributes.mapStateJSON", "attributes.optionsJSON", "attributes.panelsJSON", - "attributes.kibanaSavedObjectMeta.searchSourceJSON", + "attributes.uiStateJSON", + "attributes.visState", } ) @@ -106,7 +116,7 @@ func prepareDashboardFile(dashboardFile []byte) ([]byte, error) { } func encodeFields(ms mapStr) (mapStr, error) { - for _, field := range encodedFields { + for _, field := range fieldsToEncode { v, err := ms.getValue(field) if err == errKeyNotFound { continue @@ -242,7 +252,7 @@ func convertToKibanaObjects(dashboardFile []byte, moduleName string, datasetName } func decodeFields(ms mapStr) (mapStr, error) { - for _, field := range encodedFields { + for _, field := range fieldsToDecode { v, err := ms.getValue(field) if err == errKeyNotFound { continue @@ -250,13 +260,22 @@ func decodeFields(ms mapStr) (mapStr, error) { return nil, errors.Wrapf(err, "retrieving value failed (key: %s)", field) } - var vd interface{} - err = json.Unmarshal([]byte(v.(string)), &vd) + var target interface{} + var vd mapStr + vStr := v.(string) + err = json.Unmarshal([]byte(vStr), &vd) if err != nil { - return nil, errors.Wrapf(err, "unmarshalling value failed (key: %s)", field) + var vda []mapStr + err = json.Unmarshal([]byte(vStr), &vda) + if err != nil { + return nil, errors.Wrapf(err, "unmarshalling value failed (key: %s)", field) + } + target = vda + } else { + target = vd } - _, err = ms.put(field, vd) + _, err = ms.put(field, target) if err != nil { return nil, errors.Wrapf(err, "putting value failed (key: %s)", field) } diff --git a/dev/packages/beats/aws-0.0.1/kibana/map/0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json b/dev/packages/beats/aws-0.0.1/kibana/map/0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json index ac0269f9a..6280a6cd6 100644 --- a/dev/packages/beats/aws-0.0.1/kibana/map/0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json +++ b/dev/packages/beats/aws-0.0.1/kibana/map/0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json @@ -28,8 +28,151 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"19047c4c-18d7-4aec-b0ce-98de2828244d\",\"label\":\"Hits\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"1e82f50f-424a-4718-905b-ad45db14db62\",\"geoField\":\"source.geo.location\",\"requestType\":\"point\",\"resolution\":\"COARSE\",\"indexPatternRefName\":\"layer_1_source_index_pattern\",\"applyGlobalQuery\":true},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#167a6d\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"minSize\":4,\"maxSize\":32,\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}}},\"id\":\"1d457cd4-01be-4f96-95fd-af4ac535ebea\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\"}]", - "mapStateJSON": "{\"zoom\":3.9,\"center\":{\"lon\":13.666,\"lat\":50.97903},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"index\":\"logs-*\",\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"fileset.name\",\"value\":\"elb\",\"params\":{\"query\":\"elb\"}},\"query\":{\"match\":{\"fileset.name\":{\"query\":\"elb\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "19047c4c-18d7-4aec-b0ce-98de2828244d", + "label": "Hits", + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "1d457cd4-01be-4f96-95fd-af4ac535ebea", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyGlobalQuery": true, + "geoField": "source.geo.location", + "id": "1e82f50f-424a-4718-905b-ad45db14db62", + "indexPatternRefName": "layer_1_source_index_pattern", + "requestType": "point", + "resolution": "COARSE", + "type": "ES_GEO_GRID" + }, + "style": { + "properties": { + "fillColor": { + "options": { + "color": "Blues", + "field": { + "label": "count", + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + } + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "airfield" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "field": { + "label": "count", + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": false, + "sigma": 3 + }, + "maxSize": 32, + "minSize": 4 + }, + "type": "DYNAMIC" + }, + "lineColor": { + "options": { + "color": "#167a6d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 50.97903, + "lon": 13.666 + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "logs-*", + "key": "fileset.name", + "negate": false, + "params": { + "query": "elb" + }, + "type": "phrase", + "value": "elb" + }, + "query": { + "match": { + "fileset.name": { + "query": "elb", + "type": "phrase" + } + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "now-15m", + "to": "now" + }, + "zoom": 3.9 + }, "title": "ELB Requests Geolocation [Filebeat AWS] ECS", "uiStateJSON": { "isLayerTOCOpen": true, diff --git a/dev/packages/beats/aws-0.0.1/kibana/map/513a3d70-4482-11ea-ad63-791a5dc86f10.json b/dev/packages/beats/aws-0.0.1/kibana/map/513a3d70-4482-11ea-ad63-791a5dc86f10.json index 6d18cc352..6ed97f342 100644 --- a/dev/packages/beats/aws-0.0.1/kibana/map/513a3d70-4482-11ea-ad63-791a5dc86f10.json +++ b/dev/packages/beats/aws-0.0.1/kibana/map/513a3d70-4482-11ea-ad63-791a5dc86f10.json @@ -28,8 +28,186 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"842c201e-96d7-413d-8688-de5ee4f8a1e0\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"97903038-e08d-4451-bbd2-eb92c894bdf5\",\"type\":\"ES_SEARCH\",\"geoField\":\"destination.geo.location\",\"filterByMapBounds\":true,\"tooltipProperties\":[],\"topHitsSize\":1,\"indexPatternRefName\":\"layer_1_source_index_pattern\",\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#1EA593\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#167a6d\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":5}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}}},\"id\":\"401944dd-a371-4698-be17-bc4542e9a5d4\",\"label\":\"vpc flow action accept\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"query\":{\"query\":\"aws.vpcflow.action : \\\"ACCEPT\\\" \",\"language\":\"kuery\"}},{\"sourceDescriptor\":{\"id\":\"9c0e7cce-4f21-4bcd-bb50-ae36c0fffffb\",\"type\":\"ES_SEARCH\",\"geoField\":\"source.geo.location\",\"filterByMapBounds\":true,\"tooltipProperties\":[],\"topHitsSize\":1,\"indexPatternRefName\":\"layer_2_source_index_pattern\",\"sortField\":\"@timestamp\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"scalingType\":\"LIMIT\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#f00f0b\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#7a1a18\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":5}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}}},\"id\":\"b1d44a5c-3a04-4c80-8080-57585b02fd48\",\"label\":\"vpc flow action reject\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"query\":{\"query\":\"aws.vpcflow.action : \\\"REJECT\\\" \",\"language\":\"kuery\"}}]", - "mapStateJSON": "{\"zoom\":0.47,\"center\":{\"lon\":-108.92402,\"lat\":0},\"timeFilters\":{\"from\":\"now-15d\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "842c201e-96d7-413d-8688-de5ee4f8a1e0", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "401944dd-a371-4698-be17-bc4542e9a5d4", + "label": "vpc flow action accept", + "maxZoom": 24, + "minZoom": 0, + "query": { + "language": "kuery", + "query": "aws.vpcflow.action : \"ACCEPT\" " + }, + "sourceDescriptor": { + "applyGlobalQuery": true, + "filterByMapBounds": true, + "geoField": "destination.geo.location", + "id": "97903038-e08d-4451-bbd2-eb92c894bdf5", + "indexPatternRefName": "layer_1_source_index_pattern", + "scalingType": "LIMIT", + "sortField": "@timestamp", + "sortOrder": "desc", + "tooltipProperties": [], + "topHitsSize": 1, + "type": "ES_SEARCH" + }, + "style": { + "properties": { + "fillColor": { + "options": { + "color": "#1EA593" + }, + "type": "STATIC" + }, + "icon": { + "options": { + "value": "airfield" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 5 + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#167a6d" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + }, + { + "alpha": 0.75, + "id": "b1d44a5c-3a04-4c80-8080-57585b02fd48", + "label": "vpc flow action reject", + "maxZoom": 24, + "minZoom": 0, + "query": { + "language": "kuery", + "query": "aws.vpcflow.action : \"REJECT\" " + }, + "sourceDescriptor": { + "applyGlobalQuery": true, + "filterByMapBounds": true, + "geoField": "source.geo.location", + "id": "9c0e7cce-4f21-4bcd-bb50-ae36c0fffffb", + "indexPatternRefName": "layer_2_source_index_pattern", + "scalingType": "LIMIT", + "sortField": "@timestamp", + "sortOrder": "desc", + "tooltipProperties": [], + "topHitsSize": 1, + "type": "ES_SEARCH" + }, + "style": { + "properties": { + "fillColor": { + "options": { + "color": "#f00f0b" + }, + "type": "STATIC" + }, + "icon": { + "options": { + "value": "airfield" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 5 + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#7a1a18" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 0, + "lon": -108.92402 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "now-15d", + "to": "now" + }, + "zoom": 0.47 + }, "title": "VPC Flow Action Geo Location[Filebeat AWS]", "uiStateJSON": { "isLayerTOCOpen": false, diff --git a/dev/packages/beats/aws-0.0.1/kibana/map/dae24080-739a-11ea-a345-f985c61fe654.json b/dev/packages/beats/aws-0.0.1/kibana/map/dae24080-739a-11ea-a345-f985c61fe654.json index 5a1293e73..217376645 100644 --- a/dev/packages/beats/aws-0.0.1/kibana/map/dae24080-739a-11ea-a345-f985c61fe654.json +++ b/dev/packages/beats/aws-0.0.1/kibana/map/dae24080-739a-11ea-a345-f985c61fe654.json @@ -28,8 +28,145 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"2c7b49fb-3fb5-4e18-b27f-fabe930971f3\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"7bfe2df9-9398-4f1a-8cf7-b57aa5f3f31e\",\"geoField\":\"source.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"LIMIT\",\"topHitsSize\":1,\"type\":\"ES_SEARCH\",\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"a10fa758-30ad-4e2a-bf9d-472e133a7f17\",\"label\":\"CloudTrail Soure Location\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"joins\":[],\"query\":{\"query\":\"event.dataset:aws.cloudtrail\",\"language\":\"kuery\"}}]", - "mapStateJSON": "{\"zoom\":1.97,\"center\":{\"lon\":0,\"lat\":19.94277},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "2c7b49fb-3fb5-4e18-b27f-fabe930971f3", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "a10fa758-30ad-4e2a-bf9d-472e133a7f17", + "joins": [], + "label": "CloudTrail Soure Location", + "maxZoom": 24, + "minZoom": 0, + "query": { + "language": "kuery", + "query": "event.dataset:aws.cloudtrail" + }, + "sourceDescriptor": { + "applyGlobalQuery": true, + "filterByMapBounds": true, + "geoField": "source.geo.location", + "id": "7bfe2df9-9398-4f1a-8cf7-b57aa5f3f31e", + "indexPatternRefName": "layer_1_source_index_pattern", + "scalingType": "LIMIT", + "sortField": "", + "sortOrder": "desc", + "tooltipProperties": [], + "topHitsSize": 1, + "type": "ES_SEARCH" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "#54B399" + }, + "type": "STATIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#41937c" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 19.94277, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "now-15m", + "to": "now" + }, + "zoom": 1.97 + }, "title": "CloudTrail Source Location [Filebeat AWS]", "uiStateJSON": { "isLayerTOCOpen": true, diff --git a/dev/packages/beats/googlecloud-0.0.1/kibana/map/a97de660-73a5-11ea-a345-f985c61fe654.json b/dev/packages/beats/googlecloud-0.0.1/kibana/map/a97de660-73a5-11ea-a345-f985c61fe654.json index b3376d09c..f660b3775 100644 --- a/dev/packages/beats/googlecloud-0.0.1/kibana/map/a97de660-73a5-11ea-a345-f985c61fe654.json +++ b/dev/packages/beats/googlecloud-0.0.1/kibana/map/a97de660-73a5-11ea-a345-f985c61fe654.json @@ -28,8 +28,145 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"866b5ce1-6ca0-47db-a6f2-54c5e0dcd2f0\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"79ec6461-7561-45e4-a6a2-9d6fbd4cf986\",\"geoField\":\"source.geo.location\",\"filterByMapBounds\":true,\"scalingType\":\"LIMIT\",\"topHitsSize\":1,\"type\":\"ES_SEARCH\",\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"279da950-e9a7-4287-ab37-25906e448455\",\"label\":\"Source Locations\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"joins\":[],\"query\":{\"query\":\"event.dataset:googlecloud.audit\",\"language\":\"kuery\"}}]", - "mapStateJSON": "{\"zoom\":1.97,\"center\":{\"lon\":0,\"lat\":19.94277},\"timeFilters\":{\"from\":\"now-7d\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "866b5ce1-6ca0-47db-a6f2-54c5e0dcd2f0", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "279da950-e9a7-4287-ab37-25906e448455", + "joins": [], + "label": "Source Locations", + "maxZoom": 24, + "minZoom": 0, + "query": { + "language": "kuery", + "query": "event.dataset:googlecloud.audit" + }, + "sourceDescriptor": { + "applyGlobalQuery": true, + "filterByMapBounds": true, + "geoField": "source.geo.location", + "id": "79ec6461-7561-45e4-a6a2-9d6fbd4cf986", + "indexPatternRefName": "layer_1_source_index_pattern", + "scalingType": "LIMIT", + "sortField": "", + "sortOrder": "desc", + "tooltipProperties": [], + "topHitsSize": 1, + "type": "ES_SEARCH" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "#54B399" + }, + "type": "STATIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#41937c" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 19.94277, + "lon": 0 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "now-7d", + "to": "now" + }, + "zoom": 1.97 + }, "title": "Audit Source Locations [Filebeat GoogleCloud]", "uiStateJSON": { "isLayerTOCOpen": true, diff --git a/dev/packages/beats/o365-0.0.1/kibana/map/dbae13c0-685c-11ea-8d6a-292ef5d68366.json b/dev/packages/beats/o365-0.0.1/kibana/map/dbae13c0-685c-11ea-8d6a-292ef5d68366.json index 0d207f895..a014e3807 100644 --- a/dev/packages/beats/o365-0.0.1/kibana/map/dbae13c0-685c-11ea-8d6a-292ef5d68366.json +++ b/dev/packages/beats/o365-0.0.1/kibana/map/dbae13c0-685c-11ea-8d6a-292ef5d68366.json @@ -28,8 +28,159 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"0b910b6c-77c8-4223-892a-1ebf69b0ccb4\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"3ba31ffc-7051-44bf-96a0-a684020cd2a3\",\"geoField\":\"source.geo.location\",\"requestType\":\"point\",\"resolution\":\"FINE\",\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"color\":\"Yellow to Red\",\"colorCategory\":\"palette_0\",\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3},\"type\":\"ORDINAL\",\"useCustomColorRamp\":false}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFF\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":0}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"minSize\":8,\"maxSize\":32,\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"},\"fieldMetaOptions\":{\"isEnabled\":true,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"name\":\"doc_count\",\"origin\":\"source\"}}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}},\"isTimeAware\":true},\"id\":\"acc53b7b-3411-406b-9371-6fa62b6b9365\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\"}]", - "mapStateJSON": "{\"zoom\":2.88,\"center\":{\"lon\":16.67387,\"lat\":30.87292},\"timeFilters\":{\"from\":\"2020-02-05T03:25:59.045Z\",\"to\":\"2020-02-29T10:59:01.067Z\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"event.dataset:\\\"o365.audit\\\" \",\"language\":\"kuery\"},\"filters\":[]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "0b910b6c-77c8-4223-892a-1ebf69b0ccb4", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "acc53b7b-3411-406b-9371-6fa62b6b9365", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "applyGlobalQuery": true, + "geoField": "source.geo.location", + "id": "3ba31ffc-7051-44bf-96a0-a684020cd2a3", + "indexPatternRefName": "layer_1_source_index_pattern", + "requestType": "point", + "resolution": "FINE", + "type": "ES_GEO_GRID" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "Yellow to Red", + "colorCategory": "palette_0", + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": true, + "sigma": 3 + }, + "type": "ORDINAL", + "useCustomColorRamp": false + }, + "type": "DYNAMIC" + }, + "icon": { + "options": { + "value": "airfield" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "field": { + "name": "doc_count", + "origin": "source" + }, + "fieldMetaOptions": { + "isEnabled": true, + "sigma": 3 + }, + "maxSize": 32, + "minSize": 8 + }, + "type": "DYNAMIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "field": { + "name": "doc_count", + "origin": "source" + } + }, + "type": "DYNAMIC" + }, + "lineColor": { + "options": { + "color": "#FFF" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 0 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 30.87292, + "lon": 16.67387 + }, + "filters": [], + "query": { + "language": "kuery", + "query": "event.dataset:\"o365.audit\" " + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "2020-02-05T03:25:59.045Z", + "to": "2020-02-29T10:59:01.067Z" + }, + "zoom": 2.88 + }, "title": "Client Geo Map [Filebeat o365 audit]", "uiStateJSON": { "isLayerTOCOpen": true, diff --git a/dev/packages/beats/okta-0.0.1/kibana/map/281ca660-67b1-11ea-a76f-bf44814e437d.json b/dev/packages/beats/okta-0.0.1/kibana/map/281ca660-67b1-11ea-a76f-bf44814e437d.json index 9240fa42b..cdbbaee39 100644 --- a/dev/packages/beats/okta-0.0.1/kibana/map/281ca660-67b1-11ea-a76f-bf44814e437d.json +++ b/dev/packages/beats/okta-0.0.1/kibana/map/281ca660-67b1-11ea-a76f-bf44814e437d.json @@ -28,8 +28,166 @@ "type": "Polygon" }, "description": "", - "layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"6908e81b-1695-4445-aee4-8bc8c9f65600\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"id\":\"4b8bd321-4b90-4d97-83e0-2b12bf091f66\",\"geoField\":\"client.geo.location\",\"filterByMapBounds\":false,\"type\":\"ES_SEARCH\",\"tooltipProperties\":[],\"sortField\":\"\",\"sortOrder\":\"desc\",\"topHitsSize\":1,\"applyGlobalQuery\":true,\"indexPatternRefName\":\"layer_1_source_index_pattern\",\"scalingType\":\"LIMIT\"},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"marker\"}},\"fillColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#54B399\"}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#41937c\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"STATIC\",\"options\":{\"size\":6}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"labelText\":{\"type\":\"STATIC\",\"options\":{\"value\":\"\"}},\"labelColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#000000\"}},\"labelSize\":{\"type\":\"STATIC\",\"options\":{\"size\":14}},\"labelBorderColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#FFFFFF\"}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"labelBorderSize\":{\"options\":{\"size\":\"SMALL\"}}},\"isTimeAware\":true},\"id\":\"dc52e707-92d7-4de7-becf-a3a8bfaa2c2d\",\"label\":\"Okta \",\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\",\"query\":{\"query\":\"event.dataset : \\\"okta.system\\\" \",\"language\":\"kuery\"}}]", - "mapStateJSON": "{\"zoom\":2.75,\"center\":{\"lon\":-44.69098,\"lat\":26.54701},\"timeFilters\":{\"from\":\"now-15w\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"index\":\"logs-*\",\"key\":\"event.dataset\",\"negate\":false,\"params\":{\"query\":\"okta.system\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"event.dataset\":\"okta.system\"}}}]}", + "layerListJSON": [ + { + "alpha": 1, + "id": "6908e81b-1695-4445-aee4-8bc8c9f65600", + "label": null, + "maxZoom": 24, + "minZoom": 0, + "sourceDescriptor": { + "isAutoSelect": true, + "type": "EMS_TMS" + }, + "style": {}, + "type": "VECTOR_TILE", + "visible": true + }, + { + "alpha": 0.75, + "id": "dc52e707-92d7-4de7-becf-a3a8bfaa2c2d", + "label": "Okta ", + "maxZoom": 24, + "minZoom": 0, + "query": { + "language": "kuery", + "query": "event.dataset : \"okta.system\" " + }, + "sourceDescriptor": { + "applyGlobalQuery": true, + "filterByMapBounds": false, + "geoField": "client.geo.location", + "id": "4b8bd321-4b90-4d97-83e0-2b12bf091f66", + "indexPatternRefName": "layer_1_source_index_pattern", + "scalingType": "LIMIT", + "sortField": "", + "sortOrder": "desc", + "tooltipProperties": [], + "topHitsSize": 1, + "type": "ES_SEARCH" + }, + "style": { + "isTimeAware": true, + "properties": { + "fillColor": { + "options": { + "color": "#54B399" + }, + "type": "STATIC" + }, + "icon": { + "options": { + "value": "marker" + }, + "type": "STATIC" + }, + "iconOrientation": { + "options": { + "orientation": 0 + }, + "type": "STATIC" + }, + "iconSize": { + "options": { + "size": 6 + }, + "type": "STATIC" + }, + "labelBorderColor": { + "options": { + "color": "#FFFFFF" + }, + "type": "STATIC" + }, + "labelBorderSize": { + "options": { + "size": "SMALL" + } + }, + "labelColor": { + "options": { + "color": "#000000" + }, + "type": "STATIC" + }, + "labelSize": { + "options": { + "size": 14 + }, + "type": "STATIC" + }, + "labelText": { + "options": { + "value": "" + }, + "type": "STATIC" + }, + "lineColor": { + "options": { + "color": "#41937c" + }, + "type": "STATIC" + }, + "lineWidth": { + "options": { + "size": 1 + }, + "type": "STATIC" + }, + "symbolizeAs": { + "options": { + "value": "circle" + } + } + }, + "type": "VECTOR" + }, + "type": "VECTOR", + "visible": true + } + ], + "mapStateJSON": { + "center": { + "lat": 26.54701, + "lon": -44.69098 + }, + "filters": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "logs-*", + "key": "event.dataset", + "negate": false, + "params": { + "query": "okta.system" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "event.dataset": "okta.system" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + }, + "refreshConfig": { + "interval": 0, + "isPaused": false + }, + "timeFilters": { + "from": "now-15w", + "to": "now" + }, + "zoom": 2.75 + }, "title": "Geolocation [Filebeat Okta]", "uiStateJSON": { "isLayerTOCOpen": true, From 136996dbccc90a5990f2c8c3a44f45f223762bb5 Mon Sep 17 00:00:00 2001 From: mtojek Date: Mon, 20 Apr 2020 14:09:26 +0200 Subject: [PATCH 2/3] Address PR comments --- dev/import-beats/kibana.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/dev/import-beats/kibana.go b/dev/import-beats/kibana.go index 84f6fb24e..37369dacb 100644 --- a/dev/import-beats/kibana.go +++ b/dev/import-beats/kibana.go @@ -19,15 +19,7 @@ import ( ) var ( - fieldsToEncode = []string{ - "attributes.kibanaSavedObjectMeta.searchSourceJSON", - "attributes.optionsJSON", - "attributes.panelsJSON", - "attributes.uiStateJSON", - "attributes.visState", - } - - fieldsToDecode = []string{ + encodedFields = []string{ "attributes.kibanaSavedObjectMeta.searchSourceJSON", "attributes.layerListJSON", "attributes.mapStateJSON", @@ -124,7 +116,7 @@ func prepareDashboardFile(dashboardFile []byte) ([]byte, error) { } func encodeFields(ms mapStr) (mapStr, error) { - for _, field := range fieldsToEncode { + for _, field := range encodedFields { v, err := ms.getValue(field) if err == errKeyNotFound { continue @@ -132,6 +124,11 @@ func encodeFields(ms mapStr) (mapStr, error) { return mapStr{}, errors.Wrapf(err, "retrieving value failed (key: %s)", field) } + _, isString := v.(string) + if isString { + continue + } + ve, err := json.Marshal(v) if err != nil { return mapStr{}, errors.Wrapf(err, "marshalling value failed (key: %s)", field) @@ -260,7 +257,7 @@ func convertToKibanaObjects(dashboardFile []byte, moduleName string, datasetName } func decodeFields(ms mapStr) (mapStr, error) { - for _, field := range fieldsToDecode { + for _, field := range encodedFields { v, err := ms.getValue(field) if err == errKeyNotFound { continue From 3b22db5859169c13f261ea898d20aee32b6ddf99 Mon Sep 17 00:00:00 2001 From: mtojek Date: Tue, 21 Apr 2020 09:43:27 +0200 Subject: [PATCH 3/3] Adjust generator code --- dev/generator/main.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dev/generator/main.go b/dev/generator/main.go index cf18eb46a..e75bf0a7c 100644 --- a/dev/generator/main.go +++ b/dev/generator/main.go @@ -251,11 +251,13 @@ func writeJsonFile(v interface{}, path string) error { var ( fieldsToEncode = []string{ - "attributes.uiStateJSON", - "attributes.visState", + "attributes.kibanaSavedObjectMeta.searchSourceJSON", + "attributes.layerListJSON", + "attributes.mapStateJSON", "attributes.optionsJSON", "attributes.panelsJSON", - "attributes.kibanaSavedObjectMeta.searchSourceJSON", + "attributes.uiStateJSON", + "attributes.visState", } )