Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

import-beats: Decode fields layerListJSON, mapStateJSON #354

Merged
merged 4 commits into from
Apr 21, 2020
Merged

import-beats: Decode fields layerListJSON, mapStateJSON #354

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d9e315d
Decode fields: layerListJSON, mapStateJSON
Apr 20, 2020
64c25db
Merge branch 'master' into adjust-generated-files-2
Apr 20, 2020
136996d
Address PR comments
Apr 20, 2020
3b22db5
Adjust generator code
Apr 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions dev/generator/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,11 +251,13 @@ func writeJsonFile(v interface{}, path string) error {

var (
fieldsToEncode = []string{
"attributes.uiStateJSON",
"attributes.visState",
"attributes.kibanaSavedObjectMeta.searchSourceJSON",
"attributes.layerListJSON",
"attributes.mapStateJSON",
"attributes.optionsJSON",
"attributes.panelsJSON",
"attributes.kibanaSavedObjectMeta.searchSourceJSON",
"attributes.uiStateJSON",
"attributes.visState",
}
)

Expand Down
30 changes: 23 additions & 7 deletions dev/import-beats/kibana.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ import (

var (
encodedFields = []string{
"attributes.uiStateJSON",
"attributes.visState",
"attributes.kibanaSavedObjectMeta.searchSourceJSON",
"attributes.layerListJSON",
"attributes.mapStateJSON",
"attributes.optionsJSON",
"attributes.panelsJSON",
"attributes.kibanaSavedObjectMeta.searchSourceJSON",
"attributes.uiStateJSON",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is encoded and decode not the same list?

Could you import this list from here https://github.com/elastic/package-registry/blob/master/dev/generator/main.go#L253 so we have a single point of truth?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not the same list, because some fields are already encoded in dashboards stored in Beats (see mapStateJSON in https://github.com/elastic/beats/blob/master/x-pack/filebeat/module/o365/_meta/kibana/7/dashboard/Filebeat-O365-Audit.json#L842).

Another solution would be detecting if the field is "string" and then try to decode it. WDYT?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I see. So we basically missed it already in Beats, I wonder if we should add it there @exekias .

A check for string would be nice as it would mean we only have to keep one list and things to break if the dashboards in Beats itself are adjusted.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed!

"attributes.visState",
}
)

Expand Down Expand Up @@ -122,6 +124,11 @@ func encodeFields(ms mapStr) (mapStr, error) {
return mapStr{}, errors.Wrapf(err, "retrieving value failed (key: %s)", field)
}

_, isString := v.(string)
if isString {
continue
}

ve, err := json.Marshal(v)
if err != nil {
return mapStr{}, errors.Wrapf(err, "marshalling value failed (key: %s)", field)
Expand Down Expand Up @@ -258,13 +265,22 @@ func decodeFields(ms mapStr) (mapStr, error) {
return nil, errors.Wrapf(err, "retrieving value failed (key: %s)", field)
}

var vd interface{}
err = json.Unmarshal([]byte(v.(string)), &vd)
var target interface{}
var vd mapStr
vStr := v.(string)
err = json.Unmarshal([]byte(vStr), &vd)
if err != nil {
return nil, errors.Wrapf(err, "unmarshalling value failed (key: %s)", field)
var vda []mapStr
err = json.Unmarshal([]byte(vStr), &vda)
if err != nil {
return nil, errors.Wrapf(err, "unmarshalling value failed (key: %s)", field)
}
target = vda
} else {
target = vd
}

_, err = ms.put(field, vd)
_, err = ms.put(field, target)
if err != nil {
return nil, errors.Wrapf(err, "putting value failed (key: %s)", field)
}
Expand Down
147 changes: 145 additions & 2 deletions dev/packages/beats/aws-0.0.1/kibana/map/0edf0640-3e7e-11ea-bb0a-69c3ca1d410f.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,151 @@
"type": "Polygon"
},
"description": "",
"layerListJSON": "[{\"sourceDescriptor\":{\"type\":\"EMS_TMS\",\"isAutoSelect\":true},\"id\":\"19047c4c-18d7-4aec-b0ce-98de2828244d\",\"label\":\"Hits\",\"minZoom\":0,\"maxZoom\":24,\"alpha\":1,\"visible\":true,\"style\":{},\"type\":\"VECTOR_TILE\"},{\"sourceDescriptor\":{\"type\":\"ES_GEO_GRID\",\"id\":\"1e82f50f-424a-4718-905b-ad45db14db62\",\"geoField\":\"source.geo.location\",\"requestType\":\"point\",\"resolution\":\"COARSE\",\"indexPatternRefName\":\"layer_1_source_index_pattern\",\"applyGlobalQuery\":true},\"style\":{\"type\":\"VECTOR\",\"properties\":{\"fillColor\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"color\":\"Blues\",\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"lineColor\":{\"type\":\"STATIC\",\"options\":{\"color\":\"#167a6d\"}},\"lineWidth\":{\"type\":\"STATIC\",\"options\":{\"size\":1}},\"iconSize\":{\"type\":\"DYNAMIC\",\"options\":{\"field\":{\"label\":\"count\",\"name\":\"doc_count\",\"origin\":\"source\"},\"minSize\":4,\"maxSize\":32,\"fieldMetaOptions\":{\"isEnabled\":false,\"sigma\":3}}},\"iconOrientation\":{\"type\":\"STATIC\",\"options\":{\"orientation\":0}},\"symbolizeAs\":{\"options\":{\"value\":\"circle\"}},\"icon\":{\"type\":\"STATIC\",\"options\":{\"value\":\"airfield\"}}}},\"id\":\"1d457cd4-01be-4f96-95fd-af4ac535ebea\",\"label\":null,\"minZoom\":0,\"maxZoom\":24,\"alpha\":0.75,\"visible\":true,\"type\":\"VECTOR\"}]",
"mapStateJSON": "{\"zoom\":3.9,\"center\":{\"lon\":13.666,\"lat\":50.97903},\"timeFilters\":{\"from\":\"now-15m\",\"to\":\"now\"},\"refreshConfig\":{\"isPaused\":false,\"interval\":0},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"index\":\"logs-*\",\"alias\":null,\"negate\":false,\"disabled\":false,\"type\":\"phrase\",\"key\":\"fileset.name\",\"value\":\"elb\",\"params\":{\"query\":\"elb\"}},\"query\":{\"match\":{\"fileset.name\":{\"query\":\"elb\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}",
"layerListJSON": [
{
"alpha": 1,
"id": "19047c4c-18d7-4aec-b0ce-98de2828244d",
"label": "Hits",
"maxZoom": 24,
"minZoom": 0,
"sourceDescriptor": {
"isAutoSelect": true,
"type": "EMS_TMS"
},
"style": {},
"type": "VECTOR_TILE",
"visible": true
},
{
"alpha": 0.75,
"id": "1d457cd4-01be-4f96-95fd-af4ac535ebea",
"label": null,
"maxZoom": 24,
"minZoom": 0,
"sourceDescriptor": {
"applyGlobalQuery": true,
"geoField": "source.geo.location",
"id": "1e82f50f-424a-4718-905b-ad45db14db62",
"indexPatternRefName": "layer_1_source_index_pattern",
"requestType": "point",
"resolution": "COARSE",
"type": "ES_GEO_GRID"
},
"style": {
"properties": {
"fillColor": {
"options": {
"color": "Blues",
"field": {
"label": "count",
"name": "doc_count",
"origin": "source"
},
"fieldMetaOptions": {
"isEnabled": false,
"sigma": 3
}
},
"type": "DYNAMIC"
},
"icon": {
"options": {
"value": "airfield"
},
"type": "STATIC"
},
"iconOrientation": {
"options": {
"orientation": 0
},
"type": "STATIC"
},
"iconSize": {
"options": {
"field": {
"label": "count",
"name": "doc_count",
"origin": "source"
},
"fieldMetaOptions": {
"isEnabled": false,
"sigma": 3
},
"maxSize": 32,
"minSize": 4
},
"type": "DYNAMIC"
},
"lineColor": {
"options": {
"color": "#167a6d"
},
"type": "STATIC"
},
"lineWidth": {
"options": {
"size": 1
},
"type": "STATIC"
},
"symbolizeAs": {
"options": {
"value": "circle"
}
}
},
"type": "VECTOR"
},
"type": "VECTOR",
"visible": true
}
],
"mapStateJSON": {
"center": {
"lat": 50.97903,
"lon": 13.666
},
"filters": [
{
"$state": {
"store": "appState"
},
"meta": {
"alias": null,
"disabled": false,
"index": "logs-*",
"key": "fileset.name",
"negate": false,
"params": {
"query": "elb"
},
"type": "phrase",
"value": "elb"
},
"query": {
"match": {
"fileset.name": {
"query": "elb",
"type": "phrase"
}
}
}
}
],
"query": {
"language": "kuery",
"query": ""
},
"refreshConfig": {
"interval": 0,
"isPaused": false
},
"timeFilters": {
"from": "now-15m",
"to": "now"
},
"zoom": 3.9
},
"title": "ELB Requests Geolocation [Filebeat AWS] ECS",
"uiStateJSON": {
"isLayerTOCOpen": true,
Expand Down
Loading