From ef0857d6e2ec52595b3ce3abeb16f96088dc5753 Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 24 Aug 2017 14:13:52 -0700 Subject: [PATCH 1/6] [docs] Manually migrating Kibana index Signed-off-by: Tyler Smalley --- docs/migration.asciidoc | 1 + docs/migration/migrate_6_0.asciidoc | 8 +- docs/migration/migrate_6_0_index.asciidoc | 337 ++++++++++++++++++++++ 3 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 docs/migration/migrate_6_0_index.asciidoc diff --git a/docs/migration.asciidoc b/docs/migration.asciidoc index 14a859420e31dc..7cd10aa580ebd6 100644 --- a/docs/migration.asciidoc +++ b/docs/migration.asciidoc @@ -8,3 +8,4 @@ your application from one version of Kibana to another. -- include::migration/migrate_6_0.asciidoc[] +include::migration/migrate_6_0_index.asciidoc[] diff --git a/docs/migration/migrate_6_0.asciidoc b/docs/migration/migrate_6_0.asciidoc index 521313f230601a..37678c6b6c5417 100644 --- a/docs/migration/migrate_6_0.asciidoc +++ b/docs/migration/migrate_6_0.asciidoc @@ -4,13 +4,18 @@ This section discusses the changes that you need to be aware of when migrating your application to Kibana 6.0. +[float] +=== Types removed from Kibana index +*Details:* With the imminent {es-ref}/removal-of-types.html[removal of types in Elasticsearch], we have updated our index to not rely on types. Kibana 5.6.x can support both of these mappings to minimize downtime. A UI is provided with X-Pack to assist with the migration of the Elastic stack. + +*Impact:* You will need migrate your Kibana index either {kibana-ref}/migrating-6.0-index.html[manually], or through the Migration UI in X-Pack. + [float] === Removed option to use unsupported scripting languages *Details:* Kibana 5.x allowed users to create scripted fields using any scripting language enabled in Elasticsearch. Kibana 6.0 will only support Painless and Lucene expression based scripts. - *Impact:* You will need to migrate your groovy, python, javascript, etc. scripted fields to Painless or Lucene expressions. @@ -73,7 +78,6 @@ This is no longer the case. Now, only commas are a valid query separator: e.g. ` *Impact:* No change is required for existing Kibana index patterns. Those previously configured with this option will gracefully use the new Elasticsearch optimizations instead, as will all new index patterns. - [float] === Replace markdown parser `marked` with `markdown-it` *Details:* Starting in 6.0.0, Kibana will use `markdown-it` to parse markdown text. Kibana switched to `markdown-it` because `marked` is no longer actively maintained. Markdown-it supports CommonMark and GFM (GitHub Flavored Markdown) Tables and Strikethrough. diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc new file mode 100644 index 00000000000000..38f569c785f331 --- /dev/null +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -0,0 +1,337 @@ +[[migrating-6.0-index]] +== Migrating Kibana index to 6.0 + +[IMPORTANT] +============================================== +The following will make transformations to your Kibana index. It is recommended you backup your Kibana index prior to running any of the following commands. +============================================== + +In order to migrate to Kibana 6, your Kibana index needs to be re-indexed. This will make transformations to the data model, removing types which are being {es-ref}/removal-of-types.html[removed in Elasticsearch]. + +A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume your using the default `.kibana` index. If `kibana.index` is set to something different in your `kibana.yml` you will need to modify the commands. + +To perform this migration without downtime, you will need to be running Kibana 5.6. This is the only version which supports both of the index types. + +. Set `.kibana` index to read-only: ++ +-- +[source,js] +-------------------------------------------------- +PUT .kibana/_settings +{ + "index.blocks.write": true +} +-------------------------------------------------- +// CONSOLE +-- + +. Create `.kibana-6` index ++ +-- +[source,js] +-------------------------------------------------- +PUT .kibana-6 +{ + "settings" : { + "number_of_shards" : 1, + "index.format": 6 + }, + "mappings" : { + "doc": { + "properties": { + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "config": { + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 2048 + } + } + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE +-- + +. Re-index `.kibana` into `.kibana-6`: ++ +-- +[source,js] +-------------------------------------------------- +POST _reindex +{ + "source": { + "index": ".kibana" + }, + "dest": { + "index": ".kibana-6" + }, + "script": { + "source": "ctx._source = [ ctx._type : ctx._source ]; ctx._source.type = ctx._type; ctx._id = ctx._type + \":\" + ctx._id; ctx._type = \"doc\"; " + } +} +-------------------------------------------------- +// CONSOLE +-- + +. Alias `.kibana-6` to `.kibana` and remove legacy `.kibana` index: ++ +-- +[source,js] +-------------------------------------------------- +POST /_aliases +{ + "actions" : [ + { "add": { "index": ".kibana-6", "alias": ".kibana" } }, + { "remove_index": { "index": ".kibana" } } + ] +} +-------------------------------------------------- +// CONSOLE +-- From d0c59b8e39e60170aee4aab9289999965d205a8f Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Thu, 24 Aug 2017 20:33:31 -0700 Subject: [PATCH 2/6] Added refresh_interval Signed-off-by: Tyler Smalley --- docs/migration/migrate_6_0_index.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc index 38f569c785f331..faad9e9bfebacb 100644 --- a/docs/migration/migrate_6_0_index.asciidoc +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -34,7 +34,8 @@ PUT .kibana-6 { "settings" : { "number_of_shards" : 1, - "index.format": 6 + "index.format": 6, + "index.refresh_interval": -1 }, "mappings" : { "doc": { From 5d990c7e91e9c89d6265431ebd038b97de34f73d Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Fri, 25 Aug 2017 07:50:08 -0700 Subject: [PATCH 3/6] Reverts refresh_interval Signed-off-by: Tyler Smalley --- docs/migration/migrate_6_0_index.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc index faad9e9bfebacb..38f569c785f331 100644 --- a/docs/migration/migrate_6_0_index.asciidoc +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -34,8 +34,7 @@ PUT .kibana-6 { "settings" : { "number_of_shards" : 1, - "index.format": 6, - "index.refresh_interval": -1 + "index.format": 6 }, "mappings" : { "doc": { From 2be3db7142daaee5169ecdaa3ef161946ea503ce Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 30 Aug 2017 15:10:15 -0700 Subject: [PATCH 4/6] Feedback Signed-off-by: Tyler Smalley --- docs/migration/migrate_6_0.asciidoc | 2 +- docs/migration/migrate_6_0_index.asciidoc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/migration/migrate_6_0.asciidoc b/docs/migration/migrate_6_0.asciidoc index 37678c6b6c5417..ae7a638b34bcae 100644 --- a/docs/migration/migrate_6_0.asciidoc +++ b/docs/migration/migrate_6_0.asciidoc @@ -8,7 +8,7 @@ your application to Kibana 6.0. === Types removed from Kibana index *Details:* With the imminent {es-ref}/removal-of-types.html[removal of types in Elasticsearch], we have updated our index to not rely on types. Kibana 5.6.x can support both of these mappings to minimize downtime. A UI is provided with X-Pack to assist with the migration of the Elastic stack. -*Impact:* You will need migrate your Kibana index either {kibana-ref}/migrating-6.0-index.html[manually], or through the Migration UI in X-Pack. +*Impact:* You will need migrate your Kibana index either {kibana-ref}/migrating-6.0-index.html[manually], or through the Upgrade Assistant in X-Pack. [float] diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc index 38f569c785f331..57c7933e44b867 100644 --- a/docs/migration/migrate_6_0_index.asciidoc +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -3,12 +3,12 @@ [IMPORTANT] ============================================== -The following will make transformations to your Kibana index. It is recommended you backup your Kibana index prior to running any of the following commands. +The following will make transformations to your Kibana index. It is recommended you https://www.elastic.co/guide/en/elasticsearch/reference/5.6/modules-snapshots.html[backup your Kibana index] prior to running any of the following commands. ============================================== -In order to migrate to Kibana 6, your Kibana index needs to be re-indexed. This will make transformations to the data model, removing types which are being {es-ref}/removal-of-types.html[removed in Elasticsearch]. +In order to migrate to Kibana 6, your Kibana index needs to be reindexed. This will make transformations to the data model, removing types which are being {es-ref}/removal-of-types.html[removed in Elasticsearch]. -A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume your using the default `.kibana` index. If `kibana.index` is set to something different in your `kibana.yml` you will need to modify the commands. +A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume you're using the default `.kibana` index. If `kibana.index` is set to something different in your `kibana.yml` you will need to modify the commands. To perform this migration without downtime, you will need to be running Kibana 5.6. This is the only version which supports both of the index types. @@ -299,7 +299,7 @@ PUT .kibana-6 // CONSOLE -- -. Re-index `.kibana` into `.kibana-6`: +. Reindex `.kibana` into `.kibana-6`: + -- [source,js] From 38cc25faf53187a0057a2b8894234ab39db302ec Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 30 Aug 2017 15:29:06 -0700 Subject: [PATCH 5/6] Mention not using an alias Signed-off-by: Tyler Smalley --- docs/migration/migrate_6_0_index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc index 57c7933e44b867..8e0bed13053abd 100644 --- a/docs/migration/migrate_6_0_index.asciidoc +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -8,7 +8,7 @@ The following will make transformations to your Kibana index. It is recommended In order to migrate to Kibana 6, your Kibana index needs to be reindexed. This will make transformations to the data model, removing types which are being {es-ref}/removal-of-types.html[removed in Elasticsearch]. -A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume you're using the default `.kibana` index. If `kibana.index` is set to something different in your `kibana.yml` you will need to modify the commands. +A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume you're using the default `.kibana` index, without an alias. If `kibana.index` is set to something different in your `kibana.yml`, or you're using an alias you will need to modify the commands. To perform this migration without downtime, you will need to be running Kibana 5.6. This is the only version which supports both of the index types. From e4aee8df16c159a3f929e3b4ebfd18acaa21eeea Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Wed, 30 Aug 2017 20:44:19 -0700 Subject: [PATCH 6/6] Adds a comma Signed-off-by: Tyler Smalley --- docs/migration/migrate_6_0_index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration/migrate_6_0_index.asciidoc b/docs/migration/migrate_6_0_index.asciidoc index 8e0bed13053abd..d21ce1d426b300 100644 --- a/docs/migration/migrate_6_0_index.asciidoc +++ b/docs/migration/migrate_6_0_index.asciidoc @@ -8,7 +8,7 @@ The following will make transformations to your Kibana index. It is recommended In order to migrate to Kibana 6, your Kibana index needs to be reindexed. This will make transformations to the data model, removing types which are being {es-ref}/removal-of-types.html[removed in Elasticsearch]. -A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume you're using the default `.kibana` index, without an alias. If `kibana.index` is set to something different in your `kibana.yml`, or you're using an alias you will need to modify the commands. +A UI is available in X-Pack 5.6 to assist with the migration. The following is provided should you be required to run the migration manually. The commands assume you're using the default `.kibana` index, without an alias. If `kibana.index` is set to something different in your `kibana.yml`, or you're using an alias, you will need to modify the commands. To perform this migration without downtime, you will need to be running Kibana 5.6. This is the only version which supports both of the index types.