Merge branch 'master' into a11y_kibana_overview

.ci/Jenkinsfile_flaky

@@ -29,16 +29,20 @@ kibanaPipeline(timeoutMinutes: 180) {
       catchErrors {
         print "Agent ${agentNumberInside} - ${agentExecutions} executions"
 
-        workers.functional('flaky-test-runner', {
-          if (!IS_XPACK) {
-            kibanaPipeline.buildOss()
-            if (CI_GROUP == '1') {
-              runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
+        withEnv([
+          'IGNORE_SHIP_CI_STATS_ERROR=true',
+        ]) {
+          workers.functional('flaky-test-runner', {
+            if (!IS_XPACK) {
+              kibanaPipeline.buildOss()
+              if (CI_GROUP == '1') {
+                runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
+              }
+            } else {
+              kibanaPipeline.buildXpack()
             }
-          } else {
-            kibanaPipeline.buildXpack()
-          }
-        }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
+          }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
+        }
       }
     }
   }

.github/CODEOWNERS

@@ -223,9 +223,8 @@
 /x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-core
 /.telemetryrc.json @elastic/kibana-core
 /x-pack/.telemetryrc.json @elastic/kibana-core
-src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-core
-src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-core
-x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-core
+src/plugins/telemetry/schema/ @elastic/kibana-core @elastic/kibana-telemetry @elastic/infra-telemetry
+x-pack/plugins/telemetry_collection_xpack/schema/ @elastic/kibana-core @elastic/kibana-telemetry @elastic/infra-telemetry
 
 # Kibana Localization
 /src/dev/i18n/  @elastic/kibana-localization @elastic/kibana-core

docs/setup/install/deb.asciidoc

@@ -156,6 +156,7 @@ locations for a Debian-based system:
 | config
   | Configuration files including `kibana.yml`
   | /etc/kibana
+  | <<settings,KBN_PATH_CONF>>
  d|
 
 | data

docs/setup/install/rpm.asciidoc

@@ -149,6 +149,7 @@ locations for an RPM-based system:
 | config
   | Configuration files including `kibana.yml`
   | /etc/kibana
+  | <<settings,KBN_PATH_CONF>>
  d|
 
 | data

docs/setup/install/targz.asciidoc

@@ -134,6 +134,7 @@ important data later on.
 | config
   | Configuration files including `kibana.yml`
   | $KIBANA_HOME\config
+  | <<settings,KBN_PATH_CONF>>
  d|
 
 | data

docs/setup/install/windows.asciidoc

@@ -81,6 +81,7 @@ important data later on.
 | config
   | Configuration files including `kibana.yml`
   | $KIBANA_HOME\config
+  | <<settings,KBN_PATH_CONF>>
  d|
 
 | data

docs/setup/settings.asciidoc

@@ -5,7 +5,12 @@ The {kib} server reads properties from the `kibana.yml` file on startup. The
 location of this file differs depending on how you installed {kib}. For example,
 if you installed {kib} from an archive distribution (`.tar.gz` or `.zip`), by
 default it is in `$KIBANA_HOME/config`. By default, with package distributions
-(Debian or RPM), it is in `/etc/kibana`.
+(Debian or RPM), it is in `/etc/kibana`.  The config directory can be changed via the
+`KBN_PATH_CONF` environment variable:
+
+```
+KBN_PATH_CONF=/home/kibana/config ./bin/kibana
+```
 
 The default host and port settings configure {kib} to run on `localhost:5601`. To change this behavior and allow remote users to connect, you'll need to update your `kibana.yml` file. You can also enable SSL and set a
 variety of other options. Finally, environment variables can be injected into

rfcs/text/0013_saved_object_migrations.md

@@ -248,45 +248,49 @@ Note:
    6. Use the reindexed legacy `.kibana_pre6.5.0_001` as the source for the rest of the migration algorithm.
 3. If `.kibana` and `.kibana_7.10.0` both exists and are pointing to the same index this version's migration has already been completed.
    1. Because the same version can have plugins enabled at any point in time,
-      perform the mappings update in step (7) and migrate outdated documents
-      with step (8).
-   2. Skip to step (10) to start serving traffic.
+      migrate outdated documents with step (9) and perform the mappings update in step (10).
+   2. Skip to step (12) to start serving traffic.
 4. Fail the migration if:
    1. `.kibana` is pointing to an index that belongs to a later version of Kibana .e.g. `.kibana_7.12.0_001`
    2. (Only in 8.x) The source index contains documents that belong to an unknown Saved Object type (from a disabled plugin). Log an error explaining that the plugin that created these documents needs to be enabled again or that these objects should be deleted. See section (4.2.1.4).
-5. Mark the source index as read-only and wait for all in-flight operations to drain (requires https://github.com/elastic/elasticsearch/pull/58094). This prevents any further writes from outdated nodes. Assuming this API is similar to the existing `/<index>/_close` API, we expect to receive `"acknowledged" : true` and `"shards_acknowledged" : true`. If all shards don’t acknowledge within the timeout, retry the operation until it succeeds.
-6. Clone the source index into a new target index which has writes enabled. All nodes on the same version will use the same fixed index name e.g. `.kibana_7.10.0_001`. The `001` postfix isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`.
-   1. `POST /.kibana_n/_clone/.kibana_7.10.0_001?wait_for_active_shards=all {"settings": {"index.blocks.write": false}}`. Ignore errors if the clone already exists.
-   2. Wait for the cloning to complete `GET /_cluster/health/.kibana_7.10.0_001?wait_for_status=green&timeout=60s` If cloning doesn’t complete within the 60s timeout, log a warning for visibility and poll again.
-7. Update the mappings of the target index
+5. Set a write block on the source index. This prevents any further writes from outdated nodes.
+6. Create a new temporary index `.kibana_7.10.0_reindex_temp` with `dynamic: false` on the top-level mappings so that any kind of document can be written to the index. This allows us to write untransformed documents to the index which might have fields which have been removed from the latest mappings defined by the plugin. Define minimal mappings for the `migrationVersion` and `type` fields so that we're still able to search for outdated documents that need to be transformed.
+   1. Ignore errors if the target index already exists.
+7. Reindex the source index into the new temporary index.
+   1. Use `op_type=create` `conflicts=proceed` and `wait_for_completion=false` so that multiple instances can perform the reindex in parallel but only one write per document will succeed.
+   2. Wait for the reindex task to complete. If reindexing doesn’t complete within the 60s timeout, log a warning for visibility and poll again.
+8. Clone the temporary index into the target index `.kibana_7.10.0_001`. Since any further writes will only happen against the cloned target index this prevents a lost delete from occuring where one instance finishes the migration and deletes a document and another instance's reindex operation re-creates the deleted document.
+   1. Set a write block on the temporary index
+   2. Clone the temporary index into the target index while specifying that the target index should have writes enabled.
+   3. If the clone operation fails because the target index already exist, ignore the error and wait for the target index to become green before proceeding.
+   4. (The `001` postfix in the target index name isn't used by Kibana, but allows for re-indexing an index should this be required by an Elasticsearch upgrade. E.g. re-index `.kibana_7.10.0_001` into `.kibana_7.10.0_002` and point the `.kibana_7.10.0` alias to `.kibana_7.10.0_002`.)
+9.  Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. 
+   1. Ignore any version conflict errors.
+   2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration.
+10. Update the mappings of the target index
    1. Retrieve the existing mappings including the `migrationMappingPropertyHashes` metadata.
-   2. Update the mappings with `PUT /.kibana_7.10.0_001/_mapping`. The API deeply merges any updates so this won't remove the mappings of any plugins that were enabled in a previous version but are now disabled.
+   2. Update the mappings with `PUT /.kibana_7.10.0_001/_mapping`. The API deeply merges any updates so this won't remove the mappings of any plugins that are disabled on this instance but have been enabled on another instance that also migrated this index.
    3. Ensure that fields are correctly indexed using the target index's latest mappings `POST /.kibana_7.10.0_001/_update_by_query?conflicts=proceed`. In the future we could optimize this query by only targeting documents:
       1. That belong to a known saved object type.
-      2. Which don't have outdated migrationVersion numbers since these will be transformed anyway.
-      3. That belong to a type whose mappings were changed by comparing the `migrationMappingPropertyHashes`. (Metadata, unlike the mappings isn't commutative, so there is a small chance that the metadata hashes do not accurately reflect the latest mappings, however, this will just result in an less efficient query).
-8. Transform documents by reading batches of outdated documents from the target index then transforming and updating them with optimistic concurrency control. 
-   1. Ignore any version conflict errors.
-   2. If a document transform throws an exception, add the document to a failure list and continue trying to transform all other documents. If any failures occured, log the complete list of documents that failed to transform. Fail the migration.
-9.  Mark the migration as complete. This is done as a single atomic
+11.  Mark the migration as complete. This is done as a single atomic
     operation (requires https://github.com/elastic/elasticsearch/pull/58100)
-    to guarantees when multiple versions of Kibana are performing the
+    to guarantee that when multiple versions of Kibana are performing the
     migration in parallel, only one version will win. E.g. if 7.11 and 7.12
     are started in parallel and migrate from a 7.9 index, either 7.11 or 7.12
     should succeed and accept writes, but not both.
-   3. Checks that `.kibana` alias is still pointing to the source index
-   4. Points the `.kibana_7.10.0` and `.kibana` aliases to the target index.
-   5. If this fails with a "required alias [.kibana] does not exist" error fetch `.kibana` again:
+   1. Check that `.kibana` alias is still pointing to the source index
+   2. Point the `.kibana_7.10.0` and `.kibana` aliases to the target index.
+   3. Remove the temporary index `.kibana_7.10.0_reindex_temp`
+   4. If this fails with a "required alias [.kibana] does not exist" error or "index_not_found_exception" for the temporary index, fetch `.kibana` again:
       1. If `.kibana` is _not_ pointing to our target index fail the migration.
-      2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (10).
-10. Start serving traffic. All saved object reads/writes happen through the
+      2. If `.kibana` is pointing to our target index the migration has succeeded and we can proceed to step (12).
+12.  Start serving traffic. All saved object reads/writes happen through the
     version-specific alias `.kibana_7.10.0`.
 
 Together with the limitations, this algorithm ensures that migrations are
 idempotent. If two nodes are started simultaneously, both of them will start
 transforming documents in that version's target index, but because migrations
 are idempotent, it doesn’t matter which node’s writes win.
-
 #### Known weaknesses:
 (Also present in our existing migration algorithm since v7.4)
 When the task manager index gets reindexed a reindex script is applied.

src/dev/build/tasks/copy_source_task.ts

@@ -29,6 +29,7 @@ export const CopySource: Task = {
         '!src/cli/dev.js',
         '!src/functional_test_runner/**',
         '!src/dev/**',
+        '!src/plugins/telemetry/schema/**', // Skip telemetry schemas
         // this is the dev-only entry
         '!src/setup_node_env/index.js',
         '!**/public/**/*.{js,ts,tsx,json}',

src/plugins/discover/public/application/angular/discover.js

@@ -252,6 +252,12 @@ function discoverController($route, $scope, Promise) {
           (prop) => !_.isEqual(newStatePartial[prop], oldStatePartial[prop])
         );
 
+        if (oldStatePartial.hideChart && !newStatePartial.hideChart) {
+          // in case the histogram is hidden, no data is requested
+          // so when changing this state data needs to be fetched
+          changes.push(true);
+        }
+
         if (changes.length) {
           refetch$.next();
         }
@@ -313,6 +319,8 @@ function discoverController($route, $scope, Promise) {
     setAppState,
     data,
     stateContainer,
+    searchSessionManager,
+    refetch$,
   };
 
   const inspectorAdapters = ($scope.opts.inspectorAdapters = {
@@ -412,6 +420,9 @@ function discoverController($route, $scope, Promise) {
     if (savedSearch.grid) {
       defaultState.grid = savedSearch.grid;
     }
+    if (savedSearch.hideChart) {
+      defaultState.hideChart = savedSearch.hideChart;
+    }
 
     return defaultState;
   }
@@ -562,13 +573,6 @@ function discoverController($route, $scope, Promise) {
       });
   };
 
-  $scope.handleRefresh = function (_payload, isUpdate) {
-    if (isUpdate === false) {
-      searchSessionManager.removeSearchSessionIdFromURL({ replace: false });
-      refetch$.next();
-    }
-  };
-
   function getDimensions(aggs, timeRange) {
     const [metric, agg] = aggs;
     agg.params.timeRange = timeRange;
@@ -601,7 +605,7 @@ function discoverController($route, $scope, Promise) {
   function onResults(resp) {
     inspectorRequest.stats(getResponseInspectorStats(resp, $scope.searchSource)).ok({ json: resp });
 
-    if (getTimeField()) {
+    if (getTimeField() && !$scope.state.hideChart) {
       const tabifiedData = tabifyAggResponse($scope.opts.chartAggConfigs, resp);
       $scope.searchSource.rawResponse = resp;
       $scope.histogramData = discoverResponseHandler(
@@ -704,7 +708,7 @@ function discoverController($route, $scope, Promise) {
 
   async function setupVisualization() {
     // If no timefield has been specified we don't create a histogram of messages
-    if (!getTimeField()) return;
+    if (!getTimeField() || $scope.state.hideChart) return;
     const { interval: histogramInterval } = $scope.state;
 
     const visStateAggs = [

src/plugins/discover/public/application/angular/discover_legacy.html

@@ -17,7 +17,6 @@
     state="state"
     time-range="timeRange"
     top-nav-menu="topNavMenu"
-    update-query="handleRefresh"
     use-new-fields-api="useNewFieldsApi"
     unmapped-fields-config="unmappedFieldsConfig"
   >

src/plugins/discover/public/application/angular/discover_state.ts

@@ -44,6 +44,10 @@ export interface AppState {
    * Data Grid related state
    */
   grid?: DiscoverGridSettings;
+  /**
+   * Hide chart
+   */
+  hideChart?: boolean;
   /**
    * id of the used index pattern
    */

src/plugins/discover/public/application/components/discover.test.tsx

@@ -25,6 +25,8 @@ import { indexPatternWithTimefieldMock } from '../../__mocks__/index_pattern_wit
 import { calcFieldCounts } from '../helpers/calc_field_counts';
 import { DiscoverProps } from './types';
 import { RequestAdapter } from '../../../../inspector/common';
+import { Subject } from 'rxjs';
+import { DiscoverSearchSessionManager } from '../angular/discover_search_session';
 
 const mockNavigation = navigationPluginMock.createStartContract();
 
@@ -73,8 +75,10 @@ function getProps(indexPattern: IndexPattern): DiscoverProps {
       indexPatternList: (indexPattern as unknown) as Array<SavedObject<IndexPatternAttributes>>,
       inspectorAdapters: { requests: {} as RequestAdapter },
       navigateTo: jest.fn(),
+      refetch$: {} as Subject<undefined>,
       sampleSize: 10,
       savedSearch: savedSearchMock,
+      searchSessionManager: {} as DiscoverSearchSessionManager,
       setHeaderActionMenu: jest.fn(),
       timefield: indexPattern.timeFieldName || '',
       setAppState: jest.fn(),
@@ -86,7 +90,6 @@ function getProps(indexPattern: IndexPattern): DiscoverProps {
     rows: esHits,
     searchSource: searchSourceMock,
     state: { columns: [] },
-    updateQuery: jest.fn(),
   };
 }