Merge remote-tracking branch 'origin/master' into moving-file-data-vi…

…sualizer-UI-to-file-upload-plugin

.telemetryrc.json

@@ -2,6 +2,8 @@
   {
     "output": "src/plugins/telemetry/schema/oss_plugins.json",
     "root": "src/plugins/",
-    "exclude": []
+    "exclude": [
+      "src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts"
+    ]
   }
 ]

docs/developer/getting-started/index.asciidoc

@@ -12,8 +12,11 @@ In order to support Windows development we currently require you to use one of t
 - https://git-scm.com/download/win[Git bash] (other bash emulators like https://cmder.net/[Cmder] could work but we did not test them)
 - https://docs.microsoft.com/en-us/windows/wsl/about[WSL]
 
-Before running the steps listed below, please make sure you have installed Git bash or WSL and that 
-you are running the mentioned commands through one of them.
+As well as installing https://www.microsoft.com/en-us/download/details.aspx?id=48145[Visual C++ Redistributable for Visual Studio 2015].
+
+Before running the steps listed below, please make sure you have installed everything 
+that we require and listed above and that you are running the mentioned commands 
+through Git bash or WSL.
 
 [discrete]
 [[get-kibana-code]]

docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md

@@ -0,0 +1,15 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md)
+
+## MakeUsageFromSchema type
+
+List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to `true` then the actual value of these configs will be reoprted. If parent node or actual config path is set to `false` then the config will be reported as \[redacted\].
+
+<b>Signature:</b>
+
+```typescript
+export declare type MakeUsageFromSchema<T> = {
+    [Key in keyof T]?: T[Key] extends Maybe<object[]> ? false : T[Key] extends Maybe<any[]> ? boolean : T[Key] extends Maybe<object> ? MakeUsageFromSchema<T[Key]> | boolean : boolean;
+};
+```

docs/development/core/server/kibana-plugin-core-server.md

@@ -272,6 +272,7 @@ The plugin integrates with the core system via lifecycle events: `setup`<!-- -->
 |  [LegacyElasticsearchClientConfig](./kibana-plugin-core-server.legacyelasticsearchclientconfig.md) |  |
 |  [LifecycleResponseFactory](./kibana-plugin-core-server.lifecycleresponsefactory.md) | Creates an object containing redirection or error response with error details, HTTP headers, and other data transmitted to the client. |
 |  [LoggerConfigType](./kibana-plugin-core-server.loggerconfigtype.md) |  |
+|  [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to <code>true</code> then the actual value of these configs will be reoprted. If parent node or actual config path is set to <code>false</code> then the config will be reported as \[redacted\]. |
 |  [MetricsServiceStart](./kibana-plugin-core-server.metricsservicestart.md) | APIs to retrieves metrics gathered and exposed by the core platform. |
 |  [MIGRATION\_ASSISTANCE\_INDEX\_ACTION](./kibana-plugin-core-server.migration_assistance_index_action.md) |  |
 |  [MIGRATION\_DEPRECATION\_LEVEL](./kibana-plugin-core-server.migration_deprecation_level.md) |  |

docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md

@@ -0,0 +1,17 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [PluginConfigDescriptor](./kibana-plugin-core-server.pluginconfigdescriptor.md) &gt; [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md)
+
+## PluginConfigDescriptor.exposeToUsage property
+
+Expose non-default configs to usage collection to be sent via telemetry. set a config to `true` to report the actual changed config value. set a config to `false` to report the changed config value as \[redacted\].
+
+All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.
+
+[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md)
+
+<b>Signature:</b>
+
+```typescript
+exposeToUsage?: MakeUsageFromSchema<T>;
+```

docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md

@@ -46,5 +46,6 @@ export const config: PluginConfigDescriptor<ConfigType> = {
 |  --- | --- | --- |
 |  [deprecations](./kibana-plugin-core-server.pluginconfigdescriptor.deprecations.md) | <code>ConfigDeprecationProvider</code> | Provider for the  to apply to the plugin configuration. |
 |  [exposeToBrowser](./kibana-plugin-core-server.pluginconfigdescriptor.exposetobrowser.md) | <code>{</code><br/><code>        [P in keyof T]?: boolean;</code><br/><code>    }</code> | List of configuration properties that will be available on the client-side plugin. |
+|  [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) | <code>MakeUsageFromSchema&lt;T&gt;</code> | Expose non-default configs to usage collection to be sent via telemetry. set a config to <code>true</code> to report the actual changed config value. set a config to <code>false</code> to report the changed config value as \[redacted\].<!-- -->All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) |
 |  [schema](./kibana-plugin-core-server.pluginconfigdescriptor.schema.md) | <code>PluginConfigSchema&lt;T&gt;</code> | Schema to use to validate the plugin configuration.[PluginConfigSchema](./kibana-plugin-core-server.pluginconfigschema.md) |
 

docs/development/core/server/kibana-plugin-core-server.savedobjectsmigrationlogger.error.md

@@ -7,5 +7,5 @@
 <b>Signature:</b>
 
 ```typescript
-error: (msg: string, meta: LogMeta) => void;
+error: <Meta extends LogMeta = LogMeta>(msg: string, meta: Meta) => void;
 ```

docs/development/core/server/kibana-plugin-core-server.savedobjectsmigrationlogger.md

@@ -16,7 +16,7 @@ export interface SavedObjectsMigrationLogger
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [debug](./kibana-plugin-core-server.savedobjectsmigrationlogger.debug.md) | <code>(msg: string) =&gt; void</code> |  |
-|  [error](./kibana-plugin-core-server.savedobjectsmigrationlogger.error.md) | <code>(msg: string, meta: LogMeta) =&gt; void</code> |  |
+|  [error](./kibana-plugin-core-server.savedobjectsmigrationlogger.error.md) | <code>&lt;Meta extends LogMeta = LogMeta&gt;(msg: string, meta: Meta) =&gt; void</code> |  |
 |  [info](./kibana-plugin-core-server.savedobjectsmigrationlogger.info.md) | <code>(msg: string) =&gt; void</code> |  |
 |  [warn](./kibana-plugin-core-server.savedobjectsmigrationlogger.warn.md) | <code>(msg: string) =&gt; void</code> |  |
 |  [warning](./kibana-plugin-core-server.savedobjectsmigrationlogger.warning.md) | <code>(msg: string) =&gt; void</code> |  |

docs/settings/fleet-settings.asciidoc

@@ -37,12 +37,10 @@ See the {fleet-guide}/index.html[{fleet}] docs for more information.
 
 [cols="2*<"]
 |===
-| `xpack.fleet.agents.kibana.host`
-  | The hostname used by {agent} for accessing {kib}.
+| `xpack.fleet.agents.fleet_server.hosts`
+  | Hostnames used by {agent} for accessing {fleet-server}.
 | `xpack.fleet.agents.elasticsearch.host`
   | The hostname used by {agent} for accessing {es}.
-| `xpack.fleet.agents.tlsCheckDisabled`
-  | Set to `true` to allow {fleet} to run on a {kib} instance without TLS enabled.
 |===
 
 [NOTE]

package.json

@@ -102,7 +102,7 @@
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath/npm_module",
     "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@^8.0.0-canary.4",
     "@elastic/ems-client": "7.12.0",
-    "@elastic/eui": "32.0.4",
+    "@elastic/eui": "32.1.0",
     "@elastic/filesaver": "1.1.2",
     "@elastic/good": "^9.0.1-kibana3",
     "@elastic/maki": "6.3.0",

packages/kbn-cli-dev-mode/src/base_path_proxy_server.test.ts

@@ -8,6 +8,7 @@
 
 import { Server } from '@hapi/hapi';
 import { EMPTY } from 'rxjs';
+import moment from 'moment';
 import supertest from 'supertest';
 import {
   getServerOptions,
@@ -35,6 +36,7 @@ describe('BasePathProxyServer', () => {
     config = {
       host: '127.0.0.1',
       port: 10012,
+      shutdownTimeout: moment.duration(30, 'seconds'),
       keepaliveTimeout: 1000,
       socketTimeout: 1000,
       cors: {

packages/kbn-cli-dev-mode/src/cli_dev_mode.test.ts

@@ -108,7 +108,7 @@ it('passes correct args to sub-classes', () => {
             "bar",
             "baz",
           ],
-          "gracefulTimeout": 5000,
+          "gracefulTimeout": 30000,
           "log": <TestLog>,
           "mapLogLine": [Function],
           "script": <absolute path>/scripts/kibana,

packages/kbn-cli-dev-mode/src/cli_dev_mode.ts

@@ -44,7 +44,7 @@ Rx.merge(
   .subscribe(exitSignal$);
 
 // timeout where the server is allowed to exit gracefully
-const GRACEFUL_TIMEOUT = 5000;
+const GRACEFUL_TIMEOUT = 30000;
 
 export type SomeCliArgs = Pick<
   CliArgs,

packages/kbn-cli-dev-mode/src/config/http_config.ts

@@ -8,6 +8,7 @@
 
 import { ByteSizeValue, schema, TypeOf } from '@kbn/config-schema';
 import { ICorsConfig, IHttpConfig, ISslConfig, SslConfig, sslSchema } from '@kbn/server-http-tools';
+import { Duration } from 'moment';
 
 export const httpConfigSchema = schema.object(
   {
@@ -22,6 +23,7 @@ export const httpConfigSchema = schema.object(
     maxPayload: schema.byteSize({
       defaultValue: '1048576b',
     }),
+    shutdownTimeout: schema.duration({ defaultValue: '30s' }),
     keepaliveTimeout: schema.number({
       defaultValue: 120000,
     }),
@@ -47,6 +49,7 @@ export class HttpConfig implements IHttpConfig {
   host: string;
   port: number;
   maxPayload: ByteSizeValue;
+  shutdownTimeout: Duration;
   keepaliveTimeout: number;
   socketTimeout: number;
   cors: ICorsConfig;
@@ -57,6 +60,7 @@ export class HttpConfig implements IHttpConfig {
     this.host = rawConfig.host;
     this.port = rawConfig.port;
     this.maxPayload = rawConfig.maxPayload;
+    this.shutdownTimeout = rawConfig.shutdownTimeout;
     this.keepaliveTimeout = rawConfig.keepaliveTimeout;
     this.socketTimeout = rawConfig.socketTimeout;
     this.cors = rawConfig.cors;

packages/kbn-cli-dev-mode/src/dev_server.ts

@@ -103,7 +103,7 @@ export class DevServer {
   /**
    * Run the Kibana server
    *
-   * The observable will error if the child process failes to spawn for some reason, but if
+   * The observable will error if the child process fails to spawn for some reason, but if
    * the child process is successfully spawned then the server will be run until it completes
    * and restart when the watcher indicates it should. In order to restart the server as
    * quickly as possible we kill it with SIGKILL and spawn the process again.
@@ -146,6 +146,7 @@ export class DevServer {
     const runServer = () =>
       usingServerProcess(this.script, this.argv, (proc) => {
         this.phase$.next('starting');
+        this.ready$.next(false);
 
         // observable which emits devServer states containing lines
         // logged to stdout/stderr, completes when stdio streams complete

packages/kbn-logging/src/ecs/agent.ts

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-agent.html
+ *
+ * @internal
+ */
+export interface EcsAgent {
+  build?: { original: string };
+  ephemeral_id?: string;
+  id?: string;
+  name?: string;
+  type?: string;
+  version?: string;
+}

packages/kbn-logging/src/ecs/autonomous_system.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-as.html
+ *
+ * @internal
+ */
+export interface EcsAutonomousSystem {
+  number?: number;
+  organization?: { name: string };
+}

packages/kbn-logging/src/ecs/base.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-base.html
+ *
+ * @internal
+ */
+export interface EcsBase {
+  ['@timestamp']: string;
+  labels?: Record<string, unknown>;
+  message?: string;
+  tags?: string[];
+}

packages/kbn-logging/src/ecs/client.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EcsAutonomousSystem } from './autonomous_system';
+import { EcsGeo } from './geo';
+import { EcsNestedUser } from './user';
+
+interface NestedFields {
+  as?: EcsAutonomousSystem;
+  geo?: EcsGeo;
+  user?: EcsNestedUser;
+}
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-client.html
+ *
+ * @internal
+ */
+export interface EcsClient extends NestedFields {
+  address?: string;
+  bytes?: number;
+  domain?: string;
+  ip?: string;
+  mac?: string;
+  nat?: { ip?: string; port?: number };
+  packets?: number;
+  port?: number;
+  registered_domain?: string;
+  subdomain?: string;
+  top_level_domain?: string;
+}

packages/kbn-logging/src/ecs/cloud.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-cloud.html
+ *
+ * @internal
+ */
+export interface EcsCloud {
+  account?: { id?: string; name?: string };
+  availability_zone?: string;
+  instance?: { id?: string; name?: string };
+  machine?: { type: string };
+  project?: { id?: string; name?: string };
+  provider?: string;
+  region?: string;
+  service?: { name: string };
+}

packages/kbn-logging/src/ecs/code_signature.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-code_signature.html
+ *
+ * @internal
+ */
+export interface EcsCodeSignature {
+  exists?: boolean;
+  signing_id?: string;
+  status?: string;
+  subject_name?: string;
+  team_id?: string;
+  trusted?: boolean;
+  valid?: boolean;
+}

packages/kbn-logging/src/ecs/container.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-container.html
+ *
+ * @internal
+ */
+export interface EcsContainer {
+  id?: string;
+  image?: { name?: string; tag?: string[] };
+  labels?: Record<string, unknown>;
+  name?: string;
+  runtime?: string;
+}

packages/kbn-logging/src/ecs/destination.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { EcsAutonomousSystem } from './autonomous_system';
+import { EcsGeo } from './geo';
+import { EcsNestedUser } from './user';
+
+interface NestedFields {
+  as?: EcsAutonomousSystem;
+  geo?: EcsGeo;
+  user?: EcsNestedUser;
+}
+
+/**
+ * https://www.elastic.co/guide/en/ecs/1.9/ecs-destination.html
+ *
+ * @internal
+ */
+export interface EcsDestination extends NestedFields {
+  address?: string;
+  bytes?: number;
+  domain?: string;
+  ip?: string;
+  mac?: string;
+  nat?: { ip?: string; port?: number };
+  packets?: number;
+  port?: number;
+  registered_domain?: string;
+  subdomain?: string;
+  top_level_domain?: string;
+}