Merge branch 'master' into results-cleanup

docs/settings/search-sessions-settings.asciidoc

@@ -11,15 +11,33 @@ Configure the search session settings in your `kibana.yml` configuration file.
 [cols="2*<"]
 |===
 a| `xpack.data_enhanced.`
-`search.sessions.enabled`
+`search.sessions.enabled` {ess-icon}
 | Set to `true` (default) to enable search sessions.
 
 a| `xpack.data_enhanced.`
-`search.sessions.trackingInterval`
-| The frequency for updating the state of a search session. The default is 10s.
+`search.sessions.trackingInterval` {ess-icon}
+| The frequency for updating the state of a search session. The default is `10s`.
 
 a| `xpack.data_enhanced.`
-`search.sessions.defaultExpiration`
+`search.sessions.pageSize` {ess-icon}
+| How many search sessions {kib} processes at once while monitoring
+session progress. The default is `100`.
+
+a| `xpack.data_enhanced.`
+`search.sessions.notTouchedTimeout` {ess-icon}
+| How long {kib} stores search results from unsaved sessions,
+after the last search in the session completes. The default is `5m`.
+
+a| `xpack.data_enhanced.`
+`search.sessions.notTouchedInProgressTimeout` {ess-icon}
+| How long a search session can run after a user navigates away without saving a session. The default is `1m`.
+
+a| `xpack.data_enhanced.`
+`search.sessions.maxUpdateRetries` {ess-icon}
+| How many retries {kib} can perform while attempting to save a search session. The default is `3`.
+
+a| `xpack.data_enhanced.`
+`search.sessions.defaultExpiration` {ess-icon}
 | How long search session results are stored before they are deleted.
-Extending a search session resets the expiration by the same value. The default is 7d.
+Extending a search session resets the expiration by the same value. The default is `7d`.
 |===

src/core/server/config/ensure_valid_configuration.test.ts

@@ -16,14 +16,40 @@ describe('ensureValidConfiguration', () => {
   beforeEach(() => {
     jest.clearAllMocks();
     configService = configServiceMock.create();
-    configService.getUsedPaths.mockReturnValue(Promise.resolve(['core', 'elastic']));
+
+    configService.validate.mockResolvedValue();
+    configService.getUsedPaths.mockReturnValue(Promise.resolve([]));
   });
 
-  it('returns normally when there is no unused keys', async () => {
-    configService.getUnusedPaths.mockResolvedValue([]);
+  it('returns normally when there is no unused keys and when the config validates', async () => {
     await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
   });
 
+  it('throws when config validation fails', async () => {
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: some message]`
+    );
+  });
+
+  it('throws a `CriticalError` with the correct processExitCode value when config validation fails', async () => {
+    expect.assertions(2);
+
+    configService.validate.mockImplementation(() => {
+      throw new Error('some message');
+    });
+
+    try {
+      await ensureValidConfiguration(configService as any);
+    } catch (e) {
+      expect(e).toBeInstanceOf(CriticalError);
+      expect(e.processExitCode).toEqual(78);
+    }
+  });
+
   it('throws when there are some unused keys', async () => {
     configService.getUnusedPaths.mockResolvedValue(['some.key', 'some.other.key']);
 
@@ -44,4 +70,18 @@ describe('ensureValidConfiguration', () => {
       expect(e.processExitCode).toEqual(64);
     }
   });
+
+  it('does not throw when all unused keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'elastic.apm.enabled']);
+
+    await expect(ensureValidConfiguration(configService as any)).resolves.toBeUndefined();
+  });
+
+  it('throws when only some keys are included in the ignored paths', async () => {
+    configService.getUnusedPaths.mockResolvedValue(['dev.someDevKey', 'some.key']);
+
+    await expect(ensureValidConfiguration(configService as any)).rejects.toMatchInlineSnapshot(
+      `[Error: Unknown configuration key(s): "some.key". Check for spelling errors and ensure that expected plugins are installed.]`
+    );
+  });
 });

src/core/server/config/ensure_valid_configuration.ts

@@ -9,22 +9,27 @@
 import { ConfigService } from '@kbn/config';
 import { CriticalError } from '../errors';
 
+const ignoredPaths = ['dev.', 'elastic.apm.'];
+
+const invalidConfigExitCode = 78;
+const legacyInvalidConfigExitCode = 64;
+
 export async function ensureValidConfiguration(configService: ConfigService) {
-  await configService.validate();
+  try {
+    await configService.validate();
+  } catch (e) {
+    throw new CriticalError(e.message, 'InvalidConfig', invalidConfigExitCode, e);
+  }
 
-  const unusedConfigKeys = await configService.getUnusedPaths();
+  const unusedPaths = await configService.getUnusedPaths();
+  const unusedConfigKeys = unusedPaths.filter((unusedPath) => {
+    return !ignoredPaths.some((ignoredPath) => unusedPath.startsWith(ignoredPath));
+  });
 
   if (unusedConfigKeys.length > 0) {
     const message = `Unknown configuration key(s): ${unusedConfigKeys
       .map((key) => `"${key}"`)
       .join(', ')}. Check for spelling errors and ensure that expected plugins are installed.`;
-    throw new InvalidConfigurationError(message);
-  }
-}
-
-class InvalidConfigurationError extends CriticalError {
-  constructor(message: string) {
-    super(message, 'InvalidConfig', 64);
-    Object.setPrototypeOf(this, InvalidConfigurationError.prototype);
+    throw new CriticalError(message, 'InvalidConfig', legacyInvalidConfigExitCode);
   }
 }

src/core/server/server.ts

@@ -36,7 +36,6 @@ import { config as cspConfig } from './csp';
 import { config as elasticsearchConfig } from './elasticsearch';
 import { config as httpConfig } from './http';
 import { config as loggingConfig } from './logging';
-import { config as devConfig } from './dev';
 import { config as kibanaConfig } from './kibana_config';
 import { savedObjectsConfig, savedObjectsMigrationConfig } from './saved_objects';
 import { config as uiSettingsConfig } from './ui_settings';
@@ -303,7 +302,6 @@ export class Server {
       loggingConfig,
       httpConfig,
       pluginsConfig,
-      devConfig,
       kibanaConfig,
       savedObjectsConfig,
       savedObjectsMigrationConfig,

src/plugins/discover/kibana.json

@@ -12,7 +12,8 @@
     "urlForwarding",
     "navigation",
     "uiActions",
-    "savedObjects"
+    "savedObjects",
+    "indexPatternFieldEditor"
   ],
   "optionalPlugins": ["home", "share", "usageCollection"],
   "requiredBundles": ["kibanaUtils", "home", "kibanaReact"]

src/plugins/discover/public/application/angular/discover.js

@@ -458,6 +458,13 @@ function discoverController($route, $scope) {
     $scope.fetchStatus = fetchStatuses.COMPLETE;
   }
 
+  $scope.refreshAppState = async () => {
+    $scope.hits = [];
+    $scope.rows = [];
+    $scope.fieldCounts = {};
+    await refetch$.next();
+  };
+
   function getRequestResponder({ searchSessionId = null } = { searchSessionId: null }) {
     inspectorAdapters.requests.reset();
     const title = i18n.translate('discover.inspectorRequestDataTitle', {

src/plugins/discover/public/application/angular/discover_legacy.html

@@ -16,6 +16,7 @@
     top-nav-menu="topNavMenu"
     use-new-fields-api="useNewFieldsApi"
     unmapped-fields-config="unmappedFieldsConfig"
+    refresh-app-state="refreshAppState"
   >
   </discover>
 </discover-app>

src/plugins/discover/public/application/angular/discover_state.ts

@@ -177,7 +177,7 @@ export function getState({
     },
     uiSettings
   );
-  // todo filter source depending on fields fetchinbg flag (if no columns remain and source fetching is enabled, use default columns)
+  // todo filter source depending on fields fetching flag (if no columns remain and source fetching is enabled, use default columns)
   let previousAppState: AppState;
   const appStateContainer = createStateContainer<AppState>(initialAppState);
 

src/plugins/discover/public/application/angular/helpers/row_formatter.test.ts

@@ -90,6 +90,7 @@ describe('Row formatter', () => {
         },
         {
           'object.value': [5, 10],
+          getByName: jest.fn(),
         },
         indexPattern
       ).trim()
@@ -107,7 +108,7 @@ describe('Row formatter', () => {
     });
     const formatted = formatTopLevelObject(
       { fields: { 'a.zzz': [100], 'a.ccc': [50] } },
-      { 'a.zzz': [100], 'a.ccc': [50] },
+      { 'a.zzz': [100], 'a.ccc': [50], getByName: jest.fn() },
       indexPattern
     ).trim();
     expect(formatted.indexOf('<dt>a.ccc:</dt>')).toBeLessThan(formatted.indexOf('<dt>a.zzz:</dt>'));
@@ -134,6 +135,7 @@ describe('Row formatter', () => {
         {
           'object.value': [5, 10],
           'object.keys': ['a', 'b'],
+          getByName: jest.fn(),
         },
         indexPattern
       ).trim()
@@ -154,6 +156,7 @@ describe('Row formatter', () => {
         },
         {
           'object.value': [5, 10],
+          getByName: jest.fn(),
         },
         indexPattern
       ).trim()

src/plugins/discover/public/application/angular/helpers/row_formatter.ts

@@ -28,11 +28,13 @@ export const formatRow = (hit: Record<string, any>, indexPattern: IndexPattern)
   const highlights = hit?.highlight ?? {};
   // Keys are sorted in the hits object
   const formatted = indexPattern.formatHit(hit);
+  const fields = indexPattern.fields;
   const highlightPairs: Array<[string, unknown]> = [];
   const sourcePairs: Array<[string, unknown]> = [];
   Object.entries(formatted).forEach(([key, val]) => {
+    const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined;
     const pairs = highlights[key] ? highlightPairs : sourcePairs;
-    pairs.push([key, val]);
+    pairs.push([displayKey ? displayKey : key, val]);
   });
   return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
 };
@@ -48,9 +50,11 @@ export const formatTopLevelObject = (
   const sorted = Object.entries(fields).sort(([keyA], [keyB]) => keyA.localeCompare(keyB));
   sorted.forEach(([key, values]) => {
     const field = indexPattern.getFieldByName(key);
+    const displayKey = fields.getByName ? fields.getByName(key)?.displayName : undefined;
     const formatter = field
       ? indexPattern.getFormatterForField(field)
       : { convert: (v: string, ...rest: unknown[]) => String(v) };
+    if (!values.map) return;
     const formatted = values
       .map((val: unknown) =>
         formatter.convert(val, 'html', {
@@ -61,7 +65,7 @@ export const formatTopLevelObject = (
       )
       .join(', ');
     const pairs = highlights[key] ? highlightPairs : sourcePairs;
-    pairs.push([key, formatted]);
+    pairs.push([displayKey ? displayKey : key, formatted]);
   });
   return doTemplate({ defPairs: [...highlightPairs, ...sourcePairs] });
 };

src/plugins/discover/public/application/components/create_discover_directive.ts

@@ -28,5 +28,6 @@ export function createDiscoverDirective(reactDirective: any) {
     ['updateQuery', { watchDepth: 'reference' }],
     ['updateSavedQueryId', { watchDepth: 'reference' }],
     ['unmappedFieldsConfig', { watchDepth: 'value' }],
+    ['refreshAppState', { watchDepth: 'reference' }],
   ]);
 }

src/plugins/discover/public/application/components/discover.tsx

@@ -68,6 +68,7 @@ export function Discover({
   searchSource,
   state,
   unmappedFieldsConfig,
+  refreshAppState,
 }: DiscoverProps) {
   const [expandedDoc, setExpandedDoc] = useState<ElasticSearchHit | undefined>(undefined);
   const scrollableDesktop = useRef<HTMLDivElement>(null);
@@ -203,6 +204,12 @@ export function Discover({
     [opts, state]
   );
 
+  const onEditRuntimeField = () => {
+    if (refreshAppState) {
+      refreshAppState();
+    }
+  };
+
   const columns = useMemo(() => {
     if (!state.columns) {
       return [];
@@ -245,6 +252,7 @@ export function Discover({
                 trackUiMetric={trackUiMetric}
                 unmappedFieldsConfig={unmappedFieldsConfig}
                 useNewFieldsApi={useNewFieldsApi}
+                onEditRuntimeField={onEditRuntimeField}
               />
             </EuiFlexItem>
             <EuiHideFor sizes={['xs', 's']}>

src/plugins/discover/public/application/components/discover_grid/get_render_cell_value.tsx

@@ -77,6 +77,9 @@ export const getRenderCellValueFn = (
     const sourcePairs: Array<[string, string]> = [];
     Object.entries(innerColumns).forEach(([key, values]) => {
       const subField = indexPattern.getFieldByName(key);
+      const displayKey = indexPattern.fields.getByName
+        ? indexPattern.fields.getByName(key)?.displayName
+        : undefined;
       const formatter = subField
         ? indexPattern.getFormatterForField(subField)
         : { convert: (v: string, ...rest: unknown[]) => String(v) };
@@ -90,7 +93,7 @@ export const getRenderCellValueFn = (
         )
         .join(', ');
       const pairs = highlights[key] ? highlightPairs : sourcePairs;
-      pairs.push([key, formatted]);
+      pairs.push([displayKey ? displayKey : key, formatted]);
     });
 
     return (
@@ -130,7 +133,10 @@ export const getRenderCellValueFn = (
 
     Object.entries(formatted).forEach(([key, val]) => {
       const pairs = highlights[key] ? highlightPairs : sourcePairs;
-      pairs.push([key, val as string]);
+      const displayKey = indexPattern.fields.getByName
+        ? indexPattern.fields.getByName(key)?.displayName
+        : undefined;
+      pairs.push([displayKey ? displayKey : key, val as string]);
     });
 
     return (