Merge remote-tracking branch 'upstream/master' into np/visualize

.github/CODEOWNERS

@@ -202,7 +202,8 @@
 # Endpoint
 /x-pack/plugins/endpoint/ @elastic/endpoint-app-team
 /x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team
-/x-pack/test/functional/apps/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/functional_endpoint/ @elastic/endpoint-app-team
+/x-pack/test/functional_endpoint_ingest_failure/ @elastic/endpoint-app-team
 /x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team
 
 # SIEM

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.createsearchsource.md

@@ -0,0 +1,15 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [createSearchSource](./kibana-plugin-plugins-data-public.createsearchsource.md)
+
+## createSearchSource variable
+
+Deserializes a json string and a set of referenced objects to a `SearchSource` instance. Use this method to re-create the search source serialized using `searchSource.serialize`<!-- -->.
+
+This function is a factory function that returns the actual utility when calling it with the required service dependency (index patterns contract). A pre-wired version is also exposed in the start contract of the data plugin as part of the search service
+
+<b>Signature:</b>
+
+```typescript
+createSearchSource: (indexPatterns: Pick<import("../../index_patterns/index_patterns").IndexPatternsService, "get" | "clearCache" | "getFieldsForTimePattern" | "getFieldsForWildcard" | "getIds" | "getTitles" | "getFields" | "getCache" | "getDefault" | "make">) => (searchSourceJson: string, references: SavedObjectReference[]) => Promise<SearchSource>
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.md

@@ -102,6 +102,7 @@
 |  [castEsToKbnFieldTypeName](./kibana-plugin-plugins-data-public.castestokbnfieldtypename.md) | Get the KbnFieldType name for an esType string |
 |  [connectToQueryState](./kibana-plugin-plugins-data-public.connecttoquerystate.md) | Helper to setup two-way syncing of global data and a state container |
 |  [createSavedQueryService](./kibana-plugin-plugins-data-public.createsavedqueryservice.md) |  |
+|  [createSearchSource](./kibana-plugin-plugins-data-public.createsearchsource.md) | Deserializes a json string and a set of referenced objects to a <code>SearchSource</code> instance. Use this method to re-create the search source serialized using <code>searchSource.serialize</code>.<!-- -->This function is a factory function that returns the actual utility when calling it with the required service dependency (index patterns contract). A pre-wired version is also exposed in the start contract of the data plugin as part of the search service |
 |  [ES\_SEARCH\_STRATEGY](./kibana-plugin-plugins-data-public.es_search_strategy.md) |  |
 |  [esFilters](./kibana-plugin-plugins-data-public.esfilters.md) |  |
 |  [esKuery](./kibana-plugin-plugins-data-public.eskuery.md) |  |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.md

@@ -38,6 +38,7 @@ export declare class SearchSource
 |  [getParent()](./kibana-plugin-plugins-data-public.searchsource.getparent.md) |  | Get the parent of this SearchSource  {<!-- -->undefined\|searchSource<!-- -->} |
 |  [getSearchRequestBody()](./kibana-plugin-plugins-data-public.searchsource.getsearchrequestbody.md) |  |  |
 |  [onRequestStart(handler)](./kibana-plugin-plugins-data-public.searchsource.onrequeststart.md) |  | Add a handler that will be notified whenever requests start |
+|  [serialize()](./kibana-plugin-plugins-data-public.searchsource.serialize.md) |  | Serializes the instance to a JSON string and a set of referenced objects. Use this method to get a representation of the search source which can be stored in a saved object.<!-- -->The references returned by this function can be mixed with other references in the same object, however make sure there are no name-collisions. The references will be named <code>kibanaSavedObjectMeta.searchSourceJSON.index</code> and <code>kibanaSavedObjectMeta.searchSourceJSON.filter[&lt;number&gt;].meta.index</code>.<!-- -->Using <code>createSearchSource</code>, the instance can be re-created. |
 |  [setField(field, value)](./kibana-plugin-plugins-data-public.searchsource.setfield.md) |  |  |
 |  [setFields(newFields)](./kibana-plugin-plugins-data-public.searchsource.setfields.md) |  |  |
 |  [setParent(parent, options)](./kibana-plugin-plugins-data-public.searchsource.setparent.md) |  | Set a searchSource that this source should inherit from |

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.searchsource.serialize.md

@@ -0,0 +1,27 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [SearchSource](./kibana-plugin-plugins-data-public.searchsource.md) &gt; [serialize](./kibana-plugin-plugins-data-public.searchsource.serialize.md)
+
+## SearchSource.serialize() method
+
+Serializes the instance to a JSON string and a set of referenced objects. Use this method to get a representation of the search source which can be stored in a saved object.
+
+The references returned by this function can be mixed with other references in the same object, however make sure there are no name-collisions. The references will be named `kibanaSavedObjectMeta.searchSourceJSON.index` and `kibanaSavedObjectMeta.searchSourceJSON.filter[<number>].meta.index`<!-- -->.
+
+Using `createSearchSource`<!-- -->, the instance can be re-created.
+
+<b>Signature:</b>
+
+```typescript
+serialize(): {
+        searchSourceJSON: string;
+        references: SavedObjectReference[];
+    };
+```
+<b>Returns:</b>
+
+`{
+        searchSourceJSON: string;
+        references: SavedObjectReference[];
+    }`
+

src/legacy/core_plugins/kibana/public/discover/build_services.ts

@@ -72,6 +72,7 @@ export async function buildServices(
   const services = {
     savedObjectsClient: core.savedObjects.client,
     indexPatterns: plugins.data.indexPatterns,
+    search: plugins.data.search,
     chrome: core.chrome,
     overlays: core.overlays,
   };

src/legacy/core_plugins/kibana/public/discover/np_ready/angular/discover.html

@@ -11,7 +11,7 @@ <h1 class="euiScreenReaderOnly">{{screenTitle}}</h1>
     query="state.query"
     saved-query-id="state.savedQuery"
     screen-title="screenTitle"
-    show-date-picker="enableTimeRangeSelector"
+    show-date-picker="indexPattern.isTimeBased()"
     show-save-query="showSaveQuery"
     show-search-bar="true"
     use-default-behaviors="true"

src/legacy/core_plugins/kibana/public/management/saved_object_registry.ts

@@ -56,6 +56,7 @@ export const savedObjectManagementRegistry: ISavedObjectsManagementRegistry = {
 const services = {
   savedObjectsClient: npStart.core.savedObjects.client,
   indexPatterns: npStart.plugins.data.indexPatterns,
+  search: npStart.plugins.data.search,
   chrome: npStart.core.chrome,
   overlays: npStart.core.overlays,
 };

src/legacy/core_plugins/kibana/public/management/sections/objects/components/objects_table/components/flyout/__jest__/flyout.test.js

@@ -519,7 +519,8 @@ describe('Flyout', () => {
       expect(resolveIndexPatternConflicts).toHaveBeenCalledWith(
         component.instance().resolutions,
         mockConflictedIndexPatterns,
-        true
+        true,
+        defaultProps.indexPatterns
       );
       expect(saveObjects).toHaveBeenCalledWith(
         mockConflictedSavedObjectsLinkedToSavedSearches,

src/legacy/core_plugins/kibana/public/management/sections/objects/components/objects_table/components/flyout/flyout.js

@@ -358,7 +358,8 @@ export class Flyout extends Component {
           importCount += await resolveIndexPatternConflicts(
             resolutions,
             conflictedIndexPatterns,
-            isOverwriteAllChecked
+            isOverwriteAllChecked,
+            this.props.indexPatterns
           );
         }
         this.setState({

src/legacy/core_plugins/kibana/public/management/sections/objects/lib/resolve_saved_objects.test.ts

@@ -84,7 +84,7 @@ describe('resolveSavedObjects', () => {
         },
       } as unknown) as IndexPatternsContract;
 
-      const services = [
+      const services = ([
         {
           type: 'search',
           get: async () => {
@@ -124,7 +124,7 @@ describe('resolveSavedObjects', () => {
             };
           },
         },
-      ] as SavedObjectLoader[];
+      ] as unknown) as SavedObjectLoader[];
 
       const overwriteAll = false;
 
@@ -176,7 +176,7 @@ describe('resolveSavedObjects', () => {
         },
       } as unknown) as IndexPatternsContract;
 
-      const services = [
+      const services = ([
         {
           type: 'search',
           get: async () => {
@@ -217,7 +217,7 @@ describe('resolveSavedObjects', () => {
             };
           },
         },
-      ] as SavedObjectLoader[];
+      ] as unknown) as SavedObjectLoader[];
 
       const overwriteAll = false;
 
@@ -237,33 +237,38 @@ describe('resolveSavedObjects', () => {
 
   describe('resolveIndexPatternConflicts', () => {
     it('should resave resolutions', async () => {
-      const hydrateIndexPattern = jest.fn();
       const save = jest.fn();
 
-      const conflictedIndexPatterns = [
+      const conflictedIndexPatterns = ([
         {
           obj: {
-            searchSource: {
-              getOwnField: (field: string) => {
-                return field === 'index' ? '1' : undefined;
+            save,
+          },
+          doc: {
+            _source: {
+              kibanaSavedObjectMeta: {
+                searchSourceJSON: JSON.stringify({
+                  index: '1',
+                }),
               },
             },
-            hydrateIndexPattern,
-            save,
           },
         },
         {
           obj: {
-            searchSource: {
-              getOwnField: (field: string) => {
-                return field === 'index' ? '3' : undefined;
+            save,
+          },
+          doc: {
+            _source: {
+              kibanaSavedObjectMeta: {
+                searchSourceJSON: JSON.stringify({
+                  index: '3',
+                }),
               },
             },
-            hydrateIndexPattern,
-            save,
           },
         },
-      ];
+      ] as unknown) as Array<{ obj: SavedObject; doc: any }>;
 
       const resolutions = [
         {
@@ -282,43 +287,49 @@ describe('resolveSavedObjects', () => {
 
       const overwriteAll = false;
 
-      await resolveIndexPatternConflicts(resolutions, conflictedIndexPatterns, overwriteAll);
-      expect(hydrateIndexPattern.mock.calls.length).toBe(2);
+      await resolveIndexPatternConflicts(resolutions, conflictedIndexPatterns, overwriteAll, ({
+        get: (id: string) => Promise.resolve({ id }),
+      } as unknown) as IndexPatternsContract);
+      expect(conflictedIndexPatterns[0].obj.searchSource!.getField('index')!.id).toEqual('2');
+      expect(conflictedIndexPatterns[1].obj.searchSource!.getField('index')!.id).toEqual('4');
       expect(save.mock.calls.length).toBe(2);
       expect(save).toHaveBeenCalledWith({ confirmOverwrite: !overwriteAll });
-      expect(hydrateIndexPattern).toHaveBeenCalledWith('2');
-      expect(hydrateIndexPattern).toHaveBeenCalledWith('4');
     });
 
     it('should resolve filter index conflicts', async () => {
-      const hydrateIndexPattern = jest.fn();
       const save = jest.fn();
 
-      const conflictedIndexPatterns = [
+      const conflictedIndexPatterns = ([
         {
           obj: {
-            searchSource: {
-              getOwnField: (field: string) => {
-                return field === 'index' ? '1' : [{ meta: { index: 'filterIndex' } }];
+            save,
+          },
+          doc: {
+            _source: {
+              kibanaSavedObjectMeta: {
+                searchSourceJSON: JSON.stringify({
+                  index: '1',
+                  filter: [{ meta: { index: 'filterIndex' } }],
+                }),
               },
-              setField: jest.fn(),
             },
-            hydrateIndexPattern,
-            save,
           },
         },
         {
           obj: {
-            searchSource: {
-              getOwnField: (field: string) => {
-                return field === 'index' ? '3' : undefined;
+            save,
+          },
+          doc: {
+            _source: {
+              kibanaSavedObjectMeta: {
+                searchSourceJSON: JSON.stringify({
+                  index: '3',
+                }),
               },
             },
-            hydrateIndexPattern,
-            save,
           },
         },
-      ];
+      ] as unknown) as Array<{ obj: SavedObject; doc: any }>;
 
       const resolutions = [
         {
@@ -337,9 +348,11 @@ describe('resolveSavedObjects', () => {
 
       const overwriteAll = false;
 
-      await resolveIndexPatternConflicts(resolutions, conflictedIndexPatterns, overwriteAll);
+      await resolveIndexPatternConflicts(resolutions, conflictedIndexPatterns, overwriteAll, ({
+        get: (id: string) => Promise.resolve({ id }),
+      } as unknown) as IndexPatternsContract);
 
-      expect(conflictedIndexPatterns[0].obj.searchSource.setField).toHaveBeenCalledWith('filter', [
+      expect(conflictedIndexPatterns[0].obj.searchSource!.getField('filter')).toEqual([
         { meta: { index: 'newFilterIndex' } },
       ]);
       expect(save.mock.calls.length).toBe(2);

src/legacy/core_plugins/kibana/public/management/sections/objects/lib/resolve_saved_objects.ts

@@ -18,12 +18,17 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { OverlayStart } from 'src/core/public';
+import { cloneDeep } from 'lodash';
+import { OverlayStart, SavedObjectReference } from 'src/core/public';
 import {
   SavedObject,
   SavedObjectLoader,
 } from '../../../../../../../../plugins/saved_objects/public';
-import { IndexPatternsContract, IIndexPattern } from '../../../../../../../../plugins/data/public';
+import {
+  IndexPatternsContract,
+  IIndexPattern,
+  createSearchSource,
+} from '../../../../../../../../plugins/data/public';
 
 type SavedObjectsRawDoc = Record<string, any>;
 
@@ -126,7 +131,7 @@ async function importIndexPattern(
 async function importDocument(obj: SavedObject, doc: SavedObjectsRawDoc, overwriteAll: boolean) {
   await obj.applyESResp({
     references: doc._references || [],
-    ...doc,
+    ...cloneDeep(doc),
   });
   return await obj.save({ confirmOverwrite: !overwriteAll });
 }
@@ -160,41 +165,57 @@ async function awaitEachItemInParallel<T, R>(list: T[], op: (item: T) => R) {
 export async function resolveIndexPatternConflicts(
   resolutions: Array<{ oldId: string; newId: string }>,
   conflictedIndexPatterns: any[],
-  overwriteAll: boolean
+  overwriteAll: boolean,
+  indexPatterns: IndexPatternsContract
 ) {
   let importCount = 0;
 
-  await awaitEachItemInParallel(conflictedIndexPatterns, async ({ obj }) => {
-    // Resolve search index reference:
-    let oldIndexId = obj.searchSource.getOwnField('index');
-    // Depending on the object, this can either be the raw id or the actual index pattern object
-    if (typeof oldIndexId !== 'string') {
-      oldIndexId = oldIndexId.id;
-    }
-    let resolution = resolutions.find(({ oldId }) => oldId === oldIndexId);
-    if (resolution) {
-      const newIndexId = resolution.newId;
-      await obj.hydrateIndexPattern(newIndexId);
+  await awaitEachItemInParallel(conflictedIndexPatterns, async ({ obj, doc }) => {
+    const serializedSearchSource = JSON.parse(
+      doc._source.kibanaSavedObjectMeta?.searchSourceJSON || '{}'
+    );
+    const oldIndexId = serializedSearchSource.index;
+    let allResolved = true;
+    const inlineResolution = resolutions.find(({ oldId }) => oldId === oldIndexId);
+    if (inlineResolution) {
+      serializedSearchSource.index = inlineResolution.newId;
+    } else {
+      allResolved = false;
     }
 
     // Resolve filter index reference:
-    const filter = (obj.searchSource.getOwnField('filter') || []).map((f: any) => {
+    const filter = (serializedSearchSource.filter || []).map((f: any) => {
       if (!(f.meta && f.meta.index)) {
         return f;
       }
 
-      resolution = resolutions.find(({ oldId }) => oldId === f.meta.index);
+      const resolution = resolutions.find(({ oldId }) => oldId === f.meta.index);
       return resolution ? { ...f, ...{ meta: { ...f.meta, index: resolution.newId } } } : f;
     });
 
     if (filter.length > 0) {
-      obj.searchSource.setField('filter', filter);
+      serializedSearchSource.filter = filter;
     }
 
-    if (!resolution) {
+    const replacedReferences = (doc._references || []).map((reference: SavedObjectReference) => {
+      const resolution = resolutions.find(({ oldId }) => oldId === reference.id);
+      if (resolution) {
+        return { ...reference, id: resolution.newId };
+      } else {
+        allResolved = false;
+      }
+
+      return reference;
+    });
+
+    if (!allResolved) {
       // The user decided to skip this conflict so do nothing
       return;
     }
+    obj.searchSource = await createSearchSource(indexPatterns)(
+      JSON.stringify(serializedSearchSource),
+      replacedReferences
+    );
     if (await saveObject(obj, overwriteAll)) {
       importCount++;
     }

src/legacy/core_plugins/timelion/public/services/saved_sheets.ts

@@ -28,6 +28,7 @@ const savedObjectsClient = npStart.core.savedObjects.client;
 const services = {
   savedObjectsClient,
   indexPatterns: npStart.plugins.data.indexPatterns,
+  search: npStart.plugins.data.search,
   chrome: npStart.core.chrome,
   overlays: npStart.core.overlays,
 };