Saved Object Namespaces (elastic#23378)

* Use an instance of SavedObjectsSerializer for migrations and the repository

* Fixing spelling of serialization

* Making the serializer conditionally include and prepend id with ns

* Adding repository tests for the namespaces

* Implementing find

* Modifying the SOCs to pass the options with the namespace

* Centralizing omitting the namespace when using serializer.rawToSavedObject

* Passing the schema through to the SavedObjectRepositoryProvider

* Changing the schema to work with undefined ui exports schemas

* Adding schema tests

* Making the complimentary serialization test use the namespace

* Fixing uiExports

* Fixing some tests

* Fixing included fields for the find

* Fixing include field tests, they're checking length also...

* Updating Repository test after adding namespace to always included
fields

* Renaming UIExportsSavedObjectTypeSchema to SavedObjectsSchemaDefinition

* Completing rename... forgot to save usages

* Fixing issue with the serialization.isRawSavedObject and the trailing :

src/server/saved_objects/migrations/core/__snapshots__/build_active_mappings.test.ts.snap

@@ -23,6 +23,9 @@ Object {
         "dynamic": "true",
         "type": "object",
       },
+      "namespace": Object {
+        "type": "keyword",
+      },
       "type": Object {
         "type": "keyword",
       },

src/server/saved_objects/migrations/core/build_active_mappings.ts

@@ -71,6 +71,9 @@ function defaultMapping(): IndexMapping {
         type: {
           type: 'keyword',
         },
+        namespace: {
+          type: 'keyword',
+        },
         updated_at: {
           type: 'date',
         },

src/server/saved_objects/migrations/core/index_migrator.test.ts

@@ -19,7 +19,8 @@
 
 import _ from 'lodash';
 import sinon from 'sinon';
-import { ROOT_TYPE, SavedObjectDoc } from '../../serialization';
+import { SavedObjectsSchema } from '../../schema';
+import { ROOT_TYPE, SavedObjectDoc, SavedObjectsSerializer } from '../../serialization';
 import { CallCluster } from './call_cluster';
 import { IndexMigrator } from './index_migrator';
 
@@ -46,6 +47,7 @@ describe('IndexMigrator', () => {
           },
           foo: { type: 'text' },
           migrationVersion: { dynamic: 'true', type: 'object' },
+          namespace: { type: 'keyword' },
           type: { type: 'keyword' },
           updated_at: { type: 'date' },
         },
@@ -78,6 +80,7 @@ describe('IndexMigrator', () => {
               },
               foo: { type: 'long' },
               migrationVersion: { dynamic: 'true', type: 'object' },
+              namespace: { type: 'keyword' },
               type: { type: 'keyword' },
               updated_at: { type: 'date' },
             },
@@ -188,6 +191,7 @@ describe('IndexMigrator', () => {
               },
               foo: { type: 'text' },
               migrationVersion: { dynamic: 'true', type: 'object' },
+              namespace: { type: 'keyword' },
               type: { type: 'keyword' },
               updated_at: { type: 'date' },
             },
@@ -301,6 +305,7 @@ function defaultOpts() {
       migrationVersion: {},
       migrate: _.identity,
     },
+    serializer: new SavedObjectsSerializer(new SavedObjectsSchema()),
   };
 }
 

src/server/saved_objects/migrations/core/index_migrator.ts

@@ -149,7 +149,7 @@ async function migrateIndex(context: Context): Promise<MigrationResult> {
  */
 async function migrateSourceToDest(context: Context) {
   const { callCluster, alias, dest, source, batchSize } = context;
-  const { scrollDuration, documentMigrator, log } = context;
+  const { scrollDuration, documentMigrator, log, serializer } = context;
 
   if (!source.exists) {
     return;
@@ -174,6 +174,10 @@ async function migrateSourceToDest(context: Context) {
 
     log.debug(`Migrating saved objects ${docs.map(d => d._id).join(', ')}`);
 
-    await Index.write(callCluster, dest.indexName, migrateRawDocs(documentMigrator.migrate, docs));
+    await Index.write(
+      callCluster,
+      dest.indexName,
+      migrateRawDocs(serializer, documentMigrator.migrate, docs)
+    );
   }
 }

src/server/saved_objects/migrations/core/migrate_raw_docs.test.ts

@@ -19,13 +19,14 @@
 
 import _ from 'lodash';
 import sinon from 'sinon';
-import { ROOT_TYPE } from '../../serialization';
+import { SavedObjectsSchema } from '../../schema';
+import { ROOT_TYPE, SavedObjectsSerializer } from '../../serialization';
 import { migrateRawDocs } from './migrate_raw_docs';
 
 describe('migrateRawDocs', () => {
   test('converts raw docs to saved objects', async () => {
     const transform = sinon.spy((doc: any) => _.set(doc, 'attributes.name', 'HOI!'));
-    const result = migrateRawDocs(transform, [
+    const result = migrateRawDocs(new SavedObjectsSerializer(new SavedObjectsSchema()), transform, [
       { _id: 'a:b', _source: { type: 'a', a: { name: 'AAA' } } },
       { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
     ]);
@@ -48,7 +49,7 @@ describe('migrateRawDocs', () => {
 
   test('passes invalid docs through untouched', async () => {
     const transform = sinon.spy((doc: any) => _.set(_.cloneDeep(doc), 'attributes.name', 'TADA'));
-    const result = migrateRawDocs(transform, [
+    const result = migrateRawDocs(new SavedObjectsSerializer(new SavedObjectsSchema()), transform, [
       { _id: 'foo:b', _source: { type: 'a', a: { name: 'AAA' } } },
       { _id: 'c:d', _source: { type: 'c', c: { name: 'DDD' } } },
     ]);

src/server/saved_objects/migrations/core/migrate_raw_docs.ts

@@ -21,7 +21,7 @@
  * This file provides logic for migrating raw documents.
  */
 
-import { isRawSavedObject, RawDoc, rawToSavedObject, savedObjectToRaw } from '../../serialization';
+import { RawDoc, SavedObjectsSerializer } from '../../serialization';
 import { TransformFn } from './document_migrator';
 
 /**
@@ -32,12 +32,16 @@ import { TransformFn } from './document_migrator';
  * @param {RawDoc[]} rawDocs
  * @returns {RawDoc[]}
  */
-export function migrateRawDocs(migrateDoc: TransformFn, rawDocs: RawDoc[]): RawDoc[] {
+export function migrateRawDocs(
+  serializer: SavedObjectsSerializer,
+  migrateDoc: TransformFn,
+  rawDocs: RawDoc[]
+): RawDoc[] {
   return rawDocs.map(raw => {
-    if (isRawSavedObject(raw)) {
-      const savedObject = rawToSavedObject(raw);
+    if (serializer.isRawSavedObject(raw)) {
+      const savedObject = serializer.rawToSavedObject(raw);
       savedObject.migrationVersion = savedObject.migrationVersion || {};
-      return savedObjectToRaw(migrateDoc(savedObject));
+      return serializer.savedObjectToRaw(migrateDoc(savedObject));
     }
 
     return raw;

src/server/saved_objects/migrations/core/migration_context.ts

@@ -24,6 +24,7 @@
  * serves as a central blueprint for what migrations will end up doing.
  */
 
+import { SavedObjectsSerializer } from '../../serialization';
 import { buildActiveMappings } from './build_active_mappings';
 import { CallCluster, MappingProperties } from './call_cluster';
 import { VersionedTransformer } from './document_migrator';
@@ -39,6 +40,7 @@ export interface MigrationOpts {
   log: LogFn;
   mappingProperties: MappingProperties;
   documentMigrator: VersionedTransformer;
+  serializer: SavedObjectsSerializer;
 }
 
 export interface Context {
@@ -51,6 +53,7 @@ export interface Context {
   batchSize: number;
   pollInterval: number;
   scrollDuration: string;
+  serializer: SavedObjectsSerializer;
 }
 
 /**
@@ -74,6 +77,7 @@ export async function migrationContext(opts: MigrationOpts): Promise<Context> {
     documentMigrator: opts.documentMigrator,
     pollInterval: opts.pollInterval,
     scrollDuration: opts.scrollDuration,
+    serializer: opts.serializer,
   };
 }
 

src/server/saved_objects/migrations/kibana/__snapshots__/kibana_migrator.test.ts.snap

@@ -23,6 +23,9 @@ Object {
         "dynamic": "true",
         "type": "object",
       },
+      "namespace": Object {
+        "type": "keyword",
+      },
       "type": Object {
         "type": "keyword",
       },

src/server/saved_objects/migrations/kibana/kibana_migrator.test.ts

@@ -88,6 +88,7 @@ function mockKbnServer({ configValues }: { configValues?: any } = {}) {
       savedObjectValidations: {},
       savedObjectMigrations: {},
       savedObjectMappings: [],
+      savedObjectSchemas: {},
     },
     server: {
       config: () => ({

src/server/saved_objects/migrations/kibana/kibana_migrator.ts

@@ -22,7 +22,8 @@
  * (the shape of the mappings and documents in the index).
  */
 
-import { SavedObjectDoc } from '../../serialization';
+import { SavedObjectsSchema, SavedObjectsSchemaDefinition } from '../../schema';
+import { SavedObjectDoc, SavedObjectsSerializer } from '../../serialization';
 import { docValidator } from '../../validation';
 import { buildActiveMappings, CallCluster, IndexMigrator, LogFn, MappingProperties } from '../core';
 import { DocumentMigrator, VersionedTransformer } from '../core/document_migrator';
@@ -34,6 +35,7 @@ export interface KbnServer {
     savedObjectMappings: any[];
     savedObjectMigrations: any;
     savedObjectValidations: any;
+    savedObjectSchemas: SavedObjectsSchemaDefinition;
   };
 }
 
@@ -64,6 +66,7 @@ export class KibanaMigrator {
   private documentMigrator: VersionedTransformer;
   private mappingProperties: MappingProperties;
   private log: LogFn;
+  private serializer: SavedObjectsSerializer;
 
   /**
    * Creates an instance of KibanaMigrator.
@@ -74,6 +77,9 @@ export class KibanaMigrator {
    */
   constructor({ kbnServer }: { kbnServer: KbnServer }) {
     this.kbnServer = kbnServer;
+    this.serializer = new SavedObjectsSerializer(
+      new SavedObjectsSchema(kbnServer.uiExports.savedObjectSchemas)
+    );
     this.mappingProperties = mergeProperties(kbnServer.uiExports.savedObjectMappings || []);
     this.log = (meta: string[], message: string) => kbnServer.server.log(meta, message);
     this.documentMigrator = new DocumentMigrator({
@@ -133,6 +139,7 @@ export class KibanaMigrator {
       mappingProperties: this.mappingProperties,
       pollInterval: config.get('migrations.pollInterval'),
       scrollDuration: config.get('migrations.scrollDuration'),
+      serializer: this.serializer,
     });
 
     return migrator.migrate();

src/server/saved_objects/saved_objects_mixin.js

@@ -19,6 +19,8 @@
 
 import { createSavedObjectsService } from './service';
 import { KibanaMigrator } from './migrations';
+import { SavedObjectsSchema } from './schema';
+import { SavedObjectsSerializer } from './serialization';
 
 import {
   createBulkCreateRoute,
@@ -62,7 +64,9 @@ export function savedObjectsMixin(kbnServer, server) {
   server.route(createGetRoute(prereqs));
   server.route(createUpdateRoute(prereqs));
 
-  server.decorate('server', 'savedObjects', createSavedObjectsService(server, migrator));
+  const schema = new SavedObjectsSchema(kbnServer.uiExports.savedObjectSchemas);
+  const serializer = new SavedObjectsSerializer(schema);
+  server.decorate('server', 'savedObjects', createSavedObjectsService(server, schema, serializer, migrator));
 
   const savedObjectsClientCache = new WeakMap();
   server.decorate('request', 'getSavedObjectsClient', function () {

src/server/saved_objects/schema/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { SavedObjectsSchema, SavedObjectsSchemaDefinition } from './schema';

src/server/saved_objects/schema/schema.test.ts

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { SavedObjectsSchema } from './schema';
+
+describe('#isNamespaceAgnostic', () => {
+  it(`returns false for unknown types`, () => {
+    const schema = new SavedObjectsSchema();
+    const result = schema.isNamespaceAgnostic('bar');
+    expect(result).toBe(false);
+  });
+
+  it(`returns true for explicitly namespace agnostic type`, () => {
+    const schema = new SavedObjectsSchema({
+      foo: {
+        isNamespaceAgnostic: true,
+      },
+    });
+    const result = schema.isNamespaceAgnostic('foo');
+    expect(result).toBe(true);
+  });
+
+  it(`returns false for explicitly namespaced type`, () => {
+    const schema = new SavedObjectsSchema({
+      foo: {
+        isNamespaceAgnostic: false,
+      },
+    });
+    const result = schema.isNamespaceAgnostic('foo');
+    expect(result).toBe(false);
+  });
+});

src/server/saved_objects/schema/schema.ts

@@ -0,0 +1,47 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+interface SavedObjectsSchemaTypeDefinition {
+  isNamespaceAgnostic: boolean;
+}
+
+export interface SavedObjectsSchemaDefinition {
+  [key: string]: SavedObjectsSchemaTypeDefinition;
+}
+
+export class SavedObjectsSchema {
+  private readonly definition?: SavedObjectsSchemaDefinition;
+  constructor(schemaDefinition?: SavedObjectsSchemaDefinition) {
+    this.definition = schemaDefinition;
+  }
+
+  public isNamespaceAgnostic(type: string) {
+    // if no plugins have registered a uiExports.savedObjectSchemas,
+    // this.schema will be undefined, and no types are namespace agnostic
+    if (!this.definition) {
+      return false;
+    }
+
+    const typeSchema = this.definition[type];
+    if (!typeSchema) {
+      return false;
+    }
+    return Boolean(typeSchema.isNamespaceAgnostic);
+  }
+}