Make schema serialization deterministic and add helpers for loading s…

…chemas

Signed-off-by: lovesh <lovesh.bond@gmail.com>

README.md

@@ -1302,7 +1302,7 @@ See some of the following tests for Circom usage:
 1. [The yearly income, calculate from monthly payslips is less/greater than certain amount.](./tests/composite-proofs/msg-js-obj/r1cs/yearly-income.spec.ts).
 2. [The sum of assets is greater than the sum of liabilities where are assets and liabilities are calculated from several credentials.](./tests/composite-proofs/msg-js-obj/r1cs/assets-liabilities.spec.ts)
 3. [The blood group is not AB-](./tests/composite-proofs/msg-js-obj/r1cs/blood-group.spec.ts)
-4. [The grade is either A+, A, B+, B or C but nothing else.](./tests/composite-proofs/msg-js-obj/r1cs/grade.ts)
+4. [The grade is either A+, A, B+, B or C but nothing else.](./tests/composite-proofs/msg-js-obj/r1cs/grade.spec.ts)
 5. [Either vaccinated less than 30 days ago OR last checked negative less than 2 days ago](./tests/composite-proofs/msg-js-obj/r1cs/vaccination.spec.ts)
 6. [All receipts (used as credentials) have different receipt (credential) ids](./tests/composite-proofs/msg-js-obj/r1cs/all_receipts_different.spec.ts). This test shows using multiple circuits in a single proof.
 7. [Certain attribute is the preimage of an MiMC hash](./tests/composite-proofs/msg-js-obj/r1cs/mimc-hash.spec.ts)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docknetwork/crypto-wasm-ts",
-  "version": "0.47.0",
+  "version": "0.48.0",
   "description": "Typescript abstractions over Dock's Rust crypto library's WASM wrapper",
   "homepage": "https://github.com/docknetwork/crypto-wasm-ts",
   "main": "lib/index.js",
@@ -33,6 +33,7 @@
     "bs58": "5.0.0",
     "flat": "^5.0.2",
     "json-pointer": "^0.6.2",
+    "json-stringify-deterministic": "^1.0.11",
     "lodash": "^4.17.21",
     "lzutf8": "0.6.3",
     "semver": "^7.5.4"

src/anonymous-credentials/credential-builder-common.ts

@@ -89,7 +89,7 @@ export abstract class CredentialBuilderCommon extends Versioned {
     // Schema should be part of the credential signature to prevent the credential holder from convincing a verifier of a manipulated schema
     const s = {
       [CRYPTO_VERSION_STR]: this._version,
-      [SCHEMA_STR]: JSON.stringify(this.schema?.toJSON()),
+      [SCHEMA_STR]: this.schema?.toJsonString(),
       [SUBJECT_STR]: this._subject
     };
     for (const [k, v] of this._topLevelFields.entries()) {

src/anonymous-credentials/credential-builder.ts

@@ -42,7 +42,7 @@ export abstract class CredentialBuilder<
 > extends CredentialBuilderCommon {
   // NOTE: Follows semver and must be updated accordingly when the logic of this class changes or the
   // underlying crypto changes.
-  static VERSION = '0.3.0';
+  static VERSION = '0.4.0';
 
   _encodedAttributes?: { [key: string]: Uint8Array };
   _sig?: Signature;
@@ -107,7 +107,7 @@ export abstract class CredentialBuilder<
       } else {
         // Generate new schema
         this.schema = CredentialSchema.generateAppropriateSchema(cred, schema);
-        cred[SCHEMA_STR] = JSON.stringify(this.schema?.toJSON());
+        cred[SCHEMA_STR] = this.schema?.toJsonString();
       }
     }
     return cred;

src/anonymous-credentials/credential-common.ts

@@ -42,7 +42,7 @@ export abstract class CredentialCommon<Sig> extends Versioned {
   toJSON(): object {
     const j = {};
     j['cryptoVersion'] = this._version;
-    j['credentialSchema'] = JSON.stringify(this.schema.toJSON());
+    j['credentialSchema'] = this.schema.toJsonString();
     j['credentialSubject'] = this.subject;
     if (this.credentialStatus !== undefined) {
       j['credentialStatus'] = this.credentialStatus;

src/anonymous-credentials/credential.ts

@@ -28,7 +28,7 @@ export abstract class Credential<PublicKey, Signature, SignatureParams> extends
       [CRYPTO_VERSION_STR]: this.version,
       // Converting the schema to a JSON string rather than keeping it JSO object to avoid creating extra fields while
       // signing which makes the implementation more expensive as one sig param is needed for each field.
-      [SCHEMA_STR]: JSON.stringify(this.schema?.toJSON()),
+      [SCHEMA_STR]: this.schema?.toJsonString(),
       [SUBJECT_STR]: this.subject
     };
     for (const [k, v] of this.topLevelFields.entries()) {

src/anonymous-credentials/presentation-builder.ts

@@ -111,7 +111,7 @@ type Credential = BBSCredential | BBSPlusCredential | PSCredential;
 export class PresentationBuilder extends Versioned {
   // NOTE: Follows semver and must be updated accordingly when the logic of this class changes or the
   // underlying crypto changes.
-  static VERSION = '0.3.0';
+  static VERSION = '0.4.0';
 
   // This can specify the reason why the proof was created, or date of the proof, or self-attested attributes (as JSON string), etc
   _context?: string;
@@ -1551,7 +1551,7 @@ export class PresentationBuilder extends Versioned {
     }
     self.updatePredicateParams(paramId, param);
     let par = self.predicateParams.get(paramId);
-    let protocol;
+    let protocol: BoundCheckProtocols;
     if (par instanceof LegoProvingKey || par instanceof LegoProvingKeyUncompressed) {
       protocol = BoundCheckProtocols.Legogroth16;
     } else if (par instanceof BoundCheckBppParams || par instanceof BoundCheckBppParamsUncompressed) {

src/anonymous-credentials/presentation.ts

@@ -74,6 +74,7 @@ import {
 } from '../bound-check';
 import semver from 'semver/preload';
 import { PederCommKey, PederCommKeyUncompressed } from '../ped-com';
+import stringify from 'json-stringify-deterministic';
 
 /**
  * The context passed to the proof contains the version and the presentation spec as well. This is done to bind the
@@ -96,7 +97,9 @@ export function buildContextForProof(
       ctx = ctx.concat(Array.from(context));
     }
   }
-  ctx = ctx.concat(Array.from(te.encode(JSON.stringify(presSpec.toJSON()))));
+  // Old version used JSON.stringify
+  const specJsonStr = semver.lt(version, '0.4.0') ? JSON.stringify(presSpec.toJSON()) : stringify(presSpec.toJSON());
+  ctx = ctx.concat(Array.from(te.encode(specJsonStr)));
   return new Uint8Array(ctx);
 }
 
@@ -818,7 +821,7 @@ export class Presentation extends Versioned {
     let blindCredentialRequest, blindedAttributeCiphertexts;
     if (this.spec.blindCredentialRequest !== undefined) {
       blindCredentialRequest = deepClone(this.spec.blindCredentialRequest) as object;
-      blindCredentialRequest.schema = JSON.stringify(this.spec.blindCredentialRequest.schema.toJSON());
+      blindCredentialRequest.schema = this.spec.blindCredentialRequest.schema.toJsonString();
       blindCredentialRequest.commitment = b58.encode(this.spec.blindCredentialRequest.commitment);
       if (this.blindedAttributeCiphertexts !== undefined) {
         blindedAttributeCiphertexts = {};

src/anonymous-credentials/schema.ts

@@ -1,20 +1,23 @@
 import pointer from 'json-pointer';
+import stringify from 'json-stringify-deterministic';
 import { Versioned } from './versioned';
 import { EncodeFunc, Encoder } from '../encoder';
 import { isPositiveInteger } from '../util';
 import {
   CRYPTO_VERSION_STR,
   FlattenedSchema,
+  FULL_SCHEMA_STR,
   ID_STR,
   REV_CHECK_STR,
   REV_ID_STR,
+  SCHEMA_PROPS_STR,
   SCHEMA_STR,
   SCHEMA_TYPE_STR,
   STATUS_STR,
   SUBJECT_STR,
   TYPE_STR
 } from './types-and-consts';
-import { flattenTill2ndLastKey, isValueDate, isValueDateTime } from './util';
+import { deepClone, flattenTill2ndLastKey, isValueDate, isValueDateTime } from './util';
 import semver from 'semver/preload';
 
 /**
@@ -382,12 +385,13 @@ export interface IEmbeddedJsonSchema {
   $id?: string;
   title?: string;
   type: string;
-  properties: IJsonSchemaProperties;
+  [SCHEMA_PROPS_STR]: IJsonSchemaProperties;
   definitions?: { [key: string]: object };
 }
 
 /**
  * JSON schema that does not contain the properties but its $id property can be used to fetch the properties.
+ * Intentionally not allowing `properties` key as reconciliation will be needed in case of conflict with fetched properties 
  */
 export interface IJsonSchema {
   [META_SCHEMA_STR]: string;
@@ -420,7 +424,7 @@ export type CredVal = string | number | object | CredVal[];
 export class CredentialSchema extends Versioned {
   // NOTE: Follows semver and must be updated accordingly when the logic of this class changes or the
   // underlying crypto changes.
-  static VERSION = '0.1.0';
+  static VERSION = '0.2.0';
 
   private static readonly STR_TYPE = 'string';
   private static readonly STR_REV_TYPE = 'stringReversible';
@@ -704,7 +708,7 @@ export class CredentialSchema extends Versioned {
       // Currently only assuming support for draft-07 but other might work as well
       [META_SCHEMA_STR]: 'http://json-schema.org/draft-07/schema#',
       type: 'object',
-      properties: {
+      [SCHEMA_PROPS_STR]: {
         [SUBJECT_STR]: {
           type: 'object',
           properties: {
@@ -753,13 +757,13 @@ export class CredentialSchema extends Versioned {
   toJSON(): object {
     const embedded = this.hasEmbeddedJsonSchema();
     const j = {
-      [ID_STR]: CredentialSchema.convertToDataUri(this.jsonSchema),
+      [ID_STR]: CredentialSchema.convertToDataUri(this.jsonSchema, this.version),
       [TYPE_STR]: SCHEMA_TYPE_STR,
       parsingOptions: this.parsingOptions,
       version: this._version
     };
     if (!embedded) {
-      j['fullJsonSchema'] = CredentialSchema.convertToDataUri(this.fullJsonSchema as IEmbeddedJsonSchema);
+      j[FULL_SCHEMA_STR] = CredentialSchema.convertToDataUri(this.fullJsonSchema as IEmbeddedJsonSchema, this.version);
     }
     return j;
   }
@@ -780,6 +784,10 @@ export class CredentialSchema extends Versioned {
       if (!CredentialSchema.isEmbeddedJsonSchema(full)) {
         throw new Error(`Expected actual schema to be an embedded one but got ${full}`);
       }
+    } else {
+      if (!CredentialSchema.isEmbeddedJsonSchema(jsonSchema)) {
+        throw new Error(`Full json schema wasn't provided when a non-embedded schema was provided ${jsonSchema}`);
+      }
     }
     // Note: `parsingOptions` might still be in an incorrect format which can fail the next call
     // Note: Passing `addMissingParsingOpts` as false to recreate the exact same object that was serialized. This is important
@@ -788,8 +796,53 @@ export class CredentialSchema extends Versioned {
     return new CredentialSchema(jsonSchema, parsingOptions, false, { version: version }, full);
   }
 
-  static convertToDataUri(jsonSchema: IEmbeddedJsonSchema | IJsonSchema): string {
-    return `data:application/json;charset=utf-8,${encodeURIComponent(JSON.stringify(jsonSchema))}`;
+  /**
+   * Similar to this.fromJSON but can load an externally referenced schema if the given schema is not an embedded one.
+   * @param j
+   * @param schemaGetter
+   */
+  static async fromJSONWithPotentiallyExternalSchema(
+    j: object,
+    schemaGetter: (url: string) => Promise<IEmbeddedJsonSchema>
+  ): Promise<CredentialSchema> {
+    // @ts-ignore
+    const { id, type, parsingOptions, version } = j;
+    if (type !== SCHEMA_TYPE_STR) {
+      throw new Error(`Schema type was "${type}", expected: "${SCHEMA_TYPE_STR}"`);
+    }
+    const jsonSchema = this.convertFromDataUri(id);
+    let fullJsonSchema: IEmbeddedJsonSchema | undefined;
+    if (!CredentialSchema.isEmbeddedJsonSchema(jsonSchema)) {
+      // @ts-ignore
+      fullJsonSchema = await schemaGetter(jsonSchema.$id);
+      if (!(fullJsonSchema[SCHEMA_PROPS_STR] instanceof Object)) {
+        throw new Error(
+          `Expected the fetched schema to have key ${SCHEMA_PROPS_STR} set and as an Object but was ${fullJsonSchema[SCHEMA_PROPS_STR]}`
+        );
+      }
+    }
+    return new CredentialSchema(jsonSchema, parsingOptions, false, { version: version }, fullJsonSchema);
+  }
+
+  /**
+   * Convert to a JSON string and the string is deterministic. This is important for signing
+   */
+  toJsonString(): string {
+    // Version < 0.2.0 used JSON.stringify to create a JSON string
+    return semver.lt(this.version, '0.2.0') ? JSON.stringify(this.toJSON()) : stringify(this.toJSON());
+  }
+
+  /**
+   * Convert schema JSON to a data URI
+   * @param jsonSchema
+   * @param version - The schema version. This is needed as a different conversion to JSON function was used in
+   * older version and backward compatibility is needed.
+   */
+  static convertToDataUri(jsonSchema: IEmbeddedJsonSchema | IJsonSchema, version?: string): string {
+    // Old version used JSON.stringify
+    const newVersion = version === undefined || semver.gte(version, '0.2.0');
+    const jsonStr = newVersion ? stringify(jsonSchema) : JSON.stringify(jsonSchema);
+    return `data:application/json;charset=utf-8,${encodeURIComponent(jsonStr)}`;
   }
 
   static convertFromDataUri(embedded: string): IEmbeddedJsonSchema | IJsonSchema {
@@ -1142,7 +1195,7 @@ export class CredentialSchema extends Versioned {
   // @ts-ignore
   static generateAppropriateSchema(cred: object, schema: CredentialSchema): CredentialSchema {
     // This JSON parse and stringify is to make `newJsonSchema` a copy of `schema.jsonSchema` and not a reference
-    const newJsonSchema = JSON.parse(JSON.stringify(schema.getEmbeddedJsonSchema()));
+    const newJsonSchema = deepClone(schema.getEmbeddedJsonSchema()) as IEmbeddedJsonSchema;
     const props = newJsonSchema.properties;
     CredentialSchema.generateFromCredential(cred, props, schema.version);
     if (schema.hasEmbeddedJsonSchema()) {
@@ -1264,7 +1317,9 @@ export class CredentialSchema extends Versioned {
           schemaProps[key]['type'] == 'number'
         ) {
           if (schemaProps[key]['type'] !== typ) {
-            throw new Error(`Mismatch in credential and given schema type: ${schemaProps[key]['type']} !== ${typ}`);
+            throw new Error(
+              `Mismatch in credential and given schema type for key ${key}: ${schemaProps[key]['type']} !== ${typ}`
+            );
           }
         } else if (schemaProps[key]['type'] === 'array' && typ === 'array') {
           if (schemaProps[key]['items'].length < value.length) {
@@ -1277,10 +1332,10 @@ export class CredentialSchema extends Versioned {
             schemaProps[key]['items'] = schemaProps[key]['items'].slice(0, value.length);
           }
         } else if (schemaProps[key]['type'] === 'object' && typ === 'object') {
-          const schemaKeys = new Set([...Object.keys(schemaProps[key]['properties'])]);
+          const schemaKeys = new Set([...Object.keys(schemaProps[key][SCHEMA_PROPS_STR])]);
           const valKeys = new Set([...Object.keys(value)]);
           for (const vk of valKeys) {
-            CredentialSchema.generateFromCredential(value, schemaProps[key]['properties'], schemaVersion);
+            CredentialSchema.generateFromCredential(value, schemaProps[key][SCHEMA_PROPS_STR], schemaVersion);
           }
           // Delete extra keys not in cred
           for (const sk of schemaKeys) {
@@ -1314,11 +1369,11 @@ export class CredentialSchema extends Versioned {
   }
 
   /**
-   * Returns true if the given object is an embedded schema, i.e. it has the properties.
+   * Returns true if the given object is an embedded schema, i.e. it has the `properties` key set.
    * @param obj
    */
   static isEmbeddedJsonSchema(obj: IEmbeddedJsonSchema | IJsonSchema): boolean {
-    return obj['properties'] !== undefined;
+    return obj[SCHEMA_PROPS_STR] !== undefined;
   }
 }
 

src/anonymous-credentials/types-and-consts.ts

@@ -80,7 +80,12 @@ export type SignatureParamsClass =
 export const VERSION_STR = 'version';
 export const CRYPTO_VERSION_STR = 'cryptoVersion';
 export const SCHEMA_STR = 'credentialSchema';
+
+export const FULL_SCHEMA_STR = 'fullJsonSchema';
 export const SCHEMA_TYPE_STR = 'JsonSchemaValidator2018';
+
+export const SCHEMA_PROPS_STR = 'properties';
+
 export const SUBJECT_STR = 'credentialSubject';
 export const STATUS_STR = 'credentialStatus';
 export const TYPE_STR = 'type';

tests/anonymous-credentials/credential.spec.ts

@@ -144,9 +144,12 @@ describe(`${Scheme} Credential signing and verification`, () => {
     };
 
     const schemaRef = 'https://example.com?hash=abc123ff';
+
+    // Function that returns a schema given a reference to it. In practice, this would likely involve a network call
     async function schemaGetter(ref: string): Promise<IEmbeddedJsonSchema> {
       return schema;
     }
+
     const nonEmbeddedSchema = {
       $id: schemaRef,
       [META_SCHEMA_STR]: 'http://json-schema.org/draft-07/schema#',