feat(ref-imp): #781 - make long form use jcs SIP2 (#864)

lib/bitcoin/BitcoinClient.ts

@@ -7,9 +7,9 @@ import BitcoinOutputModel from './models/BitcoinOutputModel';
 import BitcoinTransactionModel from './models/BitcoinTransactionModel';
 import BitcoinWallet from './BitcoinWallet';
 import IBitcoinWallet from './interfaces/IBitcoinWallet';
-import nodeFetch, { FetchError, Response, RequestInit } from 'node-fetch';
+import nodeFetch, { FetchError, RequestInit, Response } from 'node-fetch';
 import ReadableStream from '../common/ReadableStream';
-import { Address, crypto, Networks, PrivateKey, Script, Transaction, Unit, Block } from 'bitcore-lib';
+import { Address, Block, crypto, Networks, PrivateKey, Script, Transaction, Unit } from 'bitcore-lib';
 import { IBlockInfo } from './BitcoinProcessor';
 
 /**

lib/core/versions/0.11.0/MongoDbOperationQueue.ts

@@ -1,7 +1,7 @@
 import ErrorCode from './ErrorCode';
 import IOperationQueue from './interfaces/IOperationQueue';
 import SidetreeError from '../../../common/SidetreeError';
-import { Binary, Collection, MongoClient, Db } from 'mongodb';
+import { Binary, Collection, Db, MongoClient } from 'mongodb';
 import QueuedOperationModel from './models/QueuedOperationModel';
 
 /**

lib/core/versions/latest/CreateOperation.ts

@@ -2,6 +2,7 @@ import DeltaModel from './models/DeltaModel';
 import Encoder from './Encoder';
 import ErrorCode from './ErrorCode';
 import JsonAsync from './util/JsonAsync';
+import JsonCanonicalizer from './util/JsonCanonicalizer';
 import Multihash from './Multihash';
 import Operation from './Operation';
 import OperationModel from './models/OperationModel';
@@ -69,6 +70,17 @@ export default class CreateOperation implements OperationModel {
     return encodedMultihash;
   }
 
+  /**
+   * Computes the DID unique suffix given the encoded suffix data object.
+   * @param suffixData the suffix data object to calculate unique suffix from
+   */
+  private static computeJcsDidUniqueSuffix (suffixData: object): string {
+    const suffixDataBuffer = JsonCanonicalizer.canonicalizeAsBuffer(suffixData);
+    const multihash = Multihash.hash(suffixDataBuffer);
+    const encodedMultihash = Encoder.encode(multihash);
+    return encodedMultihash;
+  }
+
   /**
    * Parses the given input as a create operation entry in the anchor file.
    */
@@ -85,10 +97,75 @@ export default class CreateOperation implements OperationModel {
   public static async parse (operationBuffer: Buffer): Promise<CreateOperation> {
     const operationJsonString = operationBuffer.toString();
     const operationObject = await JsonAsync.parse(operationJsonString);
-    const createOperation = await CreateOperation.parseObject(operationObject, operationBuffer, false);
+    let createOperation;
+    if (typeof operationObject.suffix_data === 'string') {
+      // TODO: SIP 2 #781 deprecates this. Should be deleted when fully switched over
+      createOperation = await CreateOperation.parseObject(operationObject, operationBuffer, false);
+    } else {
+      createOperation = CreateOperation.parseJcsObject(operationObject, operationBuffer, false);
+    }
     return createOperation;
   }
 
+  /**
+   * Parses the given operation object as a `CreateOperation`.
+   * The `operationBuffer` given is assumed to be valid and is assigned to the `operationBuffer` directly.
+   * NOTE: This method is purely intended to be used as an optimization method over the `parse` method in that
+   * JSON parsing is not required to be performed more than once when an operation buffer of an unknown operation type is given.
+   * @param operationObject The operationObject is a json object with no encoding
+   * @param operationBuffer The buffer format of the operationObject
+   * @param anchorFileMode If set to true, then `delta` and `type` properties are expected to be absent.
+   */
+  public static parseJcsObject (operationObject: any, operationBuffer: Buffer, anchorFileMode: boolean): CreateOperation {
+    let expectedPropertyCount = 3;
+    if (anchorFileMode) {
+      expectedPropertyCount = 1;
+    }
+    const properties = Object.keys(operationObject);
+    if (properties.length !== expectedPropertyCount) {
+      throw new SidetreeError(ErrorCode.CreateOperationMissingOrUnknownProperty);
+    }
+
+    CreateOperation.validateSuffixData(operationObject.suffix_data);
+    const suffixData: SuffixDataModel = {
+      deltaHash: operationObject.suffix_data.delta_hash,
+      recoveryCommitment: operationObject.suffix_data.recovery_commitment
+    };
+
+    if (operationObject.suffix_data.type !== undefined) {
+      suffixData.type = operationObject.suffix_data.type;
+    }
+
+    // For compatibility with data pruning, we have to assume that `delta` may be unavailable,
+    // thus an operation with invalid `delta` needs to be processed as an operation with unavailable `delta`,
+    // so here we let `delta` be `undefined`.
+    let delta;
+    let encodedDelta;
+    if (!anchorFileMode) {
+      if (operationObject.type !== OperationType.Create) {
+        throw new SidetreeError(ErrorCode.CreateOperationTypeIncorrect);
+      }
+
+      try {
+        Operation.validateDelta(operationObject.delta);
+        delta = {
+          patches: operationObject.delta.patches,
+          updateCommitment: operationObject.delta.update_commitment
+        };
+      } catch {
+        // For compatibility with data pruning, we have to assume that `delta` may be unavailable,
+        // thus an operation with invalid `delta` needs to be processed as an operation with unavailable `delta`,
+        // so here we let `delta` be `undefined`.
+      }
+      encodedDelta = Encoder.encode(JsonCanonicalizer.canonicalizeAsBuffer(operationObject.delta));
+    }
+
+    const didUniqueSuffix = CreateOperation.computeJcsDidUniqueSuffix(operationObject.suffix_data);
+
+    const encodedSuffixData = Encoder.encode(JsonCanonicalizer.canonicalizeAsBuffer(operationObject.suffix_data));
+    return new CreateOperation(operationBuffer, didUniqueSuffix, encodedSuffixData, suffixData, encodedDelta, delta);
+  }
+
   /**
    * Parses the given operation object as a `CreateOperation`.
    * The `operationBuffer` given is assumed to be valid and is assigned to the `operationBuffer` directly.
@@ -97,6 +174,7 @@ export default class CreateOperation implements OperationModel {
    * @param anchorFileMode If set to true, then `delta` and `type` properties are expected to be absent.
    */
   public static async parseObject (operationObject: any, operationBuffer: Buffer, anchorFileMode: boolean): Promise<CreateOperation> {
+    // TODO: SIP 2 #781 deprecates this. Should be deleted when fully switched over
     let expectedPropertyCount = 3;
     if (anchorFileMode) {
       expectedPropertyCount = 1;
@@ -132,14 +210,11 @@ export default class CreateOperation implements OperationModel {
     return new CreateOperation(operationBuffer, didUniqueSuffix, encodedSuffixData, suffixData, encodedDelta, delta);
   }
 
-  private static async parseSuffixData (suffixDataEncodedString: any): Promise<SuffixDataModel> {
-    if (typeof suffixDataEncodedString !== 'string') {
-      throw new SidetreeError(ErrorCode.CreateOperationSuffixDataMissingOrNotString);
+  private static validateSuffixData (suffixData: any): void {
+    if (typeof suffixData !== 'object') {
+      throw new SidetreeError(ErrorCode.CreateOperationSuffixDataIsNotObject);
     }
 
-    const suffixDataJsonString = Encoder.decodeAsString(suffixDataEncodedString);
-    const suffixData = await JsonAsync.parse(suffixDataJsonString);
-
     const properties = Object.keys(suffixData);
     // will have 3 if has type
     if (properties.length !== 3 && properties.length !== 2) {
@@ -166,6 +241,16 @@ export default class CreateOperation implements OperationModel {
         throw new SidetreeError(ErrorCode.CreateOperationSuffixDataTypeInvalidCharacter);
       }
     }
+  }
+
+  private static async parseSuffixData (suffixDataEncodedString: any): Promise<SuffixDataModel> {
+    if (typeof suffixDataEncodedString !== 'string') {
+      throw new SidetreeError(ErrorCode.CreateOperationSuffixDataMissingOrNotString);
+    }
+
+    const suffixDataJsonString = Encoder.decodeAsString(suffixDataEncodedString);
+    const suffixData = await JsonAsync.parse(suffixDataJsonString);
+    CreateOperation.validateSuffixData(suffixData);
 
     return {
       deltaHash: suffixData.delta_hash,

lib/core/versions/latest/Did.ts

@@ -1,6 +1,8 @@
 import CreateOperation from './CreateOperation';
 import Delta from './Delta';
+import Encoder from './Encoder';
 import ErrorCode from './ErrorCode';
+import JsonCanonicalizer from './util/JsonCanonicalizer';
 import Multihash from './Multihash';
 import OperationType from '../../enums/OperationType';
 import SidetreeError from '../../../common/SidetreeError';
@@ -35,14 +37,18 @@ export default class Did {
   private constructor (did: string, didMethodName: string) {
     this.didMethodName = didMethodName;
     const didPrefix = `did:${didMethodName}:`;
+    // TODO https://github.com/decentralized-identity/sidetree/issues/470 add network prefix to the didPrefix string
 
     if (!did.startsWith(didPrefix)) {
       throw new SidetreeError(ErrorCode.DidIncorrectPrefix);
     }
 
-    const indexOfDotChar = did.indexOf('.');
-    // If there is no 'dot', then DID can only be in short-form.
-    if (indexOfDotChar < 0) {
+    const didWithoutPrefix = did.split(didPrefix)[1];
+
+    // split by : and ?, if there is 1 element, then it's short form. Long form has 2 elements
+    // TODO: SIP 2 #781 when the ? format is deprecated, `:` will be the only seperator.
+    const didSplitLength = didWithoutPrefix.split(/:|\?/).length;
+    if (didSplitLength === 1) {
       this.isShortForm = true;
     } else {
       this.isShortForm = false;
@@ -53,7 +59,7 @@ export default class Did {
     } else {
       // Long-form can be in the form of:
       // 'did:<methodName>:<unique-portion>?-<methodName>-initial-state=<create-operation-suffix-data>.<create-operation-delta>' or
-      // 'did:<methodName>:<unique-portion>:<create-operation-suffix-data>.<create-operation-delta>'
+      // 'did:<methodName>:<unique-portion>:Base64url(JCS({suffix-data, delta}))'
 
       const indexOfQuestionMarkChar = did.indexOf('?');
       if (indexOfQuestionMarkChar > 0) {
@@ -83,18 +89,18 @@ export default class Did {
     if (!did.isShortForm) {
       // Long-form can be in the form of:
       // 'did:<methodName>:<unique-portion>?-<methodName>-initial-state=<create-operation-suffix-data>.<create-operation-delta>' or
-      // 'did:<methodName>:<unique-portion>:<create-operation-suffix-data>.<create-operation-delta>'
+      // 'did:<methodName>:<unique-portion>:Base64url(JCS({suffix-data, delta}))'
 
       const indexOfQuestionMarkChar = didString.indexOf('?');
-      let initialState;
+      let createOperation;
       if (indexOfQuestionMarkChar > 0) {
-        initialState = Did.getInitialStateFromDidStringWithQueryParameter(didString, didMethodName);
+        const initialState = Did.getInitialStateFromDidStringWithQueryParameter(didString, didMethodName);
+        createOperation = await Did.constructCreateOperationFromInitialState(initialState);
       } else {
-        initialState = Did.getInitialStateFromDidStringWithExtraColon(didString);
+        const initialStateEncodedJcs = Did.getInitialStateFromDidStringWithExtraColon(didString);
+        createOperation = Did.constructCreateOperationFromEncodedJcs(initialStateEncodedJcs);
       }
 
-      const createOperation = await Did.constructCreateOperationFromInitialState(initialState);
-
       // NOTE: we cannot use the unique suffix directly from `createOperation.didUniqueSuffix` for comparison,
       // because a given long-form DID may have been created long ago,
       // thus this version of `CreateOperation.parse()` maybe using a different hashing algorithm than that of the unique DID suffix (short-form).
@@ -152,7 +158,7 @@ export default class Did {
   }
 
   private static getInitialStateFromDidStringWithExtraColon (didString: string): string {
-    // DID example: 'did:<methodName>:<unique-portion>:<create-operation-suffix-data>.<create-operation-delta>'
+    // DID example: 'did:<methodName>:<unique-portion>:Base64url(JCS({suffix-data, delta}))'
 
     const lastColonIndex = didString.lastIndexOf(':');
 
@@ -161,7 +167,40 @@ export default class Did {
     return initialStateValue;
   }
 
+  private static constructCreateOperationFromEncodedJcs (initialStateEncodedJcs: string): CreateOperation {
+    // Initial state should be in the format base64url(JCS(initialState))
+    const initialStateDecodedJcs = Encoder.decodeAsString(initialStateEncodedJcs);
+    let initialStateObject;
+    try {
+      initialStateObject = JSON.parse(initialStateDecodedJcs);
+    } catch {
+      throw new SidetreeError(ErrorCode.DidInitialStateJcsIsNotJosn, 'Long form initial state should be encoded jcs.');
+    }
+
+    Did.validateInitialState(initialStateEncodedJcs, initialStateObject);
+
+    const createOperationRequest = {
+      type: OperationType.Create,
+      suffix_data: initialStateObject.suffix_data,
+      delta: initialStateObject.delta
+    };
+    const createOperationBuffer = Buffer.from(JSON.stringify(createOperationRequest));
+    const createOperation = CreateOperation.parseJcsObject(createOperationRequest, createOperationBuffer, false);
+    return createOperation;
+  }
+
+  /**
+   * Make sure initial state is JCS
+   */
+  private static validateInitialState (initialStateEncodedJcs: string, initialStateObject: any): void {
+    const expectedInitialState = Encoder.encode(JsonCanonicalizer.canonicalizeAsBuffer(initialStateObject));
+    if (expectedInitialState !== initialStateEncodedJcs) {
+      throw new SidetreeError(ErrorCode.DidInitialStateJcsIsNotJcs, 'Initial state object and JCS string mismatch.');
+    }
+  }
+
   private static async constructCreateOperationFromInitialState (initialState: string): Promise<CreateOperation> {
+    // TODO: SIP 2 #781 deprecates this. Should be deleted when fully switched over
     // Initial state should be in the format: <suffix-data>.<delta>
     const firstIndexOfDot = initialState.indexOf('.');
     if (firstIndexOfDot === -1) {

lib/core/versions/latest/DocumentComposer.ts

@@ -319,7 +319,7 @@ export default class DocumentComposer {
     if (typeof id !== 'string') {
       throw new SidetreeError(ErrorCode.DocumentComposerIdNotString, `ID not string: ${JSON.stringify(id)} is of type '${typeof id}'`);
     }
-    if (id.length > 20) {
+    if (id.length > 50) {
       throw new SidetreeError(ErrorCode.DocumentComposerIdTooLong);
     }
 

lib/core/versions/latest/ErrorCode.ts

@@ -31,6 +31,7 @@ export default {
   ChunkFileUnexpectedProperty: 'chunk_file_unexpected_property',
   CompressorMaxAllowedDecompressedDataSizeExceeded: 'compressor_max_allowed_decompressed_data_size_exceeded',
   CreateOperationMissingOrUnknownProperty: 'create_operation_missing_or_unknown_property',
+  CreateOperationSuffixDataIsNotObject: 'create_operation_suffix_data_is_not_object',
   CreateOperationSuffixDataMissingOrNotString: 'create_operation_suffix_data_missing_or_not_string',
   CreateOperationSuffixDataMissingOrUnknownProperty: 'create_operation_suffix_data_missing_or_unknown_property',
   CreateOperationSuffixDataTypeInvalidCharacter: 'create_operation_suffix_data_type_invalid_character',
@@ -42,10 +43,13 @@ export default {
   DeactivateOperationSignedDataMissingOrUnknownProperty: 'deactivate_operation_signed_data_missing_or_unknown_property',
   DeactivateOperationSignedDidUniqueSuffixMismatch: 'deactivate_operation_signed_did_unique_suffix_mismatch',
   DeactivateOperationTypeIncorrect: 'deactivate_operation_type_incorrect',
+  DeltaIsNotObject: 'delta_is_not_object',
   DeltaMissingOrNotString: 'delta_missing_or_not_string',
   DeltaExceedsMaximumSize: 'delta_exceeds_maximum_size',
   DeltaMissingOrUnknownProperty: 'delta_missing_or_unknown_property',
   DidIncorrectPrefix: 'did_incorrect_prefix',
+  DidInitialStateJcsIsNotJcs: 'did_initial_state_jcs_is_not_jcs',
+  DidInitialStateJcsIsNotJosn: 'did_initial_state_jcs_is_not_json',
   DidInitialStateValueContainsMoreThanOneDot: 'did_initial_state_value_contains_more_than_one_dot',
   DidInitialStateValueContainsNoDot: 'did_initial_state_value_contains_no_dot',
   DidInitialStateValueDoesNotContainTwoParts: 'did_initial_state_value_does_not_contain_two_parts',

lib/core/versions/latest/MongoDbOperationQueue.ts

@@ -1,7 +1,7 @@
 import ErrorCode from './ErrorCode';
 import IOperationQueue from './interfaces/IOperationQueue';
 import SidetreeError from '../../../common/SidetreeError';
-import { Binary, Collection, MongoClient, Db } from 'mongodb';
+import { Binary, Collection, Db, MongoClient } from 'mongodb';
 import QueuedOperationModel from './models/QueuedOperationModel';
 
 /**

lib/core/versions/latest/Multihash.ts

@@ -46,6 +46,17 @@ export default class Multihash {
     return hash;
   }
 
+  /**
+   * Canonicalize the given content, then double hashes the result using the latest supported hash algorithm, then encodes the multihash.
+   * Mainly used for testing purposes.
+   */
+  public static canonicalizeThenHashThenEncode (content: object) {
+    const canonicalizedStringBuffer = JsonCanonicalizer.canonicalizeAsBuffer(content);
+
+    const multihashEncodedString = Multihash.hashThenEncode(canonicalizedStringBuffer, ProtocolParameters.hashAlgorithmInMultihashCode);
+    return multihashEncodedString;
+  }
+
   /**
    * Canonicalize the given content, then double hashes the result using the latest supported hash algorithm, then encodes the multihash.
    * Mainly used for testing purposes.

lib/core/versions/latest/Operation.ts

@@ -43,16 +43,14 @@ export default class Operation {
   }
 
   /**
-   * Parses the given encoded delta string into an internal `DeltaModel`.
+   * validate delta and throw if invalid
+   * @param delta the delta to validate
    */
-  public static async parseDelta (deltaEncodedString: any): Promise<DeltaModel> {
-    if (typeof deltaEncodedString !== 'string') {
-      throw new SidetreeError(ErrorCode.DeltaMissingOrNotString);
+  public static validateDelta (delta: any): void {
+    if (typeof delta !== 'object') {
+      throw new SidetreeError(ErrorCode.DeltaIsNotObject);
     }
 
-    const deltaJsonString = Encoder.decodeAsString(deltaEncodedString);
-    const delta = await JsonAsync.parse(deltaJsonString);
-
     const properties = Object.keys(delta);
     if (properties.length !== 2) {
       throw new SidetreeError(ErrorCode.DeltaMissingOrUnknownProperty);
@@ -66,6 +64,20 @@ export default class Operation {
     DocumentComposer.validateDocumentPatches(delta.patches);
     const nextUpdateCommitment = Encoder.decodeAsBuffer(delta.update_commitment);
     Multihash.verifyHashComputedUsingLatestSupportedAlgorithm(nextUpdateCommitment);
+  }
+
+  /**
+   * Parses the given encoded delta string into an internal `DeltaModel`.
+   */
+  public static async parseDelta (deltaEncodedString: any): Promise<DeltaModel> {
+    if (typeof deltaEncodedString !== 'string') {
+      throw new SidetreeError(ErrorCode.DeltaMissingOrNotString);
+    }
+
+    const deltaJsonString = Encoder.decodeAsString(deltaEncodedString);
+    const delta = await JsonAsync.parse(deltaJsonString);
+
+    Operation.validateDelta(delta);
 
     return {
       patches: delta.patches,

lib/core/versions/latest/RequestHandler.ts

@@ -146,7 +146,7 @@ export default class RequestHandler implements IRequestHandler {
    * @param shortOrLongFormDid Can either be:
    *   1. A short-form DID. e.g. 'did:<methodName>:abc' or
    *   2. A long-form DID. e.g. 'did:<methodName>:<unique-portion>?-<methodName>-initial-state=<create-operation-suffix-data>.<create-operation-delta>' or
-   *                            'did:<methodName>:<unique-portion>:<create-operation-suffix-data>.<create-operation-delta>'
+   *                            'did:<methodName>:<unique-portion>:Base64url(JCS({suffix-data, delta}))'
    */
   public async handleResolveRequest (shortOrLongFormDid: string): Promise<ResponseModel> {
     try {