feat(ref-imp): #766 - Updated map (provisional index) file schema

lib/core/versions/latest/Did.ts

@@ -46,7 +46,7 @@ export default class Did {
     const didWithoutPrefix = did.split(didPrefix)[1];
 
     // split by : and ?, if there is 1 element, then it's short form. Long form has 2 elements
-    // TODO: SIP 2 #781 when the ? format is deprecated, `:` will be the only seperator.
+    // TODO: SIP 2 #781 when the ? format is deprecated, `:` will be the only separator.
     const didSplitLength = didWithoutPrefix.split(/:|\?/).length;
     if (didSplitLength === 1) {
       this.isShortForm = true;

lib/core/versions/latest/ErrorCode.ts

@@ -126,7 +126,6 @@ export default {
   MapFileHasUnknownProperty: 'map_file_has_unknown_property',
   MapFileMultipleOperationsForTheSameDid: 'map_file_multiple_operations_for_the_same_did',
   MapFileNotJson: 'map_file_not_json',
-  MapFileOperationsPropertyHasMissingOrUnknownProperty: 'map_file_operations_property_has_missing_or_unknown_property',
   MapFileProvisionalProofFileUriNotAllowed: 'map_file_provisional_proof_file_uri_not_allowed',
   MapFileUpdateOperationsNotArray: 'map_file_update_operations_not_array',
   MultihashNotLatestSupportedHashAlgorithm: 'multihash_not_latest_supported_hash_algorithm',
@@ -161,6 +160,8 @@ export default {
   UpdateOperationMissingOrUnknownProperty: 'update_operation_missing_or_unknown_property',
   UpdateOperationSignedDataHasMissingOrUnknownProperty: 'update_operation_signed_data_has_missing_or_unknown_property',
   UpdateOperationTypeIncorrect: 'update_operation_type_incorrect',
+  UpdateReferenceDidSuffixIsNotAString: 'update_reference_did_suffix_is_not_a_string',
+  UpdateReferenceRevealValueIsNotAString: 'update_reference_reveal_value_is_not_a_string',
   ValueTimeLockVerifierInvalidNumberOfOperations: 'value_time_lock_verifier_invalid_number_of_operations',
   ValueTimeLockVerifierTransactionTimeOutsideLockRange: 'value_time_lock_verifier_transaction_time_outside_lock_range',
   ValueTimeLockVerifierTransactionWriterLockOwnerMismatch: 'value_time_lock_verifier_transaction_writer_lock_owner_mismatch'

lib/core/versions/latest/MapFile.ts

@@ -5,6 +5,7 @@ import InputValidator from './InputValidator';
 import JsonAsync from './util/JsonAsync';
 import MapFileModel from './models/MapFileModel';
 import Multihash from './Multihash';
+import OperationReferenceModel from './models/OperationReferenceModel';
 import ProtocolParameters from './ProtocolParameters';
 import SidetreeError from '../../../common/SidetreeError';
 import UpdateOperation from './UpdateOperation';
@@ -19,9 +20,8 @@ export default class MapFile {
    * to keep useful metadata so that repeated computation can be avoided.
    */
   private constructor (
-    public readonly model: MapFileModel,
-    public readonly didUniqueSuffixes: string[],
-    public readonly updateOperations: UpdateOperation[]) { }
+    public model: MapFileModel,
+    public didUniqueSuffixes: string[]) { }
 
   /**
    * Parses and validates the given map file buffer.
@@ -53,11 +53,10 @@ export default class MapFile {
 
     MapFile.validateChunksProperty(mapFileModel.chunks);
 
-    const updateOperations = await MapFile.parseOperationsProperty(mapFileModel.operations);
-    const didUniqueSuffixes = updateOperations.map(operation => operation.didUniqueSuffix);
+    const didSuffixes = await MapFile.validateOperationsProperty(mapFileModel.operations);
 
     // Validate provisional proof file URI.
-    if (updateOperations.length > 0) {
+    if (didSuffixes.length > 0) {
       InputValidator.validateCasFileUri(mapFileModel.provisionalProofFileUri, 'provisional proof file URI');
     } else {
       if (mapFileModel.provisionalProofFileUri !== undefined) {
@@ -68,41 +67,67 @@ export default class MapFile {
       }
     }
 
-    const mapFile = new MapFile(mapFileModel, didUniqueSuffixes, updateOperations);
+    const mapFile = new MapFile(mapFileModel, didSuffixes);
     return mapFile;
   }
 
+  /**
+   * Removes all the update operation references from this map file.
+   */
+  public removeAllUpdateOperationReferences () {
+    delete this.model.operations;
+    delete this.model.provisionalProofFileUri;
+    this.didUniqueSuffixes = [];
+  }
+
   /**
    * Validates the given `operations` property, throws error if the property fails validation.
+   *
+   * @returns The of array of unique DID suffixes if validation succeeds.
    */
-  private static async parseOperationsProperty (operations: any): Promise<UpdateOperation[]> {
+  private static validateOperationsProperty (operations: any): string[] {
     if (operations === undefined) {
       return [];
     }
 
-    const properties = Object.keys(operations);
-    if (properties.length !== 1) {
-      throw new SidetreeError(ErrorCode.MapFileOperationsPropertyHasMissingOrUnknownProperty);
-    }
+    InputValidator.validateObjectContainsOnlyAllowedProperties(operations, ['update'], 'provisional operation references');
 
-    const updateOperations: UpdateOperation[] = [];
     if (!Array.isArray(operations.update)) {
       throw new SidetreeError(ErrorCode.MapFileUpdateOperationsNotArray);
     }
 
-    // Validate each update operation.
-    for (const operation of operations.update) {
-      const updateOperation = await UpdateOperation.parseOperationFromMapFile(operation);
-      updateOperations.push(updateOperation);
-    }
+    // Validate all update operation references.
+    MapFile.validateUpdateOperationReferences(operations.update);
 
     // Make sure no operation with same DID.
-    const didUniqueSuffixes = updateOperations.map(operation => operation.didUniqueSuffix);
-    if (ArrayMethods.hasDuplicates(didUniqueSuffixes)) {
+    const didSuffixes = (operations.update as OperationReferenceModel[]).map(operation => operation.didSuffix);
+    if (ArrayMethods.hasDuplicates(didSuffixes)) {
       throw new SidetreeError(ErrorCode.MapFileMultipleOperationsForTheSameDid);
     }
 
-    return updateOperations;
+    return didSuffixes;
+  }
+
+  private static validateUpdateOperationReferences (updateReferences: any) {
+    for (const updateReference of updateReferences) {
+      InputValidator.validateObjectContainsOnlyAllowedProperties(updateReference, ['didSuffix', 'revealValue'], 'update operation reference');
+
+      const didSuffixType = typeof updateReference.didSuffix;
+      if (didSuffixType !== 'string') {
+        throw new SidetreeError(
+          ErrorCode.UpdateReferenceDidSuffixIsNotAString,
+          `Update reference property 'didSuffix' is of type ${didSuffixType}, but needs to be a string.`
+        );
+      }
+
+      const revealValueType = typeof updateReference.revealValue;
+      if (revealValueType !== 'string') {
+        throw new SidetreeError(
+          ErrorCode.UpdateReferenceRevealValueIsNotAString,
+          `Update reference property 'revealValue' is of type ${revealValueType}, but needs to be a string.`
+        );
+      }
+    }
   }
 
   /**
@@ -134,9 +159,11 @@ export default class MapFile {
     chunkFileHash: string, provisionalProofFileHash: string | undefined, updateOperationArray: UpdateOperation[]
   ): Promise<Buffer> {
     const updateOperations = updateOperationArray.map(operation => {
+      const revealValue = Multihash.canonicalizeThenHashThenEncode(operation.signedData.updateKey);
+
       return {
         didSuffix: operation.didUniqueSuffix,
-        signedData: operation.signedDataJws.toCompactJws()
+        revealValue
       };
     });
 

lib/core/versions/latest/Operation.ts

@@ -32,7 +32,7 @@ export default class Operation {
     if (operationType === OperationType.Create) {
       return CreateOperation.parseJcsObject(operationObject, operationBuffer, isAnchorFileMode);
     } else if (operationType === OperationType.Update) {
-      return UpdateOperation.parseObject(operationObject, operationBuffer, isAnchorFileMode);
+      return UpdateOperation.parseObject(operationObject, operationBuffer);
     } else if (operationType === OperationType.Recover) {
       return RecoverOperation.parseObject(operationObject, operationBuffer, isAnchorFileMode);
     } else if (operationType === OperationType.Deactivate) {

lib/core/versions/latest/TransactionProcessor.ts

@@ -189,15 +189,17 @@ export default class TransactionProcessor implements ITransactionProcessor {
       const operationCountInAnchorFile = anchorFile.didUniqueSuffixes.length;
       const maxPaidUpdateOperationCount = paidOperationCount - operationCountInAnchorFile;
 
-      // If the actual update operation count is greater than the max paid update operation count, the map file is invalid.
-      const updateOperationCount = mapFile.updateOperations ? mapFile.updateOperations.length : 0;
+      // If the actual update operation count is greater than the max paid update operation count,
+      // we will penalize the writer by not accepting any updates.
+      const updateOperationCount = mapFile.didUniqueSuffixes.length;
       if (updateOperationCount > maxPaidUpdateOperationCount) {
-        return undefined;
+        mapFile.removeAllUpdateOperationReferences();
       }
 
-      // If we find operations for the same DID between anchor and map files, the map file is invalid.
+      // If we find operations for the same DID between anchor and map files,
+      // we will penalize the writer by not accepting any updates.
       if (!ArrayMethods.areMutuallyExclusive(anchorFile.didUniqueSuffixes, mapFile.didUniqueSuffixes)) {
-        return undefined;
+        mapFile.removeAllUpdateOperationReferences();
       }
 
       return mapFile;
@@ -268,21 +270,18 @@ export default class TransactionProcessor implements ITransactionProcessor {
     chunkFile: ChunkFileModel | undefined
   ): Promise<AnchoredOperationModel[]> {
 
+    // TODO: #766 - Pending more PR for of remainder of the operation types.
     const createOperations = anchorFile.createOperations;
     const recoverOperations = anchorFile.recoverOperations;
     const deactivateOperations = anchorFile.deactivateOperations;
-    const updateOperations = (mapFile && mapFile.updateOperations) ? mapFile.updateOperations : [];
 
-    // Add the operations in the following order of types: create, recover, update, deactivate.
+    // NOTE: this version of the protocol uses only ONE chunk file,
+    // and operations must be ordered by types with the following order: create, recover, update, deactivate.
     const operations = [];
     operations.push(...createOperations);
     operations.push(...recoverOperations);
-    operations.push(...updateOperations);
     operations.push(...deactivateOperations);
 
-    // TODO: Issue 442 - https://github.com/decentralized-identity/sidetree/issues/442
-    // Use actual operation request object instead of buffer.
-
     // Prepare proofs to compose the original operation requests.
     const proofs: (string | undefined)[] = createOperations.map(() => undefined); // Creates do not have proofs.
     if (coreProofFile !== undefined) {
@@ -291,10 +290,6 @@ export default class TransactionProcessor implements ITransactionProcessor {
       proofs.push(...recoverProofs);
       proofs.push(...deactivateProofs);
     }
-    if (provisionalProofFile !== undefined) {
-      const updateProofs = provisionalProofFile.updateProofs.map((proof) => proof.signedDataJws.toCompactJws());
-      proofs.push(...updateProofs);
-    }
 
     // NOTE: The last set of `operations` are deactivates, they don't have `delta` property.
     const anchoredOperationModels = [];
@@ -329,6 +324,59 @@ export default class TransactionProcessor implements ITransactionProcessor {
       anchoredOperationModels.push(anchoredOperationModel);
     }
 
+    const anchoredUpdateOperationModels = TransactionProcessor.composeAnchoredUpdateOperationModels(
+      transaction, anchorFile, mapFile, provisionalProofFile, chunkFile
+    );
+
+    anchoredOperationModels.push(...anchoredUpdateOperationModels);
+    return anchoredOperationModels;
+  }
+
+  private static composeAnchoredUpdateOperationModels (
+    transaction: TransactionModel,
+    anchorFile: AnchorFile,
+    mapFile: MapFile | undefined,
+    provisionalProofFile: ProvisionalProofFile | undefined,
+    chunkFile: ChunkFileModel | undefined
+  ): AnchoredOperationModel[] {
+    // If map file is undefined (in the case of batch containing only deactivates) or
+    // if map file's update operation reference count is zero (in the case of batch containing creates and/or recovers).
+    if (mapFile === undefined ||
+        mapFile.didUniqueSuffixes.length === 0) {
+      return [];
+    }
+
+    const updateDidSuffixes = mapFile.didUniqueSuffixes;
+    const updateProofs = provisionalProofFile!.updateProofs.map((proof) => proof.signedDataJws.toCompactJws());
+    const updateDeltaStartIndex = anchorFile.createOperations.length + anchorFile.recoverOperations.length;
+    const updateDeltas = chunkFile!.deltas.slice(updateDeltaStartIndex);
+
+    const anchoredOperationModels = [];
+    for (let i = 0; i < updateDeltas.length; i++) {
+      // Compose the original operation request from the files.
+      const composedRequest = {
+        type: OperationType.Update,
+        didSuffix: updateDidSuffixes[i],
+        signedData: updateProofs[i],
+        delta: updateDeltas[i]
+      };
+
+      // TODO: Issue 442 - https://github.com/decentralized-identity/sidetree/issues/442
+      // Use actual operation request object instead of buffer.
+      const operationBuffer = Buffer.from(JSON.stringify(composedRequest));
+
+      const anchoredOperationModel: AnchoredOperationModel = {
+        didUniqueSuffix: updateDidSuffixes[i],
+        type: OperationType.Update,
+        operationBuffer,
+        operationIndex: updateDeltaStartIndex + i,
+        transactionNumber: transaction.transactionNumber,
+        transactionTime: transaction.transactionTime
+      };
+
+      anchoredOperationModels.push(anchoredOperationModel);
+    }
+
     return anchoredOperationModels;
   }
 

lib/core/versions/latest/UpdateOperation.ts

@@ -51,22 +51,13 @@ export default class UpdateOperation implements OperationModel {
     this.delta = delta;
   }
 
-  /**
-   * Parses the given input as an update operation entry in the map file.
-   */
-  public static async parseOperationFromMapFile (input: any): Promise<UpdateOperation> {
-    const operationBuffer = Buffer.from(JSON.stringify(input));
-    const operation = await UpdateOperation.parseObject(input, operationBuffer, true);
-    return operation;
-  }
-
   /**
    * Parses the given buffer as a `UpdateOperation`.
    */
   public static async parse (operationBuffer: Buffer): Promise<UpdateOperation> {
     const operationJsonString = operationBuffer.toString();
     const operationObject = await JsonAsync.parse(operationJsonString);
-    const updateOperation = await UpdateOperation.parseObject(operationObject, operationBuffer, false);
+    const updateOperation = await UpdateOperation.parseObject(operationObject, operationBuffer);
     return updateOperation;
   }
 
@@ -75,13 +66,9 @@ export default class UpdateOperation implements OperationModel {
    * The `operationBuffer` given is assumed to be valid and is assigned to the `operationBuffer` directly.
    * NOTE: This method is purely intended to be used as an optimization method over the `parse` method in that
    * JSON parsing is not required to be performed more than once when an operation buffer of an unknown operation type is given.
-   * @param mapFileMode If set to true, then `delta` and `type` properties are expected to be absent.
    */
-  public static async parseObject (operationObject: any, operationBuffer: Buffer, mapFileMode: boolean): Promise<UpdateOperation> {
+  public static async parseObject (operationObject: any, operationBuffer: Buffer): Promise<UpdateOperation> {
     let expectedPropertyCount = 4;
-    if (mapFileMode) {
-      expectedPropertyCount = 2;
-    }
 
     const properties = Object.keys(operationObject);
     if (properties.length !== expectedPropertyCount) {
@@ -95,20 +82,13 @@ export default class UpdateOperation implements OperationModel {
     const signedData = Jws.parseCompactJws(operationObject.signedData);
     const signedDataModel = await UpdateOperation.parseSignedDataPayload(signedData.payload);
 
-    // If not in map file mode, we need to validate `type` and `delta` properties.
-    let delta;
-    if (!mapFileMode) {
-      if (operationObject.type !== OperationType.Update) {
-        throw new SidetreeError(ErrorCode.UpdateOperationTypeIncorrect);
-      }
-      Operation.validateDelta(operationObject.delta);
-      delta = {
-        patches: operationObject.delta.patches,
-        updateCommitment: operationObject.delta.updateCommitment
-      };
+    if (operationObject.type !== OperationType.Update) {
+      throw new SidetreeError(ErrorCode.UpdateOperationTypeIncorrect);
     }
 
-    return new UpdateOperation(operationBuffer, operationObject.didSuffix, signedData, signedDataModel, delta);
+    Operation.validateDelta(operationObject.delta);
+
+    return new UpdateOperation(operationBuffer, operationObject.didSuffix, signedData, signedDataModel, operationObject.delta);
   }
 
   /**

lib/core/versions/latest/models/MapFileModel.ts

@@ -1,13 +1,12 @@
+import OperationReferenceModel from './OperationReferenceModel';
+
 /**
  * Defines the external Map File structure.
  */
 export default interface MapFileModel {
   provisionalProofFileUri?: string;
   operations?: {
-    update: {
-      didSuffix: string,
-      signedData: string
-    }[]
+    update: OperationReferenceModel[]
   };
   chunks: {
     chunkFileUri: string

lib/core/versions/latest/models/OperationReferenceModel.ts

@@ -0,0 +1,7 @@
+/**
+ * Defines the update/recover/deactivate operation reference structure found in the `operations` property in core and provisional index file.
+ */
+export default interface OperationReferenceModel {
+  didSuffix: string,
+  revealValue: string
+}

tests/bitcoin/BitcoinProcessor.spec.ts

@@ -365,7 +365,7 @@ describe('BitcoinProcessor', () => {
     });
 
     it('should return no transaction if last processed block in DB is not found.', async () => {
-      const mockLastProcessedBlock = undefined
+      const mockLastProcessedBlock = undefined;
       blockMetadataStoreGetLastSpy.and.returnValue(Promise.resolve(mockLastProcessedBlock));
 
       const fetchedTransactions = await bitcoinProcessor.transactions();

tests/core/CoreProofFile.spec.ts

@@ -15,6 +15,9 @@ describe('CoreProofFile', async () => {
   xit('Batch writer should not write a proof file that is over the size limit.', async () => {
   });
 
+  xit('Should we check signature on observation time for all updates, recoveries, and deactivates?', async () => {
+  });
+
   describe('parse()', async () => {
     it('should parse a valid core proof file successfully.', async () => {
       const [, anyPrivateKey] = await Jwk.generateEs256kKeyPair(); // Used in multiple signed data for testing purposes.