feat(ref-imp): #783 - Fixed a crash on init + a bug that trends norma…

…lized fee to zero + minor fixes

lib/bitcoin/BitcoinProcessor.ts

@@ -449,7 +449,7 @@ export default class BitcoinProcessor {
       for (const transaction of transactions) {
         const block = blockMetaDataMap.get(transaction.transactionTime);
         if (block !== undefined) {
-          transaction.normalizedTransactionFee = block.normalizedFee;
+          transaction.normalizedTransactionFee = Math.floor(block.normalizedFee);
         } else {
           throw new RequestError(ResponseStatus.ServerError, ErrorCode.BitcoinBlockMetadataNotFound);
         }

lib/bitcoin/VersionManager.ts

@@ -39,15 +39,15 @@ export default class VersionManager {
       const version = versionModel.version;
       this.protocolParameters.set(version, versionModel.protocolParameters);
 
-      const initialNormalizedFee = versionModel.protocolParameters.initialNormalizedFee;
+      const initialNormalizedFeeInSatoshis = versionModel.protocolParameters.initialNormalizedFeeInSatoshis;
       const feeLookBackWindowInBlocks = versionModel.protocolParameters.feeLookBackWindowInBlocks;
       const feeMaxFluctuationMultiplierPerBlock = versionModel.protocolParameters.feeMaxFluctuationMultiplierPerBlock;
 
       const FeeCalculator = await this.loadDefaultExportsForVersion(version, 'NormalizedFeeCalculator');
       const feeCalculator = new FeeCalculator(
         blockMetadataStore,
         config.genesisBlockNumber,
-        initialNormalizedFee,
+        initialNormalizedFeeInSatoshis,
         feeLookBackWindowInBlocks,
         feeMaxFluctuationMultiplierPerBlock
       );

lib/bitcoin/interfaces/IFeeCalculator.ts

@@ -6,12 +6,12 @@ import BlockMetadataWithoutNormalizedFee from '../models/BlockMetadataWithoutNor
  */
 export default interface IFeeCalculator {
   /**
-   * Returns the fee for a particular block height.
+   * Returns the fee in satoshis for a particular block height.
    */
   getNormalizedFee (block: number): Promise<number>;
 
   /**
-   * Returns the block with normalized fee added
+   * Returns the block with normalized fee added.
    */
   addNormalizedFeeToBlockMetadata (blockMetadataWithoutFee: BlockMetadataWithoutNormalizedFee): Promise<BlockMetadata>;
 }

lib/bitcoin/models/ProtocolParameters.ts

@@ -5,8 +5,8 @@ export default interface ProtocolParameters {
   /** The duration for the value-time-lock */
   valueTimeLockDurationInBlocks: number;
 
-  /** The initial normalized fee */
-  initialNormalizedFee: number;
+  /** The initial normalized fee in satoshis. */
+  initialNormalizedFeeInSatoshis: number;
 
   /**
    * The look back window for normalized fee calculation

lib/bitcoin/versions/latest/NormalizedFeeCalculator.ts

@@ -8,55 +8,74 @@ import IFeeCalculator from '../../interfaces/IFeeCalculator';
  */
 export default class NormalizedFeeCalculator implements IFeeCalculator {
 
-  private blockMetadataCache: BlockMetadata[];
-  private expectedBlockHeight: number | undefined = undefined;
+  /**
+   * A cache to remember blocks in the look-back window for a particular block height
+   * which reduces calls to the block metadata store under the most common usage pattern.
+   */
+  private cachedLookBackWindow: BlockMetadata[];
+
+  /**
+   * The block height that the cached look back window is for.
+   */
+  private blockHeightOfCachedLookBackWindow: number | undefined = undefined;
+
   constructor (
     private blockMetadataStore: IBlockMetadataStore,
     private genesisBlockNumber: number,
-    private initialNormalizedFee: number,
+    private initialNormalizedFeeInSatoshis: number,
     private feeLookBackWindowInBlocks: number,
     private feeMaxFluctuationMultiplierPerBlock: number) {
-    this.blockMetadataCache = [];
+    this.cachedLookBackWindow = [];
   }
 
   /**
-   * Initializes the Bitcoin processor.
+   * Initializes the normalized fee calculator.
    */
   public async initialize () {
     console.log(`Initializing normalized fee calculator.`);
   }
 
-  /**
-   * This adds normalized fee to the block as it would calculate normalziedFee, but uses a cache to remeber previously seen blocks.
-   * Which reduces calls to the metadata store.
-   */
   public async addNormalizedFeeToBlockMetadata (blockMetadata: BlockMetadataWithoutNormalizedFee): Promise<BlockMetadata> {
+
+    // If the height of the given block does not have large enough look-back window, just use initial fee.
     if (blockMetadata.height < this.genesisBlockNumber + this.feeLookBackWindowInBlocks) {
-      return Object.assign({ normalizedFee: this.initialNormalizedFee }, blockMetadata);
+      const blockWithFee = Object.assign({ normalizedFee: this.initialNormalizedFeeInSatoshis }, blockMetadata);
+
+      // We need to push the block metadata into the look-back cache in preparation for when look-back window becomes large enough with the given block height.
+      this.cachedLookBackWindow.push(blockWithFee);
+      this.blockHeightOfCachedLookBackWindow = blockMetadata.height + 1;
+
+      return blockWithFee;
     }
-    // the cache won't work if the block is not expected, refetch the blocks and store in cache
-    if (this.expectedBlockHeight !== blockMetadata.height) {
-      this.blockMetadataCache = await this.getBlocksInLookBackWindow(blockMetadata.height);
-      this.expectedBlockHeight = blockMetadata.height;
+
+    // Code reaches here whn the look-back window is large enough.
+
+    // The cache won't work if the block is not the anticipated height, refetch the blocks and store in cache.
+    if (this.blockHeightOfCachedLookBackWindow !== blockMetadata.height) {
+      this.cachedLookBackWindow = await this.getBlocksInLookBackWindow(blockMetadata.height);
+      this.blockHeightOfCachedLookBackWindow = blockMetadata.height;
     }
-    const newFee = this.calculateNormalizedFee(this.blockMetadataCache);
-    const newBlockWithFee = Object.assign({ normalizedFee: newFee }, blockMetadata);
-    this.blockMetadataCache.push(newBlockWithFee);
-    this.blockMetadataCache.shift();
-    this.expectedBlockHeight++;
+
+    const normalizedFee = this.calculateNormalizedFee(this.cachedLookBackWindow);
+    const newBlockWithFee = Object.assign({ normalizedFee }, blockMetadata);
+    this.cachedLookBackWindow.push(newBlockWithFee);
+    this.cachedLookBackWindow.shift();
+    this.blockHeightOfCachedLookBackWindow++;
     return newBlockWithFee;
   }
 
   public async getNormalizedFee (block: number): Promise<number> {
     if (block < this.genesisBlockNumber + this.feeLookBackWindowInBlocks) {
-      return this.initialNormalizedFee;
+      return this.initialNormalizedFeeInSatoshis;
     }
     const blocksToAverage = await this.getBlocksInLookBackWindow(block);
-    return this.calculateNormalizedFee(blocksToAverage);
+
+    const rawNormalizedFee = this.calculateNormalizedFee(blocksToAverage);
+    const flooredNormalizedFee = Math.floor(rawNormalizedFee);
+    return flooredNormalizedFee;
   }
 
   private async getBlocksInLookBackWindow (block: number): Promise<BlockMetadata[]> {
-    // look back the interval
     return await this.blockMetadataStore.get(block - this.feeLookBackWindowInBlocks, block);
   }
 
@@ -69,16 +88,16 @@ export default class NormalizedFeeCalculator implements IFeeCalculator {
       totalTransactionCount += blockToAverage.transactionCount;
     }
 
-    // TODO: #926 investigate potential rounding differences between languages and implemetations
+    // TODO: #926 investigate potential rounding differences between languages and implementations
     // https://github.com/decentralized-identity/sidetree/issues/926
-    const unadjustedFee = Math.floor(totalFee / totalTransactionCount);
+    const unadjustedFee = totalFee / totalTransactionCount;
     const previousFee = blocksToAverage[blocksToAverage.length - 1].normalizedFee;
     return this.adjustFeeToWithinFluctuationRate(unadjustedFee, previousFee);
   }
 
   private adjustFeeToWithinFluctuationRate (unadjustedFee: number, previousFee: number): number {
-    const maxAllowedFee = Math.floor(previousFee * (1 + this.feeMaxFluctuationMultiplierPerBlock));
-    const minAllowedFee = Math.floor(previousFee * (1 - this.feeMaxFluctuationMultiplierPerBlock));
+    const maxAllowedFee = previousFee * (1 + this.feeMaxFluctuationMultiplierPerBlock);
+    const minAllowedFee = previousFee * (1 - this.feeMaxFluctuationMultiplierPerBlock);
 
     if (unadjustedFee > maxAllowedFee) {
       return maxAllowedFee;

lib/core/Core.ts

@@ -13,7 +13,7 @@ import ResponseModel from '../common/models/ResponseModel';
 import ResponseStatus from '../common/enums/ResponseStatus';
 import ServiceInfo from '../common/ServiceInfoProvider';
 import VersionManager from './VersionManager';
-import VersionModel from '../common/models/VersionModel';
+import VersionModel from './models/VersionModel';
 
 /**
  * The core class that is instantiated when running a Sidetree node.

lib/core/VersionManager.ts

@@ -15,7 +15,7 @@ import IVersionManager from './interfaces/IVersionManager';
 import IVersionMetadataFetcher from './interfaces/IVersionMetadataFetcher';
 import Resolver from './Resolver';
 import SidetreeError from '../common/SidetreeError';
-import VersionModel from '../common/models/VersionModel';
+import VersionModel from './models/VersionModel';
 
 /**
  * The class that handles code versioning.

lib/index.ts

@@ -4,11 +4,12 @@ import ISidetreeBitcoinConfig from './bitcoin/IBitcoinConfig';
 import ISidetreeBitcoinWallet from './bitcoin/interfaces/IBitcoinWallet';
 import ISidetreeCas from './core/interfaces/ICas';
 import SidetreeBitcoinProcessor from './bitcoin/BitcoinProcessor';
+import SidetreeBitcoinVersionModel from './bitcoin/models/BitcoinVersionModel';
 import SidetreeConfig from './core/models/Config';
 import SidetreeCore from './core/Core';
 import SidetreeResponse from './common/Response';
 import SidetreeResponseModel from './common/models/ResponseModel';
-import SidetreeVersionModel from './common/models/VersionModel';
+import SidetreeVersionModel from './core/models/VersionModel';
 
 // Core service exports.
 export {
@@ -24,5 +25,6 @@ export {
 export {
   ISidetreeBitcoinConfig,
   ISidetreeBitcoinWallet,
-  SidetreeBitcoinProcessor
+  SidetreeBitcoinProcessor,
+  SidetreeBitcoinVersionModel
 };

tests/bitcoin/BitcoinProcessor.spec.ts

@@ -37,7 +37,7 @@ function randomBlock (above: number = 0): IBlockInfo {
 
 describe('BitcoinProcessor', () => {
   const versionModels: VersionModel[] = [
-    { startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: 1, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
+    { startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: 1, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
   ];
 
   const testConfig: IBitcoinConfig = {

tests/bitcoin/VersionManager.spec.ts

@@ -9,8 +9,8 @@ describe('VersionManager', async () => {
     it('should return the correct version of fee calculator.', async () => {
 
       const versionModels: VersionModel[] = [
-        { startingBlockchainTime: 1000, version: '1000', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } },
-        { startingBlockchainTime: 2000, version: '2000', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
+        { startingBlockchainTime: 1000, version: '1000', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } },
+        { startingBlockchainTime: 2000, version: '2000', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
       ];
 
       const versionManager = new VersionManager();
@@ -51,8 +51,8 @@ describe('VersionManager', async () => {
   describe('getLockDurationInBlocks', () => {
     it('should get the correct lock duration', async () => {
       const versionModels: VersionModel[] = [
-        { startingBlockchainTime: 1000, version: '1000', protocolParameters: { valueTimeLockDurationInBlocks: 123, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } },
-        { startingBlockchainTime: 2000, version: '2000', protocolParameters: { valueTimeLockDurationInBlocks: 456, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
+        { startingBlockchainTime: 1000, version: '1000', protocolParameters: { valueTimeLockDurationInBlocks: 123, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } },
+        { startingBlockchainTime: 2000, version: '2000', protocolParameters: { valueTimeLockDurationInBlocks: 456, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }
       ];
       const versionManager = new VersionManager();
       spyOn(versionManager as any, 'loadDefaultExportsForVersion').and.callFake(async (_version: string, _className: string) => {

tests/bitcoin/lock/LockMonitor.spec.ts

@@ -31,7 +31,7 @@ describe('LockMonitor', () => {
   const mongoDbLockStore = new MongoDbLockTransactionStore('server-url', 'db');
 
   const lockDuration = 2000;
-  const versionModels: VersionModel[] = [{ startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: lockDuration, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }];
+  const versionModels: VersionModel[] = [{ startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: lockDuration, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }];
   const versionManager = new VersionManager();
   const lockResolver = new LockResolver(versionManager, bitcoinClient);
 

tests/bitcoin/lock/LockResolver.spec.ts

@@ -34,7 +34,7 @@ function createLockScriptVerifyResult (isScriptValid: boolean, owner: string | u
 }
 
 describe('LockResolver', () => {
-  const versionModels: VersionModel[] = [{ startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFee: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }];
+  const versionModels: VersionModel[] = [{ startingBlockchainTime: 0, version: 'latest', protocolParameters: { valueTimeLockDurationInBlocks: 5, initialNormalizedFeeInSatoshis: 1, feeLookBackWindowInBlocks: 1, feeMaxFluctuationMultiplierPerBlock: 1 } }];
   const versionManager = new VersionManager();
   versionManager.initialize(versionModels, { genesisBlockNumber: 0 } as any, new MockBlockMetadataStore());
 

tests/bitcoin/pof/NormalizedFeeCalculator.spec.ts

@@ -68,15 +68,15 @@ describe('NormalizedFeeCalculaor', () => {
       const actual = await normalizedFeeCalculator.addNormalizedFeeToBlockMetadata(blockMetadataWithoutFee);
       expect(actual.normalizedFee).toBeDefined();
       expect(getMetadataSpy).toHaveBeenCalled();
-      expect(normalizedFeeCalculator['blockMetadataCache'][0].height).toEqual(99);
-      expect(normalizedFeeCalculator['blockMetadataCache'][2].height).toEqual(101);
-      expect(normalizedFeeCalculator['expectedBlockHeight']).toEqual(102);
+      expect(normalizedFeeCalculator['cachedLookBackWindow'][0].height).toEqual(99);
+      expect(normalizedFeeCalculator['cachedLookBackWindow'][2].height).toEqual(101);
+      expect(normalizedFeeCalculator['blockHeightOfCachedLookBackWindow']).toEqual(102);
     });
 
     it('should calculate normalized feeand use cache', async () => {
       blockMetadataWithoutFee.height = 101;
-      normalizedFeeCalculator['expectedBlockHeight'] = 101;
-      normalizedFeeCalculator['blockMetadataCache'] = [
+      normalizedFeeCalculator['blockHeightOfCachedLookBackWindow'] = 101;
+      normalizedFeeCalculator['cachedLookBackWindow'] = [
         {
           height: 98,
           hash: 'string',
@@ -105,10 +105,10 @@ describe('NormalizedFeeCalculaor', () => {
       const getMetadataSpy = spyOn(mockMetadataStore, 'get').and.returnValue(Promise.resolve([]));
       const actual = await normalizedFeeCalculator.addNormalizedFeeToBlockMetadata(blockMetadataWithoutFee);
       expect(actual.normalizedFee).toBeDefined();
-      expect(normalizedFeeCalculator['blockMetadataCache'][0].height).toEqual(99);
-      expect(normalizedFeeCalculator['blockMetadataCache'][2].height).toEqual(101);
+      expect(normalizedFeeCalculator['cachedLookBackWindow'][0].height).toEqual(99);
+      expect(normalizedFeeCalculator['cachedLookBackWindow'][2].height).toEqual(101);
       expect(getMetadataSpy).not.toHaveBeenCalled(); // not called because there's cached data
-      expect(normalizedFeeCalculator['expectedBlockHeight']).toEqual(102);
+      expect(normalizedFeeCalculator['blockHeightOfCachedLookBackWindow']).toEqual(102);
     });
   });
 

tests/core/VersionManager.spec.ts

@@ -15,7 +15,7 @@ import OperationType from '../../lib/core/enums/OperationType';
 import Resolver from '../../lib/core/Resolver';
 import TransactionModel from '../../lib/common/models/TransactionModel';
 import VersionManager from '../../lib/core/VersionManager';
-import VersionModel from '../../lib/common/models/VersionModel';
+import VersionModel from '../../lib/core/models/VersionModel';
 
 describe('VersionManager', async () => {