Fixed IdSortable's fixed point overflowing millisecond bits and demon…

…strated base64 does not preserve lexicographic-order

src/index.ts

@@ -1,3 +1,4 @@
+export { default as Id } from './Id';
 export { default as IdRandom } from './IdRandom';
 export { default as IdDeterministic } from './IdDeterministic';
 export { default as IdSortable } from './IdSortable';

src/utils.ts

@@ -237,14 +237,24 @@ function toFixedPoint(
   size: number,
   precision?: number,
 ): [number, number] {
-  const integer = Math.trunc(floating);
+  let integer = Math.trunc(floating);
   let fractional: number;
   if (precision == null) {
     fractional = floating % 1;
   } else {
     fractional = roundPrecise(floating % 1, precision);
   }
-  const fractionalFixed = Math.round(fractional * 2 ** size);
+  // If the fractional is rounded to 1
+  // then it should be added to the integer
+  if (fractional === 1) {
+    integer += fractional;
+    fractional = 0;
+  }
+  // Floor is used to round down to a number that can be represented by the bit size
+  // if ceil or round was used, it's possible to return a number that would overflow the bit size
+  // for example if 12 bits is used, then 4096 would overflow to all zeros
+  // the maximum for 12 bit is 4095
+  const fractionalFixed = Math.floor(fractional * 2 ** size);
   return [integer, fractionalFixed];
 }
 

tests/IdSortable.test.ts

@@ -46,14 +46,19 @@ describe('IdSortable', () => {
     expect(id.equals(id_)).toBe(true);
   });
   test('ids in bytes are lexically sortable', () => {
-    const id = new IdSortable();
-    const i1 = utils.toBuffer(id.get());
-    const i2 = utils.toBuffer(id.get());
-    const i3 = utils.toBuffer(id.get());
-    const buffers = [i3, i1, i2];
-    // Comparison is done on the bytes in lexicographic order
-    buffers.sort(Buffer.compare);
-    expect(buffers).toStrictEqual([i1, i2, i3]);
+    const idGen = new IdSortable();
+    // This generating over 100,000 ids and checks that they maintain
+    // sort order for each 100 chunk of ids
+    let count = 1000;
+    while (count > 0) {
+      const idBuffers = [...utils.take(idGen, 100)];
+      const idBuffersShuffled = idBuffers.slice();
+      shuffle(idBuffersShuffled);
+      // Comparison is done on the bytes in lexicographic order
+      idBuffersShuffled.sort(Buffer.compare);
+      expect(idBuffersShuffled).toStrictEqual(idBuffers);
+      count--;
+    }
   });
   test('ids in bytes are lexically sortable with time delay', async () => {
     const id = new IdSortable();
@@ -69,29 +74,61 @@ describe('IdSortable', () => {
   });
   test('encoded id strings are lexically sortable', () => {
     const idGen = new IdSortable();
-    const idStrings = [...utils.take(idGen, 100)].map((id) => id.toString());
+    // This generating over 100,000 ids and checks that they maintain
+    // sort order for each 100 chunk of ids
+    let count = 1000;
+    while (count > 0) {
+      const idStrings = [...utils.take(idGen, 100)].map((id) => id.toString());
+      const idStringsShuffled = idStrings.slice();
+      shuffle(idStringsShuffled);
+      idStringsShuffled.sort();
+      expect(idStringsShuffled).toStrictEqual(idStrings);
+      count--;
+    }
+  });
+  test('encoded uuids are lexically sortable', () => {
+    // UUIDs are hex encoding, and the hex alphabet preserves order
+    const idGen = new IdSortable();
+    // This generating over 100,000 ids and checks that they maintain
+    // sort order for each 100 chunk of ids
+    let count = 1000;
+    while (count > 0) {
+      const idUUIDs = [...utils.take(idGen, 100)].map(utils.toUUID);
+      const idUUIDsShuffled = idUUIDs.slice();
+      shuffle(idUUIDsShuffled);
+      idUUIDsShuffled.sort();
+      expect(idUUIDsShuffled).toStrictEqual(idUUIDs);
+      count--;
+    }
+  });
+  test('encoded multibase strings may be lexically sortable (base58btc)', () => {
+    // Base58btc's alphabet preserves sort order
+    const idGen = new IdSortable();
+    // This generating over 100,000 ids and checks that they maintain
+    // sort order for each 100 chunk of ids
+    let count = 1000;
+    while (count > 0) {
+      const idStrings = [...utils.take(idGen, 100)].map((id) =>
+        utils.toMultibase(id, 'base58btc'),
+      );
+      const idStringsShuffled = idStrings.slice();
+      shuffle(idStringsShuffled);
+      idStringsShuffled.sort();
+      expect(idStringsShuffled).toStrictEqual(idStrings);
+      count--;
+    }
+  });
+  test('encoded multibase strings may not be lexically sortable (base64)', async () => {
+    // Base64's alphabet does not preserve sort order
+    const idGen = new IdSortable();
+    const idStrings = [...utils.take(idGen, 100)].map((id) =>
+      utils.toMultibase(id, 'base64'),
+    );
     const idStringsShuffled = idStrings.slice();
     shuffle(idStringsShuffled);
     idStringsShuffled.sort();
-    expect(idStringsShuffled).toStrictEqual(idStrings);
-  });
-  test('encoded uuids are lexically sortable', () => {
-    const id = new IdSortable();
-    const i1 = utils.toUUID(id.get());
-    const i2 = utils.toUUID(id.get());
-    const i3 = utils.toUUID(id.get());
-    const uuids = [i3, i2, i1];
-    uuids.sort();
-    expect(uuids).toStrictEqual([i1, i2, i3]);
-  });
-  test('encoded multibase strings are lexically sortable', () => {
-    const id = new IdSortable();
-    const i1 = utils.toMultibase(id.get(), 'base58btc');
-    const i2 = utils.toMultibase(id.get(), 'base58btc');
-    const i3 = utils.toMultibase(id.get(), 'base58btc');
-    const encodings = [i3, i2, i1];
-    encodings.sort();
-    expect(encodings).toStrictEqual([i1, i2, i3]);
+    // It will not equal
+    expect(idStringsShuffled).not.toStrictEqual(idStrings);
   });
   test('ids are monotonic within the same timestamp', () => {
     // To ensure that we it generates monotonic ids

tests/utils.test.ts

@@ -85,20 +85,98 @@ describe('utils', () => {
     // we should expect .102 to be the resulting fractional
     const fp1 = 1633860855.1015312;
     const fixed1 = utils.toFixedPoint(fp1, 12, 3);
-    expect(fixed1[1]).toBe(418);
+    expect(fixed1[1]).toBe(417);
     const fp1_ = utils.fromFixedPoint(fixed1, 12, 3);
     expect(fp1_).toBe(utils.roundPrecise(fp1, 3));
     // Also to 3 decimal places
     // expecting 0.101 now
     const fp2 = 1633860855.1014312;
     const fixed2 = utils.toFixedPoint(fp2, 12, 3);
-    expect(fixed2[1]).toBe(414);
+    expect(fixed2[1]).toBe(413);
     const fp2_ = utils.fromFixedPoint(fixed2, 12, 3);
     expect(fp2_).toBe(utils.roundPrecise(fp2, 3));
     // 0 edge case
     expect(utils.toFixedPoint(0, 12, 3)).toStrictEqual([0, 0]);
     expect(utils.fromFixedPoint([0, 0], 12, 3)).toBe(0.0);
   });
+  test('fixed point conversion when close to 1', () => {
+    // Highest number 12 digits can represent is 4095
+    // a number of 4096 would result in overflow
+    // here we test when we get close to 4096
+    // the conversion from toFixedPoint and fromFixedPoint will be lossy
+    // this is because toFixedPoint will round off precision and floor any number getting close to 4096
+    let closeTo: number;
+    let fp: [number, number];
+    // Exactly at 3 decimal points
+    closeTo = 0.999;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4091]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.999);
+    // Will round below
+    closeTo = 0.9994;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4091]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.999);
+    // Will round above to 1
+    closeTo = 0.9995;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([1, 0]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(1);
+    // Will round above to 1
+    closeTo = 0.9999;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([1, 0]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(1);
+    // Will round above to 1
+    closeTo = 0.99999;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([1, 0]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(1);
+    // Exactly at 5 decimal points
+    closeTo = 0.99999;
+    fp = utils.toFixedPoint(closeTo, 12, 5);
+    expect(fp).toStrictEqual([0, 4095]);
+    expect(utils.fromFixedPoint(fp, 12, 5)).toBe(0.99976);
+  });
+  test('fixed point conversion when close to 0', () => {
+    let closeTo: number;
+    let fp: [number, number];
+    // Exactly 3 decimal places
+    closeTo = 0.001;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.001);
+    // Will round to 0
+    closeTo = 0.0001;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 0]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0);
+    // Will round to 0
+    closeTo = 0.0004;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 0]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0);
+    // Will round to 0.001
+    closeTo = 0.0005;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.001);
+    // Will round to 0.001
+    closeTo = 0.00055;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.001);
+    // Will round to 0.001
+    closeTo = 0.0009;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 4]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.001);
+    // Will round to 0.002
+    closeTo = 0.0015;
+    fp = utils.toFixedPoint(closeTo, 12, 3);
+    expect(fp).toStrictEqual([0, 8]);
+    expect(utils.fromFixedPoint(fp, 12, 3)).toBe(0.002);
+  });
   test('multibase encoding and decoding', () => {
     const bytes = new Uint8Array([
       123, 124, 125, 126, 127, 128, 129, 130, 123, 124, 125, 126, 127, 128, 129,