fix(core): fix for duplicate data per timestamp issue

awslabs · Oct 17, 2023 · 9cca8b1 · 9cca8b1
1 parent dd0c0c6
commit 9cca8b1
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 2 deletions.
diff --git a/packages/core/src/common/intervalStructure.spec.ts b/packages/core/src/common/intervalStructure.spec.ts
@@ -1,5 +1,6 @@
 import { addInterval, intersect, isContained, mergeItems, subtractIntervals } from './intervalStructure';
 import type { IntervalStructure } from './intervalStructure';
+import { dataPointCompare } from '../data-module/data-cache/caching/caching';
 
 const INITIAL_STATE = {
   intervals: [],
@@ -439,5 +440,40 @@ describe('adds item to interval structure', () => {
         items: [[20, 30, 35, 40, 60]],
       });
     });
+
+    describe('add interval when data is object', () => {
+      it('the data has no duplicate timestamps', () => {
+        const structure: IntervalStructure<{ x: number; y: number }> = {
+          intervals: [[20, 30]],
+          items: [
+            [
+              { x: 21, y: 100 },
+              { x: 22, y: 100 },
+            ],
+          ],
+        };
+        expect(
+          addInterval(
+            structure,
+            [25, 45],
+            [
+              { x: 21, y: 100 },
+              { x: 22, y: 100 },
+              { x: 23, y: 100 },
+            ],
+            dataPointCompare
+          )
+        ).toEqual({
+          intervals: [[20, 45]],
+          items: [
+            [
+              { x: 21, y: 100 },
+              { x: 22, y: 100 },
+              { x: 23, y: 100 },
+            ],
+          ],
+        });
+      });
+    });
   });
 });
diff --git a/packages/core/src/common/intervalStructure.ts b/packages/core/src/common/intervalStructure.ts
@@ -42,6 +42,26 @@ export const subtractIntervals = (interval: Interval, intervals: Interval[]): In
     .map(toIntervalNotation);
 };
 
+// removes duplicates (by timestamp) in a data (Datapoint[][])
+const uniqByKeepLast = <T>(data: T[][], key: string) => {
+  return data.map((datum) => {
+    if (Array.isArray(datum)) {
+      return [
+        ...new Map(
+          datum.map((d, index) => {
+            if (typeof d === 'object' && !!d) {
+              return [d[key as keyof T] as unknown, d];
+            }
+            return [index, d];
+          })
+        ).values(),
+      ];
+    } else {
+      return datum;
+    }
+  });
+};
+
 /**
  * Merges together to lists of items given a way to compare items.
  *
@@ -125,8 +145,12 @@ export const addInterval = <T>(
   updatedIntervals.splice(insertIndex, overlappingIntervals.length, combinedInterval);
   const updatedItems = [...intervalStructure.items];
   updatedItems.splice(insertIndex, overlappingIntervals.length, combinedItems);
+
+  // removing duplicate timestamps in an interval
+  const updatedItemsUniqueByTimestamp = uniqByKeepLast(updatedItems, 'x');
+
   return {
     intervals: updatedIntervals,
-    items: updatedItems,
+    items: updatedItemsUniqueByTimestamp,
   };
 };
diff --git a/packages/core/src/data-module/data-cache/caching/caching.ts b/packages/core/src/data-module/data-cache/caching/caching.ts
@@ -208,7 +208,7 @@ export const getRequestInformations = ({
   return requestInformations;
 };
 
-const dataPointCompare = <T extends Primitive = number>(a: DataPoint<T>, b: DataPoint<T>) => {
+export const dataPointCompare = <T extends Primitive = number>(a: DataPoint<T>, b: DataPoint<T>) => {
   const aTime = a.x;
   const bTime = b.x;
   if (aTime !== bTime) {