⚡ Improve performance by using iterators in graph loading

.github/workflows/ci.yml

@@ -22,10 +22,13 @@ jobs:
         with:
           node-version: ${{ matrix.node }}
 
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
       - name: Install Dependencies
-        run: |
-          npm install -g pnpm
-          pnpm install
+        run: pnpm install
 
       - name: Run Lint
         run: pnpm run lint

plugins/core/src/db/graphLoader.ts

@@ -12,8 +12,8 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
 
-    const pages = pageRepo.getPagesBySnapshot(snapshotId);
-    const metrics = metricsRepo.getMetrics(snapshotId);
+    const pages = pageRepo.getPagesIteratorBySnapshot(snapshotId);
+    const metrics = metricsRepo.getMetricsIterator(snapshotId);
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     const metricsMap = new Map<number, DbMetrics>();
     for (const m of metrics) {
@@ -77,7 +77,7 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
         });
     }
 
-    const edges = edgeRepo.getEdgesBySnapshot(snapshotId);
+    const edges = edgeRepo.getEdgesIteratorBySnapshot(snapshotId);
 
     for (const e of edges) {
         const source = idMap.get(e.source_page_id);

plugins/core/src/db/repositories/EdgeRepository.ts

@@ -36,4 +36,8 @@ export class EdgeRepository {
   getEdgesBySnapshot(snapshotId: number): Edge[] {
     return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId) as Edge[];
   }
+
+  getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge> {
+    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<Edge>;
+  }
 }

plugins/core/src/db/repositories/MetricsRepository.ts

@@ -45,6 +45,10 @@ export class MetricsRepository {
     return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId) as DbMetrics[];
   }
 
+  getMetricsIterator(snapshotId: number): IterableIterator<DbMetrics> {
+    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<DbMetrics>;
+  }
+
   getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined {
     return this.getByPageStmt.get(snapshotId, pageId) as DbMetrics | undefined;
   }