chore: rewritten _performSnapshot function (#300)

Previous implementation executed 3N+2 queries for N oplog entries
(`UPDATE` timestamps, `SELECT` updated, then, per oplog entry, `SELECT`
shadow row, `UPDATE` oplog entry, `UPDATE` or `DELETE` shadow row).

This implementation always executes exactly 4 queries to achieve the
same. This lowers the amount of times we're traversing C or WASM
boundary and offloads more work to SQLite.

This shouldn't affect the performance of current "lightweight" apps, but
is likely to show significant improvements over 100+ oplog entries

.changeset/moody-comics-pretend.md

@@ -0,0 +1,5 @@
+---
+"electric-sql": patch
+---
+
+Updated snapshotting function to be more efficient when handling a large oplog

clients/typescript/src/migrators/schema.ts

@@ -9,6 +9,9 @@ export const data = {
       statements: [
         //`-- The ops log table\n`,
         `CREATE TABLE IF NOT EXISTS ${oplogTable} (\n  rowid INTEGER PRIMARY KEY AUTOINCREMENT,\n  namespace TEXT NOT NULL,\n  tablename TEXT NOT NULL,\n  optype TEXT NOT NULL,\n  primaryKey TEXT NOT NULL,\n  newRow TEXT,\n  oldRow TEXT,\n  timestamp TEXT,  clearTags TEXT DEFAULT "[]" NOT NULL\n);`,
+        // Add an index for the oplog
+        `CREATE INDEX IF NOT EXISTS ${oplogTable.namespace}._electric_table_pk_reference ON ${oplogTable.tablename} (namespace, tablename, primaryKey)`,
+        `CREATE INDEX IF NOT EXISTS ${oplogTable.namespace}._electric_timestamp ON ${oplogTable.tablename} (timestamp)`,
         //`-- Somewhere to keep our metadata\n`,
         `CREATE TABLE IF NOT EXISTS ${metaTable} (\n  key TEXT PRIMARY KEY,\n  value BLOB\n);`,
         //`-- Somewhere to track migrations\n`,

clients/typescript/src/migrators/triggers.ts

@@ -207,8 +207,14 @@ export function generateTriggers(tables: Tables): Statement[] {
 
 function joinColsForJSON(cols: string[], target?: 'new' | 'old') {
   if (typeof target === 'undefined') {
-    return cols.map((col) => `'${col}', ${col}`).join(', ')
+    return cols
+      .sort()
+      .map((col) => `'${col}', ${col}`)
+      .join(', ')
   } else {
-    return cols.map((col) => `'${col}', ${target}.${col}`).join(', ')
+    return cols
+      .sort()
+      .map((col) => `'${col}', ${target}.${col}`)
+      .join(', ')
   }
 }

clients/typescript/src/satellite/oplog.ts

@@ -269,12 +269,12 @@ export const fromTransaction = (
   relations: RelationsCache
 ): OplogEntry[] => {
   return transaction.changes.map((t) => {
-    const columnValues = t.record ? t.record : t.oldRecord
-    const pk = JSON.stringify(
+    const columnValues = t.record ? t.record : t.oldRecord!
+    const pk = primaryKeyToStr(
       Object.fromEntries(
         relations[`${t.relation.table}`].columns
           .filter((c) => c.primaryKey)
-          .map((col) => [col.name, columnValues![col.name]])
+          .map((col) => [col.name, columnValues[col.name]!])
       )
     )
 
@@ -348,7 +348,7 @@ export const getShadowPrimaryKey = (
   oplogEntry: OplogEntry | OplogEntryChanges | ShadowEntryChanges
 ): ShadowKey => {
   if ('primaryKey' in oplogEntry) {
-    return primaryKeyToStr(JSON.parse(oplogEntry.primaryKey))
+    return oplogEntry.primaryKey
   } else {
     return primaryKeyToStr(oplogEntry.primaryKeyCols)
   }
@@ -390,10 +390,28 @@ export const opLogEntryToChange = (
   }
 }
 
-export const primaryKeyToStr = (primaryKeyJson: {
+/**
+ * Convert a primary key to a string the same way our triggers do when generating oplog entries.
+ *
+ * Takes the object that contains the primary key and serializes it to JSON in a non-prettified
+ * way with column sorting.
+ *
+ * @param primaryKeyObj object representing all columns of a primary key
+ * @returns a stringified JSON with stable sorting on column names
+ */
+export const primaryKeyToStr = (primaryKeyObj: {
   [key: string]: string | number
 }): string => {
-  return Object.values(primaryKeyJson).sort().join('_')
+  const keys = Object.keys(primaryKeyObj).sort()
+  if (keys.length === 0) return '{}'
+
+  let json = '{'
+  for (const key of keys) {
+    json += JSON.stringify(key) + ':' + JSON.stringify(primaryKeyObj[key]) + ','
+  }
+
+  // Remove the last appended comma and close the object
+  return json.slice(0, -1) + '}'
 }
 
 export const generateTag = (instanceId: string, timestamp: Date): Tag => {