feat(store,): add splice events (#1354)

Co-authored-by: Kevin Ingersoll <kingersoll@gmail.com>
Co-authored-by: alvrs <alvarius@lattice.xyz>

.changeset/cyan-hats-try.md

@@ -0,0 +1,30 @@
+---
+"@latticexyz/store": major
+"@latticexyz/world": major
+---
+
+We've updated Store events to be "schemaless", meaning there is enough information in each event to only need to operate on the bytes of each record to make an update to that record without having to first decode the record by its schema. This enables new kinds of indexers and sync strategies.
+
+If you've written your own sync logic or are interacting with Store calls directly, this is a breaking change. We have a few more breaking protocol changes upcoming, so you may hold off on upgrading until those land.
+
+If you are using MUD's built-in tooling (table codegen, indexer, store sync, etc.), you don't have to make any changes except upgrading to the latest versions and deploying a fresh World.
+
+- The `data` field in each `StoreSetRecord` and `StoreEphemeralRecord` has been replaced with three new fields: `staticData`, `encodedLengths`, and `dynamicData`. This better reflects the on-chain state and makes it easier to perform modifications to the raw bytes. We recommend storing each of these fields individually in your off-chain storage of choice (indexer, client, etc.).
+
+  ```diff
+  - event StoreSetRecord(bytes32 tableId, bytes32[] keyTuple, bytes data);
+  + event StoreSetRecord(bytes32 tableId, bytes32[] keyTuple, bytes staticData, bytes32 encodedLengths, bytes dynamicData);
+
+  - event StoreEphemeralRecord(bytes32 tableId, bytes32[] keyTuple, bytes data);
+  + event StoreEphemeralRecord(bytes32 tableId, bytes32[] keyTuple, bytes staticData, bytes32 encodedLengths, bytes dynamicData);
+  ```
+
+- The `StoreSetField` event is now replaced by two new events: `StoreSpliceStaticData` and `StoreSpliceDynamicData`. Splicing allows us to perform efficient operations like push and pop, in addition to replacing a field value. We use two events because updating a dynamic-length field also requires updating the record's `encodedLengths` (aka PackedCounter).
+
+  ```diff
+  - event StoreSetField(bytes32 tableId, bytes32[] keyTuple, uint8 fieldIndex, bytes data);
+  + event StoreSpliceStaticData(bytes32 tableId, bytes32[] keyTuple, uint48 start, uint40 deleteCount, bytes data);
+  + event StoreSpliceDynamicData(bytes32 tableId, bytes32[] keyTuple, uint48 start, uint40 deleteCount, bytes data, bytes32 encodedLengths);
+  ```
+
+Similarly, Store setter methods (e.g. `setRecord`) have been updated to reflect the `data` to `staticData`, `encodedLengths`, and `dynamicData` changes. We'll be following up shortly with Store getter method changes for more gas efficient storage reads.

.changeset/fast-zebras-promise.md

@@ -0,0 +1,6 @@
+---
+"@latticexyz/common": minor
+"@latticexyz/protocol-parser": major
+---
+
+`readHex` was moved from `@latticexyz/protocol-parser` to `@latticexyz/common`

.changeset/rotten-cats-lay.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/common": minor
+---
+
+`spliceHex` was added, which has a similar API as JavaScript's [`Array.prototype.splice`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/splice), but for `Hex` strings.
+
+```ts
+spliceHex("0x123456", 1, 1, "0x0000"); // "0x12000056"
+```

.changeset/shy-sheep-wait.md

@@ -0,0 +1,9 @@
+---
+"@latticexyz/dev-tools": major
+"@latticexyz/store-sync": major
+"create-mud": minor
+---
+
+We've updated Store events to be "schemaless", meaning there is enough information in each event to only need to operate on the bytes of each record to make an update to that record without having to first decode the record by its schema. This enables new kinds of indexers and sync strategies.
+
+As such, we've replaced `blockStorageOperations$` with `storedBlockLogs$`, a stream of simplified Store event logs after they've been synced to the configured storage adapter. These logs may not reflect exactly the events that are on chain when e.g. hydrating from an indexer, but they will still allow the client to "catch up" to the on-chain state of your tables.

e2e/packages/client-vanilla/src/mud/setupNetwork.ts

@@ -33,7 +33,7 @@ export async function setupNetwork() {
     walletClient: burnerWalletClient,
   });
 
-  const { components, latestBlock$, blockStorageOperations$, waitForTransaction } = await syncToRecs({
+  const { components, latestBlock$, storedBlockLogs$, waitForTransaction } = await syncToRecs({
     world,
     config: mudConfig,
     address: networkConfig.worldAddress as Hex,
@@ -74,7 +74,7 @@ export async function setupNetwork() {
     walletClient: burnerWalletClient,
     worldContract,
     latestBlock$,
-    blockStorageOperations$,
+    storedBlockLogs$,
     waitForTransaction,
   };
 }

e2e/packages/contracts/worlds.json

@@ -1,5 +1,5 @@
 {
   "31337": {
-    "address": "0x5FbDB2315678afecb367f032d93F642f64180aa3"
+    "address": "0x3Aa5ebB10DC797CAC828524e59A333d0A371443c"
   }
 }

e2e/packages/sync-test/data/encodeTestData.test.ts

@@ -3,24 +3,36 @@ import { encodeTestData } from "./encodeTestData";
 
 describe("encodeTestData", () => {
   it("should encode numbers", () => {
-    expect(encodeTestData({ Number: [{ key: { key: 42 }, value: { value: 1337 } }] })).toStrictEqual({
-      Number: [
-        {
-          key: ["0x000000000000000000000000000000000000000000000000000000000000002a"],
-          value: "0x00000539",
-          valueSchema: "0x0004010003000000000000000000000000000000000000000000000000000000",
-        },
-      ],
-    });
+    expect(encodeTestData({ Number: [{ key: { key: 42 }, value: { value: 1337 } }] })).toMatchInlineSnapshot(`
+      {
+        "Number": [
+          {
+            "dynamicData": "0x",
+            "encodedLengths": "0x0000000000000000000000000000000000000000000000000000000000000000",
+            "fieldLayout": "0x0004010004000000000000000000000000000000000000000000000000000000",
+            "key": [
+              "0x000000000000000000000000000000000000000000000000000000000000002a",
+            ],
+            "staticData": "0x00000539",
+          },
+        ],
+      }
+    `);
 
-    expect(encodeTestData({ Vector: [{ key: { key: 1337 }, value: { x: 42, y: -69 } }] })).toStrictEqual({
-      Vector: [
-        {
-          key: ["0x0000000000000000000000000000000000000000000000000000000000000539"],
-          value: "0x0000002affffffbb",
-          valueSchema: "0x0008020023230000000000000000000000000000000000000000000000000000",
-        },
-      ],
-    });
+    expect(encodeTestData({ Vector: [{ key: { key: 1337 }, value: { x: 42, y: -69 } }] })).toMatchInlineSnapshot(`
+      {
+        "Vector": [
+          {
+            "dynamicData": "0x",
+            "encodedLengths": "0x0000000000000000000000000000000000000000000000000000000000000000",
+            "fieldLayout": "0x0008020004040000000000000000000000000000000000000000000000000000",
+            "key": [
+              "0x0000000000000000000000000000000000000000000000000000000000000539",
+            ],
+            "staticData": "0x0000002affffffbb",
+          },
+        ],
+      }
+    `);
   });
 });

e2e/packages/sync-test/data/encodeTestData.ts

@@ -1,31 +1,24 @@
 import { mapObject } from "@latticexyz/utils";
+import { encodeKey, encodeValueArgs, valueSchemaToFieldLayoutHex } from "@latticexyz/protocol-parser";
 import { Data, EncodedData } from "./types";
-import { encodeAbiParameters, encodePacked } from "viem";
 import config from "../../contracts/mud.config";
-import { abiTypesToSchema, schemaToHex } from "@latticexyz/protocol-parser";
-import { StaticAbiType } from "@latticexyz/schema-type";
 
 /**
  * Turns the typed data into encoded data in the format expected by `world.setRecord`
  */
 export function encodeTestData(testData: Data) {
   return mapObject(testData, (records, table) => {
     if (!records) return undefined;
-    const keyConfig = config.tables[table].keySchema;
+    const tableConfig = config.tables[table];
     return records.map((record) => {
-      const encodedKey = Object.entries(record.key).map(([keyName, keyValue]) => {
-        const keyType = keyConfig[keyName as keyof typeof keyConfig] as StaticAbiType;
-        return encodeAbiParameters([{ type: keyType }], [keyValue]);
-      });
-
-      const encodedValue = encodePacked(Object.values(config.tables[table].valueSchema), Object.values(record.value));
-
-      const encodedValueSchema = schemaToHex(abiTypesToSchema(Object.values(config.tables[table].valueSchema)));
+      const key = encodeKey(tableConfig.keySchema, record.key);
+      const valueArgs = encodeValueArgs(tableConfig.valueSchema, record.value);
+      const fieldLayout = valueSchemaToFieldLayoutHex(tableConfig.valueSchema);
 
       return {
-        key: encodedKey,
-        value: encodedValue,
-        valueSchema: encodedValueSchema,
+        key,
+        ...valueArgs,
+        fieldLayout,
       };
     });
   }) as EncodedData<typeof testData>;

e2e/packages/sync-test/data/expectClientData.ts

@@ -11,7 +11,7 @@ export async function expectClientData(page: Page, data: Data) {
   for (const [table, records] of Object.entries(data)) {
     for (const record of records) {
       const value = await readComponentValue(page, table, encodeEntity(config.tables[table].keySchema, record.key));
-      expect(value).toEqual(record.value);
+      expect(value).toMatchObject(record.value);
     }
   }
 }

e2e/packages/sync-test/data/setContractData.ts

@@ -16,8 +16,10 @@ export async function setContractData(page: Page, data: Data) {
         // TODO: add support for multiple namespaces after https://github.com/latticexyz/mud/issues/994 is resolved
         tableIdToHex("", table),
         record.key,
-        record.value,
-        record.valueSchema,
+        record.staticData,
+        record.encodedLengths,
+        record.dynamicData,
+        record.fieldLayout,
       ]);
 
       // Wait for transactions to be confirmed

e2e/packages/sync-test/data/types.ts

@@ -20,5 +20,5 @@ export type Datum<Table extends keyof (typeof config)["tables"] = keyof (typeof
 export type Data = { [Table in keyof (typeof config)["tables"]]?: Array<Datum<Table>> };
 
 export type EncodedData<T extends Data = Data> = {
-  [Table in keyof T]: Array<{ key: Hex[]; value: Hex; valueSchema: Hex }>;
+  [Table in keyof T]: Array<{ key: Hex[]; staticData: Hex; encodedLengths: Hex; dynamicData: Hex; fieldLayout: Hex }>;
 };

e2e/packages/sync-test/rpcSync.test.ts

@@ -1,7 +1,6 @@
 import { afterEach, beforeEach, describe, test } from "vitest";
 import type { ViteDevServer } from "vite";
 import { Browser, Page } from "@playwright/test";
-import { ExecaChildProcess } from "execa";
 import { createAsyncErrorHandler } from "./asyncErrors";
 import { deployContracts, startViteServer, startBrowserAndPage, openClientWithRootAccount } from "./setup";
 import {

e2e/packages/sync-test/setup/startBrowserAndPage.ts

@@ -14,7 +14,7 @@ export async function startBrowserAndPage(
 
   // log uncaught errors in the browser page (browser and test consoles are separate)
   page.on("pageerror", (err) => {
-    console.log(chalk.yellow("[browser page error]:"), err.message);
+    console.log(chalk.yellow("[browser page error]:"), err.message, err);
   });
 
   // log browser's console logs

e2e/packages/sync-test/vite.config.ts

@@ -1,6 +1,5 @@
 import { defineConfig } from "vitest/config";
 
-// TODO should this along with `.test.ts` be in `client-vanilla`?
 export default defineConfig({
   test: {
     environment: "jsdom",