feat(store-indexer,store-sync): filter by table and key (#1794)

Co-authored-by: alvarius <alvarius@lattice.xyz>

.changeset/empty-starfishes-lick.md

@@ -0,0 +1,16 @@
+---
+"@latticexyz/store-indexer": major
+---
+
+Removed `tableIds` filter option in favor of the more flexible `filters` option that accepts `tableId` and an optional `key0` and/or `key1` to filter data by tables and keys.
+
+If you were using an indexer client directly, you'll need to update your query:
+
+```diff
+  await indexer.findAll.query({
+    chainId,
+    address,
+-   tableIds: ['0x...'],
++   filters: [{ tableId: '0x...' }],
+  });
+```

.changeset/sharp-falcons-tie.md

@@ -0,0 +1,16 @@
+---
+"@latticexyz/store-sync": minor
+---
+
+Added a `filters` option to store sync to allow filtering client data on tables and keys. Previously, it was only possible to filter on `tableIds`, but the new filter option allows for more flexible filtering by key.
+
+If you are building a large MUD application, you can use positional keys as a way to shard data and make it possible to load only the data needed in the client for a particular section of your app. We're using this already in Sky Strife to load match-specific data into match pages without having to load data for all matches, greatly improving load time and client performance.
+
+```ts
+syncToRecs({
+  ...
+  filters: [{ tableId: '0x...', key0: '0x...' }],
+});
+```
+
+The `tableIds` option is now deprecated and will be removed in the future, but is kept here for backwards compatibility.

e2e/packages/client-vanilla/src/index.ts

@@ -1,5 +1,6 @@
 import { Component, Entity, getComponentValue } from "@latticexyz/recs";
 import { setup } from "./mud/setup";
+import { decodeEntity } from "@latticexyz/store-sync/recs";
 
 const {
   network: { components, latestBlock$, worldContract, waitForTransaction },
@@ -12,6 +13,13 @@ _window.waitForTransaction = waitForTransaction;
 _window.getComponentValue = (componentName: keyof typeof components, entity: Entity) =>
   getComponentValue(components[componentName] as Component, entity);
 
+_window.getEntities = (componentName: keyof typeof components) => Array.from(components[componentName].entities());
+
+_window.getKeys = (componentName: keyof typeof components) =>
+  Array.from(components[componentName].entities()).map((entity) =>
+    decodeEntity(components[componentName].metadata.keySchema, entity)
+  );
+
 // Update block number in the UI
 latestBlock$.subscribe((block) => {
   const element = document.querySelector("#block");

e2e/packages/client-vanilla/src/mud/setupNetwork.ts

@@ -1,10 +1,10 @@
-import { createPublicClient, http, createWalletClient, Hex, parseEther, ClientConfig } from "viem";
+import { createPublicClient, http, createWalletClient, Hex, parseEther, ClientConfig, stringToHex } from "viem";
 import { createFaucetService } from "@latticexyz/services/faucet";
 import { encodeEntity, syncToRecs } from "@latticexyz/store-sync/recs";
 import { getNetworkConfig } from "./getNetworkConfig";
 import { world } from "./world";
 import IWorldAbi from "contracts/out/IWorld.sol/IWorld.abi.json";
-import { createBurnerAccount, getContract, transportObserver } from "@latticexyz/common";
+import { createBurnerAccount, getContract, resourceToHex, transportObserver } from "@latticexyz/common";
 import mudConfig from "contracts/mud.config";
 
 export type SetupNetworkResult = Awaited<ReturnType<typeof setupNetwork>>;
@@ -40,6 +40,21 @@ export async function setupNetwork() {
     publicClient,
     startBlock: BigInt(networkConfig.initialBlockNumber),
     indexerUrl: networkConfig.indexerUrl ?? undefined,
+    filters: Object.entries(mudConfig.tables).map(([, table]) => {
+      const tableId = resourceToHex({
+        type: table.offchainOnly ? "offchainTable" : "table",
+        namespace: mudConfig.namespace,
+        name: table.name,
+      });
+      if (table.name === mudConfig.tables.Position.name) {
+        return {
+          tableId,
+          key0: stringToHex("map1", { size: 32 }),
+        };
+      } else {
+        return { tableId };
+      }
+    }),
   });
 
   // Request drip from faucet

e2e/packages/contracts/mud.config.ts

@@ -37,5 +37,15 @@ export default mudConfig({
         value: "bool",
       },
     },
+    Position: {
+      keySchema: {
+        zone: "bytes32",
+        x: "int32",
+        y: "int32",
+      },
+      valueSchema: {
+        player: "address",
+      },
+    },
   },
 });

e2e/packages/contracts/script/PostDeploy.s.sol

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: MIT
+pragma solidity >=0.8.21;
+
+import { Script } from "forge-std/Script.sol";
+import { console } from "forge-std/console.sol";
+import { StoreSwitch } from "@latticexyz/store/src/StoreSwitch.sol";
+import { IWorld } from "../src/codegen/world/IWorld.sol";
+
+import { Position } from "../src/codegen/index.sol";
+
+contract PostDeploy is Script {
+  function run(address worldAddress) external {
+    StoreSwitch.setStoreAddress(worldAddress);
+
+    uint256 deployerPrivateKey = vm.envUint("PRIVATE_KEY");
+    vm.startBroadcast(deployerPrivateKey);
+
+    // Set up a bunch of position data so we can demonstrate filtering
+    Position.set({ zone: "map1", x: 1, y: 1, player: msg.sender });
+    Position.set({ zone: "map1", x: 2, y: -2, player: msg.sender });
+    Position.set({ zone: "map2", x: 0, y: -99, player: msg.sender });
+    Position.set({ zone: "map2", x: 0, y: 99, player: msg.sender });
+    Position.set({ zone: "map99", x: 99, y: 99, player: msg.sender });
+
+    vm.stopBroadcast();
+  }
+}