Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(store-sync): sync to sqlite (#1185)
- Loading branch information
Showing
24 changed files
with
1,722 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| --- | ||
| "@latticexyz/store-sync": minor | ||
| --- | ||
|
|
||
| `blockLogsToStorage(sqliteStorage(...))` converts block logs to SQLite operations. You can use it like: | ||
|
|
||
| ```ts | ||
| import { drizzle } from "drizzle-orm/better-sqlite3"; | ||
| import Database from "better-sqlite3"; | ||
| import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; | ||
| import { createPublicClient } from "viem"; | ||
| import { blockLogsToStorage } from "@latticexyz/store-sync"; | ||
| import { sqliteStorage } from "@latticexyz/store-sync/sqlite"; | ||
|
|
||
| const database = drizzle(new Database('store.db')) as any as BaseSQLiteDatabase<"sync", void>; | ||
| const publicClient = createPublicClient({ ... }); | ||
|
|
||
| blockLogs$ | ||
| .pipe( | ||
| concatMap(blockLogsToStorage(sqliteStorage({ database, publicClient }))), | ||
| tap(({ blockNumber, operations }) => { | ||
| console.log("stored", operations.length, "operations for block", blockNumber); | ||
| }) | ||
| ) | ||
| .subscribe(); | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| import { SchemaAbiType, SchemaAbiTypeToPrimitiveType, StaticAbiType } from "@latticexyz/schema-type"; | ||
| import { Address, Hex } from "viem"; | ||
| // TODO: move these type helpers into store? | ||
| import { Key, Value } from "@latticexyz/store-cache"; | ||
| import { GetLogsResult, GroupLogsByBlockNumberResult } from "@latticexyz/block-logs-stream"; | ||
| import { StoreEventsAbi, StoreConfig } from "@latticexyz/store"; | ||
|
|
||
| export type ChainId = number; | ||
| export type WorldId = `${ChainId}:${Address}`; | ||
|
|
||
| export type TableNamespace = string; | ||
| export type TableName = string; | ||
|
|
||
| export type KeySchema = Record<string, StaticAbiType>; | ||
| export type ValueSchema = Record<string, SchemaAbiType>; | ||
|
|
||
| export type SchemaToPrimitives<TSchema extends ValueSchema> = { | ||
| [key in keyof TSchema]: SchemaAbiTypeToPrimitiveType<TSchema[key]>; | ||
| }; | ||
|
|
||
| export type TableRecord<TKeySchema extends KeySchema = KeySchema, TValueSchema extends ValueSchema = ValueSchema> = { | ||
| key: SchemaToPrimitives<TKeySchema>; | ||
| value: SchemaToPrimitives<TValueSchema>; | ||
| }; | ||
|
|
||
| export type Table = { | ||
| address: Address; | ||
| tableId: Hex; | ||
| namespace: TableNamespace; | ||
| name: TableName; | ||
| keySchema: KeySchema; | ||
| valueSchema: ValueSchema; | ||
| }; | ||
|
|
||
| export type StoreEventsLog = GetLogsResult<StoreEventsAbi>[number]; | ||
| export type BlockLogs = GroupLogsByBlockNumberResult<StoreEventsLog>[number]; | ||
|
|
||
| export type BaseStorageOperation = { | ||
| log: StoreEventsLog; | ||
| namespace: string; | ||
| name: string; | ||
| }; | ||
|
|
||
| export type SetRecordOperation<TConfig extends StoreConfig> = BaseStorageOperation & { | ||
| type: "SetRecord"; | ||
| } & { | ||
| [TTable in keyof TConfig["tables"]]: { | ||
| name: TTable & string; | ||
| key: Key<TConfig, TTable>; | ||
| value: Value<TConfig, TTable>; | ||
| }; | ||
| }[keyof TConfig["tables"]]; | ||
|
|
||
| export type SetFieldOperation<TConfig extends StoreConfig> = BaseStorageOperation & { | ||
| type: "SetField"; | ||
| } & { | ||
| [TTable in keyof TConfig["tables"]]: { | ||
| name: TTable & string; | ||
| key: Key<TConfig, TTable>; | ||
| } & { | ||
| [TValue in keyof Value<TConfig, TTable>]: { | ||
| fieldName: TValue & string; | ||
| fieldValue: Value<TConfig, TTable>[TValue]; | ||
| }; | ||
| }[keyof Value<TConfig, TTable>]; | ||
| }[keyof TConfig["tables"]]; | ||
|
|
||
| export type DeleteRecordOperation<TConfig extends StoreConfig> = BaseStorageOperation & { | ||
| type: "DeleteRecord"; | ||
| } & { | ||
| [TTable in keyof TConfig["tables"]]: { | ||
| name: TTable & string; | ||
| key: Key<TConfig, TTable>; | ||
| }; | ||
| }[keyof TConfig["tables"]]; | ||
|
|
||
| export type StorageOperation<TConfig extends StoreConfig> = | ||
| | SetFieldOperation<TConfig> | ||
| | SetRecordOperation<TConfig> | ||
| | DeleteRecordOperation<TConfig>; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,2 @@ | ||
| export * from "./blockLogsToStorage"; | ||
| export * from "./common"; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| import { schemaAbiTypeToDefaultValue } from "@latticexyz/schema-type"; | ||
| import { ValueSchema, SchemaToPrimitives } from "./common"; | ||
|
|
||
| export function schemaToDefaults<TSchema extends ValueSchema>(schema: TSchema): SchemaToPrimitives<TSchema> { | ||
| return Object.fromEntries( | ||
| Object.entries(schema).map(([key, abiType]) => [key, schemaAbiTypeToDefaultValue[abiType]]) | ||
| ) as SchemaToPrimitives<TSchema>; | ||
| } |
Oops, something went wrong.