feat(store-indexer): add experimental/local SQL API endpoint (#3676)

Co-authored-by: Kevin Ingersoll <kingersoll@gmail.com>
Co-authored-by: alvarius <alvarius@lattice.xyz>

Author: 3 people

.changeset/yellow-otters-swim.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-indexer": patch
+---
+
+Added experimental SQL API endpoint `/q` to the SQLite indexer. This is only intended for local development purposes and should not be used in production.

packages/store-indexer/package.json

@@ -61,12 +61,14 @@
     "@sentry/utils": "^7.86.0",
     "@trpc/client": "10.34.0",
     "@trpc/server": "10.34.0",
+    "@types/koa-bodyparser": "^4.3.12",
     "accepts": "^1.3.8",
     "better-sqlite3": "^8.6.0",
     "debug": "^4.3.4",
     "dotenv": "^16.0.3",
     "drizzle-orm": "^0.28.5",
     "koa": "^2.15.4",
+    "koa-bodyparser": "^4.4.1",
     "koa-compose": "^4.1.0",
     "postgres": "3.3.5",
     "prom-client": "^15.1.2",

packages/store-indexer/src/bin/sqlite-indexer.ts

@@ -7,6 +7,7 @@ import { drizzle } from "drizzle-orm/better-sqlite3";
 import Database from "better-sqlite3";
 import Koa from "koa";
 import cors from "@koa/cors";
+import bodyParser from "koa-bodyparser";
 import { createKoaMiddleware } from "trpc-koa-adapter";
 import { createAppRouter } from "@latticexyz/store-sync/trpc-indexer";
 import { chainState, schemaVersion, syncToSqlite } from "@latticexyz/store-sync/sqlite";
@@ -28,6 +29,11 @@ const env = parseEnv(
     z.object({
       SQLITE_FILENAME: z.string().default("indexer.db"),
       SENTRY_DSN: z.string().optional(),
+      ENABLE_UNSAFE_QUERY_API: z
+        .string()
+        .optional()
+        .default("false")
+        .transform((val) => val === "true"),
     }),
   ),
 );
@@ -121,6 +127,7 @@ if (env.SENTRY_DSN) {
 }
 
 server.use(cors());
+server.use(bodyParser());
 server.use(
   healthcheck({
     isReady: () => isCaughtUp,
@@ -136,7 +143,7 @@ server.use(
   }),
 );
 server.use(helloWorld());
-server.use(apiRoutes(database));
+server.use(apiRoutes({ database, enableUnsafeQueryApi: env.ENABLE_UNSAFE_QUERY_API }));
 
 server.use(
   createKoaMiddleware({
@@ -150,3 +157,12 @@ server.use(
 
 server.listen({ host: env.HOST, port: env.PORT });
 console.log(`sqlite indexer frontend listening on http://${env.HOST}:${env.PORT}`);
+
+if (env.ENABLE_UNSAFE_QUERY_API) {
+  console.warn("\n\n⚠️  SECURITY WARNING ⚠️");
+  console.warn("=========================\n");
+  console.warn("UNSAFE QUERY API IS ENABLED");
+  console.warn("DO NOT USE IN PRODUCTION");
+  console.warn("This will expose your database to public access");
+  console.warn("\n=========================\n\n");
+}

packages/store-indexer/src/koa-middleware/sentry.ts

@@ -12,7 +12,11 @@ export function errorHandler(): Koa.Middleware {
     } catch (err) {
       Sentry.withScope((scope) => {
         scope.addEventProcessor((event) => {
-          return Sentry.addRequestDataToEvent(event, ctx.request);
+          return Sentry.addRequestDataToEvent(event, {
+            ...ctx.request,
+            body: ctx.request.body as string | Record<string, unknown> | undefined,
+            query: ctx.request.query as Record<string, unknown> | undefined,
+          });
         });
         Sentry.captureException(err);
       });
@@ -27,10 +31,10 @@ export function requestHandler(): Koa.Middleware {
       const hub = Sentry.getCurrentHub();
       hub.configureScope((scope) =>
         scope.addEventProcessor((event) =>
-          Sentry.addRequestDataToEvent(event, ctx.request, {
-            include: {
-              user: false,
-            },
+          Sentry.addRequestDataToEvent(event, {
+            ...ctx.request,
+            body: ctx.request.body as string | Record<string, unknown> | undefined,
+            query: ctx.request.query as Record<string, unknown> | undefined,
           }),
         ),
       );

packages/store-indexer/src/sqlite/apiRoutes.ts

@@ -1,3 +1,5 @@
+import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
+import { sql } from "drizzle-orm";
 import { Middleware } from "koa";
 import Router from "@koa/router";
 import compose from "koa-compose";
@@ -7,10 +9,14 @@ import { debug } from "../debug";
 import { createBenchmark } from "@latticexyz/common";
 import { compress } from "../koa-middleware/compress";
 import { getTablesWithRecords } from "./getTablesWithRecords";
-import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core";
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware {
+type Props = {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  database: BaseSQLiteDatabase<"sync", any>;
+  enableUnsafeQueryApi?: boolean;
+};
+
+export function apiRoutes({ database, enableUnsafeQueryApi = false }: Props): Middleware {
   const router = new Router();
 
   router.get("/api/logs", compress(), async (ctx) => {
@@ -44,5 +50,50 @@ export function apiRoutes(database: BaseSQLiteDatabase<"sync", any>): Middleware
     }
   });
 
+  router.post("/q", async (ctx) => {
+    if (!enableUnsafeQueryApi) {
+      ctx.status = 404;
+      ctx.body = JSON.stringify({ error: "Query endpoint is not enabled" });
+      return;
+    }
+
+    try {
+      const queries = Array.isArray(ctx.request.body) ? ctx.request.body : [];
+      if (queries.length === 0) {
+        ctx.status = 400;
+        ctx.body = JSON.stringify({ error: "No queries provided" });
+        return;
+      }
+
+      const result = [];
+      for (const { query } of queries) {
+        const data = database.all(sql.raw(query)) as Record<string, unknown>[];
+        if (!data || !Array.isArray(data)) {
+          throw new Error("Invalid query result");
+        }
+
+        if (data.length === 0) {
+          result.push([]);
+          continue;
+        }
+
+        if (!data[0]) {
+          throw new Error("Invalid row data");
+        }
+
+        const columns = Object.keys(data[0]).map((key) => key.replaceAll("_", "").toLowerCase());
+        const rows = data.map((row) => Object.values(row).map((value) => value?.toString() ?? ""));
+        result.push([columns, ...rows]);
+      }
+
+      ctx.status = 200;
+      ctx.body = JSON.stringify({ result });
+    } catch (error) {
+      const errorMessage = error instanceof Error ? error.message : "An unknown error occurred";
+      ctx.status = 400;
+      ctx.body = JSON.stringify({ error: errorMessage });
+    }
+  });
+
   return compose([router.routes(), router.allowedMethods()]) as Middleware;
 }