feat(store-indexer): format sql queries (#3687)

Co-authored-by: alvarius <alvarius@lattice.xyz>

Author: karooolis

.changeset/happy-melons-own.md

@@ -0,0 +1,5 @@
+---
+"@latticexyz/store-indexer": patch
+---
+
+The local SQLite indexer now automatically converts camelCase column names to snake_case to comply with the SQL API.

packages/store-indexer/package.json

@@ -45,7 +45,7 @@
     "start:sqlite": "tsx src/bin/sqlite-indexer",
     "start:sqlite:local": "SQLITE_FILENAME=anvil.db RPC_HTTP_URL=http://127.0.0.1:8545 pnpm start:sqlite",
     "start:sqlite:testnet": "SQLITE_FILENAME=testnet.db RPC_HTTP_URL=https://rpc.holesky.redstone.xyz pnpm start:sqlite",
-    "test": "tsc --noEmit",
+    "test": "tsc --noEmit && vitest --run --passWithNoTests",
     "test:ci": "pnpm run test"
   },
   "dependencies": {
@@ -70,6 +70,7 @@
     "koa": "^2.15.4",
     "koa-bodyparser": "^4.4.1",
     "koa-compose": "^4.1.0",
+    "node-sql-parser": "^5.3.3",
     "postgres": "3.3.5",
     "prom-client": "^15.1.2",
     "rxjs": "7.5.5",

packages/store-indexer/src/sqlite/apiRoutes.ts

@@ -9,6 +9,7 @@ import { debug } from "../debug";
 import { createBenchmark } from "@latticexyz/common";
 import { compress } from "../koa-middleware/compress";
 import { getTablesWithRecords } from "./getTablesWithRecords";
+import { formatSqlQuery } from "./formatSqlQuery";
 
 type Props = {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -67,7 +68,8 @@ export function apiRoutes({ database, enableUnsafeQueryApi = false }: Props): Mi
 
       const result = [];
       for (const { query } of queries) {
-        const data = database.all(sql.raw(query)) as Record<string, unknown>[];
+        const formattedQuery = formatSqlQuery(query);
+        const data = database.all(sql.raw(formattedQuery)) as Record<string, unknown>[];
         if (!data || !Array.isArray(data)) {
           throw new Error("Invalid query result");
         }

packages/store-indexer/src/sqlite/formatSqlQuery.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, it } from "vitest";
+import { formatSqlQuery } from "./formatSqlQuery";
+
+describe("formatSqlQuery", () => {
+  it("should convert camelCase column names to snake_case", () => {
+    const input = "SELECT userId, firstName, lastName FROM users";
+    const expected = 'SELECT "user_id", "first_name", "last_name" FROM "users"';
+    expect(formatSqlQuery(input)).toBe(expected);
+  });
+
+  it("should not modify non-camelCase column names", () => {
+    const input = "SELECT user_id, first_name, last_name FROM users";
+    const expected = 'SELECT "user_id", "first_name", "last_name" FROM "users"';
+    expect(formatSqlQuery(input)).toBe(expected);
+  });
+
+  it("should not modify camelCase words that are not column names", () => {
+    const input = "SELECT * FROM users WHERE firstName LIKE '%NameSurname%'";
+    const expected = 'SELECT * FROM "users" WHERE "first_name" LIKE \'%NameSurname%\'';
+    expect(formatSqlQuery(input)).toBe(expected);
+  });
+
+  it("should not modify table names", () => {
+    const input = "SELECT * FROM usersTable";
+    const expected = 'SELECT * FROM "usersTable"';
+    expect(formatSqlQuery(input)).toBe(expected);
+
+    const input2 = "SELECT * FROM users_table";
+    const expected2 = 'SELECT * FROM "users_table"';
+    expect(formatSqlQuery(input2)).toBe(expected2);
+  });
+});

packages/store-indexer/src/sqlite/formatSqlQuery.ts

@@ -0,0 +1,50 @@
+import sqlParser from "node-sql-parser";
+
+type AstNode = {
+  type?: string;
+  [key: string]: unknown;
+};
+
+/**
+ * Transforms camelCase identifiers to snake_case in SQL queries
+ *
+ * @param sqlQuery The SQL query to transform
+ * @returns The transformed SQL query
+ */
+export function formatSqlQuery(sqlQuery: string): string {
+  const parser = new sqlParser.Parser();
+  const ast = parser.astify(sqlQuery);
+
+  updateNode(ast);
+
+  return parser.sqlify(ast, { database: "sqlite" });
+}
+
+function updateNode(node: unknown): void {
+  if (Array.isArray(node)) {
+    node.forEach((item) => updateNode(item));
+    return;
+  }
+
+  if (node && typeof node === "object") {
+    const astNode = node as AstNode;
+
+    if (
+      astNode.type === "column_ref" &&
+      typeof astNode.column === "string" &&
+      /^[a-z]+[A-Z][a-z]*$/.test(astNode.column)
+    ) {
+      astNode.column = camelToSnakeCase(astNode.column);
+    }
+
+    for (const [, value] of Object.entries(astNode)) {
+      if (typeof value === "object" && value !== null) {
+        updateNode(value);
+      }
+    }
+  }
+}
+
+function camelToSnakeCase(str: string): string {
+  return str.replace(/([a-z]+)([A-Z][a-z]*)/g, "$1_$2").toLowerCase();
+}