From 175aa328fc35f643e95743f0b55994bde013c53b Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Fri, 1 Nov 2024 16:22:11 +0100 Subject: [PATCH 01/13] cli: refactor rollup configuration --- packages/cli/build.config.ts | 21 +- packages/cli/package.json | 27 ++- packages/cli/playground/apibara.config.ts | 10 +- .../playground/indexers/starknet.indexer.ts | 8 +- packages/cli/playground/tsconfig.json | 11 - packages/cli/runtime-meta.d.ts | 2 + packages/cli/runtime-meta.mjs | 7 + packages/cli/src/cli/commands/build.ts | 9 +- packages/cli/src/cli/commands/prepare.ts | 2 - packages/cli/src/config/index.ts | 9 +- packages/cli/src/core/apibara.ts | 4 + packages/cli/src/core/build/build.ts | 2 + packages/cli/src/core/build/prepare.ts | 4 +- packages/cli/src/core/build/prod.ts | 9 +- packages/cli/src/core/build/types.ts | 4 +- packages/cli/src/core/config/defaults.ts | 2 +- packages/cli/src/core/config/loader.ts | 1 + .../resolvers/runtime-config.resolver.ts | 2 +- packages/cli/src/core/config/update.ts | 5 +- packages/cli/src/core/scan.ts | 40 ++++ packages/cli/src/internal/citty/index.ts | 1 - packages/cli/src/internal/consola/index.ts | 1 - packages/cli/src/rollup/config.ts | 206 ++++-------------- packages/cli/src/rollup/plugins/config.ts | 10 + packages/cli/src/rollup/plugins/indexers.ts | 16 ++ packages/cli/src/runtime/dev.ts | 2 + packages/cli/src/runtime/index.ts | 1 + packages/cli/src/runtime/internal/app.ts | 37 ++++ packages/cli/src/runtime/start.ts | 48 ++++ packages/cli/src/types/apibara.ts | 8 + packages/cli/src/types/config.ts | 34 ++- packages/cli/src/types/virtual/config.d.ts | 3 + packages/cli/src/types/virtual/indexers.d.ts | 10 + packages/cli/tsconfig.json | 4 +- pnpm-lock.yaml | 63 +++++- 35 files changed, 369 insertions(+), 254 deletions(-) delete mode 100644 packages/cli/playground/tsconfig.json create mode 100644 packages/cli/runtime-meta.d.ts create mode 100644 packages/cli/runtime-meta.mjs create mode 100644 packages/cli/src/core/scan.ts delete mode 100644 packages/cli/src/internal/citty/index.ts delete mode 100644 packages/cli/src/internal/consola/index.ts create mode 100644 packages/cli/src/rollup/plugins/config.ts create mode 100644 packages/cli/src/rollup/plugins/indexers.ts create mode 100644 packages/cli/src/runtime/dev.ts create mode 100644 packages/cli/src/runtime/index.ts create mode 100644 packages/cli/src/runtime/internal/app.ts create mode 100644 packages/cli/src/runtime/start.ts create mode 100644 packages/cli/src/types/virtual/config.d.ts create mode 100644 packages/cli/src/types/virtual/indexers.d.ts diff --git a/packages/cli/build.config.ts b/packages/cli/build.config.ts index b36675ec..36c5ac7a 100644 --- a/packages/cli/build.config.ts +++ b/packages/cli/build.config.ts @@ -2,9 +2,17 @@ import { fileURLToPath } from "node:url"; import { resolve } from "pathe"; import { defineBuildConfig } from "unbuild"; -const modules = ["cli", "config", "core", "rollup", "types", "hooks "]; +const modules = [ + "cli", + "config", + "core", + "rollup", + "types", + "hooks", + "runtime", +]; -// @ts-ignore The 'import.meta' meta-property is only allowed when the '--module' option is 'es2020', 'es2022', 'esnext', 'system', 'node16', or 'nodenext'. +// @ts-ignore const srcDir = fileURLToPath(new URL("src", import.meta.url)); export default defineBuildConfig({ @@ -15,13 +23,13 @@ export default defineBuildConfig({ { input: "./src/rollup/index.ts" }, { input: "./src/types/index.ts" }, { input: "./src/hooks/index.ts" }, - { input: "./src/internal/consola/index.ts" }, - { input: "./src/internal/citty/index.ts" }, + { input: "./src/runtime/", outDir: "./dist/runtime", format: "esm" }, ], clean: true, outDir: "./dist", declaration: true, alias: { + "apibara/runtime/meta": resolve(srcDir, "../runtime-meta.mjs"), ...Object.fromEntries( modules.map((module) => [ `apibara/${module}`, @@ -29,5 +37,8 @@ export default defineBuildConfig({ ]), ), }, - externals: [...modules.map((module) => `apibara/${module}`)], + externals: [ + "apibara/runtime/meta", + ...modules.map((module) => `apibara/${module}`), + ], }); diff --git a/packages/cli/package.json b/packages/cli/package.json index 3452dd3f..2d2ede5f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -32,13 +32,21 @@ "types": "./dist/hooks/index.d.ts", "import": "./dist/hooks/index.mjs" }, - "./internal/consola": { - "import": "./dist/internal/consola/index.mjs", - "types": "./dist/internal/consola/index.d.ts" + "./runtime": { + "types": "./dist/runtime/index.d.ts", + "import": "./dist/runtime/index.mjs" }, - "./internal/citty": { - "import": "./dist/internal/citty/index.mjs", - "types": "./dist/internal/citty/index.d.ts" + "./runtime/meta": { + "types": "./runtime-meta.d.ts", + "import": "./runtime-meta.mjs" + }, + "./runtime/*": { + "types": "./dist/runtime/*.d.ts", + "import": "./dist/runtime/*.mjs" + }, + "./dist/runtime/*": { + "types": "./dist/runtime/*.d.ts", + "import": "./dist/runtime/*.mjs" } }, "bin": { @@ -55,13 +63,14 @@ "typecheck": "tsc --noEmit", "lint:fix": "pnpm lint --write", "format": "biome format . --write", - "playground": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts", + "playground": "JITI_ESM_RESOLVE=1 CONSOLA_LEVEL=debug NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts", "playground:prepare": "pnpm playground prepare --dir playground", "playground:dev": "pnpm playground dev --dir playground", "playground:build": "pnpm playground build --dir playground", "playground:start": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./playground/.apibara/build/main.mjs" }, "devDependencies": { + "@apibara/starknet": "workspace:*", "@types/fs-extra": "^11.0.4", "@types/node": "^20.14.0", "jiti": "^1.21.0", @@ -71,14 +80,13 @@ "vitest": "^1.6.0" }, "dependencies": { - "@apibara/evm": "workspace:*", "@apibara/indexer": "workspace:*", "@apibara/protocol": "workspace:*", - "@apibara/starknet": "workspace:*", "@rollup/plugin-commonjs": "^26.0.1", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", "@rollup/plugin-typescript": "^11.1.6", + "@rollup/plugin-virtual": "^3.0.2", "c12": "^1.11.1", "chokidar": "^3.6.0", "citty": "^0.1.6", @@ -92,6 +100,7 @@ "perfect-debounce": "^1.0.0", "pkg-types": "^1.1.3", "rollup": "^4.18.1", + "rollup-plugin-esbuild": "^6.1.1", "tslib": "^2.6.3", "untyped": "^1.4.2" } diff --git a/packages/cli/playground/apibara.config.ts b/packages/cli/playground/apibara.config.ts index 284df071..7a48bd10 100644 --- a/packages/cli/playground/apibara.config.ts +++ b/packages/cli/playground/apibara.config.ts @@ -1,10 +1,15 @@ -import { defaultSink } from "@apibara/indexer"; import { defineConfig } from "apibara/config"; export default defineConfig({ runtimeConfig: { test: 123, check: "something", + nested: { + test: 456, + }, + fromEnv: { + nodeEnv: process.env.NODE_ENV, + }, }, presets: { dev: { @@ -13,7 +18,4 @@ export default defineConfig({ }, }, }, - sink: { - default: () => defaultSink(), - }, }); diff --git a/packages/cli/playground/indexers/starknet.indexer.ts b/packages/cli/playground/indexers/starknet.indexer.ts index 57ce82a0..d07b74fc 100644 --- a/packages/cli/playground/indexers/starknet.indexer.ts +++ b/packages/cli/playground/indexers/starknet.indexer.ts @@ -7,10 +7,10 @@ import { hash } from "starknet"; export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { consola.log("--> Starknet Indexer Runtime Config: ", runtimeConfig); return defineIndexer(StarknetStream)({ - streamUrl: "http://mainnet-v2.starknet.a5a.ch:7007", + streamUrl: "https://starknet.preview.apibara.org", finality: "accepted", startingCursor: { - orderKey: 80_000n, + orderKey: 800_000n, }, filter: { events: [ @@ -23,7 +23,9 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { ], }, async transform({ block: { header, events } }) { - consola.info("Transforming block ", header?.blockNumber); + consola.info( + `Got block ${header?.blockNumber} with ${events.length} events`, + ); }, hooks: { "handler:after": ({ endCursor }) => { diff --git a/packages/cli/playground/tsconfig.json b/packages/cli/playground/tsconfig.json deleted file mode 100644 index 8247ce67..00000000 --- a/packages/cli/playground/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": ["../../../tsconfig.json", "./.apibara/types/tsconfig.json"], - "compilerOptions": { - "outDir": "dist", - "declarationDir": "dist", - "noEmit": false, - "rootDir": ".", - "types": ["node"] - }, - "include": ["indexers/", "apibara.config.ts"] -} diff --git a/packages/cli/runtime-meta.d.ts b/packages/cli/runtime-meta.d.ts new file mode 100644 index 00000000..8d161dd7 --- /dev/null +++ b/packages/cli/runtime-meta.d.ts @@ -0,0 +1,2 @@ +export declare const pkgDir: string; +export declare const runtimeDir: string; diff --git a/packages/cli/runtime-meta.mjs b/packages/cli/runtime-meta.mjs new file mode 100644 index 00000000..b820745f --- /dev/null +++ b/packages/cli/runtime-meta.mjs @@ -0,0 +1,7 @@ +import { fileURLToPath } from "node:url"; + +export const pkgDir = fileURLToPath(new URL(".", import.meta.url)); + +export const runtimeDir = fileURLToPath( + new URL("dist/runtime/", import.meta.url), +); diff --git a/packages/cli/src/cli/commands/build.ts b/packages/cli/src/cli/commands/build.ts index 840d2925..2f34ba2a 100644 --- a/packages/cli/src/cli/commands/build.ts +++ b/packages/cli/src/cli/commands/build.ts @@ -1,7 +1,7 @@ import { build, createApibara, prepare, writeTypes } from "apibara/core"; +import { runtimeDir } from "apibara/runtime/meta"; import { defineCommand } from "citty"; -import consola from "consola"; -import { resolve } from "pathe"; +import { join, resolve } from "pathe"; import { commonArgs } from "../common"; export default defineCommand({ @@ -13,11 +13,14 @@ export default defineCommand({ ...commonArgs, }, async run({ args }) { - consola.start("Building"); const rootDir = resolve((args.dir || args._dir || ".") as string); const apibara = await createApibara({ rootDir, }); + apibara.logger.start("Building"); + + apibara.options.entry = join(runtimeDir, "start.mjs"); + await prepare(apibara); await writeTypes(apibara); await build(apibara); diff --git a/packages/cli/src/cli/commands/prepare.ts b/packages/cli/src/cli/commands/prepare.ts index d0c7816e..a129b618 100644 --- a/packages/cli/src/cli/commands/prepare.ts +++ b/packages/cli/src/cli/commands/prepare.ts @@ -1,7 +1,6 @@ import { createApibara, writeTypes } from "apibara/core"; import {} from "apibara/types"; import { defineCommand } from "citty"; -import consola from "consola"; import { resolve } from "pathe"; import { commonArgs } from "../common"; @@ -14,7 +13,6 @@ export default defineCommand({ ...commonArgs, }, async run({ args }) { - consola.start("Preparing Types"); const rootDir = resolve((args.dir || ".") as string); const apibara = await createApibara({ rootDir }); await writeTypes(apibara); diff --git a/packages/cli/src/config/index.ts b/packages/cli/src/config/index.ts index 8ff321d8..552d64cd 100644 --- a/packages/cli/src/config/index.ts +++ b/packages/cli/src/config/index.ts @@ -1,10 +1,11 @@ import type { ApibaraConfig, DeepPartial } from "apibara/types"; export function defineConfig< - // biome-ignore lint/complexity/noBannedTypes: - T extends Record>> = {}, - // biome-ignore lint/complexity/noBannedTypes: - R extends Record = {}, + T extends Record>> = Record< + string, + never + >, + R extends Record = Record, >(config: ApibaraConfig): ApibaraConfig { return config; } diff --git a/packages/cli/src/core/apibara.ts b/packages/cli/src/core/apibara.ts index 664c6ff9..c6358552 100644 --- a/packages/cli/src/core/apibara.ts +++ b/packages/cli/src/core/apibara.ts @@ -8,6 +8,7 @@ import consola from "consola"; import { createHooks } from "hookable"; import { loadOptions } from "./config/loader"; import { updateApibaraConfig } from "./config/update"; +import { scanIndexers } from "./scan"; export async function createApibara( config: ApibaraConfig = {}, @@ -20,6 +21,7 @@ export async function createApibara( // create apibara context const apibara: Apibara = { options, + indexers: [], hooks: createHooks(), close: () => apibara.hooks.callHook("close"), logger: consola.withTag("apibara"), @@ -30,5 +32,7 @@ export async function createApibara( apibara.hooks.addHooks(apibara.options.hooks); + await scanIndexers(apibara); + return apibara; } diff --git a/packages/cli/src/core/build/build.ts b/packages/cli/src/core/build/build.ts index a9cfed39..ffcf579f 100644 --- a/packages/cli/src/core/build/build.ts +++ b/packages/cli/src/core/build/build.ts @@ -5,7 +5,9 @@ import { buildProduction } from "./prod"; export async function build(apibara: Apibara) { const rollupConfig = getRollupConfig(apibara); + await apibara.hooks.callHook("rollup:before", apibara, rollupConfig); + return apibara.options.dev ? await watchDev(apibara, rollupConfig) : await buildProduction(apibara, rollupConfig); diff --git a/packages/cli/src/core/build/prepare.ts b/packages/cli/src/core/build/prepare.ts index 3f34de79..4e11c21d 100644 --- a/packages/cli/src/core/build/prepare.ts +++ b/packages/cli/src/core/build/prepare.ts @@ -1,12 +1,12 @@ import fsp from "node:fs/promises"; import type { Apibara } from "apibara/types"; -import consola from "consola"; import fse from "fs-extra"; export async function prepare(apibara: Apibara) { await prepareDir(apibara.options.buildDir); await prepareDir(apibara.options.outputDir); - consola.success("Output directory cleaned"); + + apibara.logger.success("Output directory cleaned"); } async function prepareDir(dir: string) { diff --git a/packages/cli/src/core/build/prod.ts b/packages/cli/src/core/build/prod.ts index 2b051151..0a67de74 100644 --- a/packages/cli/src/core/build/prod.ts +++ b/packages/cli/src/core/build/prod.ts @@ -1,16 +1,14 @@ import type { Apibara, RollupConfig } from "apibara/types"; -import consola from "consola"; import { type OutputOptions, rollup } from "rollup"; export async function buildProduction( apibara: Apibara, rollupConfig: RollupConfig, ) { + apibara.logger.start("Building indexers"); try { - // Create a bundle const bundle = await rollup(rollupConfig); - // Generate output if (Array.isArray(rollupConfig.output)) { for (const outputOptions of rollupConfig.output) { await bundle.write(outputOptions); @@ -21,12 +19,11 @@ export async function buildProduction( throw new Error("No output options specified in Rollup config"); } - // Close the bundle await bundle.close(); - consola.success("Build completed successfully!"); + apibara.logger.success("Build succeeded!"); } catch (error) { - console.error("Build failed:", error); + apibara.logger.error("Build failed", error); throw error; } } diff --git a/packages/cli/src/core/build/types.ts b/packages/cli/src/core/build/types.ts index 979d9638..cba6c1d8 100644 --- a/packages/cli/src/core/build/types.ts +++ b/packages/cli/src/core/build/types.ts @@ -1,12 +1,12 @@ import fsp from "node:fs/promises"; import type { Apibara } from "apibara/types"; -import consola from "consola"; import defu from "defu"; import { dirname, isAbsolute, join, relative, resolve } from "pathe"; import type { TSConfig } from "pkg-types"; import { type JSValue, generateTypes, resolveSchema } from "untyped"; export async function writeTypes(apibara: Apibara) { + apibara.logger.start("Preparing Types"); const typesDir = resolve(apibara.options.buildDir, "types"); const config = [ @@ -131,7 +131,7 @@ declare module "apibara/types" {`, }), ); - consola.success("Types generated"); + apibara.logger.success("Types generated"); } const RELATIVE_RE = /^\.{1,2}\//; diff --git a/packages/cli/src/core/config/defaults.ts b/packages/cli/src/core/config/defaults.ts index 0c421c8c..a5c32ce9 100644 --- a/packages/cli/src/core/config/defaults.ts +++ b/packages/cli/src/core/config/defaults.ts @@ -1,8 +1,8 @@ -// import { defaultSink } from "@apibara/indexer"; import type { ApibaraConfig } from "apibara/types"; export const ApibaraDefaults: ApibaraConfig = { rootDir: ".", + indexersDir: "indexers", runtimeConfig: {}, hooks: {}, diff --git a/packages/cli/src/core/config/loader.ts b/packages/cli/src/core/config/loader.ts index ee3d285b..09ea9eb6 100644 --- a/packages/cli/src/core/config/loader.ts +++ b/packages/cli/src/core/config/loader.ts @@ -53,6 +53,7 @@ async function _loadUserConfig( options._config = configOverrides; options._c12 = loadedConfig; + if (dev) { options.dev = dev; } diff --git a/packages/cli/src/core/config/resolvers/runtime-config.resolver.ts b/packages/cli/src/core/config/resolvers/runtime-config.resolver.ts index 45d7a579..960647a3 100644 --- a/packages/cli/src/core/config/resolvers/runtime-config.resolver.ts +++ b/packages/cli/src/core/config/resolvers/runtime-config.resolver.ts @@ -1,6 +1,6 @@ import type { ApibaraOptions } from "apibara/types"; export async function resolveRuntimeConfigOptions(options: ApibaraOptions) { - options.runtimeConfig = { ...options.runtimeConfig, default: "value" }; + options.runtimeConfig = { ...options.runtimeConfig }; process.env.APIBARA_RUNTIME_CONFIG = JSON.stringify(options.runtimeConfig); } diff --git a/packages/cli/src/core/config/update.ts b/packages/cli/src/core/config/update.ts index 54d70a27..52127e9e 100644 --- a/packages/cli/src/core/config/update.ts +++ b/packages/cli/src/core/config/update.ts @@ -1,10 +1,9 @@ import type { Apibara, ApibaraDynamicConfig } from "apibara/types"; -import consola from "consola"; export async function updateApibaraConfig( apibara: Apibara, - config: ApibaraDynamicConfig, + _config: ApibaraDynamicConfig, ) { await apibara.hooks.callHook("rollup:reload"); - consola.success("Apibara config hot reloaded!"); + apibara.logger.success("Apibara config hot reloaded!"); } diff --git a/packages/cli/src/core/scan.ts b/packages/cli/src/core/scan.ts new file mode 100644 index 00000000..4cd22492 --- /dev/null +++ b/packages/cli/src/core/scan.ts @@ -0,0 +1,40 @@ +import type { Apibara } from "apibara/types"; +import { existsSync, readdirSync } from "fs-extra"; +import { basename, join } from "pathe"; + +const INDEXER_EXTENSIONS = [".indexer.ts", ".indexer.js"]; + +export async function scanIndexers(apibara: Apibara) { + apibara.logger.debug("Scanning indexers"); + + const indexersDir = join( + apibara.options.rootDir, + apibara.options.indexersDir, + ); + + if (!existsSync(indexersDir)) { + throw new Error(`Indexers directory not found: ${indexersDir}`); + } + + apibara.indexers = []; + + for (const file of readdirSync(indexersDir)) { + const indexerName = indexerNameFromFile(file); + if (indexerName) { + apibara.indexers.push({ + name: indexerName, + indexer: join(indexersDir, file), + }); + } + } + + apibara.logger.debug(`Found ${apibara.indexers.length} indexers`); +} + +function indexerNameFromFile(file: string) { + for (const extension of INDEXER_EXTENSIONS) { + if (file.endsWith(extension)) { + return basename(file, extension); + } + } +} diff --git a/packages/cli/src/internal/citty/index.ts b/packages/cli/src/internal/citty/index.ts deleted file mode 100644 index aacf5a43..00000000 --- a/packages/cli/src/internal/citty/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "citty"; diff --git a/packages/cli/src/internal/consola/index.ts b/packages/cli/src/internal/consola/index.ts deleted file mode 100644 index e0c4d21e..00000000 --- a/packages/cli/src/internal/consola/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "consola"; diff --git a/packages/cli/src/rollup/config.ts b/packages/cli/src/rollup/config.ts index f821ee9d..06cd82fc 100644 --- a/packages/cli/src/rollup/config.ts +++ b/packages/cli/src/rollup/config.ts @@ -1,17 +1,16 @@ -import { existsSync } from "node:fs"; import { builtinModules } from "node:module"; import commonjs from "@rollup/plugin-commonjs"; import json from "@rollup/plugin-json"; import { nodeResolve } from "@rollup/plugin-node-resolve"; -import typescript from "@rollup/plugin-typescript"; import type { Apibara, RollupConfig } from "apibara/types"; -import fsExtra from "fs-extra"; -import { basename, join } from "pathe"; +import { join } from "pathe"; +import type { OutputPluginOption } from "rollup"; +import esbuild from "rollup-plugin-esbuild"; -export const getRollupConfig = ( - apibara: Apibara, - dev = false, -): RollupConfig => { +import { appConfig } from "./plugins/config"; +import { indexers } from "./plugins/indexers"; + +export function getRollupConfig(apibara: Apibara): RollupConfig { const extensions: string[] = [ ".ts", ".mjs", @@ -22,124 +21,8 @@ export const getRollupConfig = ( ".jsx", ]; - const indexerDir = join(apibara.options.rootDir, "indexers"); - const configPath = join(apibara.options.rootDir, "./apibara.config.ts"); - - // Check if the indexers directory and config file exist - if (!existsSync(indexerDir)) { - throw new Error(`Indexers directory not found: ${indexerDir}`); - } - if (!existsSync(configPath)) { - throw new Error(`Config file not found: ${configPath}`); - } - - // Check if the indexers directory exists and is not empty - let indexerFiles: string[] = []; - try { - indexerFiles = fsExtra - .readdirSync(indexerDir) - .filter((file) => file.endsWith(".indexer.ts")); - if (indexerFiles.length === 0) { - console.warn(`No indexer files found in ${indexerDir}`); - } - } catch (error) { - console.error(`Error reading indexers directory: ${error}`); - } - - const indexerImports = indexerFiles - .map( - (file, index) => - `import indexer${index} from '${join(indexerDir, file)}';`, - ) - .join("\n"); - - // Generate main.ts content - const mainContent = ` -import { createClient } from "@apibara/protocol"; -import { createIndexer, run } from "@apibara/indexer"; -import { consola as _consola } from "apibara/internal/consola"; -import { defineCommand, runMain } from "apibara/internal/citty"; -import config from './${configPath}'; - -const consola = _consola.withTag("Apibara | "); - -${indexerImports} - -const indexers = { - ${indexerFiles - .map( - (file, index) => `'${basename(file, ".indexer.ts")}': indexer${index},`, - ) - .join("\n ")} -}; - -const command = defineCommand({ - meta: { - name: "run-indexers", - description: "Run Apibara indexers", - }, - args: { - indexers: { - type: "string", - description: "Comma-separated list of indexers to run", - }, - preset: { - type: "string", - description: "Preset to use", - }, - sink: { - type: "string", - description: "Sink to use", - }, - }, - async run({ args }) { - const selectedIndexers = args.indexers ? args.indexers.split(',') : Object.keys(indexers); - const preset = args.preset || config.preset || 'default'; - const sinkName = args.sink || 'default'; - - // Apply preset - let runtimeConfig = { ...config.runtimeConfig }; - if (preset && config.presets && config.presets[preset]) { - runtimeConfig = { ...runtimeConfig, ...config.presets[preset].runtimeConfig }; - } - - // Get sink function - const sinkFunction = config.sink?.[sinkName]; - if (!sinkFunction) { - throw new Error(\`Sink \${sinkName} not found\`); - } - - await Promise.all(selectedIndexers.map(async (name) => { - if (!indexers[name]) { - console.error(\`Indexer \${name} not found\`); - return; - } - - const indexerConfig = indexers[name] - const indexer = typeof indexerConfig === 'function' - ? await createIndexer(indexerConfig(runtimeConfig)) - : createIndexer(indexerConfig); - - const client = createClient(indexer.streamConfig, indexer.options.streamUrl); - const sink = sinkFunction(); - - try { - consola.log("Running Indexer: ", name); - await run(client, indexer, sink); - } catch (error) { - consola.error(\`Error in indexer \${name}:\`, error); - } - })); - }, -}); - -runMain(command); -`; - - return { - input: { - main: "virtual:main.ts", - }, + const rollupConfig: RollupConfig & { plugins: OutputPluginOption[] } = { + input: apibara.options.entry, output: { dir: join(apibara.options.outputDir || "./.apibara/build"), format: "esm", @@ -155,41 +38,12 @@ runMain(command); return relativePath.includes("node_modules"); }, }, - plugins: [ - { - name: "virtual", - resolveId(id) { - if (id === "virtual:main.ts") { - return id; - } - return null; - }, - load(id) { - if (id === "virtual:main.ts") { - return mainContent; - } - return null; - }, - }, - nodeResolve({ - extensions, - preferBuiltins: true, - }), - commonjs(), - json(), - typescript({ - tsconfig: join(apibara.options.rootDir, "./tsconfig.json"), - compilerOptions: { - outDir: join(apibara.options.outputDir || "./.apibara/build"), - declarationDir: join(apibara.options.outputDir || "./.apibara/build"), - noEmit: false, - types: ["node"], - }, - }), - ], + plugins: [], onwarn(warning, rollupWarn) { if ( - !["CIRCULAR_DEPENDENCY", "EVAL"].includes(warning.code || "") && + !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes( + warning.code || "", + ) && !warning.message.includes("Unsupported source map comment") && !warning.message.includes("@__PURE__") ) { @@ -197,13 +51,29 @@ runMain(command); } }, treeshake: true, - external: [ - ...builtinModules, - "@apibara/indexer", - "@apibara/protocol", - "@apibara/evm", - "@apibara/starknet", - "@apibara/beaconchain", - ], + external: [...builtinModules], }; -}; + + rollupConfig.plugins.push(commonjs()); + rollupConfig.plugins.push(json()); + + rollupConfig.plugins.push( + nodeResolve({ + extensions, + preferBuiltins: true, + mainFields: ["main"], + }), + ); + + rollupConfig.plugins.push( + esbuild({ + target: "es2022", + sourceMap: apibara.options.sourceMap, + }), + ); + + rollupConfig.plugins.push(indexers(apibara)); + rollupConfig.plugins.push(appConfig(apibara)); + + return rollupConfig; +} diff --git a/packages/cli/src/rollup/plugins/config.ts b/packages/cli/src/rollup/plugins/config.ts new file mode 100644 index 00000000..6601532f --- /dev/null +++ b/packages/cli/src/rollup/plugins/config.ts @@ -0,0 +1,10 @@ +import virtual from "@rollup/plugin-virtual"; +import type { Apibara } from "apibara/types"; + +export function appConfig(apibara: Apibara) { + return virtual({ + "#apibara-internal-virtual/config": ` + export const config = ${JSON.stringify(apibara.options, null, 2)}; + `, + }); +} diff --git a/packages/cli/src/rollup/plugins/indexers.ts b/packages/cli/src/rollup/plugins/indexers.ts new file mode 100644 index 00000000..047f86e8 --- /dev/null +++ b/packages/cli/src/rollup/plugins/indexers.ts @@ -0,0 +1,16 @@ +import virtual from "@rollup/plugin-virtual"; +import type { Apibara } from "apibara/types"; + +export function indexers(apibara: Apibara) { + const indexers = [...new Set(apibara.indexers)]; + + return virtual({ + "#apibara-internal-virtual/indexers": ` + ${indexers.map((i) => `import ${i.name} from '${i.indexer}';`).join("\n")} + + export const indexers = [ + ${indexers.map((i) => `{ name: "${i.name}", indexer: ${i.name} }`).join(",\n")} + ]; + `, + }); +} diff --git a/packages/cli/src/runtime/dev.ts b/packages/cli/src/runtime/dev.ts new file mode 100644 index 00000000..0128e1bb --- /dev/null +++ b/packages/cli/src/runtime/dev.ts @@ -0,0 +1,2 @@ +console.log("Inside DEV"); +export default {}; diff --git a/packages/cli/src/runtime/index.ts b/packages/cli/src/runtime/index.ts new file mode 100644 index 00000000..f06163a0 --- /dev/null +++ b/packages/cli/src/runtime/index.ts @@ -0,0 +1 @@ +export { createIndexer } from "./internal/app"; diff --git a/packages/cli/src/runtime/internal/app.ts b/packages/cli/src/runtime/internal/app.ts new file mode 100644 index 00000000..e468f064 --- /dev/null +++ b/packages/cli/src/runtime/internal/app.ts @@ -0,0 +1,37 @@ +import { createIndexer as _createIndexer } from "@apibara/indexer"; + +import { config } from "#apibara-internal-virtual/config"; +import { indexers } from "#apibara-internal-virtual/indexers"; + +export function createIndexer(indexerName: string, preset?: string) { + let runtimeConfig: Record = { ...config.runtimeConfig }; + + if (preset) { + if (config.presets === undefined) { + throw new Error( + `Specified preset "${preset}" but no presets were defined`, + ); + } + + if (config.presets[preset] === undefined) { + throw new Error(`Specified preset "${preset}" but it was not defined`); + } + + const presetValue = config.presets[preset] as { + runtimeConfig: Record; + }; + runtimeConfig = { ...runtimeConfig, ...presetValue.runtimeConfig }; + } + + const indexerDefinition = indexers.find((i) => i.name === indexerName); + + if (indexerDefinition === undefined) { + throw new Error( + `Specified indexer "${indexerName}" but it was not defined`, + ); + } + + return typeof indexerDefinition.indexer === "function" + ? _createIndexer(indexerDefinition.indexer(runtimeConfig)) + : _createIndexer(indexerDefinition.indexer); +} diff --git a/packages/cli/src/runtime/start.ts b/packages/cli/src/runtime/start.ts new file mode 100644 index 00000000..c446fed3 --- /dev/null +++ b/packages/cli/src/runtime/start.ts @@ -0,0 +1,48 @@ +import { run } from "@apibara/indexer"; +import { createClient } from "@apibara/protocol"; +import { defineCommand, runMain } from "citty"; +import { createIndexer } from "./internal/app"; + +const startCommand = defineCommand({ + meta: { + name: "start", + description: "Start the indexer", + }, + args: { + indexer: { + type: "string", + description: "Indexer name", + required: true, + }, + preset: { + type: "string", + description: "Preset to use", + }, + }, + async run({ args }) { + const { indexer, preset } = args; + + const indexerInstance = createIndexer(indexer, preset); + + const client = createClient( + indexerInstance.streamConfig, + indexerInstance.options.streamUrl, + ); + + await run(client, indexerInstance); + }, +}); + +export const mainCli = defineCommand({ + meta: { + name: "indexer-runner", + description: "Run an indexer", + }, + subCommands: { + start: () => startCommand, + }, +}); + +runMain(mainCli); + +export default {}; diff --git a/packages/cli/src/types/apibara.ts b/packages/cli/src/types/apibara.ts index 3dbe087f..a805d2fb 100644 --- a/packages/cli/src/types/apibara.ts +++ b/packages/cli/src/types/apibara.ts @@ -3,9 +3,17 @@ import type { Hookable } from "hookable"; import type { ApibaraDynamicConfig, ApibaraOptions } from "./config"; import type { ApibaraHooks } from "./hooks"; +export type IndexerDefinition = { + // Name of the indexer. + name: string; + // Path to the indexer file. + indexer: string; +}; + export interface Apibara { options: ApibaraOptions; hooks: Hookable; + indexers: IndexerDefinition[]; logger: ConsolaInstance; close: () => Promise; updateConfig: (config: ApibaraDynamicConfig) => void | Promise; diff --git a/packages/cli/src/types/config.ts b/packages/cli/src/types/config.ts index 7910c18c..224fb3d3 100644 --- a/packages/cli/src/types/config.ts +++ b/packages/cli/src/types/config.ts @@ -1,4 +1,3 @@ -import type { Sink } from "@apibara/indexer"; import type { C12InputConfig, ConfigWatcher, @@ -16,16 +15,13 @@ import type { RollupConfig } from "./rollup"; * Apibara Config type (apibara.config) */ export interface ApibaraConfig< - // biome-ignore lint/complexity/noBannedTypes: - T extends Record>> = {}, - // biome-ignore lint/complexity/noBannedTypes: - R extends Record = {}, + T extends Record>> = Record< + string, + never + >, + R extends Record = Record, > extends DeepPartial, "preset" | "presets" | "dev">>, C12InputConfig> { - sink?: { - default: () => Sink; - [key: string]: () => Sink; - }; runtimeConfig?: R; presets?: T; preset?: keyof T; @@ -42,10 +38,11 @@ export interface LoadConfigOptions { } export interface ApibaraOptions< - // biome-ignore lint/complexity/noBannedTypes: - T extends Record>> = {}, - // biome-ignore lint/complexity/noBannedTypes: - R extends Record = {}, + T extends Record>> = Record< + string, + never + >, + R extends Record = Record, > { // Internal _config: ApibaraConfig; @@ -53,12 +50,6 @@ export interface ApibaraOptions< | ResolvedConfig> | ConfigWatcher>; - // Sink - sink: { - default: () => Sink; - [key: string]: () => Sink; - }; - // Presets presets?: T; preset?: keyof T; @@ -69,16 +60,19 @@ export interface ApibaraOptions< rootDir: string; buildDir: string; outputDir: string; + indexersDir: string; + // Dev dev: boolean; watchOptions: WatchOptions; // Hooks hooks: NestedHooks; + // Rollup rollupConfig?: RollupConfig; + sourceMap?: boolean; entry: string; - minify: boolean; // Advanced typescript: { diff --git a/packages/cli/src/types/virtual/config.d.ts b/packages/cli/src/types/virtual/config.d.ts new file mode 100644 index 00000000..f9f5016c --- /dev/null +++ b/packages/cli/src/types/virtual/config.d.ts @@ -0,0 +1,3 @@ +import type { ApibaraConfig } from "apibara/types"; + +export const config: ApibaraConfig = {}; diff --git a/packages/cli/src/types/virtual/indexers.d.ts b/packages/cli/src/types/virtual/indexers.d.ts new file mode 100644 index 00000000..c9283b5b --- /dev/null +++ b/packages/cli/src/types/virtual/indexers.d.ts @@ -0,0 +1,10 @@ +import type { IndexerWithStreamConfig } from "@apibara/indexer"; +import type { ApibaraRuntimeConfig } from "apibara/types"; + +export type IndexerConstructor = + | (( + runtimeConfig: ApibaraRuntimeConfig, + ) => IndexerWithStreamConfig) + | IndexerWithStreamConfig; + +export const indexers: { name: string; indexer: IndexerConstructor }[] = []; diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 3db8de70..8a34b40e 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -13,8 +13,8 @@ "apibara/rollup": ["./src/rollup"], "apibara/types": ["./src/types"], "apibara/hooks": ["./src/hooks"], - "apibara/internal/consola": ["./src/internal/consola"], - "apibara/internal/citty": ["./src/internal/citty"] + "apibara/runtime/meta": ["./runtime-meta"], + "#apibara-internal-virtual/*": ["./src/types/virtual/*"] } }, "include": ["src/"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e34ab0c..3e5f37d1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -288,18 +288,12 @@ importers: packages/cli: dependencies: - '@apibara/evm': - specifier: workspace:* - version: link:../evm '@apibara/indexer': specifier: workspace:* version: link:../indexer '@apibara/protocol': specifier: workspace:* version: link:../protocol - '@apibara/starknet': - specifier: workspace:* - version: link:../starknet '@rollup/plugin-commonjs': specifier: ^26.0.1 version: 26.0.1(rollup@4.18.1) @@ -312,6 +306,9 @@ importers: '@rollup/plugin-typescript': specifier: ^11.1.6 version: 11.1.6(rollup@4.18.1)(tslib@2.6.3)(typescript@5.6.2) + '@rollup/plugin-virtual': + specifier: ^3.0.2 + version: 3.0.2(rollup@4.18.1) c12: specifier: ^1.11.1 version: 1.11.1 @@ -351,6 +348,9 @@ importers: rollup: specifier: ^4.18.1 version: 4.18.1 + rollup-plugin-esbuild: + specifier: ^6.1.1 + version: 6.1.1(esbuild@0.23.0)(rollup@4.18.1) tslib: specifier: ^2.6.3 version: 2.6.3 @@ -358,6 +358,9 @@ importers: specifier: ^1.4.2 version: 1.4.2 devDependencies: + '@apibara/starknet': + specifier: workspace:* + version: link:../starknet '@types/fs-extra': specifier: ^11.0.4 version: 11.0.4 @@ -1516,6 +1519,15 @@ packages: tslib: optional: true + '@rollup/plugin-virtual@3.0.2': + resolution: {integrity: sha512-10monEYsBp3scM4/ND4LNH5Rxvh3e/cVeL3jWTgZ2SrQ+BmUoQcopVQvnaMcOnykb1VkxUFuDAN+0FnpTFRy2A==} + engines: {node: '>=14.0.0'} + peerDependencies: + rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0 + peerDependenciesMeta: + rollup: + optional: true + '@rollup/pluginutils@5.1.0': resolution: {integrity: sha512-XTIWOPPcpvyKI6L1NHo0lFlCyznUEyPmPY1mc3KpPVDYulHSTvyeLNVW00QTLIAFNhR3kYnJTQHeGqU4M3n09g==} engines: {node: '>=14.0.0'} @@ -2148,6 +2160,9 @@ packages: error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + es-module-lexer@1.5.4: + resolution: {integrity: sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==} + esbuild@0.19.11: resolution: {integrity: sha512-HJ96Hev2hX/6i5cDVwcqiJBBtuo9+FeIJOtZ9W1kA5M6AMJRHUZlpYZ1/SbEwtO0ioNAW8rUooVpC/WehY2SfA==} engines: {node: '>=12'} @@ -2274,6 +2289,9 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} + get-tsconfig@4.8.1: + resolution: {integrity: sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==} + giget@1.2.3: resolution: {integrity: sha512-8EHPljDvs7qKykr6uw8b+lqLiUc/vUg+KVTI0uND4s63TdsZM2Xus3mflvF0DDG9SiM4RlCkFGL+7aAjRmV7KA==} hasBin: true @@ -3104,6 +3122,9 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@1.22.8: resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} hasBin: true @@ -3119,6 +3140,13 @@ packages: rollup: ^3.29.4 || ^4 typescript: ^4.5 || ^5.0 + rollup-plugin-esbuild@6.1.1: + resolution: {integrity: sha512-CehMY9FAqJD5OUaE/Mi1r5z0kNeYxItmRO2zG4Qnv2qWKF09J2lTy5GUzjJR354ZPrLkCj4fiBN41lo8PzBUhw==} + engines: {node: '>=14.18.0'} + peerDependencies: + esbuild: '>=0.18.0' + rollup: ^1.20.0 || ^2.0.0 || ^3.0.0 || ^4.0.0 + rollup@3.29.4: resolution: {integrity: sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw==} engines: {node: '>=14.18.0', npm: '>=8.0.0'} @@ -4365,6 +4393,10 @@ snapshots: rollup: 4.18.1 tslib: 2.6.3 + '@rollup/plugin-virtual@3.0.2(rollup@4.18.1)': + optionalDependencies: + rollup: 4.18.1 + '@rollup/pluginutils@5.1.0(rollup@3.29.4)': dependencies: '@types/estree': 1.0.5 @@ -4931,6 +4963,8 @@ snapshots: dependencies: is-arrayish: 0.2.1 + es-module-lexer@1.5.4: {} + esbuild@0.19.11: optionalDependencies: '@esbuild/aix-ppc64': 0.19.11 @@ -5123,6 +5157,10 @@ snapshots: get-stream@8.0.1: {} + get-tsconfig@4.8.1: + dependencies: + resolve-pkg-maps: 1.0.0 + giget@1.2.3: dependencies: citty: 0.1.6 @@ -5935,6 +5973,8 @@ snapshots: resolve-from@4.0.0: {} + resolve-pkg-maps@1.0.0: {} + resolve@1.22.8: dependencies: is-core-module: 2.13.1 @@ -5951,6 +5991,17 @@ snapshots: optionalDependencies: '@babel/code-frame': 7.24.6 + rollup-plugin-esbuild@6.1.1(esbuild@0.23.0)(rollup@4.18.1): + dependencies: + '@rollup/pluginutils': 5.1.0(rollup@4.18.1) + debug: 4.3.4 + es-module-lexer: 1.5.4 + esbuild: 0.23.0 + get-tsconfig: 4.8.1 + rollup: 4.18.1 + transitivePeerDependencies: + - supports-color + rollup@3.29.4: optionalDependencies: fsevents: 2.3.3 From 2c7a5197b47e08a695184b91a6423071d24b6b88 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Fri, 1 Nov 2024 16:36:28 +0100 Subject: [PATCH 02/13] cli: add start command to build and start production indexer --- packages/cli/src/cli/commands/start.ts | 54 ++++++++++++++++++++++++++ packages/cli/src/cli/index.ts | 1 + 2 files changed, 55 insertions(+) create mode 100644 packages/cli/src/cli/commands/start.ts diff --git a/packages/cli/src/cli/commands/start.ts b/packages/cli/src/cli/commands/start.ts new file mode 100644 index 00000000..107b4336 --- /dev/null +++ b/packages/cli/src/cli/commands/start.ts @@ -0,0 +1,54 @@ +import { spawn } from "node:child_process"; +import { build, createApibara, prepare, writeTypes } from "apibara/core"; +import { runtimeDir } from "apibara/runtime/meta"; +import { defineCommand } from "citty"; +import { join, resolve } from "pathe"; +import { commonArgs } from "../common"; + +export default defineCommand({ + meta: { + name: "start", + description: "Start one indexer", + }, + args: { + ...commonArgs, + indexer: { + type: "string", + description: "The indexer to start", + required: true, + }, + preset: { + type: "string", + description: "The preset to use", + }, + }, + async run({ args }) { + const rootDir = resolve((args.dir || args._dir || ".") as string); + const apibara = await createApibara({ + rootDir, + }); + + apibara.logger.start("Building"); + + apibara.options.entry = join(runtimeDir, "start.mjs"); + + await prepare(apibara); + await writeTypes(apibara); + await build(apibara); + await apibara.close(); + + const { indexer, preset } = args; + + const childArgs = [ + resolve(apibara.options.outputDir || "./.apibara/build", "start.mjs"), + "start", + "--indexer", + indexer, + ...(preset ? ["--preset", preset] : []), + ]; + + spawn("node", childArgs, { + stdio: "inherit", + }); + }, +}); diff --git a/packages/cli/src/cli/index.ts b/packages/cli/src/cli/index.ts index c370cc15..92991954 100755 --- a/packages/cli/src/cli/index.ts +++ b/packages/cli/src/cli/index.ts @@ -9,6 +9,7 @@ export const mainCli = defineCommand({ subCommands: { dev: () => import("./commands/dev").then((r) => r.default), build: () => import("./commands/build").then((r) => r.default), + start: () => import("./commands/start").then((r) => r.default), prepare: () => import("./commands/prepare").then((r) => r.default), }, }); From b46b3cedf63b5b91955c9b77de65bec478fc1bfa Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Fri, 1 Nov 2024 16:48:25 +0100 Subject: [PATCH 03/13] cli: update dev command --- .../playground/indexers/starknet.indexer.ts | 5 -- packages/cli/src/cli/commands/dev.ts | 32 ++++++---- packages/cli/src/runtime/dev.ts | 64 ++++++++++++++++++- packages/cli/src/runtime/internal/app.ts | 2 + 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/packages/cli/playground/indexers/starknet.indexer.ts b/packages/cli/playground/indexers/starknet.indexer.ts index d07b74fc..a65f34b8 100644 --- a/packages/cli/playground/indexers/starknet.indexer.ts +++ b/packages/cli/playground/indexers/starknet.indexer.ts @@ -27,10 +27,5 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) { `Got block ${header?.blockNumber} with ${events.length} events`, ); }, - hooks: { - "handler:after": ({ endCursor }) => { - consola.info("Handler After ", endCursor?.orderKey); - }, - }, }); } diff --git a/packages/cli/src/cli/commands/dev.ts b/packages/cli/src/cli/commands/dev.ts index eeea73bc..a9ca46e6 100644 --- a/packages/cli/src/cli/commands/dev.ts +++ b/packages/cli/src/cli/commands/dev.ts @@ -1,17 +1,15 @@ import { type ChildProcess, spawn } from "node:child_process"; import { build, createApibara, prepare, writeTypes } from "apibara/core"; +import { runtimeDir } from "apibara/runtime/meta"; import type { Apibara } from "apibara/types"; import { defineCommand } from "citty"; -import consola from "consola"; -import { resolve } from "pathe"; +import { join, resolve } from "pathe"; import { commonArgs } from "../common"; // Hot module reloading key regex // for only runtimeConfig.* keys const hmrKeyRe = /^runtimeConfig\./; -let childProcess: ChildProcess | undefined; - export default defineCommand({ meta: { name: "dev", @@ -33,18 +31,21 @@ export default defineCommand({ }, }, async run({ args }) { - consola.start("Starting dev server"); const rootDir = resolve((args.dir || args._dir || ".") as string); + let apibara: Apibara; + let childProcess: ChildProcess | undefined; const reload = async () => { if (apibara) { - consola.info("Restarting dev server"); + apibara.logger.info("Restarting dev server"); if ("unwatch" in apibara.options._c12) { await apibara.options._c12.unwatch(); } + await apibara.close(); } + apibara = await createApibara( { rootDir, @@ -59,10 +60,11 @@ export default defineCommand({ return; // No changes } - consola.info( - `Nitro config updated: + apibara.logger.info( + `Config updated: ${diff.map((entry) => ` ${entry.toString()}`).join("\n")}`, ); + await (diff.every((e) => hmrKeyRe.test(e.key)) ? apibara.updateConfig(newConfig.config || {}) // Hot reload : reload()); // Full reload @@ -71,23 +73,27 @@ export default defineCommand({ }, true, ); + apibara.hooks.hookOnce("restart", reload); + apibara.options.entry = join(runtimeDir, "dev.mjs"); + await prepare(apibara); await writeTypes(apibara); await build(apibara); apibara.hooks.hook("dev:reload", () => { if (childProcess) { - consola.start("Restarting indexers"); + apibara.logger.start("Restarting indexers"); childProcess.kill(); } else { - consola.success("Dev server started"); - consola.success("Starting indexers"); + apibara.logger.success("Dev server started"); + apibara.logger.success("Starting indexers"); } const childArgs = [ - resolve(apibara.options.outputDir || "./.apibara/build", "main.mjs"), + resolve(apibara.options.outputDir || "./.apibara/build", "dev.mjs"), + "start", ...(args.indexers ? ["--indexers", args.indexers] : []), ...(args.preset ? ["--preset", args.preset] : []), ...(args.sink ? ["--sink", args.sink] : []), @@ -99,7 +105,7 @@ export default defineCommand({ childProcess.on("close", (code) => { if (code !== null) { - consola.log(`Indexers process exited with code ${code}`); + apibara.logger.log(`Indexers process exited with code ${code}`); } }); }); diff --git a/packages/cli/src/runtime/dev.ts b/packages/cli/src/runtime/dev.ts index 0128e1bb..01a937dd 100644 --- a/packages/cli/src/runtime/dev.ts +++ b/packages/cli/src/runtime/dev.ts @@ -1,2 +1,64 @@ -console.log("Inside DEV"); +import { run } from "@apibara/indexer"; +import { createClient } from "@apibara/protocol"; +import { defineCommand, runMain } from "citty"; +import { availableIndexers, createIndexer } from "./internal/app"; + +const startCommand = defineCommand({ + meta: { + name: "start", + description: "Start the indexer", + }, + args: { + indexers: { + type: "string", + description: "Which indexers to run", + }, + preset: { + type: "string", + description: "Preset to use", + }, + }, + async run({ args }) { + const { indexers: indexersArgs, preset } = args; + + let selectedIndexers = availableIndexers; + if (indexersArgs) { + selectedIndexers = indexersArgs.split(","); + } + + for (const indexer of selectedIndexers) { + if (!availableIndexers.includes(indexer)) { + throw new Error( + `Specified indexer "${indexer}" but it was not defined`, + ); + } + } + + await Promise.all( + selectedIndexers.map(async (indexer) => { + const indexerInstance = createIndexer(indexer, preset); + + const client = createClient( + indexerInstance.streamConfig, + indexerInstance.options.streamUrl, + ); + + await run(client, indexerInstance); + }), + ); + }, +}); + +export const mainCli = defineCommand({ + meta: { + name: "indexer-dev-runner", + description: "Run indexer in dev mode", + }, + subCommands: { + start: () => startCommand, + }, +}); + +runMain(mainCli); + export default {}; diff --git a/packages/cli/src/runtime/internal/app.ts b/packages/cli/src/runtime/internal/app.ts index e468f064..dd841d40 100644 --- a/packages/cli/src/runtime/internal/app.ts +++ b/packages/cli/src/runtime/internal/app.ts @@ -3,6 +3,8 @@ import { createIndexer as _createIndexer } from "@apibara/indexer"; import { config } from "#apibara-internal-virtual/config"; import { indexers } from "#apibara-internal-virtual/indexers"; +export const availableIndexers = indexers.map((i) => i.name); + export function createIndexer(indexerName: string, preset?: string) { let runtimeConfig: Record = { ...config.runtimeConfig }; From 7c386299e696387c38d1f3ce1469487b92dd8eda Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Fri, 1 Nov 2024 17:04:14 +0100 Subject: [PATCH 04/13] cli: stop indexers on dev rebuild --- packages/cli/src/cli/commands/dev.ts | 12 +++++++++--- packages/cli/src/core/build/dev.ts | 1 + packages/cli/src/types/hooks.ts | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/cli/commands/dev.ts b/packages/cli/src/cli/commands/dev.ts index a9ca46e6..83de37e8 100644 --- a/packages/cli/src/cli/commands/dev.ts +++ b/packages/cli/src/cli/commands/dev.ts @@ -82,13 +82,19 @@ export default defineCommand({ await writeTypes(apibara); await build(apibara); + apibara.hooks.hook("dev:restart", () => { + if (childProcess) { + apibara.logger.info("Change detected, stopping indexers to restart"); + childProcess.kill(); + childProcess = undefined; + } + }); + apibara.hooks.hook("dev:reload", () => { if (childProcess) { - apibara.logger.start("Restarting indexers"); childProcess.kill(); } else { - apibara.logger.success("Dev server started"); - apibara.logger.success("Starting indexers"); + apibara.logger.success("Restarting indexers"); } const childArgs = [ diff --git a/packages/cli/src/core/build/dev.ts b/packages/cli/src/core/build/dev.ts index ff0546b2..5a3787fe 100644 --- a/packages/cli/src/core/build/dev.ts +++ b/packages/cli/src/core/build/dev.ts @@ -53,6 +53,7 @@ function startRollupWatcher(apibara: Apibara, rollupConfig: RollupConfig) { switch (event.code) { // The watcher is (re)starting case "START": { + apibara.hooks.callHook("dev:restart"); return; } diff --git a/packages/cli/src/types/hooks.ts b/packages/cli/src/types/hooks.ts index bc03c15b..0134960d 100644 --- a/packages/cli/src/types/hooks.ts +++ b/packages/cli/src/types/hooks.ts @@ -4,6 +4,7 @@ import type { RollupConfig } from "./rollup"; export interface ApibaraHooks { "rollup:before": (apibara: Apibara, rollupConfig: RollupConfig) => void; compiled: (apibara: Apibara) => void; + "dev:restart": () => void; "dev:reload": () => void; "rollup:reload": () => void; restart: () => void; From fb19cfe988a076d548bcd5e64d70a70d8600759e Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 19:50:24 +0100 Subject: [PATCH 05/13] nix: fix biome on nix --- flake.nix | 1 + package.json | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 50f870cb..6075dfc6 100644 --- a/flake.nix +++ b/flake.nix @@ -13,6 +13,7 @@ in { devShells.default = pkgs.mkShell { + MADNESS_ALLOW_LDD = "1"; nativeBuildInputs = with pkgs; [ protobuf nodejs diff --git a/package.json b/package.json index 5d5e5958..32e7caa7 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,8 @@ "test": "turbo run test", "test:ci": "turbo run test:ci", "test:postgres": "docker run --rm -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:16", - "lint": "turbo run lint --parallel", - "lint:fix": "turbo run lint:fix --parallel", - "format": "turbo run format --parallel", + "lint": "pnpm run -r lint", + "lint:fix": "pnpm run -r lint:fix", "release": "pnpm build" }, "repository": { From 8d11ff4a5a435942b8775baf91e6d22200b06057 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 19:51:43 +0100 Subject: [PATCH 06/13] cli: fix commonjs issue --- examples/cli/package.json | 8 +++++++- packages/cli/src/core/scan.ts | 6 +++--- packages/indexer/package.json | 5 ++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/cli/package.json b/examples/cli/package.json index 61a635ba..34d0d56d 100644 --- a/examples/cli/package.json +++ b/examples/cli/package.json @@ -3,7 +3,13 @@ "private": true, "version": "1.0.0", "description": "", - "scripts": {}, + "scripts": { + "build": "apibara build", + "dev": "apibara dev", + "start": "apibara start", + "lint": "biome check .", + "lint:fix": "pnpm lint --write" + }, "keywords": [], "author": "", "license": "ISC", diff --git a/packages/cli/src/core/scan.ts b/packages/cli/src/core/scan.ts index 4cd22492..bba11051 100644 --- a/packages/cli/src/core/scan.ts +++ b/packages/cli/src/core/scan.ts @@ -1,5 +1,5 @@ import type { Apibara } from "apibara/types"; -import { existsSync, readdirSync } from "fs-extra"; +import fse from "fs-extra"; import { basename, join } from "pathe"; const INDEXER_EXTENSIONS = [".indexer.ts", ".indexer.js"]; @@ -12,13 +12,13 @@ export async function scanIndexers(apibara: Apibara) { apibara.options.indexersDir, ); - if (!existsSync(indexersDir)) { + if (!fse.existsSync(indexersDir)) { throw new Error(`Indexers directory not found: ${indexersDir}`); } apibara.indexers = []; - for (const file of readdirSync(indexersDir)) { + for (const file of fse.readdirSync(indexersDir)) { const indexerName = indexerNameFromFile(file); if (indexerName) { apibara.indexers.push({ diff --git a/packages/indexer/package.json b/packages/indexer/package.json index 80c27b88..f783f3cd 100644 --- a/packages/indexer/package.json +++ b/packages/indexer/package.json @@ -67,12 +67,11 @@ }, "scripts": { "build": "unbuild", - "lint": "biome check .", "typecheck": "tsc --noEmit", + "lint": "biome check .", "lint:fix": "pnpm lint --write", "test": "vitest", - "test:ci": "vitest run", - "format": "biome format . --write" + "test:ci": "vitest run" }, "devDependencies": { "@types/better-sqlite3": "^7.6.11", From 933a8e699d91b0c8f7c4734b2de3bdcacdbb4b1f Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 20:00:01 +0100 Subject: [PATCH 07/13] cli: start command won't build the indexer --- packages/cli/package.json | 2 +- packages/cli/src/cli/commands/start.ts | 29 ++++++++++++++++---------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 2d2ede5f..2b11150e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -67,7 +67,7 @@ "playground:prepare": "pnpm playground prepare --dir playground", "playground:dev": "pnpm playground dev --dir playground", "playground:build": "pnpm playground build --dir playground", - "playground:start": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./playground/.apibara/build/main.mjs" + "playground:start": "pnpm playground start --dir playground --indexer starknet" }, "devDependencies": { "@apibara/starknet": "workspace:*", diff --git a/packages/cli/src/cli/commands/start.ts b/packages/cli/src/cli/commands/start.ts index 107b4336..598490f3 100644 --- a/packages/cli/src/cli/commands/start.ts +++ b/packages/cli/src/cli/commands/start.ts @@ -1,8 +1,8 @@ import { spawn } from "node:child_process"; -import { build, createApibara, prepare, writeTypes } from "apibara/core"; -import { runtimeDir } from "apibara/runtime/meta"; +import { createApibara } from "apibara/core"; import { defineCommand } from "citty"; -import { join, resolve } from "pathe"; +import fse from "fs-extra"; +import { resolve } from "pathe"; import { commonArgs } from "../common"; export default defineCommand({ @@ -23,24 +23,31 @@ export default defineCommand({ }, }, async run({ args }) { + const { indexer, preset } = args; const rootDir = resolve((args.dir || args._dir || ".") as string); + const apibara = await createApibara({ rootDir, }); - apibara.logger.start("Building"); + apibara.logger.start( + `Starting indexer ${indexer}${preset ? ` with preset ${preset}` : ""}`, + ); - apibara.options.entry = join(runtimeDir, "start.mjs"); + const outputDir = apibara.options.outputDir || "./.apibara/build"; + const entry = resolve(outputDir, "start.mjs"); - await prepare(apibara); - await writeTypes(apibara); - await build(apibara); - await apibara.close(); + if (!fse.existsSync(entry)) { + apibara.logger.error( + `Output directory ${outputDir} does not exist. Try building the indexer with "apibara build" first.`, + ); + return process.exit(1); + } - const { indexer, preset } = args; + await apibara.close(); const childArgs = [ - resolve(apibara.options.outputDir || "./.apibara/build", "start.mjs"), + entry, "start", "--indexer", indexer, From b80ea2a2023ca003e98d971b22b3fa2b0c869a5d Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 20:17:58 +0100 Subject: [PATCH 08/13] cli: improve build command output --- examples/cli/indexers/starknet.indexer.ts | 7 +------ packages/cli/src/cli/commands/build.ts | 1 - packages/cli/src/cli/commands/dev.ts | 5 ++++- packages/cli/src/core/build/prepare.ts | 5 ++++- packages/cli/src/core/build/prod.ts | 9 ++++++++- packages/cli/src/core/build/types.ts | 4 ++-- packages/cli/src/core/path.ts | 11 +++++++++++ 7 files changed, 30 insertions(+), 12 deletions(-) create mode 100644 packages/cli/src/core/path.ts diff --git a/examples/cli/indexers/starknet.indexer.ts b/examples/cli/indexers/starknet.indexer.ts index ce8f95db..0dc983f4 100644 --- a/examples/cli/indexers/starknet.indexer.ts +++ b/examples/cli/indexers/starknet.indexer.ts @@ -9,7 +9,7 @@ export default function (runtimeConfig: ApibaraRuntimeConfig) { streamUrl: "https://starknet.preview.apibara.org", finality: "accepted", startingCursor: { - orderKey: 80_000n, + orderKey: 800_000n, }, filter: { events: [ @@ -24,10 +24,5 @@ export default function (runtimeConfig: ApibaraRuntimeConfig) { async transform({ block: { header } }) { console.log("Transforming block ", header?.blockNumber); }, - hooks: { - "handler:after": ({ endCursor }) => { - console.log("Handler After ", endCursor?.orderKey); - }, - }, }); } diff --git a/packages/cli/src/cli/commands/build.ts b/packages/cli/src/cli/commands/build.ts index 2f34ba2a..ad22437a 100644 --- a/packages/cli/src/cli/commands/build.ts +++ b/packages/cli/src/cli/commands/build.ts @@ -17,7 +17,6 @@ export default defineCommand({ const apibara = await createApibara({ rootDir, }); - apibara.logger.start("Building"); apibara.options.entry = join(runtimeDir, "start.mjs"); diff --git a/packages/cli/src/cli/commands/dev.ts b/packages/cli/src/cli/commands/dev.ts index 83de37e8..49d3ad8b 100644 --- a/packages/cli/src/cli/commands/dev.ts +++ b/packages/cli/src/cli/commands/dev.ts @@ -3,6 +3,7 @@ import { build, createApibara, prepare, writeTypes } from "apibara/core"; import { runtimeDir } from "apibara/runtime/meta"; import type { Apibara } from "apibara/types"; import { defineCommand } from "citty"; +import { colors } from "consola/utils"; import { join, resolve } from "pathe"; import { commonArgs } from "../common"; @@ -111,7 +112,9 @@ export default defineCommand({ childProcess.on("close", (code) => { if (code !== null) { - apibara.logger.log(`Indexers process exited with code ${code}`); + apibara.logger.log( + `Indexers process exited with code ${colors.red(code)}`, + ); } }); }); diff --git a/packages/cli/src/core/build/prepare.ts b/packages/cli/src/core/build/prepare.ts index 4e11c21d..1d2e67dd 100644 --- a/packages/cli/src/core/build/prepare.ts +++ b/packages/cli/src/core/build/prepare.ts @@ -1,12 +1,15 @@ import fsp from "node:fs/promises"; import type { Apibara } from "apibara/types"; import fse from "fs-extra"; +import { prettyPath } from "../path"; export async function prepare(apibara: Apibara) { await prepareDir(apibara.options.buildDir); await prepareDir(apibara.options.outputDir); - apibara.logger.success("Output directory cleaned"); + apibara.logger.success( + `Output directory ${prettyPath(apibara.options.outputDir)} cleaned`, + ); } async function prepareDir(dir: string) { diff --git a/packages/cli/src/core/build/prod.ts b/packages/cli/src/core/build/prod.ts index 0a67de74..3b97ca63 100644 --- a/packages/cli/src/core/build/prod.ts +++ b/packages/cli/src/core/build/prod.ts @@ -1,11 +1,15 @@ import type { Apibara, RollupConfig } from "apibara/types"; +import { colors } from "consola/utils"; import { type OutputOptions, rollup } from "rollup"; export async function buildProduction( apibara: Apibara, rollupConfig: RollupConfig, ) { - apibara.logger.start("Building indexers"); + apibara.logger.start( + `Building ${colors.cyan(apibara.indexers.length)} indexers`, + ); + try { const bundle = await rollup(rollupConfig); @@ -22,6 +26,9 @@ export async function buildProduction( await bundle.close(); apibara.logger.success("Build succeeded!"); + apibara.logger.info( + `You can start the indexers with ${colors.cyan("apibara start")}`, + ); } catch (error) { apibara.logger.error("Build failed", error); throw error; diff --git a/packages/cli/src/core/build/types.ts b/packages/cli/src/core/build/types.ts index cba6c1d8..ba969830 100644 --- a/packages/cli/src/core/build/types.ts +++ b/packages/cli/src/core/build/types.ts @@ -4,9 +4,9 @@ import defu from "defu"; import { dirname, isAbsolute, join, relative, resolve } from "pathe"; import type { TSConfig } from "pkg-types"; import { type JSValue, generateTypes, resolveSchema } from "untyped"; +import { prettyPath } from "../path"; export async function writeTypes(apibara: Apibara) { - apibara.logger.start("Preparing Types"); const typesDir = resolve(apibara.options.buildDir, "types"); const config = [ @@ -131,7 +131,7 @@ declare module "apibara/types" {`, }), ); - apibara.logger.success("Types generated"); + apibara.logger.success(`Types written to ${prettyPath(typesDir)}`); } const RELATIVE_RE = /^\.{1,2}\//; diff --git a/packages/cli/src/core/path.ts b/packages/cli/src/core/path.ts new file mode 100644 index 00000000..27e1b3ff --- /dev/null +++ b/packages/cli/src/core/path.ts @@ -0,0 +1,11 @@ +import { colors } from "consola/utils"; +import { relative } from "pathe"; + +/** Return a (possibly highlighted) path relative to the current working directory. + * + * From nitrojs/nitro. + */ +export function prettyPath(path: string, highlight = true) { + const rel = relative(process.cwd(), path); + return highlight ? colors.cyan(rel) : rel; +} From 19c8973248bba39ba93b1225077beec17ef45eb5 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 20:27:00 +0100 Subject: [PATCH 09/13] ci: remove useless check --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 063dcfb4..aa45857c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,8 +37,6 @@ jobs: run: pnpm install --strict-peer-dependencies=false - name: Run lint run: pnpm lint - - name: Run format - run: pnpm format - name: Run build run: pnpm build - name: Run typecheck From ee9d3ea40930779848c38c1f8dcc39270099bc14 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sat, 2 Nov 2024 20:38:08 +0100 Subject: [PATCH 10/13] ci: exclude example cli from build --- .github/workflows/build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa45857c..25466fa2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,10 +38,11 @@ jobs: - name: Run lint run: pnpm lint - name: Run build - run: pnpm build + # Exclude examples/cli since it requires the cli to be built. + run: pnpm build --filter=!./examples/cli - name: Run typecheck run: pnpm typecheck - name: Run test - run: pnpm test:ci + run: pnpm test:ci --filter=!./examples/cli - name: Check change files run: pnpm beachball check From c4c53a88049205c1ae2b2bd2e6828af573fcfc1c Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Fri, 1 Nov 2024 17:07:54 +0100 Subject: [PATCH 11/13] cli: add changelog --- ...ibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json | 7 +++++++ change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json create mode 100644 change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json diff --git a/change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json b/change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json new file mode 100644 index 00000000..34848b06 --- /dev/null +++ b/change/@apibara-indexer-843e6831-307c-49b1-bf59-833803a4a27f.json @@ -0,0 +1,7 @@ +{ + "type": "none", + "comment": "Remove unused scripts", + "packageName": "@apibara/indexer", + "email": "francesco@ceccon.me", + "dependentChangeType": "none" +} diff --git a/change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json b/change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json new file mode 100644 index 00000000..06d018fe --- /dev/null +++ b/change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json @@ -0,0 +1,7 @@ +{ + "type": "prerelease", + "comment": "Refactor dev, build, and start commands", + "packageName": "apibara", + "email": "francesco@ceccon.me", + "dependentChangeType": "patch" +} From 44326f44dd6ed8ac33682bdea513257c04f23736 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sun, 3 Nov 2024 11:14:25 +0100 Subject: [PATCH 12/13] Add Javascript example --- .github/workflows/build.yml | 4 +-- examples/cli-js/apibara.config.mjs | 15 +++++++++++ examples/cli-js/indexers/starknet.indexer.js | 24 +++++++++++++++++ examples/cli-js/package.json | 27 ++++++++++++++++++++ examples/cli/apibara.config.ts | 4 --- pnpm-lock.yaml | 25 ++++++++++++++++++ 6 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 examples/cli-js/apibara.config.mjs create mode 100644 examples/cli-js/indexers/starknet.indexer.js create mode 100644 examples/cli-js/package.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25466fa2..35752844 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,10 +39,10 @@ jobs: run: pnpm lint - name: Run build # Exclude examples/cli since it requires the cli to be built. - run: pnpm build --filter=!./examples/cli + run: pnpm build --filter=!./examples/cli --filter=!./examples/cli-js - name: Run typecheck run: pnpm typecheck - name: Run test - run: pnpm test:ci --filter=!./examples/cli + run: pnpm test:ci --filter=!./examples/cli --filter=!./examples/cli-js - name: Check change files run: pnpm beachball check diff --git a/examples/cli-js/apibara.config.mjs b/examples/cli-js/apibara.config.mjs new file mode 100644 index 00000000..698e5782 --- /dev/null +++ b/examples/cli-js/apibara.config.mjs @@ -0,0 +1,15 @@ +import { defineConfig } from "apibara/config"; + +export default defineConfig({ + runtimeConfig: { + test: 123, + check: "something", + }, + presets: { + dev: { + runtimeConfig: { + test: 999, + }, + }, + }, +}); diff --git a/examples/cli-js/indexers/starknet.indexer.js b/examples/cli-js/indexers/starknet.indexer.js new file mode 100644 index 00000000..762dd8fd --- /dev/null +++ b/examples/cli-js/indexers/starknet.indexer.js @@ -0,0 +1,24 @@ +import { defineIndexer } from "@apibara/indexer"; +import { StarknetStream } from "@apibara/starknet"; +import { hash } from "starknet"; + +export default defineIndexer(StarknetStream)({ + streamUrl: "https://starknet.preview.apibara.org", + finality: "accepted", + startingCursor: { + orderKey: 800_000n, + }, + filter: { + events: [ + { + address: + "0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7", + keys: [hash.getSelectorFromName("Transfer")], + includeReceipt: true, + }, + ], + }, + async transform({ block: { header } }) { + console.log("Transforming block ", header?.blockNumber); + }, +}); diff --git a/examples/cli-js/package.json b/examples/cli-js/package.json new file mode 100644 index 00000000..b8b617ad --- /dev/null +++ b/examples/cli-js/package.json @@ -0,0 +1,27 @@ +{ + "name": "example-cli-js", + "private": true, + "version": "1.0.0", + "description": "", + "scripts": { + "build": "apibara build", + "dev": "apibara dev", + "start": "apibara start", + "lint": "biome check .", + "lint:fix": "pnpm lint --write" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@types/node": "^20.5.2", + "typescript": "^5.6.2" + }, + "dependencies": { + "@apibara/indexer": "workspace:*", + "@apibara/protocol": "workspace:*", + "@apibara/starknet": "workspace:*", + "apibara": "workspace:*", + "starknet": "^6.11.0" + } +} diff --git a/examples/cli/apibara.config.ts b/examples/cli/apibara.config.ts index 284df071..698e5782 100644 --- a/examples/cli/apibara.config.ts +++ b/examples/cli/apibara.config.ts @@ -1,4 +1,3 @@ -import { defaultSink } from "@apibara/indexer"; import { defineConfig } from "apibara/config"; export default defineConfig({ @@ -13,7 +12,4 @@ export default defineConfig({ }, }, }, - sink: { - default: () => defaultSink(), - }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3e5f37d1..a688743c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,31 @@ importers: specifier: ^5.6.2 version: 5.6.2 + examples/cli-js: + dependencies: + '@apibara/indexer': + specifier: workspace:* + version: link:../../packages/indexer + '@apibara/protocol': + specifier: workspace:* + version: link:../../packages/protocol + '@apibara/starknet': + specifier: workspace:* + version: link:../../packages/starknet + apibara: + specifier: workspace:* + version: link:../../packages/cli + starknet: + specifier: ^6.11.0 + version: 6.11.0 + devDependencies: + '@types/node': + specifier: ^20.5.2 + version: 20.14.0 + typescript: + specifier: ^5.6.2 + version: 5.6.2 + examples/evm-client: dependencies: '@apibara/evm': From 7039ff0c93fccb2bcf0dbf65a2d8b497d1d971b9 Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Sun, 3 Nov 2024 19:14:35 +0100 Subject: [PATCH 13/13] cli: improve support for cjs modules like better-sqlite3 --- examples/cli/indexers/starknet.indexer.ts | 5 ++ examples/cli/package.json | 2 + examples/indexer/package.json | 2 +- packages/cli/package.json | 1 + packages/cli/src/rollup/config.ts | 76 +++++++++++++-------- packages/cli/src/rollup/plugins/esm-shim.ts | 69 +++++++++++++++++++ packages/cli/src/types/config.ts | 2 + packages/indexer/package.json | 4 +- pnpm-lock.yaml | 63 +++++++++++------ 9 files changed, 170 insertions(+), 54 deletions(-) create mode 100644 packages/cli/src/rollup/plugins/esm-shim.ts diff --git a/examples/cli/indexers/starknet.indexer.ts b/examples/cli/indexers/starknet.indexer.ts index 0dc983f4..5a9dfee1 100644 --- a/examples/cli/indexers/starknet.indexer.ts +++ b/examples/cli/indexers/starknet.indexer.ts @@ -1,16 +1,21 @@ import { defineIndexer } from "@apibara/indexer"; +import { sqlitePersistence } from "@apibara/indexer/plugins/persistence"; import { StarknetStream } from "@apibara/starknet"; import type { ApibaraRuntimeConfig } from "apibara/types"; +import Database from "better-sqlite3"; import { hash } from "starknet"; export default function (runtimeConfig: ApibaraRuntimeConfig) { console.log("--> Starknet Indexer Runtime Config: ", runtimeConfig); + const database = new Database(":memory:"); + return defineIndexer(StarknetStream)({ streamUrl: "https://starknet.preview.apibara.org", finality: "accepted", startingCursor: { orderKey: 800_000n, }, + plugins: [sqlitePersistence({ database })], filter: { events: [ { diff --git a/examples/cli/package.json b/examples/cli/package.json index 34d0d56d..18b6159b 100644 --- a/examples/cli/package.json +++ b/examples/cli/package.json @@ -14,6 +14,7 @@ "author": "", "license": "ISC", "devDependencies": { + "@types/better-sqlite3": "^7.6.11", "@types/node": "^20.5.2", "typescript": "^5.6.2" }, @@ -22,6 +23,7 @@ "@apibara/protocol": "workspace:*", "@apibara/starknet": "workspace:*", "apibara": "workspace:*", + "better-sqlite3": "^11.5.0", "starknet": "^6.11.0" } } diff --git a/examples/indexer/package.json b/examples/indexer/package.json index 275dbd82..6b1fb1de 100644 --- a/examples/indexer/package.json +++ b/examples/indexer/package.json @@ -19,7 +19,7 @@ "@opentelemetry/sdk-node": "^0.52.0", "@opentelemetry/sdk-trace-base": "^1.25.0", "@opentelemetry/semantic-conventions": "^1.25.0", - "better-sqlite3": "^11.1.2", + "better-sqlite3": "^11.5.0", "citty": "^0.1.6", "consola": "^3.2.3", "csv-stringify": "^6.5.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 2b11150e..a1e10a3e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -96,6 +96,7 @@ "fs-extra": "^11.2.0", "hookable": "^5.5.3", "klona": "^2.0.6", + "magic-string": "^0.30.12", "pathe": "^1.1.2", "perfect-debounce": "^1.0.0", "pkg-types": "^1.1.3", diff --git a/packages/cli/src/rollup/config.ts b/packages/cli/src/rollup/config.ts index 06cd82fc..8be71cf7 100644 --- a/packages/cli/src/rollup/config.ts +++ b/packages/cli/src/rollup/config.ts @@ -7,9 +7,13 @@ import { join } from "pathe"; import type { OutputPluginOption } from "rollup"; import esbuild from "rollup-plugin-esbuild"; +import defu from "defu"; import { appConfig } from "./plugins/config"; +import { esmShim } from "./plugins/esm-shim"; import { indexers } from "./plugins/indexers"; +const runtimeDependencies = ["better-sqlite3"]; + export function getRollupConfig(apibara: Apibara): RollupConfig { const extensions: string[] = [ ".ts", @@ -21,42 +25,54 @@ export function getRollupConfig(apibara: Apibara): RollupConfig { ".jsx", ]; - const rollupConfig: RollupConfig & { plugins: OutputPluginOption[] } = { - input: apibara.options.entry, - output: { - dir: join(apibara.options.outputDir || "./.apibara/build"), - format: "esm", - exports: "auto", - entryFileNames: "[name].mjs", - chunkFileNames: "chunks/[name]-[hash].mjs", - generatedCode: { - constBindings: true, + const rollupConfig: RollupConfig & { plugins: OutputPluginOption[] } = defu( + // biome-ignore lint/suspicious/noExplicitAny: apibara.options.rollupConfig is typed + apibara.options.rollupConfig as any, + { + input: apibara.options.entry, + output: { + dir: join(apibara.options.outputDir || "./.apibara/build"), + format: "esm", + exports: "auto", + entryFileNames: "[name].mjs", + chunkFileNames: "chunks/[name]-[hash].mjs", + generatedCode: { + constBindings: true, + }, + sourcemap: true, + sourcemapExcludeSources: true, + sourcemapIgnoreList(relativePath, sourcemapPath) { + return relativePath.includes("node_modules"); + }, }, - sourcemap: true, - sourcemapExcludeSources: true, - sourcemapIgnoreList(relativePath, sourcemapPath) { - return relativePath.includes("node_modules"); + plugins: [], + onwarn(warning, rollupWarn) { + if ( + !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes( + warning.code || "", + ) && + !warning.message.includes("Unsupported source map comment") && + !warning.message.includes("@__PURE__") + ) { + rollupWarn(warning); + } }, + treeshake: true, + external: [...builtinModules, ...runtimeDependencies], }, - plugins: [], - onwarn(warning, rollupWarn) { - if ( - !["CIRCULAR_DEPENDENCY", "EVAL", "THIS_IS_UNDEFINED"].includes( - warning.code || "", - ) && - !warning.message.includes("Unsupported source map comment") && - !warning.message.includes("@__PURE__") - ) { - rollupWarn(warning); - } - }, - treeshake: true, - external: [...builtinModules], - }; + ); - rollupConfig.plugins.push(commonjs()); + rollupConfig.plugins.push(esmShim()); rollupConfig.plugins.push(json()); + rollupConfig.plugins.push( + commonjs({ + strictRequires: true, + requireReturnsDefault: "auto", + ...apibara.options.commonJS, + }), + ); + rollupConfig.plugins.push( nodeResolve({ extensions, diff --git a/packages/cli/src/rollup/plugins/esm-shim.ts b/packages/cli/src/rollup/plugins/esm-shim.ts new file mode 100644 index 00000000..dd51ada8 --- /dev/null +++ b/packages/cli/src/rollup/plugins/esm-shim.ts @@ -0,0 +1,69 @@ +import MagicString from "magic-string"; +import type { Plugin } from "rollup"; + +/** + * An alternative to @rollup/plugin-esm-shim + */ +export function esmShim(): Plugin { + const ESMShim = ` +// -- Shims -- +import cjsUrl from 'node:url'; +import cjsPath from 'node:path'; +const __filename = cjsUrl.fileURLToPath(import.meta.url); +const __dirname = cjsPath.dirname(__filename); +// -- End Shims -- +`; + + const CJSyntaxRegex = /__filename|__dirname/; + + return { + name: "esm-shim", + + renderChunk(code, _chunk, opts) { + if (opts.format === "es") { + if (code.includes(ESMShim) || !CJSyntaxRegex.test(code)) { + return null; + } + + let endIndexOfLastImport = -1; + + // Find the last import statement and its ending index + for (const match of code.matchAll(/^import\s.*';$/gm)) { + if (match.length === 0 || typeof match.index !== "number") { + continue; + } + + endIndexOfLastImport = match.index + match[0].length; + } + + const s = new MagicString(code); + s.appendRight(endIndexOfLastImport, ESMShim); + + const sourceMap = s.generateMap({ + includeContent: true, + }); + + let sourcesContent: string[] | undefined; + if (Array.isArray(sourceMap.sourcesContent)) { + sourcesContent = []; + for (let i = 0; i < sourceMap.sourcesContent.length; i++) { + const content = sourceMap.sourcesContent[i]; + if (typeof content === "string") { + sourcesContent.push(content); + } + } + } + + return { + code: s.toString(), + map: { + ...sourceMap, + sourcesContent, + }, + }; + } + + return null; + }, + }; +} diff --git a/packages/cli/src/types/config.ts b/packages/cli/src/types/config.ts index 224fb3d3..2b8c6637 100644 --- a/packages/cli/src/types/config.ts +++ b/packages/cli/src/types/config.ts @@ -1,3 +1,4 @@ +import type { RollupCommonJSOptions } from "@rollup/plugin-commonjs"; import type { C12InputConfig, ConfigWatcher, @@ -73,6 +74,7 @@ export interface ApibaraOptions< rollupConfig?: RollupConfig; sourceMap?: boolean; entry: string; + commonJS?: RollupCommonJSOptions; // Advanced typescript: { diff --git a/packages/indexer/package.json b/packages/indexer/package.json index f783f3cd..f291d976 100644 --- a/packages/indexer/package.json +++ b/packages/indexer/package.json @@ -77,7 +77,7 @@ "@types/better-sqlite3": "^7.6.11", "@types/node": "^20.14.0", "@types/pg": "^8.11.10", - "better-sqlite3": "^11.1.2", + "better-sqlite3": "^11.5.0", "csv-stringify": "^6.5.0", "drizzle-orm": "^0.35.2", "pg": "^8.12.0", @@ -95,7 +95,7 @@ "unctx": "^2.3.1" }, "peerDependencies": { - "better-sqlite3": "^11.1.2", + "better-sqlite3": "^11.5.0", "csv-stringify": "^6.5.0", "drizzle-orm": "^0.35.2", "postgres-range": "^1.1.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a688743c..81f2e05a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -60,10 +60,16 @@ importers: apibara: specifier: workspace:* version: link:../../packages/cli + better-sqlite3: + specifier: ^11.5.0 + version: 11.5.0 starknet: specifier: ^6.11.0 version: 6.11.0 devDependencies: + '@types/better-sqlite3': + specifier: ^7.6.11 + version: 7.6.11 '@types/node': specifier: ^20.5.2 version: 20.14.0 @@ -154,8 +160,8 @@ importers: specifier: ^1.25.0 version: 1.25.0 better-sqlite3: - specifier: ^11.1.2 - version: 11.1.2 + specifier: ^11.5.0 + version: 11.5.0 citty: specifier: ^0.1.6 version: 0.1.6 @@ -167,7 +173,7 @@ importers: version: 6.5.0 drizzle-orm: specifier: ^0.35.0 - version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.1.2)(pg@8.13.0)(postgres@3.4.4) + version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.5.0)(pg@8.13.0)(postgres@3.4.4) postgres: specifier: ^3.4.4 version: 3.4.4 @@ -250,7 +256,7 @@ importers: version: 6.5.0 drizzle-orm: specifier: ^0.35.0 - version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.1.2)(pg@8.13.0)(postgres@3.4.4) + version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.5.0)(pg@8.13.0)(postgres@3.4.4) postgres: specifier: ^3.4.4 version: 3.4.4 @@ -361,6 +367,9 @@ importers: klona: specifier: ^2.0.6 version: 2.0.6 + magic-string: + specifier: ^0.30.12 + version: 0.30.12 pathe: specifier: ^1.1.2 version: 1.1.2 @@ -482,14 +491,14 @@ importers: specifier: ^8.11.10 version: 8.11.10 better-sqlite3: - specifier: ^11.1.2 - version: 11.1.2 + specifier: ^11.5.0 + version: 11.5.0 csv-stringify: specifier: ^6.5.0 version: 6.5.0 drizzle-orm: specifier: ^0.35.2 - version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.1.2)(pg@8.13.0)(postgres@3.4.4) + version: 0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.5.0)(pg@8.13.0)(postgres@3.4.4) pg: specifier: ^8.12.0 version: 8.13.0 @@ -1287,6 +1296,9 @@ packages: '@jridgewell/sourcemap-codec@1.4.15': resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + '@jridgewell/sourcemap-codec@1.5.0': + resolution: {integrity: sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==} + '@jridgewell/trace-mapping@0.3.25': resolution: {integrity: sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==} @@ -1811,8 +1823,8 @@ packages: engines: {node: '>=14.0.0'} hasBin: true - better-sqlite3@11.1.2: - resolution: {integrity: sha512-gujtFwavWU4MSPT+h9B+4pkvZdyOUkH54zgLdIrMmmmd4ZqiBIrRNBzNzYVFO417xo882uP5HBu4GjOfaSrIQw==} + better-sqlite3@11.5.0: + resolution: {integrity: sha512-e/6eggfOutzoK0JWiU36jsisdWoHOfN9iWiW/SieKvb7SAa6aGNmBM/UKyp+/wWSXpLlWNN8tCPwoDNPhzUvuQ==} binary-extensions@2.3.0: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} @@ -2559,6 +2571,9 @@ packages: magic-string@0.30.10: resolution: {integrity: sha512-iIRwTIf0QKV3UAnYK4PU8uiEc4SRh5jX0mwpIwETPpHdhVM4f53RSwS/vXvN1JhGX+Cs7B8qIq3d6AH49O5fAQ==} + magic-string@0.30.12: + resolution: {integrity: sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==} + mdn-data@2.0.28: resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} @@ -4123,6 +4138,8 @@ snapshots: '@jridgewell/sourcemap-codec@1.4.15': {} + '@jridgewell/sourcemap-codec@1.5.0': {} + '@jridgewell/trace-mapping@0.3.25': dependencies: '@jridgewell/resolve-uri': 3.1.1 @@ -4353,7 +4370,7 @@ snapshots: estree-walker: 2.0.2 glob: 8.1.0 is-reference: 1.2.1 - magic-string: 0.30.10 + magic-string: 0.30.12 optionalDependencies: rollup: 3.29.4 @@ -4364,7 +4381,7 @@ snapshots: estree-walker: 2.0.2 glob: 10.4.5 is-reference: 1.2.1 - magic-string: 0.30.10 + magic-string: 0.30.12 optionalDependencies: rollup: 4.18.1 @@ -4405,7 +4422,7 @@ snapshots: '@rollup/plugin-replace@5.0.5(rollup@3.29.4)': dependencies: '@rollup/pluginutils': 5.1.0(rollup@3.29.4) - magic-string: 0.30.10 + magic-string: 0.30.12 optionalDependencies: rollup: 3.29.4 @@ -4567,7 +4584,7 @@ snapshots: '@vitest/snapshot@1.6.0': dependencies: - magic-string: 0.30.10 + magic-string: 0.30.12 pathe: 1.1.2 pretty-format: 29.7.0 @@ -4665,7 +4682,7 @@ snapshots: transitivePeerDependencies: - typescript - better-sqlite3@11.1.2: + better-sqlite3@11.5.0: dependencies: bindings: 1.5.0 prebuild-install: 7.1.2 @@ -4959,12 +4976,12 @@ snapshots: dotenv@16.4.5: {} - drizzle-orm@0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.1.2)(pg@8.13.0)(postgres@3.4.4): + drizzle-orm@0.35.2(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.11)(@types/pg@8.11.10)(better-sqlite3@11.5.0)(pg@8.13.0)(postgres@3.4.4): optionalDependencies: '@opentelemetry/api': 1.9.0 '@types/better-sqlite3': 7.6.11 '@types/pg': 8.11.10 - better-sqlite3: 11.1.2 + better-sqlite3: 11.5.0 pg: 8.13.0 postgres: 3.4.4 @@ -5424,6 +5441,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + magic-string@0.30.12: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.0 + mdn-data@2.0.28: {} mdn-data@2.0.30: {} @@ -6010,7 +6031,7 @@ snapshots: rollup-plugin-dts@6.1.1(rollup@3.29.4)(typescript@5.6.2): dependencies: - magic-string: 0.30.10 + magic-string: 0.30.12 rollup: 3.29.4 typescript: 5.6.2 optionalDependencies: @@ -6289,7 +6310,7 @@ snapshots: globby: 13.2.2 hookable: 5.5.3 jiti: 1.21.6 - magic-string: 0.30.10 + magic-string: 0.30.12 mkdist: 1.5.1(typescript@5.6.2) mlly: 1.7.0 pathe: 1.1.2 @@ -6498,7 +6519,7 @@ snapshots: debug: 4.3.4 execa: 8.0.1 local-pkg: 0.5.0 - magic-string: 0.30.10 + magic-string: 0.30.12 pathe: 1.1.2 picocolors: 1.0.1 std-env: 3.7.0 @@ -6531,7 +6552,7 @@ snapshots: debug: 4.3.4 execa: 8.0.1 local-pkg: 0.5.0 - magic-string: 0.30.10 + magic-string: 0.30.12 pathe: 1.1.2 picocolors: 1.0.1 std-env: 3.7.0 @@ -6564,7 +6585,7 @@ snapshots: debug: 4.3.4 execa: 8.0.1 local-pkg: 0.5.0 - magic-string: 0.30.10 + magic-string: 0.30.12 pathe: 1.1.2 picocolors: 1.0.1 std-env: 3.7.0