Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,12 @@ jobs:
run: pnpm install --strict-peer-dependencies=false
- name: Run lint
run: pnpm lint
- name: Run format
run: pnpm format
- name: Run build
run: pnpm build
# Exclude examples/cli since it requires the cli to be built.
run: pnpm build --filter=!./examples/cli --filter=!./examples/cli-js
- name: Run typecheck
run: pnpm typecheck
- name: Run test
run: pnpm test:ci
run: pnpm test:ci --filter=!./examples/cli --filter=!./examples/cli-js
- name: Check change files
run: pnpm beachball check
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"type": "none",
"comment": "Remove unused scripts",
"packageName": "@apibara/indexer",
"email": "francesco@ceccon.me",
"dependentChangeType": "none"
}
7 changes: 7 additions & 0 deletions change/apibara-ed31dbbf-efbd-44c6-8eb0-f729dde0acb6.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"type": "prerelease",
"comment": "Refactor dev, build, and start commands",
"packageName": "apibara",
"email": "francesco@ceccon.me",
"dependentChangeType": "patch"
}
15 changes: 15 additions & 0 deletions examples/cli-js/apibara.config.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import { defineConfig } from "apibara/config";

export default defineConfig({
runtimeConfig: {
test: 123,
check: "something",
},
presets: {
dev: {
runtimeConfig: {
test: 999,
},
},
},
});
24 changes: 24 additions & 0 deletions examples/cli-js/indexers/starknet.indexer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
import { defineIndexer } from "@apibara/indexer";
import { StarknetStream } from "@apibara/starknet";
import { hash } from "starknet";

export default defineIndexer(StarknetStream)({
streamUrl: "https://starknet.preview.apibara.org",
finality: "accepted",
startingCursor: {
orderKey: 800_000n,
},
filter: {
events: [
{
address:
"0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7",
keys: [hash.getSelectorFromName("Transfer")],
includeReceipt: true,
},
],
},
async transform({ block: { header } }) {
console.log("Transforming block ", header?.blockNumber);
},
Comment on lines +21 to +23
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Enhance transform function with structured logging and error handling.

The current transform function uses basic console.log and lacks error handling. Consider using a structured logger and adding proper error handling.

Here's a suggested improvement:

-  async transform({ block: { header } }) {
-    console.log("Transforming block ", header?.blockNumber);
+  async transform({ block: { header, events } }) {
+    try {
+      const blockNumber = header?.blockNumber;
+      if (!blockNumber) {
+        throw new Error('Block number is undefined');
+      }
+
+      console.log({
+        level: 'info',
+        message: 'Transforming block',
+        blockNumber: blockNumber.toString(),
+        eventsCount: events?.length ?? 0,
+      });
+
+      // Process events here
+    } catch (error) {
+      console.error({
+        level: 'error',
+        message: 'Failed to transform block',
+        error: error.message,
+        blockNumber: header?.blockNumber?.toString(),
+      });
+      throw error; // Re-throw to ensure the indexer handles the error
+    }
   },
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
async transform({ block: { header } }) {
console.log("Transforming block ", header?.blockNumber);
},
async transform({ block: { header, events } }) {
try {
const blockNumber = header?.blockNumber;
if (!blockNumber) {
throw new Error('Block number is undefined');
}
console.log({
level: 'info',
message: 'Transforming block',
blockNumber: blockNumber.toString(),
eventsCount: events?.length ?? 0,
});
// Process events here
} catch (error) {
console.error({
level: 'error',
message: 'Failed to transform block',
error: error.message,
blockNumber: header?.blockNumber?.toString(),
});
throw error; // Re-throw to ensure the indexer handles the error
}
},

});
27 changes: 27 additions & 0 deletions examples/cli-js/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"name": "example-cli-js",
"private": true,
"version": "1.0.0",
"description": "",
"scripts": {
"build": "apibara build",
"dev": "apibara dev",
"start": "apibara start",
"lint": "biome check .",
"lint:fix": "pnpm lint --write"
},
"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@types/node": "^20.5.2",
"typescript": "^5.6.2"
},
"dependencies": {
"@apibara/indexer": "workspace:*",
"@apibara/protocol": "workspace:*",
"@apibara/starknet": "workspace:*",
"apibara": "workspace:*",
"starknet": "^6.11.0"
}
}
4 changes: 0 additions & 4 deletions examples/cli/apibara.config.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { defaultSink } from "@apibara/indexer";
import { defineConfig } from "apibara/config";

export default defineConfig({
Expand All @@ -13,7 +12,4 @@ export default defineConfig({
},
},
},
sink: {
default: () => defaultSink(),
},
});
12 changes: 6 additions & 6 deletions examples/cli/indexers/starknet.indexer.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
import { defineIndexer } from "@apibara/indexer";
import { sqlitePersistence } from "@apibara/indexer/plugins/persistence";
import { StarknetStream } from "@apibara/starknet";
import type { ApibaraRuntimeConfig } from "apibara/types";
import Database from "better-sqlite3";
import { hash } from "starknet";

export default function (runtimeConfig: ApibaraRuntimeConfig) {
console.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
const database = new Database(":memory:");

return defineIndexer(StarknetStream)({
streamUrl: "https://starknet.preview.apibara.org",
finality: "accepted",
startingCursor: {
orderKey: 80_000n,
orderKey: 800_000n,
},
plugins: [sqlitePersistence({ database })],
filter: {
events: [
{
Expand All @@ -24,10 +29,5 @@ export default function (runtimeConfig: ApibaraRuntimeConfig) {
async transform({ block: { header } }) {
console.log("Transforming block ", header?.blockNumber);
},
hooks: {
"handler:after": ({ endCursor }) => {
console.log("Handler After ", endCursor?.orderKey);
},
},
});
}
10 changes: 9 additions & 1 deletion examples/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,18 @@
"private": true,
"version": "1.0.0",
"description": "",
"scripts": {},
"scripts": {
"build": "apibara build",
"dev": "apibara dev",
"start": "apibara start",
"lint": "biome check .",
"lint:fix": "pnpm lint --write"
},
Comment on lines +6 to +12
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Based on the search results, I can now generate the final response since I have enough information to verify the documentation of the commands.

Commands need documentation

The new scripts (build, dev, start, lint) are not properly documented. While the CLI commands have basic descriptions in the code, there is no user-facing documentation:

  • The @apibara/cli package README is empty with just "TODO" placeholders
  • The CLI help text only provides minimal descriptions
  • No documentation found in docs directory
🔗 Analysis chain

LGTM! Verify command documentation.

The new scripts align well with the PR objectives for refactoring the build, dev, and start commands. The addition of linting scripts with Biome is also a good practice for code quality.

Let's verify if these commands are documented:


Let me try another search strategy to verify the documentation of these commands.


Let me try one more search to find CLI command documentation in the source code.

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Check if these commands are documented in README or CLI help
fd -t f "README.md" -X cat {} | rg -A 5 "(?:build|dev|start) command"

Length of output: 449


Script:

#!/bin/bash
# Search for documentation in different locations
echo "=== Searching in README files ==="
fd -t f "README" -X cat {}

echo -e "\n=== Searching in docs directory ==="
fd -t f . -e md -p docs/ -X cat {}

echo -e "\n=== Searching for CLI help text ==="
ast-grep --pattern 'program
  .command($_)
  .description($_)'

Length of output: 5760


Script:

#!/bin/bash
# Search for CLI command definitions
echo "=== Searching for command definitions ==="
rg -A 3 "\.command\(['\"](?:build|dev|start)['\"].*\)" --type ts

echo -e "\n=== Checking CLI package for documentation ==="
fd -t f . -p "packages/cli" -X cat {}

Length of output: 44122

"keywords": [],
"author": "",
"license": "ISC",
"devDependencies": {
"@types/better-sqlite3": "^7.6.11",
"@types/node": "^20.5.2",
"typescript": "^5.6.2"
},
Expand All @@ -16,6 +23,7 @@
"@apibara/protocol": "workspace:*",
"@apibara/starknet": "workspace:*",
"apibara": "workspace:*",
"better-sqlite3": "^11.5.0",
"starknet": "^6.11.0"
}
}
2 changes: 1 addition & 1 deletion examples/indexer/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"@opentelemetry/sdk-node": "^0.52.0",
"@opentelemetry/sdk-trace-base": "^1.25.0",
"@opentelemetry/semantic-conventions": "^1.25.0",
"better-sqlite3": "^11.1.2",
"better-sqlite3": "^11.5.0",
"citty": "^0.1.6",
"consola": "^3.2.3",
"csv-stringify": "^6.5.0",
Expand Down
1 change: 1 addition & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
in
{
devShells.default = pkgs.mkShell {
MADNESS_ALLOW_LDD = "1";
nativeBuildInputs = with pkgs; [
protobuf
nodejs
Expand Down
5 changes: 2 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,8 @@
"test": "turbo run test",
"test:ci": "turbo run test:ci",
"test:postgres": "docker run --rm -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres postgres:16",
"lint": "turbo run lint --parallel",
"lint:fix": "turbo run lint:fix --parallel",
"format": "turbo run format --parallel",
"lint": "pnpm run -r lint",
"lint:fix": "pnpm run -r lint:fix",
"release": "pnpm build"
},
"repository": {
Expand Down
21 changes: 16 additions & 5 deletions packages/cli/build.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,17 @@ import { fileURLToPath } from "node:url";
import { resolve } from "pathe";
import { defineBuildConfig } from "unbuild";

const modules = ["cli", "config", "core", "rollup", "types", "hooks "];
const modules = [
"cli",
"config",
"core",
"rollup",
"types",
"hooks",
"runtime",
];

// @ts-ignore The 'import.meta' meta-property is only allowed when the '--module' option is 'es2020', 'es2022', 'esnext', 'system', 'node16', or 'nodenext'.
// @ts-ignore
const srcDir = fileURLToPath(new URL("src", import.meta.url));

export default defineBuildConfig({
Expand All @@ -15,19 +23,22 @@ export default defineBuildConfig({
{ input: "./src/rollup/index.ts" },
{ input: "./src/types/index.ts" },
{ input: "./src/hooks/index.ts" },
{ input: "./src/internal/consola/index.ts" },
{ input: "./src/internal/citty/index.ts" },
{ input: "./src/runtime/", outDir: "./dist/runtime", format: "esm" },
],
clean: true,
outDir: "./dist",
declaration: true,
alias: {
"apibara/runtime/meta": resolve(srcDir, "../runtime-meta.mjs"),
...Object.fromEntries(
modules.map((module) => [
`apibara/${module}`,
resolve(srcDir, `${module}/index.ts`),
]),
),
},
externals: [...modules.map((module) => `apibara/${module}`)],
externals: [
"apibara/runtime/meta",
...modules.map((module) => `apibara/${module}`),
],
});
30 changes: 20 additions & 10 deletions packages/cli/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,21 @@
"types": "./dist/hooks/index.d.ts",
"import": "./dist/hooks/index.mjs"
},
"./internal/consola": {
"import": "./dist/internal/consola/index.mjs",
"types": "./dist/internal/consola/index.d.ts"
"./runtime": {
"types": "./dist/runtime/index.d.ts",
"import": "./dist/runtime/index.mjs"
},
"./internal/citty": {
"import": "./dist/internal/citty/index.mjs",
"types": "./dist/internal/citty/index.d.ts"
"./runtime/meta": {
"types": "./runtime-meta.d.ts",
"import": "./runtime-meta.mjs"
},
"./runtime/*": {
"types": "./dist/runtime/*.d.ts",
"import": "./dist/runtime/*.mjs"
},
"./dist/runtime/*": {
"types": "./dist/runtime/*.d.ts",
"import": "./dist/runtime/*.mjs"
Comment on lines +35 to +49
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Runtime exports configuration needs adjustment

The exports configuration in package.json references files in the dist directory, but the source files are currently only present in src/runtime. The runtime-meta.* files are correctly placed at the package root, but:

  • ./dist/runtime/*.d.ts and ./dist/runtime/*.mjs paths don't exist yet
  • The source files exist at:
    • src/runtime/dev.ts
    • src/runtime/index.ts
    • src/runtime/start.ts
    • src/runtime/internal/app.ts

Either:

  1. The build process needs to be set up to generate the dist files, or
  2. The exports configuration needs to be updated to match the current structure
🔗 Analysis chain

LGTM! Verify runtime directory structure.

The new runtime-related exports are well-structured and consistent with the existing pattern. The wildcard exports will enable flexible runtime module loading as needed for the command refactoring.

Let's verify the runtime directory structure matches the exports configuration:

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Description: Verify runtime directory structure exists as expected

# Check if runtime directory and type definitions exist
fd -t d "runtime$" "packages/cli/(src|dist)"
fd -t f ".*\.d\.ts$" -p "packages/cli/dist/runtime"

Length of output: 322


Script:

#!/bin/bash
# Let's try a different approach to verify the runtime-related files and structure

# Check src directory structure
fd -t d "runtime" "packages/cli/src"

# Check for runtime-related source files
fd -t f ".*\.(ts|js|mjs)$" "packages/cli/src/runtime"

# Check for runtime meta files
fd "runtime-meta" "packages/cli"

# List all files in packages/cli to understand the structure
ls -la packages/cli/

Length of output: 1103

}
},
"bin": {
Expand All @@ -55,13 +63,14 @@
"typecheck": "tsc --noEmit",
"lint:fix": "pnpm lint --write",
"format": "biome format . --write",
"playground": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts",
"playground": "JITI_ESM_RESOLVE=1 CONSOLA_LEVEL=debug NODE_OPTIONS=\"--enable-source-maps\" jiti ./src/cli/index.ts",
"playground:prepare": "pnpm playground prepare --dir playground",
"playground:dev": "pnpm playground dev --dir playground",
"playground:build": "pnpm playground build --dir playground",
"playground:start": "JITI_ESM_RESOLVE=1 NODE_OPTIONS=\"--enable-source-maps\" jiti ./playground/.apibara/build/main.mjs"
"playground:start": "pnpm playground start --dir playground --indexer starknet"
},
"devDependencies": {
"@apibara/starknet": "workspace:*",
"@types/fs-extra": "^11.0.4",
"@types/node": "^20.14.0",
"jiti": "^1.21.0",
Expand All @@ -71,14 +80,13 @@
"vitest": "^1.6.0"
},
"dependencies": {
"@apibara/evm": "workspace:*",
"@apibara/indexer": "workspace:*",
"@apibara/protocol": "workspace:*",
"@apibara/starknet": "workspace:*",
"@rollup/plugin-commonjs": "^26.0.1",
"@rollup/plugin-json": "^6.1.0",
"@rollup/plugin-node-resolve": "^15.2.3",
"@rollup/plugin-typescript": "^11.1.6",
"@rollup/plugin-virtual": "^3.0.2",
"c12": "^1.11.1",
"chokidar": "^3.6.0",
"citty": "^0.1.6",
Expand All @@ -88,10 +96,12 @@
"fs-extra": "^11.2.0",
"hookable": "^5.5.3",
"klona": "^2.0.6",
"magic-string": "^0.30.12",
"pathe": "^1.1.2",
"perfect-debounce": "^1.0.0",
"pkg-types": "^1.1.3",
"rollup": "^4.18.1",
"rollup-plugin-esbuild": "^6.1.1",
"tslib": "^2.6.3",
"untyped": "^1.4.2"
}
Expand Down
10 changes: 6 additions & 4 deletions packages/cli/playground/apibara.config.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
import { defaultSink } from "@apibara/indexer";
import { defineConfig } from "apibara/config";

export default defineConfig({
runtimeConfig: {
test: 123,
check: "something",
nested: {
test: 456,
},
fromEnv: {
nodeEnv: process.env.NODE_ENV,
},
Comment on lines +10 to +12
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Consider adding type safety and validation for environment variables.

The direct use of process.env.NODE_ENV could lead to runtime issues if the environment variable is undefined. Consider:

  1. Adding type safety
  2. Providing a default value
  3. Validating allowed values
    fromEnv: {
-      nodeEnv: process.env.NODE_ENV,
+      nodeEnv: process.env.NODE_ENV ?? 'development',
    },

You might also want to add a type guard:

const validateNodeEnv = (env: string | undefined): env is 'development' | 'production' | 'test' => {
  return env === 'development' || env === 'production' || env === 'test';
};

// Usage
nodeEnv: validateNodeEnv(process.env.NODE_ENV) 
  ? process.env.NODE_ENV 
  : 'development',

},
presets: {
dev: {
Expand All @@ -13,7 +18,4 @@ export default defineConfig({
},
},
},
sink: {
default: () => defaultSink(),
},
});
13 changes: 5 additions & 8 deletions packages/cli/playground/indexers/starknet.indexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import { hash } from "starknet";
export default function indexer(runtimeConfig: ApibaraRuntimeConfig) {
consola.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
return defineIndexer(StarknetStream)({
streamUrl: "http://mainnet-v2.starknet.a5a.ch:7007",
streamUrl: "https://starknet.preview.apibara.org",
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Update required: Old Starknet URL found in examples

The new Starknet stream URL https://starknet.preview.apibara.org is consistently used in the playground and CLI examples, but the old URL http://mainnet-v2.starknet.a5a.ch:7007 is still present in:

  • examples/starknet-indexer/src/main.ts

This file needs to be updated to maintain consistency across the codebase and avoid confusion.

🔗 Analysis chain

Verify the new Starknet stream URL configuration.

The migration to https://starknet.preview.apibara.org improves security by using HTTPS. However, we should ensure this preview endpoint is ready for use.

Run the following script to check for any other instances of the old URL that might need updating:

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Search for any remaining instances of the old URL
rg "mainnet-v2\.starknet\.a5a\.ch:7007"

Length of output: 136


Script:

#!/bin/bash
# Let's check if there are any other Starknet stream URL configurations
rg -A 2 "streamUrl.*starknet" 

# Also check for any documentation or configuration files mentioning Starknet URLs
rg -t md -t yaml -t json "starknet.*\.org|starknet.*\.ch"

Length of output: 585

finality: "accepted",
startingCursor: {
orderKey: 80_000n,
orderKey: 800_000n,
},
filter: {
events: [
Expand All @@ -23,12 +23,9 @@ export default function indexer(runtimeConfig: ApibaraRuntimeConfig) {
],
},
async transform({ block: { header, events } }) {
consola.info("Transforming block ", header?.blockNumber);
},
hooks: {
"handler:after": ({ endCursor }) => {
consola.info("Handler After ", endCursor?.orderKey);
},
consola.info(
`Got block ${header?.blockNumber} with ${events.length} events`,
);
Comment on lines +26 to +28
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Migrate to apibara.logger for consistent logging.

According to the broader changes in this PR, logging is being centralized through apibara.logger. However, this file still uses consola.

Consider updating the logging implementation:

-import consola from "consola";
+import { logger } from "apibara";

 export default function indexer(runtimeConfig: ApibaraRuntimeConfig) {
-  consola.log("--> Starknet Indexer Runtime Config: ", runtimeConfig);
+  logger.debug("Starknet Indexer Runtime Config: ", runtimeConfig);
   // ...
-      consola.info(
+      logger.info(
         `Got block ${header?.blockNumber} with ${events.length} events`,
       );

Committable suggestion skipped: line range outside the PR's diff.

},
});
}
11 changes: 0 additions & 11 deletions packages/cli/playground/tsconfig.json

This file was deleted.

2 changes: 2 additions & 0 deletions packages/cli/runtime-meta.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
export declare const pkgDir: string;
export declare const runtimeDir: string;
Loading
Loading