From 132d8751af00df770cff9bf10e3ab3fc2614572b Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 8 Aug 2026 15:27:23 +0200 Subject: [PATCH] harden Tinybird JWT limits and cache --- .env.example | 3 + CONTRIBUTING.md | 1 + apps/alerting/alchemy.run.ts | 1 + apps/api/alchemy.run.ts | 1 + apps/api/src/platform/Env.ts | 35 +++++++++ .../src/services/auth/tinybird-jwt.test.ts | 24 ++++++- apps/api/src/services/auth/tinybird-jwt.ts | 15 +++- .../TinybirdOrgTokenService.test.ts | 71 +++++++++++++++++-- .../integrations/TinybirdOrgTokenService.ts | 22 ++++++ docs/self-hosted-clickhouse.md | 2 + 10 files changed, 165 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 1753224d5..2f8c0f998 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,9 @@ TINYBIRD_TOKEN=your-tinybird-token # per-org read JWTs and are intentionally independent from the API token. # TINYBIRD_SIGNING_KEY=your-tinybird-jwt-signing-key # TINYBIRD_WORKSPACE_ID=your-tinybird-workspace-id +# Optional Tinybird-side ceiling shared by raw SQL from the API and alerting, +# with an independent rate-limit bucket for each org. Unset means no JWT RPS limit. +# TINYBIRD_RAW_SQL_JWT_RPS_LIMIT=100 # ClickHouse CLICKHOUSE_URL=http://localhost:9000 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 485126836..11c813de7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -188,6 +188,7 @@ Default URL: `http://localhost:3472` | `TINYBIRD_HOST` | yes | Placeholder OK when using `CLICKHOUSE_URL` | | `TINYBIRD_TOKEN` | yes | Placeholder OK when using `CLICKHOUSE_URL` | | `TINYBIRD_SIGNING_KEY` / `TINYBIRD_WORKSPACE_ID` | with Tinybird raw SQL | Explicit JWT signing configuration; never derived from `TINYBIRD_TOKEN` | +| `TINYBIRD_RAW_SQL_JWT_RPS_LIMIT` | optional | Positive integer; Tinybird-enforced raw-SQL ceiling with a separate bucket per org | | `CLICKHOUSE_URL` | recommended | `http://localhost:8123` for local ClickHouse stack | | `CLICKHOUSE_PROVIDER` | optional | `tinybird` (default) or `clickhouse`; set `clickhouse` for env-level vanilla/self-managed ClickHouse | | `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` / `CLICKHOUSE_DATABASE` | with CH | Match docker-compose (`maple` / `maple` / `default`) | diff --git a/apps/alerting/alchemy.run.ts b/apps/alerting/alchemy.run.ts index f0480c262..6dd62d23b 100644 --- a/apps/alerting/alchemy.run.ts +++ b/apps/alerting/alchemy.run.ts @@ -85,6 +85,7 @@ export const createAlertingWorker = ({ stage, mapleDb }: CreateAlertingWorkerOpt // Tinybird-scoped raw SQL" (same bindings as the api worker). ...optionalSecret("TINYBIRD_SIGNING_KEY"), ...optionalPlain("TINYBIRD_WORKSPACE_ID"), + ...optionalPlain("TINYBIRD_RAW_SQL_JWT_RPS_LIMIT"), MAPLE_AUTH_MODE: process.env.MAPLE_AUTH_MODE?.trim() || "self_hosted", MAPLE_DEFAULT_ORG_ID: process.env.MAPLE_DEFAULT_ORG_ID?.trim() || "default", MAPLE_INGEST_KEY_ENCRYPTION_KEY: Redacted.make(requireEnv("MAPLE_INGEST_KEY_ENCRYPTION_KEY")), diff --git a/apps/api/alchemy.run.ts b/apps/api/alchemy.run.ts index 86161c435..d59f5d6fa 100644 --- a/apps/api/alchemy.run.ts +++ b/apps/api/alchemy.run.ts @@ -243,6 +243,7 @@ export const createMapleApi = ({ stage, domains }: CreateMapleApiOptions) => TINYBIRD_TOKEN: Redacted.make(requireEnv("TINYBIRD_TOKEN")), ...optionalSecret("TINYBIRD_SIGNING_KEY"), ...optionalPlain("TINYBIRD_WORKSPACE_ID"), + ...optionalPlain("TINYBIRD_RAW_SQL_JWT_RPS_LIMIT"), ...optionalPlain("CLICKHOUSE_URL"), CLICKHOUSE_PROVIDER: process.env.CLICKHOUSE_PROVIDER?.trim() || "tinybird", ...optionalPlain("CLICKHOUSE_USER"), diff --git a/apps/api/src/platform/Env.ts b/apps/api/src/platform/Env.ts index 53710d5c7..dd610bd22 100644 --- a/apps/api/src/platform/Env.ts +++ b/apps/api/src/platform/Env.ts @@ -13,6 +13,8 @@ export interface EnvShape { readonly TINYBIRD_TOKEN: Redacted.Redacted readonly TINYBIRD_SIGNING_KEY: Option.Option> readonly TINYBIRD_WORKSPACE_ID: Option.Option + /** Optional Tinybird-enforced RPS ceiling, bucketed independently per org. */ + readonly TINYBIRD_RAW_SQL_JWT_RPS_LIMIT: Option.Option readonly CLICKHOUSE_URL: Option.Option readonly CLICKHOUSE_PROVIDER: string readonly CLICKHOUSE_USER: string @@ -100,6 +102,7 @@ const envConfig = Config.all({ TINYBIRD_TOKEN: Config.redacted("TINYBIRD_TOKEN"), TINYBIRD_SIGNING_KEY: optionalRedacted("TINYBIRD_SIGNING_KEY"), TINYBIRD_WORKSPACE_ID: optionalString("TINYBIRD_WORKSPACE_ID"), + TINYBIRD_RAW_SQL_JWT_RPS_LIMIT: Config.option(Config.number("TINYBIRD_RAW_SQL_JWT_RPS_LIMIT")), CLICKHOUSE_URL: optionalString("CLICKHOUSE_URL"), CLICKHOUSE_PROVIDER: stringWithDefault("CLICKHOUSE_PROVIDER", "tinybird"), CLICKHOUSE_USER: stringWithDefault("CLICKHOUSE_USER", "default"), @@ -244,6 +247,38 @@ const makeEnv = Effect.gen(function* () { ) } + const hasTinybirdSigningKey = Option.isSome(env.TINYBIRD_SIGNING_KEY) + const hasTinybirdWorkspaceId = Option.isSome(env.TINYBIRD_WORKSPACE_ID) + if (hasTinybirdSigningKey !== hasTinybirdWorkspaceId) { + return yield* Effect.die( + new EnvValidationError({ + message: + "TINYBIRD_SIGNING_KEY and TINYBIRD_WORKSPACE_ID must be configured together for Tinybird raw SQL", + }), + ) + } + + if ( + Option.isSome(env.TINYBIRD_RAW_SQL_JWT_RPS_LIMIT) && + (!Number.isSafeInteger(env.TINYBIRD_RAW_SQL_JWT_RPS_LIMIT.value) || + env.TINYBIRD_RAW_SQL_JWT_RPS_LIMIT.value <= 0) + ) { + return yield* Effect.die( + new EnvValidationError({ + message: "TINYBIRD_RAW_SQL_JWT_RPS_LIMIT must be a positive integer when configured", + }), + ) + } + + if ( + Option.isSome(env.TINYBIRD_SIGNING_KEY) && + Redacted.value(env.TINYBIRD_TOKEN) === Redacted.value(env.TINYBIRD_SIGNING_KEY.value) + ) { + yield* Effect.logWarning( + "TINYBIRD_TOKEN and TINYBIRD_SIGNING_KEY are identical; use a least-privilege runtime token instead of the workspace admin token", + ) + } + const authMode = env.MAPLE_AUTH_MODE.toLowerCase() if (authMode !== "clerk" && Option.isNone(env.MAPLE_ROOT_PASSWORD)) { diff --git a/apps/api/src/services/auth/tinybird-jwt.test.ts b/apps/api/src/services/auth/tinybird-jwt.test.ts index 1944ae76b..589dbcfb9 100644 --- a/apps/api/src/services/auth/tinybird-jwt.test.ts +++ b/apps/api/src/services/auth/tinybird-jwt.test.ts @@ -27,7 +27,8 @@ describe("mintOrgReadJwt", () => { scopes: ReadonlyArray<{ type: string; resource: string; filter: string }> } assert.strictEqual(decoded.workspace_id, "ws-uuid-123") - assert.strictEqual(decoded.name, "maple-raw-sql") + assert.match(decoded.name, /^maple-raw-sql-[0-9a-f]{16}$/) + assert.notInclude(decoded.name, "org_abc") assert.strictEqual(decoded.exp, 1_600) assert.deepStrictEqual(decoded.scopes, [ { type: "DATASOURCES:READ", resource: "traces", filter: "OrgId = 'org_abc'" }, @@ -39,6 +40,27 @@ describe("mintOrgReadJwt", () => { assert.strictEqual(signature, expected) }) + it("adds an org-isolated Tinybird rate-limit bucket when configured", () => { + const tokenFor = (orgId: string) => + mintOrgReadJwt({ + signingKey: SIGNING_KEY, + workspaceId: "ws-uuid-123", + orgId, + datasourceNames: ["traces"], + nowSeconds: 1_000, + ttlSeconds: 600, + rpsLimit: 25, + }) + const decode = (token: string) => + decodePart(token.split(".")[1]) as { name: string; limits: { rps: number } } + + const a = decode(tokenFor("org_a")) + const b = decode(tokenFor("org_b")) + assert.deepStrictEqual(a.limits, { rps: 25 }) + assert.deepStrictEqual(b.limits, { rps: 25 }) + assert.notStrictEqual(a.name, b.name) + }) + it("escapes single quotes in the org id to prevent filter injection", () => { const jwt = mintOrgReadJwt({ signingKey: SIGNING_KEY, diff --git a/apps/api/src/services/auth/tinybird-jwt.ts b/apps/api/src/services/auth/tinybird-jwt.ts index db39d3ee4..fe8593395 100644 --- a/apps/api/src/services/auth/tinybird-jwt.ts +++ b/apps/api/src/services/auth/tinybird-jwt.ts @@ -1,4 +1,4 @@ -import { createHmac } from "node:crypto" +import { createHash, createHmac } from "node:crypto" import { escapeClickHouseString } from "@maple/query-engine/sql" // --------------------------------------------------------------------------- @@ -36,11 +36,21 @@ export interface MintOrgReadJwtInput { readonly nowSeconds: number /** Token lifetime in seconds. */ readonly ttlSeconds: number + /** Optional Tinybird-enforced request ceiling for this org's token bucket. */ + readonly rpsLimit?: number } const base64url = (input: string | Buffer): string => (Buffer.isBuffer(input) ? input : Buffer.from(input, "utf8")).toString("base64url") +/** + * Tinybird groups JWT rate limits by `name`. Keep the bucket stable per org so + * one noisy tenant cannot consume every org's allowance, while hashing the org + * id keeps customer identifiers out of Tinybird's token-name analytics. + */ +const tokenNameForOrg = (orgId: string): string => + `maple-raw-sql-${createHash("sha256").update(orgId).digest("hex").slice(0, 16)}` + /** * Mint a per-org Tinybird read JWT scoped to `datasourceNames`, each filtered to * `OrgId = ''`. HS256, signed with the workspace admin token. @@ -57,9 +67,10 @@ export function mintOrgReadJwt(input: MintOrgReadJwtInput): string { const payload = base64url( JSON.stringify({ workspace_id: input.workspaceId, - name: "maple-raw-sql", + name: tokenNameForOrg(input.orgId), exp: input.nowSeconds + input.ttlSeconds, scopes, + ...(input.rpsLimit === undefined ? {} : { limits: { rps: input.rpsLimit } }), }), ) const signingInput = `${header}.${payload}` diff --git a/apps/api/src/services/integrations/TinybirdOrgTokenService.test.ts b/apps/api/src/services/integrations/TinybirdOrgTokenService.test.ts index 7e4da446e..5070fafaa 100644 --- a/apps/api/src/services/integrations/TinybirdOrgTokenService.test.ts +++ b/apps/api/src/services/integrations/TinybirdOrgTokenService.test.ts @@ -1,9 +1,9 @@ import { assert, describe, it } from "@effect/vitest" -import { ConfigProvider, Effect, Layer } from "effect" +import { Cause, ConfigProvider, Effect, Exit, Layer } from "effect" import { OrgId } from "@maple/domain" import { Schema } from "effect" import { TestClock } from "effect/testing" -import { TinybirdOrgTokenService } from "./TinybirdOrgTokenService" +import { JWT_CACHE_MAX_ENTRIES, TinybirdOrgTokenService } from "./TinybirdOrgTokenService" import { Env } from "@/platform/Env" const SIGNING_KEY = "explicit-test-signing-key" @@ -36,7 +36,9 @@ const decodePayload = (jwt: string) => JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString("utf8")) as { workspace_id: string exp: number + name: string scopes: ReadonlyArray<{ resource: string; filter: string }> + limits?: { rps: number } } describe("TinybirdOrgTokenService", () => { @@ -84,6 +86,35 @@ describe("TinybirdOrgTokenService", () => { }).pipe(Effect.provide(layer)), ) + it.effect("applies the configured per-org Tinybird RPS limit", () => { + const limitedLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({ TINYBIRD_RAW_SQL_JWT_RPS_LIMIT: "25" })), + ) + return Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const a = decodePayload(yield* svc.getOrgReadToken(asOrgId("org_a"))) + const b = decodePayload(yield* svc.getOrgReadToken(asOrgId("org_b"))) + assert.deepStrictEqual(a.limits, { rps: 25 }) + assert.deepStrictEqual(b.limits, { rps: 25 }) + assert.notStrictEqual(a.name, b.name) + }).pipe(Effect.provide(limitedLayer)) + }) + + it.effect("bounds the per-isolate token cache with LRU eviction", () => + Effect.gen(function* () { + const svc = yield* TinybirdOrgTokenService + const first = yield* svc.getOrgReadToken(asOrgId("org_0")) + for (let i = 1; i <= JWT_CACHE_MAX_ENTRIES; i++) { + yield* svc.getOrgReadToken(asOrgId(`org_${i}`)) + } + yield* TestClock.setTime(1_000) + const reminted = yield* svc.getOrgReadToken(asOrgId("org_0")) + assert.notStrictEqual(reminted, first) + assert.isAbove(decodePayload(reminted).exp, decodePayload(first).exp) + }).pipe(Effect.provide(layer)), + ) + it.effect("returns a typed error when signing configuration is missing", () => { const missingLayer = TinybirdOrgTokenService.layer.pipe( Layer.provide(Env.layer), @@ -97,15 +128,41 @@ describe("TinybirdOrgTokenService", () => { }).pipe(Effect.provide(missingLayer)) }) - it.effect("returns a typed error for an empty workspace id", () => { + it.effect("fails startup when only one Tinybird JWT setting is configured", () => { const malformedLayer = TinybirdOrgTokenService.layer.pipe( Layer.provide(Env.layer), Layer.provide(testConfig({ TINYBIRD_WORKSPACE_ID: "" })), ) return Effect.gen(function* () { - const svc = yield* TinybirdOrgTokenService - const error = yield* Effect.flip(svc.getOrgReadToken(asOrgId("org_a"))) - assert.strictEqual(error.reason, "MissingWorkspaceId") - }).pipe(Effect.provide(malformedLayer)) + yield* TinybirdOrgTokenService + }).pipe( + Effect.provide(malformedLayer), + Effect.exit, + Effect.map((exit) => { + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) { + assert.match(String(Cause.squash(exit.cause)), /must be configured together/) + } + }), + ) + }) + + it.effect("fails startup for a non-positive JWT RPS limit", () => { + const malformedLayer = TinybirdOrgTokenService.layer.pipe( + Layer.provide(Env.layer), + Layer.provide(testConfig({ TINYBIRD_RAW_SQL_JWT_RPS_LIMIT: "0" })), + ) + return Effect.gen(function* () { + yield* TinybirdOrgTokenService + }).pipe( + Effect.provide(malformedLayer), + Effect.exit, + Effect.map((exit) => { + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) { + assert.match(String(Cause.squash(exit.cause)), /must be a positive integer/) + } + }), + ) }) }) diff --git a/apps/api/src/services/integrations/TinybirdOrgTokenService.ts b/apps/api/src/services/integrations/TinybirdOrgTokenService.ts index 5e22e7c05..df5f14809 100644 --- a/apps/api/src/services/integrations/TinybirdOrgTokenService.ts +++ b/apps/api/src/services/integrations/TinybirdOrgTokenService.ts @@ -19,6 +19,12 @@ import { Env } from "@/platform/Env" const JWT_TTL_SECONDS = 600 /** Re-mint this many seconds before true expiry (must exceed the executor's 30s client cache). */ const JWT_REFRESH_SKEW_SECONDS = 60 +/** + * Each token carries one scope per OrgId-bearing datasource, so an unbounded + * org map can consume meaningful isolate memory. Eviction only causes a cheap + * local re-mint; it does not invalidate the previously issued JWT. + */ +export const JWT_CACHE_MAX_ENTRIES = 512 export interface TinybirdOrgTokenServiceShape { /** A Tinybird read JWT scoped to `orgId` across every OrgId-bearing datasource. */ @@ -44,6 +50,16 @@ export class TinybirdOrgTokenService extends Context.Service< // Per-instance (per-isolate) cache. `expiresAt` is the re-mint deadline in ms. const cache = new Map() + const pruneCache = (nowMs: number) => { + for (const [orgId, entry] of cache) { + if (entry.expiresAt <= nowMs) cache.delete(orgId) + } + while (cache.size >= JWT_CACHE_MAX_ENTRIES) { + const oldestOrgId = cache.keys().next().value + if (oldestOrgId === undefined) break + cache.delete(oldestOrgId) + } + } const getOrgReadToken = Effect.fn("TinybirdOrgTokenService.getOrgReadToken")(function* ( orgId: OrgId, @@ -51,10 +67,15 @@ export class TinybirdOrgTokenService extends Context.Service< const nowMs = yield* Clock.currentTimeMillis const cached = cache.get(orgId) if (cached !== undefined && cached.expiresAt > nowMs) { + // Map iteration order is insertion order; refresh it on access so the + // fixed-size cache evicts the least-recently-used org. + cache.delete(orgId) + cache.set(orgId, cached) yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", true) return cached.token } yield* Effect.annotateCurrentSpan("maple.tinybird.jwt.cache_hit", false) + pruneCache(nowMs) if (Option.isNone(env.TINYBIRD_SIGNING_KEY)) { return yield* new TinybirdOrgTokenError({ reason: "MissingSigningKey", @@ -84,6 +105,7 @@ export class TinybirdOrgTokenService extends Context.Service< datasourceNames, nowSeconds: Math.floor(nowMs / 1000), ttlSeconds: JWT_TTL_SECONDS, + rpsLimit: Option.getOrUndefined(env.TINYBIRD_RAW_SQL_JWT_RPS_LIMIT), }), catch: () => new TinybirdOrgTokenError({ diff --git a/docs/self-hosted-clickhouse.md b/docs/self-hosted-clickhouse.md index 2667553ea..5c7442b56 100644 --- a/docs/self-hosted-clickhouse.md +++ b/docs/self-hosted-clickhouse.md @@ -30,6 +30,8 @@ removes Tinybird-restricted query settings. For a vanilla/self-managed server, s `CLICKHOUSE_PROVIDER=clickhouse`; Maple then preserves `CLICKHOUSE_PASSWORD` for raw SQL. Tinybird raw SQL also requires explicit `TINYBIRD_SIGNING_KEY` and `TINYBIRD_WORKSPACE_ID` values; Maple never derives either from the API token. +Set `TINYBIRD_RAW_SQL_JWT_RPS_LIMIT` to a positive integer to add an optional +Tinybird-enforced request ceiling; Maple gives each org an independent bucket. Env-level vanilla ClickHouse raw SQL is enabled only when `MAPLE_AUTH_MODE=self_hosted`, where the deployment is single-org. Hosted multi-org deployments fail closed unless