Skip to content
Merged
14 changes: 7 additions & 7 deletions apps/sim/background/knowledge-processing.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createLogger } from '@sim/logger'
import { task } from '@trigger.dev/sdk'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { processDocumentAsync } from '@/lib/knowledge/documents/service'

const logger = createLogger('TriggerKnowledgeProcessing')
Expand All @@ -23,16 +23,16 @@ export type DocumentProcessingPayload = {

export const processDocument = task({
id: 'knowledge-process-document',
maxDuration: env.KB_CONFIG_MAX_DURATION || 600,
maxDuration: envNumber(env.KB_CONFIG_MAX_DURATION, 600),
machine: 'large-1x', // 2 vCPU, 2GB RAM - needed for large PDF processing
retry: {
maxAttempts: env.KB_CONFIG_MAX_ATTEMPTS || 3,
factor: env.KB_CONFIG_RETRY_FACTOR || 2,
minTimeoutInMs: env.KB_CONFIG_MIN_TIMEOUT || 1000,
maxTimeoutInMs: env.KB_CONFIG_MAX_TIMEOUT || 10000,
maxAttempts: envNumber(env.KB_CONFIG_MAX_ATTEMPTS, 3),
factor: envNumber(env.KB_CONFIG_RETRY_FACTOR, 2),
minTimeoutInMs: envNumber(env.KB_CONFIG_MIN_TIMEOUT, 1000),
maxTimeoutInMs: envNumber(env.KB_CONFIG_MAX_TIMEOUT, 10000),
},
queue: {
concurrencyLimit: env.KB_CONFIG_CONCURRENCY_LIMIT || 20,
concurrencyLimit: envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20),
name: 'document-processing-queue',
},
run: async (payload: DocumentProcessingPayload) => {
Expand Down
10 changes: 5 additions & 5 deletions apps/sim/lib/billing/subscriptions/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import {
isTeam,
} from '@/lib/billing/plan-helpers'
import { parseEnterpriseSubscriptionMetadata } from '@/lib/billing/types'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'

export const ENTITLED_SUBSCRIPTION_STATUSES = ['active', 'past_due'] as const

Expand Down Expand Up @@ -52,28 +52,28 @@ export function hasUsableSubscriptionAccess(
* Get the free tier limit from env or fallback to default
*/
export function getFreeTierLimit(): number {
return env.FREE_TIER_COST_LIMIT || DEFAULT_FREE_CREDITS
return envNumber(env.FREE_TIER_COST_LIMIT, DEFAULT_FREE_CREDITS)
}

/**
* Get the pro tier limit from env or fallback to default
*/
export function getProTierLimit(): number {
return env.PRO_TIER_COST_LIMIT || DEFAULT_PRO_TIER_COST_LIMIT
return envNumber(env.PRO_TIER_COST_LIMIT, DEFAULT_PRO_TIER_COST_LIMIT)
}

/**
* Get the team tier limit per seat from env or fallback to default
*/
export function getTeamTierLimitPerSeat(): number {
return env.TEAM_TIER_COST_LIMIT || DEFAULT_TEAM_TIER_COST_LIMIT
return envNumber(env.TEAM_TIER_COST_LIMIT, DEFAULT_TEAM_TIER_COST_LIMIT)
}

/**
* Get the enterprise tier limit per seat from env or fallback to default
*/
export function getEnterpriseTierLimitPerSeat(): number {
return env.ENTERPRISE_TIER_COST_LIMIT || DEFAULT_ENTERPRISE_TIER_COST_LIMIT
return envNumber(env.ENTERPRISE_TIER_COST_LIMIT, DEFAULT_ENTERPRISE_TIER_COST_LIMIT)
}

export function checkEnterprisePlan(subscription: any): boolean {
Expand Down
4 changes: 2 additions & 2 deletions apps/sim/lib/billing/threshold-billing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ import {
} from '@/lib/billing/subscriptions/utils'
import { toDecimal, toNumber } from '@/lib/billing/utils/decimal'
import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-handlers'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { enqueueOutboxEvent } from '@/lib/core/outbox/service'

const logger = createLogger('ThresholdBilling')

const OVERAGE_THRESHOLD = env.OVERAGE_THRESHOLD_DOLLARS || DEFAULT_OVERAGE_THRESHOLD
const OVERAGE_THRESHOLD = envNumber(env.OVERAGE_THRESHOLD_DOLLARS, DEFAULT_OVERAGE_THRESHOLD)

export async function checkAndBillOverageThreshold(userId: string): Promise<void> {
try {
Expand Down
14 changes: 3 additions & 11 deletions apps/sim/lib/copilot/request/session/buffer.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { sleep } from '@sim/utils/helpers'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { getRedisClient } from '@/lib/core/config/redis'
import {
type PersistedStreamEventEnvelope,
Expand Down Expand Up @@ -40,19 +40,11 @@ export type StreamConfig = {

export function getStreamConfig(): StreamConfig {
return {
ttlSeconds: parsePositiveNumber(env.COPILOT_STREAM_TTL_SECONDS, DEFAULT_TTL_SECONDS),
eventLimit: parsePositiveNumber(env.COPILOT_STREAM_EVENT_LIMIT, DEFAULT_EVENT_LIMIT),
ttlSeconds: envNumber(env.COPILOT_STREAM_TTL_SECONDS, DEFAULT_TTL_SECONDS, { min: 1 }),
eventLimit: envNumber(env.COPILOT_STREAM_EVENT_LIMIT, DEFAULT_EVENT_LIMIT, { min: 1 }),
}
}

function parsePositiveNumber(value: number | string | undefined, fallback: number) {
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
return value
}
const parsed = Number(value)
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback
}

async function withRedisRetry<T>(
metadata: RedisOperationMetadata,
operation: (redis: NonNullable<ReturnType<typeof getRedisClient>>) => Promise<T>
Expand Down
34 changes: 34 additions & 0 deletions apps/sim/lib/core/config/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,16 @@ export const env = createEnv({
ENTERPRISE_STORAGE_LIMIT_GB: z.number().optional().default(500), // Default storage limit in GB for enterprise tier (can be overridden per org)
BILLING_ENABLED: z.boolean().optional(), // Enable billing enforcement and usage tracking

// Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans.
FREE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on free tier (default: 3)
FREE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on free tier (default: 1000)
PRO_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on pro tier (default: 25)
PRO_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on pro tier (default: 5000)
TEAM_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on team tier (default: 100)
TEAM_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on team tier (default: 10000)
ENTERPRISE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on enterprise tier (default: 10000)
ENTERPRISE_TABLE_ROWS_LIMIT: z.number().optional(), // Max rows per table on enterprise tier (default: 1000000)

// Credit-tier Stripe prices (monthly)
STRIPE_PRICE_TIER_25_MO: z.string().min(1).optional(), // Pro: $25/mo (6,000 credits)
STRIPE_PRICE_TIER_100_MO: z.string().min(1).optional(), // Max: $100/mo (25,000 credits)
Expand Down Expand Up @@ -504,3 +514,27 @@ export const isFalsy = (value: string | boolean | number | undefined) =>
typeof value === 'string' ? value.toLowerCase() === 'false' || value === '0' : value === false

export { getEnv }

/**
* Coerce an env-derived value to a finite number ≥ `min`, falling back to the
* provided default when the value is unset, empty, non-finite, or below `min`.
* `min` defaults to `0` so configs like `KB_CONFIG_DELAY_BETWEEN_BATCHES=0`
* (meaning "no delay / max throughput") are honored. Pass `min: 1` for configs
* where zero is invalid (e.g. Redis TTLs, capacity limits).
*
* `createEnv` is configured with `skipValidation: true`, so values declared as
* `z.number()` arrive as raw strings when sourced from `process.env` or Helm.
* Use this helper anywhere a numeric env override is consumed to normalize the
* type at the boundary instead of relying on JS implicit coercion.
*/
export function envNumber(
value: number | string | undefined | null,
fallback: number,
options: { min?: number } = {}
): number {
const min = options.min ?? 0
if (typeof value === 'number' && Number.isFinite(value) && value >= min) return value
if (value === undefined || value === null || value === '') return fallback
const parsed = Number(value)
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback
}
4 changes: 2 additions & 2 deletions apps/sim/lib/knowledge/documents/document-processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import {
TokenChunker,
} from '@/lib/chunkers'
import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { parseBuffer, parseFile } from '@/lib/file-parsers'
import type { FileParseMetadata } from '@/lib/file-parsers/types'
import { resolveParserExtension } from '@/lib/knowledge/documents/parser-extension'
Expand All @@ -30,7 +30,7 @@ const TIMEOUTS = {
MISTRAL_OCR_API: 120000,
} as const

const MAX_CONCURRENT_CHUNKS = env.KB_CONFIG_CHUNK_CONCURRENCY
const MAX_CONCURRENT_CHUNKS = envNumber(env.KB_CONFIG_CHUNK_CONCURRENCY, 10)

type OCRResult = {
success: boolean
Expand Down
15 changes: 8 additions & 7 deletions apps/sim/lib/knowledge/documents/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import {
import { recordUsage } from '@/lib/billing/core/usage-log'
import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing'
import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/feature-flags'
import { processDocument } from '@/lib/knowledge/documents/document-processor'
import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types'
Expand All @@ -54,12 +54,12 @@ import { calculateCost } from '@/providers/utils'
const logger = createLogger('DocumentService')

const TIMEOUTS = {
OVERALL_PROCESSING: (env.KB_CONFIG_MAX_DURATION || 600) * 1000,
OVERALL_PROCESSING: envNumber(env.KB_CONFIG_MAX_DURATION, 600) * 1000,
} as const

const LARGE_DOC_CONFIG = {
MAX_CHUNKS_PER_BATCH: 500,
MAX_EMBEDDING_BATCH: env.KB_CONFIG_BATCH_SIZE || 2000,
MAX_EMBEDDING_BATCH: envNumber(env.KB_CONFIG_BATCH_SIZE, 2000),
MAX_FILE_SIZE: 100 * 1024 * 1024,
MAX_CHUNKS_PER_DOCUMENT: 100000,
}
Expand All @@ -78,10 +78,11 @@ function withTimeout<T>(
}

const PROCESSING_CONFIG = {
maxConcurrentDocuments: Math.max(1, Math.floor((env.KB_CONFIG_CONCURRENCY_LIMIT || 20) / 5)) || 4,
batchSize: Math.max(1, Math.floor((env.KB_CONFIG_BATCH_SIZE || 20) / 2)) || 10,
delayBetweenBatches: (env.KB_CONFIG_DELAY_BETWEEN_BATCHES || 100) * 2,
delayBetweenDocuments: (env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS || 50) * 2,
maxConcurrentDocuments:
Math.max(1, Math.floor(envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20) / 5)) || 4,
batchSize: Math.max(1, Math.floor(envNumber(env.KB_CONFIG_BATCH_SIZE, 20) / 2)) || 10,
delayBetweenBatches: envNumber(env.KB_CONFIG_DELAY_BETWEEN_BATCHES, 100) * 2,
Comment thread
waleedlatif1 marked this conversation as resolved.
delayBetweenDocuments: envNumber(env.KB_CONFIG_DELAY_BETWEEN_DOCUMENTS, 50) * 2,
}

export function getProcessingConfig() {
Expand Down
4 changes: 2 additions & 2 deletions apps/sim/lib/knowledge/embeddings.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { createLogger } from '@sim/logger'
import { getBYOKKey } from '@/lib/api-key/byok'
import { getRotatingApiKey } from '@/lib/core/config/api-keys'
import { env } from '@/lib/core/config/env'
import { env, envNumber } from '@/lib/core/config/env'
import { isRetryableError, retryWithExponentialBackoff } from '@/lib/knowledge/documents/utils'
import {
DEFAULT_EMBEDDING_MODEL,
Expand All @@ -15,7 +15,7 @@ import { batchByTokenLimit, estimateTokenCount } from '@/lib/tokenization'
const logger = createLogger('EmbeddingUtils')

const MAX_TOKENS_PER_REQUEST = 8000
const MAX_CONCURRENT_BATCHES = env.KB_CONFIG_CONCURRENCY_LIMIT || 50
const MAX_CONCURRENT_BATCHES = envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 50)
const EMBEDDING_REQUEST_TIMEOUT_MS = 60_000

export type { EmbeddingModelInfo } from '@/lib/knowledge/embedding-models'
Expand Down
10 changes: 6 additions & 4 deletions apps/sim/lib/table/billing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { createLogger } from '@sim/logger'
import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription'
import { getPlanTypeForLimits } from '@/lib/billing/plan-helpers'
import { getWorkspaceBilledAccountUserId } from '@/lib/workspaces/utils'
import { type PlanName, TABLE_PLAN_LIMITS, type TablePlanLimits } from './constants'
import { getTablePlanLimits, type PlanName, type TablePlanLimits } from './constants'

const logger = createLogger('TableBilling')

Expand All @@ -22,18 +22,20 @@ const logger = createLogger('TableBilling')
* @returns Table limits based on the workspace's billing plan
*/
export async function getWorkspaceTableLimits(workspaceId: string): Promise<TablePlanLimits> {
const planLimits = getTablePlanLimits()

try {
const billedAccountUserId = await getWorkspaceBilledAccountUserId(workspaceId)

if (!billedAccountUserId) {
logger.warn('No billed account found for workspace, using free tier limits', { workspaceId })
return TABLE_PLAN_LIMITS.free
return planLimits.free
}

const subscription = await getHighestPrioritySubscription(billedAccountUserId)
const planName = getPlanTypeForLimits(subscription?.plan) as PlanName

const limits = TABLE_PLAN_LIMITS[planName] ?? TABLE_PLAN_LIMITS.free
const limits = planLimits[planName] ?? planLimits.free

logger.info('Retrieved workspace table limits', {
workspaceId,
Expand All @@ -48,7 +50,7 @@ export async function getWorkspaceTableLimits(workspaceId: string): Promise<Tabl
workspaceId,
error,
})
return TABLE_PLAN_LIMITS.free
return planLimits.free
}
}

Expand Down
53 changes: 50 additions & 3 deletions apps/sim/lib/table/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
* Limits and constants for user-defined tables.
*/

import { env, envNumber } from '@/lib/core/config/env'

export const TABLE_LIMITS = {
MAX_TABLES_PER_WORKSPACE: 100,
MAX_ROWS_PER_TABLE: 10000,
Expand All @@ -24,9 +26,11 @@ export const TABLE_LIMITS = {
} as const

/**
* Plan-based table limits.
* Default plan-based table limits. Each value can be overridden via env vars
* (see `getTablePlanLimits`) so self-hosted deployments can raise the free-tier
* caps that apply when billing is disabled.
*/
export const TABLE_PLAN_LIMITS = {
export const DEFAULT_TABLE_PLAN_LIMITS = {
free: {
maxTables: 3,
maxRowsPerTable: 1000,
Expand All @@ -45,13 +49,56 @@ export const TABLE_PLAN_LIMITS = {
},
} as const

export type PlanName = keyof typeof TABLE_PLAN_LIMITS
export type PlanName = keyof typeof DEFAULT_TABLE_PLAN_LIMITS

export interface TablePlanLimits {
maxTables: number
maxRowsPerTable: number
}

export type TablePlanLimitsByPlan = Record<PlanName, TablePlanLimits>

/**
* Returns plan-based table limits, applying env var overrides on top of the
* defaults. When no override is set the value falls back to the hosted-default
* constant so behavior is unchanged for the hosted product.
*/
export function getTablePlanLimits(): TablePlanLimitsByPlan {
return {
free: {
maxTables: envNumber(env.FREE_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.free.maxTables),
maxRowsPerTable: envNumber(
env.FREE_TABLE_ROWS_LIMIT,
DEFAULT_TABLE_PLAN_LIMITS.free.maxRowsPerTable
),
},
pro: {
maxTables: envNumber(env.PRO_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.pro.maxTables),
maxRowsPerTable: envNumber(
env.PRO_TABLE_ROWS_LIMIT,
DEFAULT_TABLE_PLAN_LIMITS.pro.maxRowsPerTable
),
},
team: {
maxTables: envNumber(env.TEAM_TABLES_LIMIT, DEFAULT_TABLE_PLAN_LIMITS.team.maxTables),
maxRowsPerTable: envNumber(
env.TEAM_TABLE_ROWS_LIMIT,
DEFAULT_TABLE_PLAN_LIMITS.team.maxRowsPerTable
),
},
enterprise: {
maxTables: envNumber(
env.ENTERPRISE_TABLES_LIMIT,
DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxTables
),
maxRowsPerTable: envNumber(
env.ENTERPRISE_TABLE_ROWS_LIMIT,
DEFAULT_TABLE_PLAN_LIMITS.enterprise.maxRowsPerTable
),
},
}
}
Comment thread
waleedlatif1 marked this conversation as resolved.

export const COLUMN_TYPES = ['string', 'number', 'boolean', 'date', 'json'] as const

export const NAME_PATTERN = /^[a-z_][a-z0-9_]*$/i
Expand Down
11 changes: 11 additions & 0 deletions helm/sim/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,17 @@ app:
EXECUTION_TIMEOUT_ASYNC_TEAM: "5400" # Team tier async timeout (90 minutes)
EXECUTION_TIMEOUT_ASYNC_ENTERPRISE: "5400" # Enterprise tier async timeout (90 minutes)

# Table Feature Limits (per workspace, per plan)
# Apply when billing is disabled (free tier defaults) or for billed plans
FREE_TABLES_LIMIT: "3" # Max user tables per workspace on free tier
FREE_TABLE_ROWS_LIMIT: "1000" # Max rows per table on free tier
PRO_TABLES_LIMIT: "25" # Max user tables per workspace on pro tier
PRO_TABLE_ROWS_LIMIT: "5000" # Max rows per table on pro tier
TEAM_TABLES_LIMIT: "100" # Max user tables per workspace on team tier
TEAM_TABLE_ROWS_LIMIT: "10000" # Max rows per table on team tier
ENTERPRISE_TABLES_LIMIT: "10000" # Max user tables per workspace on enterprise tier
ENTERPRISE_TABLE_ROWS_LIMIT: "1000000" # Max rows per table on enterprise tier

# Isolated-VM Worker Pool Configuration
IVM_POOL_SIZE: "4" # Max worker processes in pool
IVM_MAX_CONCURRENT: "10000" # Max concurrent executions globally
Expand Down
11 changes: 11 additions & 0 deletions packages/testing/src/mocks/env.mock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ export function createEnvMock(overrides: Record<string, string | undefined> = {}
typeof value === 'string'
? value.toLowerCase() === 'false' || value === '0'
: value === false,
envNumber: (
value: number | string | undefined | null,
fallback: number,
options: { min?: number } = {}
): number => {
const min = options.min ?? 0
if (typeof value === 'number' && Number.isFinite(value) && value >= min) return value
if (value === undefined || value === null || value === '') return fallback
const parsed = Number(value)
return Number.isFinite(parsed) && parsed >= min ? parsed : fallback
},
}
}

Expand Down
Loading