diff --git a/apps/event-worker/Dockerfile b/apps/event-worker/Dockerfile index 0b7a9c9fb..0f7f43b81 100644 --- a/apps/event-worker/Dockerfile +++ b/apps/event-worker/Dockerfile @@ -32,6 +32,8 @@ COPY packages/db/package.json ./packages/db/package.json COPY packages/validators/package.json ./packages/validators/package.json COPY packages/logger/package.json ./packages/logger/package.json COPY packages/job-dispatch/package.json ./packages/job-dispatch/package.json +COPY packages/release-manager/package.json ./packages/release-manager/package.json +COPY packages/rule-engine/package.json ./packages/rule-engine/package.json COPY packages/secrets/package.json ./packages/secrets/package.json COPY apps/event-worker/package.json ./apps/event-worker/package.json diff --git a/apps/event-worker/package.json b/apps/event-worker/package.json index d8b39a671..9be882694 100644 --- a/apps/event-worker/package.json +++ b/apps/event-worker/package.json @@ -37,6 +37,7 @@ "js-yaml": "^4.1.0", "lodash": "catalog:", "ms": "^2.1.3", + "redis-semaphore": "^5.6.2", "semver": "catalog:", "ts-is-present": "^1.2.2", "uuid": "^10.0.0", diff --git a/apps/event-worker/src/index.ts b/apps/event-worker/src/index.ts index ee1249c16..4db9fc238 100644 --- a/apps/event-worker/src/index.ts +++ b/apps/event-worker/src/index.ts @@ -2,17 +2,23 @@ import { logger } from "@ctrlplane/logger"; import { createDispatchExecutionJobWorker } from "./job-dispatch/index.js"; import { redis } from "./redis.js"; +import { createReleaseNewVersionWorker } from "./releases/new-version/index.js"; import { createResourceScanWorker } from "./resource-scan/index.js"; const resourceScanWorker = createResourceScanWorker(); const dispatchExecutionJobWorker = createDispatchExecutionJobWorker(); +const releaseNewVersionWorker = createReleaseNewVersionWorker(); const shutdown = () => { logger.warn("Exiting..."); - resourceScanWorker.close(); - dispatchExecutionJobWorker.close(); - redis.quit(); - process.exit(0); + Promise.all([ + resourceScanWorker.close(), + dispatchExecutionJobWorker.close(), + releaseNewVersionWorker.close(), + ]).then(async () => { + await redis.quit(); + process.exit(0); + }); }; process.on("SIGTERM", shutdown); diff --git a/apps/event-worker/src/releases/evaluate/index.ts b/apps/event-worker/src/releases/evaluate/index.ts index 5c1a9dc2e..8c894bbf4 100644 --- a/apps/event-worker/src/releases/evaluate/index.ts +++ b/apps/event-worker/src/releases/evaluate/index.ts @@ -7,22 +7,30 @@ import { evaluate } from "@ctrlplane/rule-engine"; import { createCtx, getApplicablePolicies } from "@ctrlplane/rule-engine/db"; import { Channel } from "@ctrlplane/validators/events"; +import { ReleaseRepositoryMutex } from "../mutex.js"; + export const createReleaseEvaluateWorker = () => new Worker(Channel.ReleaseEvaluate, async (job) => { job.log( `Evaluating release for deployment ${job.data.deploymentId} and resource ${job.data.resourceId}`, ); - const ctx = await createCtx(db, job.data); - if (ctx == null) { - job.log( - `Resource ${job.data.resourceId} not found for deployment ${job.data.deploymentId} and environment ${job.data.environmentId}`, - ); - return; - } + const mutex = await ReleaseRepositoryMutex.lock(job.data); - const { workspaceId } = ctx.resource; - const policy = await getApplicablePolicies(db, workspaceId, job.data); - const result = await evaluate(policy, [], ctx); - console.log(result); + try { + const ctx = await createCtx(db, job.data); + if (ctx == null) { + job.log( + `Resource ${job.data.resourceId} not found for deployment ${job.data.deploymentId} and environment ${job.data.environmentId}`, + ); + return; + } + + const { workspaceId } = ctx.resource; + const policy = await getApplicablePolicies(db, workspaceId, job.data); + const result = await evaluate(policy, [], ctx); + console.log(result); + } finally { + await mutex.unlock(); + } }); diff --git a/apps/event-worker/src/releases/mutex.ts b/apps/event-worker/src/releases/mutex.ts new file mode 100644 index 000000000..89c9fd867 --- /dev/null +++ b/apps/event-worker/src/releases/mutex.ts @@ -0,0 +1,30 @@ +import type { ReleaseRepository } from "@ctrlplane/rule-engine"; +import type { Mutex as RedisMutex } from "redis-semaphore"; +import { Mutex as RedisSemaphoreMutex } from "redis-semaphore"; + +import { redis } from "../redis.js"; + +export class ReleaseRepositoryMutex { + static async lock(repo: ReleaseRepository) { + const mutex = new ReleaseRepositoryMutex(repo); + await mutex.lock(); + return mutex; + } + + private mutex: RedisMutex; + + constructor(repo: ReleaseRepository) { + const key = `release-repository-mutex-${repo.deploymentId}-${repo.resourceId}-${repo.environmentId}`; + this.mutex = new RedisSemaphoreMutex(redis, key, {}); + } + + async lock(): Promise { + if (this.mutex.isAcquired) throw new Error("Mutex is already locked"); + await this.mutex.acquire(); + } + + async unlock(): Promise { + if (!this.mutex.isAcquired) throw new Error("Mutex is not locked"); + await this.mutex.release(); + } +} diff --git a/apps/event-worker/src/releases/new-version/index.ts b/apps/event-worker/src/releases/new-version/index.ts new file mode 100644 index 000000000..a7e4d0063 --- /dev/null +++ b/apps/event-worker/src/releases/new-version/index.ts @@ -0,0 +1,26 @@ +import type { ReleaseNewVersionEvent } from "@ctrlplane/validators/events"; +import { Worker } from "bullmq"; +import _ from "lodash"; + +import { eq } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import * as schema from "@ctrlplane/db/schema"; +import { Channel } from "@ctrlplane/validators/events"; + +import { getSystemResources } from "./system-resources.js"; + +export const createReleaseNewVersionWorker = () => + new Worker(Channel.ReleaseNewVersion, async (job) => { + const version = await db.query.deploymentVersion.findFirst({ + where: eq(schema.deploymentVersion.id, job.data.versionId), + with: { deployment: true }, + }); + + if (version == null) throw new Error("Version not found"); + + const { deployment } = version; + const { systemId } = deployment; + + const impactedResources = await getSystemResources(db, systemId); + console.log(impactedResources.length); + }); diff --git a/apps/event-worker/src/releases/new-version/system-resources.ts b/apps/event-worker/src/releases/new-version/system-resources.ts new file mode 100644 index 000000000..1f21e454e --- /dev/null +++ b/apps/event-worker/src/releases/new-version/system-resources.ts @@ -0,0 +1,37 @@ +import type { Tx } from "@ctrlplane/db"; +import _ from "lodash"; + +import { and, eq } from "@ctrlplane/db"; +import * as schema from "@ctrlplane/db/schema"; + +/** + * Retrieves all resources for a given system by using environment selectors + */ +export const getSystemResources = async (tx: Tx, systemId: string) => { + const system = await tx.query.system.findFirst({ + where: eq(schema.system.id, systemId), + with: { environments: true }, + }); + + if (system == null) throw new Error("System not found"); + + const { environments } = system; + + // Simplify the chained operations with standard Promise.all + const resources = await Promise.all( + environments.map(async (env) => { + const res = await tx + .select() + .from(schema.resource) + .where( + and( + eq(schema.resource.workspaceId, system.workspaceId), + schema.resourceMatchesMetadata(tx, env.resourceSelector), + ), + ); + return res.map((r) => ({ ...r, environment: env })); + }), + ).then((arrays) => arrays.flat()); + + return resources; +}; diff --git a/apps/event-worker/src/releases/variable-change/index.ts b/apps/event-worker/src/releases/variable-change/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/db/src/schema/deployment-variables.ts b/packages/db/src/schema/deployment-variables.ts index 455307470..f32883ada 100644 --- a/packages/db/src/schema/deployment-variables.ts +++ b/packages/db/src/schema/deployment-variables.ts @@ -2,7 +2,7 @@ import type { ResourceCondition } from "@ctrlplane/validators/resources"; import type { VariableConfigType } from "@ctrlplane/validators/variables"; import type { InferInsertModel, InferSelectModel } from "drizzle-orm"; import type { AnyPgColumn, ColumnsWithTable } from "drizzle-orm/pg-core"; -import { sql } from "drizzle-orm"; +import { relations, sql } from "drizzle-orm"; import { foreignKey, jsonb, @@ -116,3 +116,44 @@ export const deploymentVariableSet = pgTable( }, (t) => ({ uniq: uniqueIndex().on(t.deploymentId, t.variableSetId) }), ); + +export const deploymentVariableRelationships = relations( + deploymentVariable, + ({ one, many }) => ({ + deployment: one(deployment, { + fields: [deploymentVariable.deploymentId], + references: [deployment.id], + }), + + defaultValue: one(deploymentVariableValue, { + fields: [deploymentVariable.defaultValueId], + references: [deploymentVariableValue.id], + }), + + values: many(deploymentVariableValue), + }), +); + +export const deploymentVariableValueRelationships = relations( + deploymentVariableValue, + ({ one }) => ({ + variable: one(deploymentVariable, { + fields: [deploymentVariableValue.variableId], + references: [deploymentVariable.id], + }), + }), +); + +export const deploymentVariableSetRelationships = relations( + deploymentVariableSet, + ({ one }) => ({ + deployment: one(deployment, { + fields: [deploymentVariableSet.deploymentId], + references: [deployment.id], + }), + variableSet: one(variableSet, { + fields: [deploymentVariableSet.variableSetId], + references: [variableSet.id], + }), + }), +); diff --git a/packages/db/src/schema/deployment-version.ts b/packages/db/src/schema/deployment-version.ts index 24d11fd09..c4caba148 100644 --- a/packages/db/src/schema/deployment-version.ts +++ b/packages/db/src/schema/deployment-version.ts @@ -17,6 +17,7 @@ import { not, notExists, or, + relations, sql, } from "drizzle-orm"; import { @@ -286,3 +287,50 @@ export function deploymentVersionMatchesCondition( ? undefined : buildCondition(tx, condition); } + +export const deploymentVersionRelations = relations( + deploymentVersion, + ({ one, many }) => ({ + deployment: one(deployment, { + fields: [deploymentVersion.deploymentId], + references: [deployment.id], + }), + metadata: many(deploymentVersionMetadata), + dependencies: many(versionDependency), + channels: many(deploymentVersionChannel), + }), +); + +export const deploymentVersionChannelRelations = relations( + deploymentVersionChannel, + ({ one }) => ({ + deployment: one(deployment, { + fields: [deploymentVersionChannel.deploymentId], + references: [deployment.id], + }), + }), +); + +export const versionDependencyRelations = relations( + versionDependency, + ({ one }) => ({ + version: one(deploymentVersion, { + fields: [versionDependency.versionId], + references: [deploymentVersion.id], + }), + deployment: one(deployment, { + fields: [versionDependency.deploymentId], + references: [deployment.id], + }), + }), +); + +export const deploymentVersionMetadataRelations = relations( + deploymentVersionMetadata, + ({ one }) => ({ + version: one(deploymentVersion, { + fields: [deploymentVersionMetadata.versionId], + references: [deploymentVersion.id], + }), + }), +); diff --git a/packages/db/src/schema/release.ts b/packages/db/src/schema/release.ts index bbac57ce1..c55c41d31 100644 --- a/packages/db/src/schema/release.ts +++ b/packages/db/src/schema/release.ts @@ -1,3 +1,4 @@ +import { relations } from "drizzle-orm"; import { boolean, json, @@ -61,3 +62,45 @@ export const releaseJob = pgTable("release_job", { .notNull() .defaultNow(), }); + +export const releaseRelations = relations(release, ({ one, many }) => ({ + version: one(deploymentVersion, { + fields: [release.versionId], + references: [deploymentVersion.id], + }), + resource: one(resource, { + fields: [release.resourceId], + references: [resource.id], + }), + deployment: one(deployment, { + fields: [release.deploymentId], + references: [deployment.id], + }), + environment: one(environment, { + fields: [release.environmentId], + references: [environment.id], + }), + variables: many(releaseVariable), + jobs: many(releaseJob), +})); + +export const releaseVariableRelations = relations( + releaseVariable, + ({ one }) => ({ + release: one(release, { + fields: [releaseVariable.releaseId], + references: [release.id], + }), + }), +); + +export const releaseJobRelations = relations(releaseJob, ({ one }) => ({ + release: one(release, { + fields: [releaseJob.releaseId], + references: [release.id], + }), + job: one(job, { + fields: [releaseJob.jobId], + references: [job.id], + }), +})); diff --git a/packages/release-manager/README.md b/packages/release-manager/README.md new file mode 100644 index 000000000..febe729f5 --- /dev/null +++ b/packages/release-manager/README.md @@ -0,0 +1,137 @@ +# @ctrlplane/release-manager + +A streamlined solution for managing context-specific software releases with +variable resolution and version tracking. + +## Overview + +The release-manager package provides functionality to manage software releases +across different resources and environments. It handles variable resolution from +multiple sources and ensures releases are only created when necessary (when +variables or versions change). + +## Key Components + +### ReleaseManager + +The main entry point for the package. It coordinates variable retrieval and +release creation. + +```typescript +// Use the static factory method for creating a manager +const manager = await ReleaseManager.usingDatabase({ + deploymentId: "deployment-123", + environmentId: "production", + resourceId: "resource-456", + db: dbTransaction, // Optional transaction object +}); + +// Create a release for a specific version +const { created, release } = await manager.upsertRelease("v1.0.0"); + +// Set a release as the desired release +await manager.setDesiredRelease(release.id); +``` + +### Variable System + +Handles the retrieval and resolution of variables from multiple sources with a +clear priority order: + +1. **Resource Variables**: Specific to a resource +2. **Deployment Variables**: Associated with deployments, matched to resources + via selectors +3. **System Variable Sets**: Global variables for environments + +```typescript +// Get all variables for the current context +const variables = await manager.getCurrentVariables(); +``` + +### Repository Layer + +Manages database interactions for releases with a clean interface: + +```typescript +// The ReleaseManager uses these methods internally +const repository = new DatabaseReleaseRepository(db); + +// Create a release directly +const release = await repository.create({ + deploymentId: "deployment-123", + environmentId: "production", + resourceId: "resource-456", + versionId: "v1.0.0", + variables: [...resolvedVariables], +}); + +// Ensure a release exists (create only if needed) +const { created, release } = await repository.upsert( + { deploymentId, environmentId, resourceId }, + versionId, + variables +); +``` + +## How It Works + +1. **Variable Resolution**: When a release is requested, the system fetches + variables from all available providers (resource, deployment, and system). + +2. **Release Creation Logic**: The system checks if a release with the exact + same version and variables already exists: + + - If it exists, it returns the existing release + - If not, it creates a new release with the current variables + +3. **Version Release Flow**: When a new version is released to a deployment: + - The system identifies all applicable resources + - For each resource, it creates a release with the resolved variables + - Optionally marks the release as the "desired" release for the resource + +## Testing + +```bash +# Run tests +pnpm test + +# Check types +pnpm typecheck +``` + +## Integration Example + +```typescript +import { db } from "@ctrlplane/db/client"; +import { ReleaseManager } from "@ctrlplane/release-manager"; + +// Create a release manager for a specific context +const releaseManager = await ReleaseManager.usingDatabase({ + deploymentId: "my-app", + environmentId: "production", + resourceId: "web-server-1", +}); + +// Create or get an existing release for version "2.0.0" +// All variables will be automatically resolved +const { created, release } = await releaseManager.upsertRelease("2.0.0", { + setAsDesired: true, // Mark as the desired release +}); + +console.log(`Release ${created ? "created" : "already exists"}: ${release.id}`); +console.log(`Variables resolved: ${release.variables.length}`); +``` + +## Release New Version Flow + +```typescript +import { db } from "@ctrlplane/db/client"; +import { releaseNewVersion } from "@ctrlplane/release-manager"; + +// Create releases for all resources matching a deployment version +await releaseNewVersion(db, "version-123"); +``` + +## License + +Parent Repository: [ctrlplanedev/ctrlplane](https://github.com/ctrlplane/ctrlplane) diff --git a/packages/release-manager/eslint.config.js b/packages/release-manager/eslint.config.js new file mode 100644 index 000000000..d09a7dae7 --- /dev/null +++ b/packages/release-manager/eslint.config.js @@ -0,0 +1,13 @@ +import baseConfig, { requireJsSuffix } from "@ctrlplane/eslint-config/base"; + +/** @type {import('typescript-eslint').Config} */ +export default [ + { + ignores: ["dist/**"], + rules: { + "@typescript-eslint/require-await": "off", + }, + }, + ...requireJsSuffix, + ...baseConfig, +]; diff --git a/packages/release-manager/package.json b/packages/release-manager/package.json new file mode 100644 index 000000000..d22b0e6d2 --- /dev/null +++ b/packages/release-manager/package.json @@ -0,0 +1,39 @@ +{ + "name": "@ctrlplane/release-manager", + "private": true, + "version": "0.1.0", + "type": "module", + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./dist/index.js" + } + }, + "license": "", + "scripts": { + "build": "tsc", + "dev": "tsc --watch", + "test": "vitest", + "clean": "rm -rf .turbo node_modules", + "format": "prettier --check . --ignore-path ../../.gitignore", + "lint": "eslint", + "typecheck": "tsc --noEmit --emitDeclarationOnly false" + }, + "dependencies": { + "@ctrlplane/db": "workspace:*", + "@ctrlplane/validators": "workspace:*", + "lodash": "catalog:", + "zod": "catalog:" + }, + "devDependencies": { + "@ctrlplane/eslint-config": "workspace:*", + "@ctrlplane/prettier-config": "workspace:*", + "@ctrlplane/tsconfig": "workspace:*", + "@types/node": "catalog:node22", + "eslint": "catalog:", + "prettier": "catalog:", + "typescript": "catalog:", + "vitest": "^2.1.9" + }, + "prettier": "@ctrlplane/prettier-config" +} diff --git a/packages/release-manager/src/index.ts b/packages/release-manager/src/index.ts new file mode 100644 index 000000000..449dad5c9 --- /dev/null +++ b/packages/release-manager/src/index.ts @@ -0,0 +1,4 @@ +export * from "./types.js"; +export * from "./variables/variables.js"; +export * from "./manager.js"; +export * from "./repositories/release-repository.js"; diff --git a/packages/release-manager/src/manager.ts b/packages/release-manager/src/manager.ts new file mode 100644 index 000000000..8b50b369d --- /dev/null +++ b/packages/release-manager/src/manager.ts @@ -0,0 +1,97 @@ +import type { Tx } from "@ctrlplane/db"; + +import { db } from "@ctrlplane/db/client"; + +import type { ReleaseRepository } from "./repositories/types.js"; +import type { ReleaseIdentifier } from "./types.js"; +import { DatabaseReleaseRepository } from "./repositories/release-repository.js"; +import { VariableManager } from "./variables/variables.js"; + +/** + * Options for configuring a ReleaseManager instance + */ +type ReleaseManagerOptions = ReleaseIdentifier & { + /** Repository for managing releases */ + repository: ReleaseRepository; + /** Manager for handling variables */ + variableManager: VariableManager; +}; + +/** + * Options for creating a database-backed ReleaseManager + */ +export type DatabaseReleaseManagerOptions = ReleaseIdentifier & { + /** Optional database transaction */ + db?: Tx; +}; + +/** + * Manages the lifecycle of releases including creation, updates and desired + * state + */ +export class ReleaseManager { + /** + * Creates a new ReleaseManager instance backed by the database + * @param options Configuration options including database connection + * @returns A configured ReleaseManager instance + */ + static async usingDatabase(options: DatabaseReleaseManagerOptions) { + const variableManager = await VariableManager.database(options); + const repository = new DatabaseReleaseRepository(options.db ?? db); + const manager = new ReleaseManager({ + ...options, + variableManager, + repository, + }); + return manager; + } + + private constructor(private readonly options: ReleaseManagerOptions) {} + + /** + * Gets the repository used by this manager + */ + get repository() { + return this.options.repository; + } + + /** + * Gets the variable manager used by this manager + */ + get variableManager() { + return this.options.variableManager; + } + + /** + * Upserts a release for the given version + * @param versionId The ID of the version to ensure + * @param opts Optional settings for the ensure operation + * @param opts.setAsDesired Whether to set this as the desired release + * @returns Object containing whether a new release was created and the + * release details + */ + async upsertRelease(versionId: string, opts?: { setAsDesired?: boolean }) { + const variables = await this.variableManager.getVariables(); + + // Use the repository directly to ensure the release + const { created, release } = await this.repository.upsert( + this.options, + versionId, + variables, + ); + + if (opts?.setAsDesired) await this.setDesiredRelease(release.id); + return { created, release }; + } + + /** + * Sets the desired release for this resource + * @param desiredReleaseId The ID of the release to set as desired + */ + async setDesiredRelease(desiredReleaseId: string) { + await this.repository.setDesired({ + ...this.options, + desiredReleaseId, + }); + } +} diff --git a/packages/release-manager/src/release-new-verion.ts b/packages/release-manager/src/release-new-verion.ts new file mode 100644 index 000000000..09058327b --- /dev/null +++ b/packages/release-manager/src/release-new-verion.ts @@ -0,0 +1,76 @@ +// import type { Tx } from "@ctrlplane/db"; +// import _ from "lodash"; + +// import { and, eq, takeFirst } from "@ctrlplane/db"; +// import * as schema from "@ctrlplane/db/schema"; + +// import { ReleaseManager } from "./manager.js"; + +// /** +// * Retrieves system resources for a given system +// */ +// const getSystemResources = async (tx: Tx, systemId: string) => { +// const system = await tx.query.system.findFirst({ +// where: eq(schema.system.id, systemId), +// with: { environments: true }, +// }); + +// if (system == null) throw new Error("System not found"); + +// const { environments } = system; + +// // Simplify the chained operations with standard Promise.all +// const resources = await Promise.all( +// environments.map(async (env) => { +// const res = await tx +// .select() +// .from(schema.resource) +// .where( +// and( +// eq(schema.resource.workspaceId, system.workspaceId), +// schema.resourceMatchesMetadata(tx, env.resourceSelector), +// ), +// ); +// return res.map((r) => ({ ...r, environment: env })); +// }), +// ).then((arrays) => arrays.flat()); + +// return resources; +// }; + +// /** +// * Releases a new version for a deployment across all resources +// */ +// export const releaseNewVersion = async (tx: Tx, versionId: string) => { +// // Get deployment, version and system in a single query +// const { +// deployment_version: version, +// deployment, +// system, +// } = await tx +// .select() +// .from(schema.deploymentVersion) +// .innerJoin( +// schema.deployment, +// eq(schema.deploymentVersion.deploymentId, schema.deployment.id), +// ) +// .innerJoin(schema.system, eq(schema.deployment.systemId, schema.system.id)) +// .where(eq(schema.deploymentVersion.id, versionId)) +// .then(takeFirst); + +// // Get all resources for this system +// const resources = await getSystemResources(tx, system.id); + +// // Create a release manager for each resource and ensure the release +// const releaseManagers = resources.map( +// (r) => +// new ReleaseManager({ +// db: tx, +// deploymentId: deployment.id, +// environmentId: r.environment.id, +// resourceId: r.id, +// }), +// ); + +// await Promise.all(releaseManagers.map((rm) => rm.ensureRelease(version.id))); +// }; diff --git a/packages/release-manager/src/repositories/release-repository.ts b/packages/release-manager/src/repositories/release-repository.ts new file mode 100644 index 000000000..7609f747e --- /dev/null +++ b/packages/release-manager/src/repositories/release-repository.ts @@ -0,0 +1,134 @@ +import type { Tx } from "@ctrlplane/db"; +import _ from "lodash"; + +import { + and, + buildConflictUpdateColumns, + desc, + eq, + takeFirst, + takeFirstOrNull, +} from "@ctrlplane/db"; +import { db as dbClient } from "@ctrlplane/db/client"; +import * as schema from "@ctrlplane/db/schema"; + +import type { Release, ReleaseIdentifier, ReleaseWithId } from "../types.js"; +import type { MaybeVariable } from "../variables/types.js"; +import type { ReleaseRepository } from "./types.js"; + +/** + * Enhanced repository that combines database operations with business logic + * for managing releases + */ +export class DatabaseReleaseRepository implements ReleaseRepository { + constructor(private readonly db: Tx = dbClient) {} + + /** + * Get the latest release for a specific resource, deployment, and environment + */ + async getLatest(options: ReleaseIdentifier) { + return this.db.query.release + .findFirst({ + where: and( + eq(schema.release.resourceId, options.resourceId), + eq(schema.release.deploymentId, options.deploymentId), + eq(schema.release.environmentId, options.environmentId), + ), + with: { + variables: true, + }, + orderBy: desc(schema.release.createdAt), + }) + .then((r) => r ?? null); + } + + /** + * Create a new release with the given details + */ + async create(release: Release) { + const dbRelease = await this.db + .insert(schema.release) + .values(release) + .returning() + .then(takeFirst); + + return { + ...release, + ...dbRelease, + }; + } + + /** + * Create a new release with variables for a specific version + */ + async createForVersion( + options: ReleaseIdentifier, + versionId: string, + variables: MaybeVariable[], + ): Promise { + const release: Release = { + ...options, + versionId, + variables: _.compact(variables), + }; + + return this.create(release); + } + + async upsert( + options: ReleaseIdentifier, + versionId: string, + variables: MaybeVariable[], + ): Promise<{ created: boolean; release: ReleaseWithId }> { + const latestRelease = await this.getLatest(options); + + // Convert releases to comparable objects + const latestR = { + versionId: latestRelease?.versionId, + variables: _(latestRelease?.variables ?? []) + .map((v) => [v.key, v.value]) + .fromPairs() + .value(), + }; + + const newR = { + versionId, + variables: _(variables) + .compact() + .map((v) => [v.key, v.value]) + .fromPairs() + .value(), + }; + + const isSame = latestRelease != null && _.isEqual(latestR, newR); + return isSame + ? { created: false, release: latestRelease } + : { + created: true, + release: await this.createForVersion(options, versionId, variables), + }; + } + + async setDesired(options: ReleaseIdentifier & { desiredReleaseId: string }) { + await this.db + .insert(schema.resourceRelease) + .values({ + environmentId: options.environmentId, + deploymentId: options.deploymentId, + resourceId: options.resourceId, + desiredReleaseId: options.desiredReleaseId, + }) + .onConflictDoUpdate({ + target: [ + schema.resourceRelease.environmentId, + schema.resourceRelease.deploymentId, + schema.resourceRelease.resourceId, + ], + set: buildConflictUpdateColumns(schema.resourceRelease, [ + "desiredReleaseId", + ]), + }) + .returning() + .then(takeFirstOrNull); + } +} diff --git a/packages/release-manager/src/repositories/types.ts b/packages/release-manager/src/repositories/types.ts new file mode 100644 index 000000000..98ed6e433 --- /dev/null +++ b/packages/release-manager/src/repositories/types.ts @@ -0,0 +1,42 @@ +import type { Release, ReleaseIdentifier, ReleaseWithId } from "../types.js"; +import type { MaybeVariable } from "../variables/types.js"; + +export interface ReleaseRepository { + /** + * Get the latest release for a specific resource, deployment, and environment + */ + getLatest( + options: ReleaseIdentifier, + ): Promise<(ReleaseWithId & { variables: MaybeVariable[] }) | null>; + + /** + * Create a new release with the given details + */ + create(release: Release): Promise; + + /** + * Create a new release with variables for a specific version + */ + createForVersion( + options: ReleaseIdentifier, + versionId: string, + variables: MaybeVariable[], + ): Promise; + + /** + * Ensure a release exists for the given version and variables + * Creates a new release only if necessary + */ + upsert( + options: ReleaseIdentifier, + versionId: string, + variables: MaybeVariable[], + ): Promise<{ created: boolean; release: ReleaseWithId }>; + + /** + * Set a specific release as the desired release + */ + setDesired( + options: ReleaseIdentifier & { desiredReleaseId: string }, + ): Promise; +} diff --git a/packages/release-manager/src/types.ts b/packages/release-manager/src/types.ts new file mode 100644 index 000000000..c6e7adfbb --- /dev/null +++ b/packages/release-manager/src/types.ts @@ -0,0 +1,16 @@ +import type { Variable } from "./variables/types.js"; + +export type ReleaseIdentifier = { + environmentId: string; + deploymentId: string; + resourceId: string; +}; + +export type Release = ReleaseIdentifier & { + versionId: string; + variables: Variable[]; +}; + +export type ReleaseWithId = Release & { id: string }; + +export type ReleaseQueryOptions = ReleaseIdentifier; diff --git a/packages/release-manager/src/variables/db-variable-providers.ts b/packages/release-manager/src/variables/db-variable-providers.ts new file mode 100644 index 000000000..5440962f5 --- /dev/null +++ b/packages/release-manager/src/variables/db-variable-providers.ts @@ -0,0 +1,183 @@ +import type { Tx } from "@ctrlplane/db"; +import type { VariableSetValue } from "@ctrlplane/db/schema"; + +import { and, asc, eq, takeFirstOrNull } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import { + deploymentVariable, + deploymentVariableValue, + resource, + resourceMatchesMetadata, + resourceVariable, + variableSetEnvironment, + variableSetValue, +} from "@ctrlplane/db/schema"; + +import type { ReleaseIdentifier } from "../types.js"; +import type { MaybeVariable, Variable, VariableProvider } from "./types.js"; + +export type DatabaseResourceVariableOptions = Pick< + ReleaseIdentifier, + "resourceId" +> & { + db?: Tx; +}; + +export class DatabaseResourceVariableProvider implements VariableProvider { + private db: Tx; + private variables: Promise | null = null; + + constructor(private options: DatabaseResourceVariableOptions) { + this.db = options.db ?? db; + } + + private async loadVariables() { + const variables = await this.db.query.resourceVariable.findMany({ + where: and(eq(resourceVariable.resourceId, this.options.resourceId)), + }); + return variables.map((v) => ({ + id: v.id, + key: v.key, + value: v.value, + sensitive: v.sensitive, + })); + } + + private getVariablesPromise() { + return (this.variables ??= this.loadVariables()); + } + + async getVariable(key: string): Promise { + const variables = await this.getVariablesPromise(); + return variables.find((v) => v.key === key) ?? null; + } +} + +export type DatabaseDeploymentVariableOptions = Pick< + ReleaseIdentifier, + "resourceId" | "deploymentId" +> & { + db?: Tx; +}; + +type DeploymentVariableValue = { + value: any; + resourceSelector: any; +}; + +type DeploymentVariable = { + id: string; + key: string; + defaultValue: DeploymentVariableValue | null; + values: DeploymentVariableValue[]; +}; + +export class DatabaseDeploymentVariableProvider implements VariableProvider { + private db: Tx; + private variables: Promise | null = null; + + constructor(private options: DatabaseDeploymentVariableOptions) { + this.db = options.db ?? db; + } + + private loadVariables() { + return this.db.query.deploymentVariable.findMany({ + where: eq(deploymentVariable.deploymentId, this.options.deploymentId), + with: { + defaultValue: true, + values: { orderBy: [asc(deploymentVariableValue.value)] }, + }, + }); + } + + private getVariables() { + return (this.variables ??= this.loadVariables()); + } + + async getVariable(key: string): Promise { + const variables = await this.getVariables(); + const variable = variables.find((v) => v.key === key) ?? null; + if (variable == null) return null; + + for (const value of variable.values) { + const res = await this.db + .select() + .from(resource) + .where( + and( + eq(resource.id, this.options.resourceId), + resourceMatchesMetadata(this.db, value.resourceSelector), + ), + ) + .then(takeFirstOrNull); + + if (res != null) + return { + id: variable.id, + key, + sensitive: false, + ...value, + }; + } + + if (variable.defaultValue != null) + return { + id: variable.id, + key, + sensitive: false, + ...variable.defaultValue, + }; + + return null; + } +} + +export type DatabaseSystemVariableSetOptions = Pick< + ReleaseIdentifier, + "environmentId" +> & { + db?: Tx; +}; + +export class DatabaseSystemVariableSetProvider implements VariableProvider { + private db: Tx; + private variables: Promise | null = null; + + constructor(private options: DatabaseSystemVariableSetOptions) { + this.db = options.db ?? db; + } + + private loadVariables() { + return this.db + .select() + .from(variableSetValue) + .innerJoin( + variableSetEnvironment, + eq( + variableSetValue.variableSetId, + variableSetEnvironment.variableSetId, + ), + ) + .where( + eq(variableSetEnvironment.environmentId, this.options.environmentId), + ) + .orderBy(asc(variableSetValue.value)) + .then((rows) => rows.map((r) => r.variable_set_value)); + } + + private getVariables() { + return (this.variables ??= this.loadVariables()); + } + + async getVariable(key: string): Promise { + const variables = await this.getVariables(); + const variable = variables.find((v) => v.key === key) ?? null; + if (variable == null) return null; + return { + id: variable.id, + key, + value: variable.value, + sensitive: false, + }; + } +} diff --git a/packages/release-manager/src/variables/types.ts b/packages/release-manager/src/variables/types.ts new file mode 100644 index 000000000..71487618a --- /dev/null +++ b/packages/release-manager/src/variables/types.ts @@ -0,0 +1,23 @@ +import type { ReleaseIdentifier } from "src/types"; + +export type Variable = { + id: string; + key: string; + value: T; + sensitive: boolean; +}; + +export type MaybePromise = T | Promise; +export type MaybeVariable = Variable | null; + +export type VariableProvider = { + getVariable(key: string): MaybePromise; +}; + +export type VariableProviderFactory = { + create(options: VariableProviderOptions): VariableProvider; +}; + +export type VariableProviderOptions = ReleaseIdentifier & { + db?: any; +}; diff --git a/packages/release-manager/src/variables/variables.ts b/packages/release-manager/src/variables/variables.ts new file mode 100644 index 000000000..2fd3118db --- /dev/null +++ b/packages/release-manager/src/variables/variables.ts @@ -0,0 +1,71 @@ +import type { Tx } from "@ctrlplane/db"; + +import { eq } from "@ctrlplane/db"; +import { db } from "@ctrlplane/db/client"; +import * as schema from "@ctrlplane/db/schema"; + +import type { + MaybeVariable, + VariableProvider, + VariableProviderOptions, +} from "./types.js"; +import { + DatabaseDeploymentVariableProvider, + DatabaseResourceVariableProvider, + DatabaseSystemVariableSetProvider, +} from "./db-variable-providers.js"; + +const getDeploymentVariableKeys = async (options: { + deploymentId: string; + db?: Tx; +}): Promise => { + const tx = options.db ?? db; + return tx + .select({ key: schema.deploymentVariable.key }) + .from(schema.deploymentVariable) + .where(eq(schema.deploymentVariable.deploymentId, options.deploymentId)) + .then((results) => results.map((r) => r.key)); +}; + +type VariableManagerOptions = VariableProviderOptions & { + keys: string[]; +}; + +export class VariableManager { + static async database(options: VariableProviderOptions) { + const providers = [ + new DatabaseSystemVariableSetProvider(options), + new DatabaseResourceVariableProvider(options), + new DatabaseDeploymentVariableProvider(options), + ]; + + const keys = await getDeploymentVariableKeys(options); + return new VariableManager({ ...options, keys }, providers); + } + + private constructor( + private options: VariableManagerOptions, + private variableProviders: VariableProvider[], + ) {} + + getProviders() { + return [...this.variableProviders]; + } + + addProvider(provider: VariableProvider) { + this.variableProviders.push(provider); + return this; + } + + async getVariables(): Promise { + return Promise.all(this.options.keys.map((key) => this.getVariable(key))); + } + + async getVariable(key: string): Promise { + for (const provider of this.variableProviders) { + const variable = await provider.getVariable(key); + if (variable) return variable; + } + return null; + } +} diff --git a/packages/release-manager/tsconfig.json b/packages/release-manager/tsconfig.json new file mode 100644 index 000000000..e02676b57 --- /dev/null +++ b/packages/release-manager/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "@ctrlplane/tsconfig/internal-package.json", + "compilerOptions": { + "outDir": "dist", + "baseUrl": ".", + "incremental": true, + "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json" + }, + "include": ["*.ts", "src"], + "exclude": ["node_modules"] +} diff --git a/packages/release-manager/vitest.config.ts b/packages/release-manager/vitest.config.ts new file mode 100644 index 000000000..8a7a111b2 --- /dev/null +++ b/packages/release-manager/vitest.config.ts @@ -0,0 +1,15 @@ +import { resolve } from "path"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["src/**/__tests__/**/*.test.ts"], + exclude: ["**/node_modules/**", "**/dist/**"], + }, + resolve: { + alias: { + "@": resolve(__dirname, "./src"), + }, + }, +}); diff --git a/packages/validators/src/events/index.ts b/packages/validators/src/events/index.ts index 57a46449b..d234e19e0 100644 --- a/packages/validators/src/events/index.ts +++ b/packages/validators/src/events/index.ts @@ -7,6 +7,7 @@ export enum Channel { DispatchJob = "dispatch-job", ResourceScan = "resource-scan", ReleaseEvaluate = "release-evaluate", + ReleaseNewVersion = "release-new-version", } export const resourceScanEvent = z.object({ resourceProviderId: z.string() }); @@ -26,3 +27,6 @@ export const releaseEvaluateEvent = z.object({ resourceId: z.string(), }); export type ReleaseEvaluateEvent = z.infer; + +export const releaseNewVersionEvent = z.object({ versionId: z.string() }); +export type ReleaseNewVersionEvent = z.infer; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 166bd393b..9241a0e4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -199,6 +199,9 @@ importers: ms: specifier: ^2.1.3 version: 2.1.3 + redis-semaphore: + specifier: ^5.6.2 + version: 5.6.2(ioredis@5.4.1) semver: specifier: 'catalog:' version: 7.7.1 @@ -1230,6 +1233,46 @@ importers: specifier: 'catalog:' version: 5.8.2 + packages/release-manager: + dependencies: + '@ctrlplane/db': + specifier: workspace:* + version: link:../db + '@ctrlplane/validators': + specifier: workspace:* + version: link:../validators + lodash: + specifier: 'catalog:' + version: 4.17.21 + zod: + specifier: 'catalog:' + version: 3.24.2 + devDependencies: + '@ctrlplane/eslint-config': + specifier: workspace:* + version: link:../../tooling/eslint + '@ctrlplane/prettier-config': + specifier: workspace:* + version: link:../../tooling/prettier + '@ctrlplane/tsconfig': + specifier: workspace:* + version: link:../../tooling/typescript + '@types/node': + specifier: catalog:node22 + version: 22.13.10 + eslint: + specifier: 'catalog:' + version: 9.11.1(jiti@2.3.3) + prettier: + specifier: 'catalog:' + version: 3.5.3 + typescript: + specifier: 'catalog:' + version: 5.8.2 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@22.13.10)(jsdom@25.0.1)(terser@5.36.0) + packages/rule-engine: dependencies: '@ctrlplane/db': @@ -2168,9 +2211,6 @@ packages: '@dabh/diagnostics@2.0.3': resolution: {integrity: sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==} - '@date-fns/tz@1.1.2': - resolution: {integrity: sha512-Xmg2cPmOPQieCLAdf62KtFPU9y7wbQDq1OAzrs/bEQFvhtCPXDiks1CHDE/sTXReRfh/MICVkw/vY6OANHUGiA==} - '@date-fns/tz@1.2.0': resolution: {integrity: sha512-LBrd7MiJZ9McsOgxqWX7AaxrDjcFVjWH/tIKJd7pnR7McaslGYOP1QmmiBXdJH/H/yLCT+rcQ7FaPBUxRGUtrg==} @@ -10050,6 +10090,15 @@ packages: resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} engines: {node: '>=4'} + redis-semaphore@5.6.2: + resolution: {integrity: sha512-Oh1zOqNa51VC14mwYcmdOyjHpb+y8N1ieqpGxITjkrqPiO8IoCYiXGrSyKEmXH5+UEsl/7OAnju2e0x1TY5Jhg==} + engines: {node: '>= 14.17.0'} + peerDependencies: + ioredis: ^4.1.0 || ^5 + peerDependenciesMeta: + ioredis: + optional: true + redis@4.7.0: resolution: {integrity: sha512-zvmkHEAdGMn+hMRXuMBtu4Vo5P6rHQjLoHftu+lBqq8ZTA3RCVC/WzD790bkKKiNFp7d5/9PcSD19fJyyRvOdQ==} @@ -12537,8 +12586,6 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 - '@date-fns/tz@1.1.2': {} - '@date-fns/tz@1.2.0': {} '@discoveryjs/json-ext@0.5.7': {} @@ -21272,7 +21319,7 @@ snapshots: react-day-picker@9.2.1(react@19.0.0): dependencies: - '@date-fns/tz': 1.1.2 + '@date-fns/tz': 1.2.0 date-fns: 4.1.0 react: 19.0.0 @@ -21571,6 +21618,14 @@ snapshots: dependencies: redis-errors: 1.2.0 + redis-semaphore@5.6.2(ioredis@5.4.1): + dependencies: + debug: 4.4.0 + optionalDependencies: + ioredis: 5.4.1 + transitivePeerDependencies: + - supports-color + redis@4.7.0: dependencies: '@redis/bloom': 1.2.0(@redis/client@1.6.0)