diff --git a/.github/workflows/apps-event-queue.yaml b/.github/workflows/apps-event-queue.yaml new file mode 100644 index 000000000..6fbf6881c --- /dev/null +++ b/.github/workflows/apps-event-queue.yaml @@ -0,0 +1,81 @@ +name: Apps / Event Queue + +on: + pull_request: + branches: ["*"] + paths: + - apps/event-queue/** + - packages/logger/** + - packages/events/** + - .github/workflows/apps-event-queue.yaml + - pnpm-lock.yaml + push: + branches: ["main"] + paths: + - apps/event-queue/** + - packages/logger/** + - packages/events/** + - .github/workflows/apps-event-queue.yaml + - pnpm-lock.yaml + +jobs: + build: + runs-on: ubuntu-latest + + permissions: + contents: read + id-token: write + + strategy: + matrix: + platform: [linux/amd64] + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Check if Docker Hub secrets are available + run: | + if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then + echo "DOCKERHUB_LOGIN=false" >> $GITHUB_ENV + else + echo "DOCKERHUB_LOGIN=true" >> $GITHUB_ENV + fi + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: env.DOCKERHUB_LOGIN == 'true' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ctrlplane/event-queue + tags: | + type=sha,format=short,prefix= + + - name: Build + uses: docker/build-push-action@v6 + if: github.ref != 'refs/heads/main' + with: + push: false + file: apps/event-queue/Dockerfile + platforms: ${{ matrix.platform }} + tags: ${{ steps.meta.outputs.tags }} + + - name: Build and Push + uses: docker/build-push-action@v6 + if: github.ref == 'refs/heads/main' && env.DOCKERHUB_LOGIN == 'true' + with: + push: true + file: apps/event-queue/Dockerfile + platforms: ${{ matrix.platform }} + tags: ${{ steps.meta.outputs.tags }} \ No newline at end of file diff --git a/apps/event-queue/Dockerfile b/apps/event-queue/Dockerfile new file mode 100644 index 000000000..11e6c049a --- /dev/null +++ b/apps/event-queue/Dockerfile @@ -0,0 +1,55 @@ +# https://github.com/WhiskeySockets/Baileys/issues/1003#issuecomment-2306467419 +ARG NODE_VERSION=20 +FROM node:${NODE_VERSION}-alpine + +RUN apk add --no-cache libc6-compat python3 make g++ curl + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" + +RUN npm install -g turbo +RUN npm install -g corepack@latest +RUN corepack enable pnpm + +WORKDIR /app + +COPY .gitignore .gitignore +COPY turbo.json turbo.json +RUN pnpm add -g turbo + +COPY package.json package.json +COPY .npmrc .npmrc +COPY pnpm-*.yaml . + +COPY tooling/prettier/package.json ./tooling/prettier/package.json +COPY tooling/eslint/package.json ./tooling/eslint/package.json +COPY tooling/typescript/package.json ./tooling/typescript/package.json + +COPY packages/logger/package.json ./packages/logger/package.json +COPY packages/events/package.json ./packages/events/package.json +COPY packages/secrets/package.json ./packages/secrets/package.json +COPY packages/validators/package.json ./packages/validators/package.json +COPY packages/db/package.json ./packages/db/package.json +COPY packages/job-dispatch/package.json ./packages/job-dispatch/package.json +COPY packages/rule-engine/package.json ./packages/rule-engine/package.json + +COPY apps/event-queue/package.json ./apps/event-queue/package.json + +RUN pnpm install --frozen-lockfile + +COPY . . + +RUN turbo build --filter=...@ctrlplane/event-queue + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nodejs +USER nodejs + +ENV NODE_ENV=production +ENV NODE_MAX_OLD_SPACE_SIZE=4096 + +EXPOSE 3123 + +WORKDIR apps/event-queue/dist/ + +CMD ["node", "index.js"] \ No newline at end of file diff --git a/apps/event-queue/README.md b/apps/event-queue/README.md new file mode 100644 index 000000000..b671f5707 --- /dev/null +++ b/apps/event-queue/README.md @@ -0,0 +1,9 @@ +# Event Queue + +This is a simple event queue that listens to the `ctrlplane-events` topic and logs the events to the console. + +## Running + +```bash +pnpm dev +``` diff --git a/apps/event-queue/eslint.config.js b/apps/event-queue/eslint.config.js new file mode 100644 index 000000000..dd9824eb4 --- /dev/null +++ b/apps/event-queue/eslint.config.js @@ -0,0 +1,8 @@ +import baseConfig, { requireJsSuffix } from "@ctrlplane/eslint-config/base"; + +/** @type {import('typescript-eslint').Config} */ +export default [ + { ignores: [".nitro/**", ".output/**", "dist/**"] }, + ...requireJsSuffix, + ...baseConfig, +]; diff --git a/apps/event-queue/package.json b/apps/event-queue/package.json new file mode 100644 index 000000000..e7dbef6c8 --- /dev/null +++ b/apps/event-queue/package.json @@ -0,0 +1,33 @@ +{ + "name": "@ctrlplane/event-queue", + "private": true, + "type": "module", + "scripts": { + "clean": "rm -rf .turbo node_modules", + "dev": "pnpm with-env tsx watch --clear-screen=false src/index.ts", + "lint": "eslint", + "build": "tsc", + "format": "prettier --check . --ignore-path ../../.gitignore", + "typecheck": "tsc --noEmit", + "with-env": "dotenv -e ../../.env --" + }, + "dependencies": { + "@ctrlplane/events": "workspace:*", + "@ctrlplane/logger": "workspace:*", + "@t3-oss/env-core": "catalog:", + "dotenv": "^16.4.5", + "kafkajs": "^2.2.4", + "zod": "catalog:" + }, + "devDependencies": { + "@ctrlplane/eslint-config": "workspace:^", + "@ctrlplane/prettier-config": "workspace:^", + "@ctrlplane/tsconfig": "workspace:*", + "@types/node": "catalog:node22", + "eslint": "catalog:", + "prettier": "catalog:", + "tsx": "catalog:", + "typescript": "catalog:" + }, + "prettier": "@ctrlplane/prettier-config" +} diff --git a/apps/event-queue/src/config.ts b/apps/event-queue/src/config.ts new file mode 100644 index 000000000..3000a968d --- /dev/null +++ b/apps/event-queue/src/config.ts @@ -0,0 +1,26 @@ +import { createEnv } from "@t3-oss/env-core"; +import dotenv from "dotenv"; +import { z } from "zod"; + +dotenv.config(); + +export const env = createEnv({ + server: { + NODE_ENV: z + .enum(["development", "production", "test"]) + .default("development"), + KAFKA_BROKERS: z + .string() + .default("localhost:9092") + .transform((val) => + val + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ) + .refine((arr) => arr.length > 0, { + message: "KAFKA_BROKERS must be a non-empty list of brokers", + }), + }, + runtimeEnv: process.env, +}); diff --git a/apps/event-queue/src/index.ts b/apps/event-queue/src/index.ts new file mode 100644 index 000000000..0986f7f36 --- /dev/null +++ b/apps/event-queue/src/index.ts @@ -0,0 +1,33 @@ +import { Kafka } from "kafkajs"; + +import { logger } from "@ctrlplane/logger"; + +import { env } from "./config.js"; + +const kafka = new Kafka({ + clientId: "ctrlplane-events", + brokers: env.KAFKA_BROKERS, +}); + +const consumer = kafka.consumer({ groupId: "ctrlplane-events" }); + +export const start = async () => { + logger.info("Starting event queue", { brokers: env.KAFKA_BROKERS }); + await consumer.connect(); + await consumer.subscribe({ topic: "ctrlplane-events", fromBeginning: true }); + logger.info("Subscribed to ctrlplane-events topic"); + + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + logger.info("Received event", { + topic, + partition, + message: message.value?.toString() ?? "No message", + }); + + return Promise.resolve(); + }, + }); +}; + +start(); diff --git a/apps/event-queue/tsconfig.json b/apps/event-queue/tsconfig.json new file mode 100644 index 000000000..34755271a --- /dev/null +++ b/apps/event-queue/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "@ctrlplane/tsconfig/base.json", + "compilerOptions": { + "target": "ESNext", + "outDir": "dist", + "noEmit": false, + "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json", + "baseUrl": ".", + "esModuleInterop": true, + "importsNotUsedAsValues": "remove" + }, + "include": ["src", "*.ts"], + "exclude": ["node_modules"] +} diff --git a/apps/event-worker/Dockerfile b/apps/event-worker/Dockerfile index a3ab5e2f8..8d5bba4b1 100644 --- a/apps/event-worker/Dockerfile +++ b/apps/event-worker/Dockerfile @@ -50,7 +50,7 @@ RUN adduser --system --uid 1001 nodejs USER nodejs ENV NODE_ENV=production -ENV NODE_MAX_OLD_SPACE_SIZE=4096 +ENV NODE_OPTIONS="--max-old-space-size=4096" EXPOSE 3123 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85d9dffd2..ba44b6517 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -134,6 +134,52 @@ importers: specifier: ^5.7.3 version: 5.8.3 + apps/event-queue: + dependencies: + '@ctrlplane/events': + specifier: workspace:* + version: link:../../packages/events + '@ctrlplane/logger': + specifier: workspace:* + version: link:../../packages/logger + '@t3-oss/env-core': + specifier: 'catalog:' + version: 0.11.1(typescript@5.8.2)(zod@3.24.2) + dotenv: + specifier: ^16.4.5 + version: 16.6.1 + kafkajs: + specifier: ^2.2.4 + version: 2.2.4 + zod: + specifier: 'catalog:' + version: 3.24.2 + devDependencies: + '@ctrlplane/eslint-config': + specifier: workspace:^ + version: link:../../tooling/eslint + '@ctrlplane/prettier-config': + specifier: workspace:^ + version: link:../../tooling/prettier + '@ctrlplane/tsconfig': + specifier: workspace:* + version: link:../../tooling/typescript + '@types/node': + specifier: catalog:node22 + version: 22.13.10 + eslint: + specifier: 'catalog:' + version: 9.11.1(jiti@2.3.3) + prettier: + specifier: 'catalog:' + version: 3.5.3 + tsx: + specifier: 'catalog:' + version: 4.19.2 + typescript: + specifier: 'catalog:' + version: 5.8.2 + apps/event-worker: dependencies: '@aws-sdk/client-ec2': @@ -7680,10 +7726,6 @@ packages: resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==} engines: {node: '>=12'} - dotenv@16.5.0: - resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==} - engines: {node: '>=12'} - dotenv@16.6.1: resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} engines: {node: '>=12'} @@ -17372,7 +17414,7 @@ snapshots: '@types/cors@2.8.19': dependencies: - '@types/node': 22.13.10 + '@types/node': 22.16.2 '@types/d3-array@3.2.1': {} @@ -17668,7 +17710,7 @@ snapshots: '@types/ws@8.18.1': dependencies: - '@types/node': 22.13.10 + '@types/node': 22.16.2 '@typescript-eslint/eslint-plugin@8.13.0(@typescript-eslint/parser@8.13.0(eslint@9.11.1(jiti@2.3.3))(typescript@5.8.2))(eslint@9.11.1(jiti@2.3.3))(typescript@5.8.2)': dependencies: @@ -18951,7 +18993,7 @@ snapshots: dotenv-cli@7.4.4: dependencies: cross-spawn: 7.0.6 - dotenv: 16.5.0 + dotenv: 16.6.1 dotenv-expand: 10.0.0 minimist: 1.2.8 @@ -18959,8 +19001,6 @@ snapshots: dotenv@16.0.3: {} - dotenv@16.5.0: {} - dotenv@16.6.1: {} drange@1.1.1: {}