chore: init kafka ts consumer

.github/workflows/apps-event-queue.yaml

@@ -0,0 +1,81 @@
+name: Apps / Event Queue
+
+on:
+  pull_request:
+    branches: ["*"]
+    paths:
+      - apps/event-queue/**
+      - packages/logger/**
+      - packages/events/**
+      - .github/workflows/apps-event-queue.yaml
+      - pnpm-lock.yaml
+  push:
+    branches: ["main"]
+    paths:
+      - apps/event-queue/**
+      - packages/logger/**
+      - packages/events/**
+      - .github/workflows/apps-event-queue.yaml
+      - pnpm-lock.yaml
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      id-token: write
+
+    strategy:
+      matrix:
+        platform: [linux/amd64]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Check if Docker Hub secrets are available
+        run: |
+          if [ -z "${{ secrets.DOCKERHUB_USERNAME }}" ] || [ -z "${{ secrets.DOCKERHUB_TOKEN }}" ]; then
+            echo "DOCKERHUB_LOGIN=false" >> $GITHUB_ENV
+          else
+            echo "DOCKERHUB_LOGIN=true" >> $GITHUB_ENV
+          fi
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: env.DOCKERHUB_LOGIN == 'true'
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ctrlplane/event-queue
+          tags: |
+            type=sha,format=short,prefix=
+
+      - name: Build
+        uses: docker/build-push-action@v6
+        if: github.ref != 'refs/heads/main'
+        with:
+          push: false
+          file: apps/event-queue/Dockerfile
+          platforms: ${{ matrix.platform }}
+          tags: ${{ steps.meta.outputs.tags }}
+
+      - name: Build and Push
+        uses: docker/build-push-action@v6
+        if: github.ref == 'refs/heads/main' && env.DOCKERHUB_LOGIN == 'true'
+        with:
+          push: true
+          file: apps/event-queue/Dockerfile
+          platforms: ${{ matrix.platform }}
+          tags: ${{ steps.meta.outputs.tags }}

apps/event-queue/Dockerfile

@@ -0,0 +1,55 @@
+# https://github.com/WhiskeySockets/Baileys/issues/1003#issuecomment-2306467419
+ARG NODE_VERSION=20
+FROM node:${NODE_VERSION}-alpine
+
+RUN apk add --no-cache libc6-compat python3 make g++ curl
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+
+RUN npm install -g turbo
+RUN npm install -g corepack@latest
+RUN corepack enable pnpm
+
+WORKDIR /app
+
+COPY .gitignore .gitignore
+COPY turbo.json turbo.json
+RUN pnpm add -g turbo
+
+COPY package.json package.json
+COPY .npmrc .npmrc
+COPY pnpm-*.yaml .
+
+COPY tooling/prettier/package.json ./tooling/prettier/package.json
+COPY tooling/eslint/package.json ./tooling/eslint/package.json
+COPY tooling/typescript/package.json ./tooling/typescript/package.json
+
+COPY packages/logger/package.json ./packages/logger/package.json
+COPY packages/events/package.json ./packages/events/package.json
+COPY packages/secrets/package.json ./packages/secrets/package.json
+COPY packages/validators/package.json ./packages/validators/package.json
+COPY packages/db/package.json ./packages/db/package.json
+COPY packages/job-dispatch/package.json ./packages/job-dispatch/package.json
+COPY packages/rule-engine/package.json ./packages/rule-engine/package.json
+
+COPY apps/event-queue/package.json ./apps/event-queue/package.json
+
+RUN pnpm install --frozen-lockfile
+
+COPY . .
+
+RUN turbo build --filter=...@ctrlplane/event-queue
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nodejs
+USER nodejs
+
+ENV NODE_ENV=production
+ENV NODE_MAX_OLD_SPACE_SIZE=4096
+
+EXPOSE 3123
+
+WORKDIR apps/event-queue/dist/
+
+CMD ["node", "index.js"]

apps/event-queue/README.md

@@ -0,0 +1,9 @@
+# Event Queue
+
+This is a simple event queue that listens to the `ctrlplane-events` topic and logs the events to the console.
+
+## Running
+
+```bash
+pnpm dev
+```

apps/event-queue/eslint.config.js

@@ -0,0 +1,8 @@
+import baseConfig, { requireJsSuffix } from "@ctrlplane/eslint-config/base";
+
+/** @type {import('typescript-eslint').Config} */
+export default [
+  { ignores: [".nitro/**", ".output/**", "dist/**"] },
+  ...requireJsSuffix,
+  ...baseConfig,
+];

apps/event-queue/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@ctrlplane/event-queue",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "clean": "rm -rf .turbo node_modules",
+    "dev": "pnpm with-env tsx watch --clear-screen=false src/index.ts",
+    "lint": "eslint",
+    "build": "tsc",
+    "format": "prettier --check . --ignore-path ../../.gitignore",
+    "typecheck": "tsc --noEmit",
+    "with-env": "dotenv -e ../../.env --"
+  },
+  "dependencies": {
+    "@ctrlplane/events": "workspace:*",
+    "@ctrlplane/logger": "workspace:*",
+    "@t3-oss/env-core": "catalog:",
+    "dotenv": "^16.4.5",
+    "kafkajs": "^2.2.4",
+    "zod": "catalog:"
+  },
+  "devDependencies": {
+    "@ctrlplane/eslint-config": "workspace:^",
+    "@ctrlplane/prettier-config": "workspace:^",
+    "@ctrlplane/tsconfig": "workspace:*",
+    "@types/node": "catalog:node22",
+    "eslint": "catalog:",
+    "prettier": "catalog:",
+    "tsx": "catalog:",
+    "typescript": "catalog:"
+  },
+  "prettier": "@ctrlplane/prettier-config"
+}

apps/event-queue/src/config.ts

@@ -0,0 +1,26 @@
+import { createEnv } from "@t3-oss/env-core";
+import dotenv from "dotenv";
+import { z } from "zod";
+
+dotenv.config();
+
+export const env = createEnv({
+  server: {
+    NODE_ENV: z
+      .enum(["development", "production", "test"])
+      .default("development"),
+    KAFKA_BROKERS: z
+      .string()
+      .default("localhost:9092")
+      .transform((val) =>
+        val
+          .split(",")
+          .map((s) => s.trim())
+          .filter(Boolean),
+      )
+      .refine((arr) => arr.length > 0, {
+        message: "KAFKA_BROKERS must be a non-empty list of brokers",
+      }),
+  },
+  runtimeEnv: process.env,
+});

apps/event-queue/src/index.ts

@@ -0,0 +1,33 @@
+import { Kafka } from "kafkajs";
+
+import { logger } from "@ctrlplane/logger";
+
+import { env } from "./config.js";
+
+const kafka = new Kafka({
+  clientId: "ctrlplane-events",
+  brokers: env.KAFKA_BROKERS,
+});
+
+const consumer = kafka.consumer({ groupId: "ctrlplane-events" });
+
+export const start = async () => {
+  logger.info("Starting event queue", { brokers: env.KAFKA_BROKERS });
+  await consumer.connect();
+  await consumer.subscribe({ topic: "ctrlplane-events", fromBeginning: true });
+  logger.info("Subscribed to ctrlplane-events topic");
+
+  await consumer.run({
+    eachMessage: async ({ topic, partition, message }) => {
+      logger.info("Received event", {
+        topic,
+        partition,
+        message: message.value?.toString() ?? "No message",
+      });
+
+      return Promise.resolve();
+    },
+  });
+};
+
+start();

apps/event-queue/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "@ctrlplane/tsconfig/base.json",
+  "compilerOptions": {
+    "target": "ESNext",
+    "outDir": "dist",
+    "noEmit": false,
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json",
+    "baseUrl": ".",
+    "esModuleInterop": true,
+    "importsNotUsedAsValues": "remove"
+  },
+  "include": ["src", "*.ts"],
+  "exclude": ["node_modules"]
+}

apps/event-worker/Dockerfile

@@ -50,8 +50,8 @@
 USER nodejs
 
 ENV NODE_ENV=production
-ENV NODE_MAX_OLD_SPACE_SIZE=4096
+ENV NODE_OPTIONS="--max-old-space-size=4096"
 
 EXPOSE 3123
 
 CMD node -r ./apps/event-worker/dist/instrumentation-node.js --max-old-space-size=${NODE_MAX_OLD_SPACE_SIZE} apps/event-worker/dist/index.js