feat: packages worker temporal scaffold

backend/.env.dist.local

@@ -150,6 +150,9 @@ CROWD_TEMPORAL_NAMESPACE=default
 CROWD_TEMPORAL_ENCRYPTION_KEY_ID=local
 CROWD_TEMPORAL_ENCRYPTION_KEY=FweBMRnGCLshER8FlSvNusQA6G3MRUKt
 
+# Temporal — packages namespace
+CROWD_PACKAGES_TEMPORAL_NAMESPACE=default
+
 # Seach sync api
 CROWD_SEARCH_SYNC_API_URL=http://localhost:8083
 

scripts/builders/packages-worker.env

@@ -0,0 +1,4 @@
+DOCKERFILE="./services/docker/Dockerfile.packages-worker"
+CONTEXT="../"
+REPO="sjc.ocir.io/axbydjxa5zuh/packages-worker"
+SERVICES="packages-worker"

scripts/services/docker/Dockerfile.packages-worker

@@ -1,22 +1,26 @@
-FROM node:20-alpine as builder
+FROM node:20-bookworm-slim as builder
 
-RUN apk add --no-cache python3 make g++
+RUN apt-get update && apt-get install -y python3 make g++ --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /usr/crowd/app
 RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate
 
 COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./
 RUN pnpm fetch
 
-COPY ./services ./services
-RUN pnpm i --frozen-lockfile
+COPY ./services/base.tsconfig.json ./services/
+COPY ./services/archetypes ./services/archetypes
+COPY ./services/libs ./services/libs
+COPY ./services/apps/packages_worker ./services/apps/packages_worker
+RUN pnpm i --frozen-lockfile --filter @crowd/packages-worker...
 
-FROM node:20-alpine as runner
+FROM node:20-bookworm-slim as runner
 
 WORKDIR /usr/crowd/app
-RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate
+RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /usr/crowd/app/node_modules ./node_modules
 COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json
 COPY --from=builder /usr/crowd/app/services/libs ./services/libs
+COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes
 COPY --from=builder /usr/crowd/app/services/apps/packages_worker/ ./services/apps/packages_worker

scripts/services/docker/Dockerfile.packages-worker.dockerignore

@@ -0,0 +1,20 @@
+**/.git
+**/node_modules
+**/venv*
+**/.webpack
+**/.serverless
+**/.env
+**/.env.*
+**/.idea
+**/.vscode
+**/dist
+.vscode/
+.github/
+frontend/
+scripts/
+.flake8
+*.md
+Makefile
+backend/
+services/apps/
+!services/apps/packages_worker

scripts/services/packages-worker.yaml

@@ -4,6 +4,8 @@ x-env-args: &env-args
   DOCKER_BUILDKIT: 1
   NODE_ENV: docker
   SERVICE: packages-worker
+  CROWD_TEMPORAL_TASKQUEUE: packages-worker
+  CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE}
   SHELL: /bin/sh
   SUPPRESS_NO_CONFIG_WARNING: 'true'
 

services/apps/packages_worker/package.json

@@ -2,22 +2,26 @@
   "name": "@crowd/packages-worker",
   "private": true,
   "scripts": {
-    "start:packages-worker": "SERVICE=packages-worker tsx src/bin/packages-worker.ts",
+    "start:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker tsx src/bin/packages-worker.ts",
     "start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts",
-    "dev:packages-worker": "SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
+    "dev:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
-    "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
+    "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts",
     "dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts",
     "lint": "npx eslint --ext .ts src --max-warnings=0",
     "format": "npx prettier --write \"src/**/*.ts\"",
     "format-check": "npx prettier --check .",
     "tsc-check": "tsc --noEmit"
   },
   "dependencies": {
+    "@crowd/archetype-standard": "workspace:*",
+    "@crowd/archetype-worker": "workspace:*",
     "@crowd/common": "workspace:*",
     "@crowd/data-access-layer": "workspace:*",
     "@crowd/database": "workspace:*",
     "@crowd/logging": "workspace:*",
+    "@temporalio/client": "~1.11.8",
+    "@temporalio/workflow": "~1.11.8",
     "tsx": "^4.7.1",
     "typescript": "^5.6.3"
   },

services/apps/packages_worker/src/activities.ts

@@ -0,0 +1 @@
+export { sayHiNpm } from './npm/activities'

services/apps/packages_worker/src/bin/github-repos-enricher.ts

@@ -32,7 +32,7 @@ const main = async () => {
   log.info('Connected to packages-db.')
 
   log.info(
-    { tokens: config.tokens.length, pageSize: config.pageSize, batchSize: config.batchSize },
+    { tokens: config.tokens.length, batchSize: config.batchSize },
     'Starting enrichment loop',
   )
 

services/apps/packages_worker/src/bin/packages-worker.ts

@@ -1,53 +1,8 @@
-import fs from 'fs'
-import path from 'path'
+import { scheduleNpmIngest } from '../npm/schedule'
+import { svc } from '../service'
 
-import { getServiceLogger } from '@crowd/logging'
-
-import { getPackagesDb } from '../db'
-
-const log = getServiceLogger()
-
-const liveFilePath = path.join(__dirname, '../tmp/packages-worker-live.tmp')
-const readyFilePath = path.join(__dirname, '../tmp/packages-worker-ready.tmp')
-
-let shuttingDown = false
-
-const shutdown = async () => {
-  if (shuttingDown) return
-  shuttingDown = true
-  log.info('Shutting down packages-worker...')
-  process.exit(0)
-}
-
-process.on('SIGINT', shutdown)
-process.on('SIGTERM', shutdown)
-
-const main = async () => {
-  log.info('packages-worker starting...')
-
-  const qx = await getPackagesDb()
-  await qx.selectOne('SELECT 1')
-  log.info('Connected to packages-db.')
-
-  // Create tmp directory for health probe files
-  fs.mkdirSync(path.dirname(liveFilePath), { recursive: true })
-
-  setInterval(async () => {
-    if (shuttingDown) return
-    try {
-      await Promise.all([
-        fs.promises.open(liveFilePath, 'a').then((f) => f.close()),
-        fs.promises.open(readyFilePath, 'a').then((f) => f.close()),
-      ])
-    } catch (err) {
-      log.warn({ err }, 'Failed to write health probe files')
-    }
-  }, 5000)
-
-  log.info('packages-worker started, idle.')
-}
-
-main().catch((err) => {
-  log.error({ err }, 'packages-worker fatal error')
-  process.exit(1)
+setImmediate(async () => {
+  await svc.init()
+  await scheduleNpmIngest()
+  await svc.start()
 })

services/apps/packages_worker/src/config.ts

@@ -20,7 +20,10 @@ export function getPackagesDbConfig() {
 
 export function getEnricherConfig() {
   const rawTokens = process.env.ENRICHER_GITHUB_TOKENS ?? ''
-  const tokens = rawTokens.split(',').map((t) => t.trim()).filter(Boolean)
+  const tokens = rawTokens
+    .split(',')
+    .map((t) => t.trim())
+    .filter(Boolean)
 
   return {
     tokens,

services/apps/packages_worker/src/db.ts

@@ -1,5 +1,5 @@
-import { getDbConnection } from '@crowd/database'
 import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
+import { getDbConnection } from '@crowd/database'
 
 import { getPackagesDbConfig } from './config'
 

services/apps/packages_worker/src/enricher/fetchLightRepo.ts

@@ -67,7 +67,8 @@ export async function fetchLightRepo(url: string, token: string): Promise<LightR
 
   if (json.errors?.length) {
     const err = json.errors[0]
-    if (err.type === 'RATE_LIMITED') throw new FetchError('RATE_LIMIT', `RATE_LIMITED for ${url}`, resetMs)
+    if (err.type === 'RATE_LIMITED')
+      throw new FetchError('RATE_LIMIT', `RATE_LIMITED for ${url}`, resetMs)
     if (err.type === 'NOT_FOUND') throw new FetchError('NOT_FOUND', `NOT_FOUND for ${url}`)
     throw new FetchError('TRANSIENT', `GraphQL error for ${url}: ${err.message ?? err.type}`)
   }
@@ -82,7 +83,9 @@ export async function fetchLightRepo(url: string, token: string): Promise<LightR
     name,
     description: repo.description ?? null,
     primaryLanguage: repo.primaryLanguage?.name ?? null,
-    topics: (repo.repositoryTopics?.nodes ?? []).map((n: { topic: { name: string } }) => n.topic.name),
+    topics: (repo.repositoryTopics?.nodes ?? []).map(
+      (n: { topic: { name: string } }) => n.topic.name,
+    ),
     stars: repo.stargazerCount ?? 0,
     forks: repo.forkCount ?? 0,
     watchers: repo.watchers?.totalCount ?? 0,