diff --git a/backend/.env.dist.local b/backend/.env.dist.local index d9126fe135..14c770b4dc 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -150,6 +150,9 @@ CROWD_TEMPORAL_NAMESPACE=default CROWD_TEMPORAL_ENCRYPTION_KEY_ID=local CROWD_TEMPORAL_ENCRYPTION_KEY=FweBMRnGCLshER8FlSvNusQA6G3MRUKt +# Temporal — packages namespace +CROWD_PACKAGES_TEMPORAL_NAMESPACE=default + # Seach sync api CROWD_SEARCH_SYNC_API_URL=http://localhost:8083 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96db6643f6..56850aa78a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1294,6 +1294,12 @@ importers: services/apps/packages_worker: dependencies: + '@crowd/archetype-standard': + specifier: workspace:* + version: link:../../archetypes/standard + '@crowd/archetype-worker': + specifier: workspace:* + version: link:../../archetypes/worker '@crowd/common': specifier: workspace:* version: link:../../libs/common @@ -1306,6 +1312,12 @@ importers: '@crowd/logging': specifier: workspace:* version: link:../../libs/logging + '@temporalio/client': + specifier: ~1.11.8 + version: 1.11.8 + '@temporalio/workflow': + specifier: ~1.11.8 + version: 1.11.8 tsx: specifier: ^4.7.1 version: 4.7.3 @@ -6103,6 +6115,7 @@ packages: dottie@2.0.6: resolution: {integrity: sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. dtrace-provider@0.8.8: resolution: {integrity: sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==} @@ -6858,11 +6871,11 @@ packages: glob@6.0.4: resolution: {integrity: sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + deprecated: Glob versions prior to v9 are no longer supported glob@7.2.3: resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + deprecated: Glob versions prior to v9 are no longer supported global-directory@4.0.1: resolution: {integrity: sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q==} @@ -9819,11 +9832,12 @@ packages: uuid@3.3.2: resolution: {integrity: sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==} - deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true uuid@9.0.1: @@ -10264,8 +10278,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10459,11 +10473,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': + '@aws-sdk/client-sso-oidc@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10502,7 +10516,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso@3.556.0': @@ -10678,11 +10691,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0': + '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10721,6 +10734,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.985.0': @@ -10886,7 +10900,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -11063,7 +11077,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0 + '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -11375,7 +11389,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -12181,7 +12195,7 @@ snapshots: '@grpc/proto-loader@0.7.13': dependencies: lodash.camelcase: 4.3.0 - long: 5.2.3 + long: 5.3.2 protobufjs: 7.5.3 yargs: 17.7.2 @@ -13557,7 +13571,7 @@ snapshots: '@temporalio/common': 1.11.8 '@temporalio/proto': 1.11.8 abort-controller: 3.0.0 - long: 5.2.3 + long: 5.3.2 uuid: 9.0.1 '@temporalio/common@1.11.8': @@ -18182,7 +18196,7 @@ snapshots: '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 '@types/node': 20.12.7 - long: 5.2.3 + long: 5.3.2 protobufjs@7.5.3: dependencies: @@ -18197,7 +18211,7 @@ snapshots: '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.0 '@types/node': 20.12.7 - long: 5.2.3 + long: 5.3.2 proxy-addr@2.0.7: dependencies: diff --git a/scripts/builders/packages-worker.env b/scripts/builders/packages-worker.env new file mode 100644 index 0000000000..9172b5d217 --- /dev/null +++ b/scripts/builders/packages-worker.env @@ -0,0 +1,4 @@ +DOCKERFILE="./services/docker/Dockerfile.packages-worker" +CONTEXT="../" +REPO="sjc.ocir.io/axbydjxa5zuh/packages-worker" +SERVICES="packages-worker" diff --git a/scripts/services/docker/Dockerfile.packages-worker b/scripts/services/docker/Dockerfile.packages-worker index 491ebd7665..c01bf60266 100644 --- a/scripts/services/docker/Dockerfile.packages-worker +++ b/scripts/services/docker/Dockerfile.packages-worker @@ -1,6 +1,6 @@ -FROM node:20-alpine as builder +FROM node:20-bookworm-slim as builder -RUN apk add --no-cache python3 make g++ +RUN apt-get update && apt-get install -y python3 make g++ --no-install-recommends && rm -rf /var/lib/apt/lists/* WORKDIR /usr/crowd/app RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate @@ -8,15 +8,19 @@ RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare p COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ RUN pnpm fetch -COPY ./services ./services -RUN pnpm i --frozen-lockfile +COPY ./services/base.tsconfig.json ./services/ +COPY ./services/archetypes ./services/archetypes +COPY ./services/libs ./services/libs +COPY ./services/apps/packages_worker ./services/apps/packages_worker +RUN pnpm i --frozen-lockfile --filter @crowd/packages-worker... -FROM node:20-alpine as runner +FROM node:20-bookworm-slim as runner WORKDIR /usr/crowd/app -RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* COPY --from=builder /usr/crowd/app/node_modules ./node_modules COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json COPY --from=builder /usr/crowd/app/services/libs ./services/libs +COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes COPY --from=builder /usr/crowd/app/services/apps/packages_worker/ ./services/apps/packages_worker diff --git a/scripts/services/docker/Dockerfile.packages-worker.dockerignore b/scripts/services/docker/Dockerfile.packages-worker.dockerignore new file mode 100644 index 0000000000..17864cafa3 --- /dev/null +++ b/scripts/services/docker/Dockerfile.packages-worker.dockerignore @@ -0,0 +1,20 @@ +**/.git +**/node_modules +**/venv* +**/.webpack +**/.serverless +**/.env +**/.env.* +**/.idea +**/.vscode +**/dist +.vscode/ +.github/ +frontend/ +scripts/ +.flake8 +*.md +Makefile +backend/ +services/apps/ +!services/apps/packages_worker diff --git a/scripts/services/packages-worker.yaml b/scripts/services/packages-worker.yaml index cb2ecdbf0f..3a84159018 100644 --- a/scripts/services/packages-worker.yaml +++ b/scripts/services/packages-worker.yaml @@ -4,6 +4,8 @@ x-env-args: &env-args DOCKER_BUILDKIT: 1 NODE_ENV: docker SERVICE: packages-worker + CROWD_TEMPORAL_TASKQUEUE: packages-worker + CROWD_TEMPORAL_NAMESPACE: ${CROWD_PACKAGES_TEMPORAL_NAMESPACE} SHELL: /bin/sh SUPPRESS_NO_CONFIG_WARNING: 'true' diff --git a/services/apps/packages_worker/package.json b/services/apps/packages_worker/package.json index 8ee8dceeae..8c822bd7b1 100644 --- a/services/apps/packages_worker/package.json +++ b/services/apps/packages_worker/package.json @@ -2,11 +2,11 @@ "name": "@crowd/packages-worker", "private": true, "scripts": { - "start:packages-worker": "SERVICE=packages-worker tsx src/bin/packages-worker.ts", + "start:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker tsx src/bin/packages-worker.ts", "start:github-repos-enricher": "SERVICE=github-repos-enricher tsx src/bin/github-repos-enricher.ts", - "dev:packages-worker": "SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts", + "dev:packages-worker": "CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts", "dev:github-repos-enricher": "SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts", - "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts", + "dev:packages-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=packages-worker CROWD_TEMPORAL_NAMESPACE=$CROWD_PACKAGES_TEMPORAL_NAMESPACE SERVICE=packages-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9233 src/bin/packages-worker.ts", "dev:github-repos-enricher:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=github-repos-enricher LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9234 src/bin/github-repos-enricher.ts", "lint": "npx eslint --ext .ts src --max-warnings=0", "format": "npx prettier --write \"src/**/*.ts\"", @@ -14,10 +14,14 @@ "tsc-check": "tsc --noEmit" }, "dependencies": { + "@crowd/archetype-standard": "workspace:*", + "@crowd/archetype-worker": "workspace:*", "@crowd/common": "workspace:*", "@crowd/data-access-layer": "workspace:*", "@crowd/database": "workspace:*", "@crowd/logging": "workspace:*", + "@temporalio/client": "~1.11.8", + "@temporalio/workflow": "~1.11.8", "tsx": "^4.7.1", "typescript": "^5.6.3" }, diff --git a/services/apps/packages_worker/src/activities.ts b/services/apps/packages_worker/src/activities.ts new file mode 100644 index 0000000000..b86e9ae448 --- /dev/null +++ b/services/apps/packages_worker/src/activities.ts @@ -0,0 +1 @@ +export { sayHiNpm } from './npm/activities' diff --git a/services/apps/packages_worker/src/bin/github-repos-enricher.ts b/services/apps/packages_worker/src/bin/github-repos-enricher.ts index 8de28a7aaa..184a52918d 100644 --- a/services/apps/packages_worker/src/bin/github-repos-enricher.ts +++ b/services/apps/packages_worker/src/bin/github-repos-enricher.ts @@ -32,7 +32,7 @@ const main = async () => { log.info('Connected to packages-db.') log.info( - { tokens: config.tokens.length, pageSize: config.pageSize, batchSize: config.batchSize }, + { tokens: config.tokens.length, batchSize: config.batchSize }, 'Starting enrichment loop', ) diff --git a/services/apps/packages_worker/src/bin/packages-worker.ts b/services/apps/packages_worker/src/bin/packages-worker.ts index c97ff2843a..7715c934f7 100644 --- a/services/apps/packages_worker/src/bin/packages-worker.ts +++ b/services/apps/packages_worker/src/bin/packages-worker.ts @@ -1,53 +1,8 @@ -import fs from 'fs' -import path from 'path' +import { scheduleNpmIngest } from '../npm/schedule' +import { svc } from '../service' -import { getServiceLogger } from '@crowd/logging' - -import { getPackagesDb } from '../db' - -const log = getServiceLogger() - -const liveFilePath = path.join(__dirname, '../tmp/packages-worker-live.tmp') -const readyFilePath = path.join(__dirname, '../tmp/packages-worker-ready.tmp') - -let shuttingDown = false - -const shutdown = async () => { - if (shuttingDown) return - shuttingDown = true - log.info('Shutting down packages-worker...') - process.exit(0) -} - -process.on('SIGINT', shutdown) -process.on('SIGTERM', shutdown) - -const main = async () => { - log.info('packages-worker starting...') - - const qx = await getPackagesDb() - await qx.selectOne('SELECT 1') - log.info('Connected to packages-db.') - - // Create tmp directory for health probe files - fs.mkdirSync(path.dirname(liveFilePath), { recursive: true }) - - setInterval(async () => { - if (shuttingDown) return - try { - await Promise.all([ - fs.promises.open(liveFilePath, 'a').then((f) => f.close()), - fs.promises.open(readyFilePath, 'a').then((f) => f.close()), - ]) - } catch (err) { - log.warn({ err }, 'Failed to write health probe files') - } - }, 5000) - - log.info('packages-worker started, idle.') -} - -main().catch((err) => { - log.error({ err }, 'packages-worker fatal error') - process.exit(1) +setImmediate(async () => { + await svc.init() + await scheduleNpmIngest() + await svc.start() }) diff --git a/services/apps/packages_worker/src/config.ts b/services/apps/packages_worker/src/config.ts index 4faf7444bf..84c11fb134 100644 --- a/services/apps/packages_worker/src/config.ts +++ b/services/apps/packages_worker/src/config.ts @@ -20,7 +20,10 @@ export function getPackagesDbConfig() { export function getEnricherConfig() { const rawTokens = process.env.ENRICHER_GITHUB_TOKENS ?? '' - const tokens = rawTokens.split(',').map((t) => t.trim()).filter(Boolean) + const tokens = rawTokens + .split(',') + .map((t) => t.trim()) + .filter(Boolean) return { tokens, diff --git a/services/apps/packages_worker/src/db.ts b/services/apps/packages_worker/src/db.ts index c300b8b0a2..f314861d8b 100644 --- a/services/apps/packages_worker/src/db.ts +++ b/services/apps/packages_worker/src/db.ts @@ -1,5 +1,5 @@ -import { getDbConnection } from '@crowd/database' import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { getDbConnection } from '@crowd/database' import { getPackagesDbConfig } from './config' diff --git a/services/apps/packages_worker/src/enricher/fetchLightRepo.ts b/services/apps/packages_worker/src/enricher/fetchLightRepo.ts index ebd1dd87f4..5d57f372ef 100644 --- a/services/apps/packages_worker/src/enricher/fetchLightRepo.ts +++ b/services/apps/packages_worker/src/enricher/fetchLightRepo.ts @@ -67,7 +67,8 @@ export async function fetchLightRepo(url: string, token: string): Promise n.topic.name), + topics: (repo.repositoryTopics?.nodes ?? []).map( + (n: { topic: { name: string } }) => n.topic.name, + ), stars: repo.stargazerCount ?? 0, forks: repo.forkCount ?? 0, watchers: repo.watchers?.totalCount ?? 0, diff --git a/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts b/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts index b80d84b0bd..19c64155ed 100644 --- a/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts +++ b/services/apps/packages_worker/src/enricher/runEnrichmentLoop.ts @@ -1,7 +1,8 @@ -import { getServiceChildLogger } from '@crowd/logging' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceChildLogger } from '@crowd/logging' import { getEnricherConfig } from '../config' + import { fetchLightRepo, parseGithubUrl } from './fetchLightRepo' import { FetchError, LightRepoResult } from './types' import { updateEnrichedRepos } from './updateEnrichedRepos' @@ -10,10 +11,7 @@ const log = getServiceChildLogger('github-repos-enricher') const MAX_RETRIES = 3 -async function fetchWithRetries( - url: string, - token: string, -): Promise { +async function fetchWithRetries(url: string, token: string): Promise { for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) { try { return await fetchLightRepo(url, token) @@ -98,9 +96,8 @@ async function processPage( await new Promise((r) => setTimeout(r, initialPark)) } - while (true) { + while (nextIdx < validUrls.length) { const idx = nextIdx++ - if (idx >= validUrls.length) break const url = validUrls[idx] try { @@ -171,7 +168,13 @@ export async function runEnrichmentLoop( continue } - const { fetched, failed, flushed } = await processPage(urls, config.tokens, parkedUntil, config, qx) + const { fetched, failed, flushed } = await processPage( + urls, + config.tokens, + parkedUntil, + config, + qx, + ) log.info( `Page ${pageNum}: read=${urls.length} fetched=${fetched} failed=${failed} flushed=${flushed}`, diff --git a/services/apps/packages_worker/src/enricher/updateEnrichedRepos.ts b/services/apps/packages_worker/src/enricher/updateEnrichedRepos.ts index de18cb4c7b..ae0a6eb2ae 100644 --- a/services/apps/packages_worker/src/enricher/updateEnrichedRepos.ts +++ b/services/apps/packages_worker/src/enricher/updateEnrichedRepos.ts @@ -1,11 +1,14 @@ -import { getServiceChildLogger } from '@crowd/logging' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' +import { getServiceChildLogger } from '@crowd/logging' import { LightRepoResult } from './types' const log = getServiceChildLogger('github-repos-enricher:update') -export async function updateEnrichedRepos(qx: QueryExecutor, rows: LightRepoResult[]): Promise { +export async function updateEnrichedRepos( + qx: QueryExecutor, + rows: LightRepoResult[], +): Promise { if (rows.length === 0) return for (const r of rows) { diff --git a/services/apps/packages_worker/src/npm/activities.ts b/services/apps/packages_worker/src/npm/activities.ts new file mode 100644 index 0000000000..e916ad2fba --- /dev/null +++ b/services/apps/packages_worker/src/npm/activities.ts @@ -0,0 +1,8 @@ +import { getServiceChildLogger } from '@crowd/logging' + +const log = getServiceChildLogger('npm') + +export async function sayHiNpm(): Promise { + log.info('hi from npm activity') + return 'hi from npm' +} diff --git a/services/apps/packages_worker/src/npm/schedule.ts b/services/apps/packages_worker/src/npm/schedule.ts new file mode 100644 index 0000000000..1c7a3b0f5c --- /dev/null +++ b/services/apps/packages_worker/src/npm/schedule.ts @@ -0,0 +1,40 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +import { svc } from '../service' +import { npmHello } from '../workflows' + +export async function scheduleNpmIngest(): Promise { + const { temporal } = svc + if (!temporal) throw new Error('Temporal client not initialized') + + try { + await temporal.schedule.create({ + scheduleId: 'npm-registry-ingest', + spec: { + cronExpressions: ['15 3 * * *'], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 hour', + }, + action: { + type: 'startWorkflow', + workflowType: npmHello, + taskQueue: 'packages-worker', + workflowExecutionTimeout: '1 hour', + retry: { + initialInterval: '30 seconds', + backoffCoefficient: 2, + maximumAttempts: 3, + }, + args: [], + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Schedule npm-registry-ingest already registered.') + } else { + throw err + } + } +} diff --git a/services/apps/packages_worker/src/npm/workflows.ts b/services/apps/packages_worker/src/npm/workflows.ts new file mode 100644 index 0000000000..06796494ca --- /dev/null +++ b/services/apps/packages_worker/src/npm/workflows.ts @@ -0,0 +1,11 @@ +import { proxyActivities } from '@temporalio/workflow' + +import type * as activities from './activities' + +const { sayHiNpm } = proxyActivities({ + startToCloseTimeout: '1 minute', +}) + +export async function npmHello(): Promise { + await sayHiNpm() +} diff --git a/services/apps/packages_worker/src/service.ts b/services/apps/packages_worker/src/service.ts new file mode 100644 index 0000000000..6bbb449bda --- /dev/null +++ b/services/apps/packages_worker/src/service.ts @@ -0,0 +1,15 @@ +import { Config } from '@crowd/archetype-standard' +import { Options, ServiceWorker } from '@crowd/archetype-worker' + +const config: Config = { + envvars: [], + producer: { enabled: false }, + temporal: { enabled: true }, + redis: { enabled: false }, +} + +const options: Options = { + postgres: { enabled: false }, // packages-db is managed via getPackagesDb() +} + +export const svc = new ServiceWorker(config, options) diff --git a/services/apps/packages_worker/src/workflows/index.ts b/services/apps/packages_worker/src/workflows/index.ts new file mode 100644 index 0000000000..5bd3dd2623 --- /dev/null +++ b/services/apps/packages_worker/src/workflows/index.ts @@ -0,0 +1 @@ +export { npmHello } from '../npm/workflows'