Collapse worker into server and various refactor

.gitignore

@@ -39,3 +39,5 @@ yarn-error.log*
 *.tfstate
 *.tfstate.*
 kubeconfig
+
+*.tsbuildinfo

Dockerfile

@@ -26,7 +26,6 @@ ADD package.json pnpm-lock.yaml pnpm-workspace.yaml packages .
 ADD apps/cli/package.json ./apps/cli/package.json
 ADD apps/web/package.json ./apps/web/package.json
 ADD apps/server/package.json ./apps/server/package.json
-ADD apps/worker/package.json ./apps/worker/package.json
 ADD packages/tsconfig/package.json ./packages/tsconfig/package.json
 ADD packages/design/package.json ./packages/design/package.json
 
@@ -74,12 +73,12 @@ FROM base-env as worker
 COPY --from=prod-deps /app/node_modules /app/node_modules
 COPY --from=build /app/ /app/
 
-WORKDIR /app/apps/worker
+WORKDIR /app/apps/server
 
 ARG VERSION
 ENV VERSION $VERSION
 
-CMD ["pnpm", "start"]
+CMD ["pnpm", "start:worker"]
 
 # cli service
 FROM base-env as cli
@@ -106,4 +105,4 @@ WORKDIR /app/apps/server
 ARG VERSION
 ENV VERSION $VERSION
 
-CMD ["pnpm", "start"]
+CMD ["pnpm", "start:api"]

Makefile

@@ -7,6 +7,10 @@ reset-db:
 	$(MAKE) drop-db
 	$(MAKE) create-db
 
+reset-test-db:
+	$(MAKE) drop-db-test
+	$(MAKE) create-db-test
+
 drop-db: drop-db-dev drop-db-test
 
 drop-db-dev:

apps/cli/package.json

@@ -3,13 +3,13 @@
   "name": "@peated/cli",
   "scripts": {
     "build": "tsc",
-    "cli": "esno --require dotenv/config ./src/index.ts",
-    "db": "esno --require dotenv/config ./src/index.ts db",
-    "bottles": "esno --require dotenv/config ./src/index.ts bottles",
-    "entities": "esno --require dotenv/config ./src/index.ts entities",
-    "tastings": "esno --require dotenv/config ./src/index.ts tastings",
-    "mocks": "esno --require dotenv/config ./src/bin/index.ts mocks",
-    "users": "esno --require dotenv/config ./src/bin/index.ts users",
+    "cli": "tsx --require dotenv/config ./src/index.ts",
+    "db": "tsx --require dotenv/config ./src/index.ts db",
+    "bottles": "tsx --require dotenv/config ./src/index.ts bottles",
+    "entities": "tsx --require dotenv/config ./src/index.ts entities",
+    "tastings": "tsx --require dotenv/config ./src/index.ts tastings",
+    "mocks": "tsx --require dotenv/config ./src/bin/index.ts mocks",
+    "users": "tsx --require dotenv/config ./src/bin/index.ts users",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
@@ -18,23 +18,21 @@
     "@sentry/node": "~8.0.0-alpha.7",
     "@sentry/profiling-node": "~8.0.0-alpha.7",
     "@sentry/utils": "~8.0.0-alpha.7",
-    "@trpc/client": "^10.45.0",
-    "@trpc/server": "^10.45.0",
+    "@trpc/client": "^10.45.2",
+    "@trpc/server": "^10.45.2",
     "@types/bcrypt": "^5.0.2",
-    "@types/node": "^18.19.3",
-    "@types/pg": "^8.10.9",
+    "@types/node": "^20.12.5",
+    "@types/pg": "^8.11.4",
     "bcrypt": "^5.1.1",
-    "commander": "^10.0.1",
-    "dotenv": "^16.3.1",
-    "dotenv-cli": "^7.3.0",
-    "drizzle-kit": "^0.20.9",
-    "drizzle-orm": "^0.29.2",
-    "esbuild": "^0.19.10",
-    "esbuild-node-tsc": "^2.0.5",
-    "esno": "^4.7.0",
-    "pg": "^8.11.3",
+    "commander": "^12.0.0",
+    "dotenv": "^16.4.5",
+    "dotenv-cli": "^7.4.1",
+    "drizzle-kit": "^0.20.14",
+    "drizzle-orm": "^0.30.7",
+    "pg": "^8.11.5",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.3.3"
+    "tsx": "^4.7.2",
+    "typescript": "~5.4.4"
   },
   "volta": {
     "extends": "../../package.json"

apps/cli/src/commands/bottles.ts

@@ -1,7 +1,7 @@
 import program from "@peated/cli/program";
 import { db } from "@peated/server/db";
 import { bottles, reviews, tastings } from "@peated/server/db/schema";
-import { pushJob } from "@peated/server/jobs";
+import { pushJob } from "@peated/server/jobs/client";
 import { findEntity } from "@peated/server/lib/bottleFinder";
 import { formatCategoryName } from "@peated/server/lib/format";
 import { createCaller } from "@peated/server/trpc/router";

apps/cli/src/commands/entities.ts

@@ -6,7 +6,7 @@ import {
   entities,
   tastings,
 } from "@peated/server/db/schema";
-import { pushJob } from "@peated/server/jobs";
+import { pushJob } from "@peated/server/jobs/client";
 import { sql } from "drizzle-orm";
 
 const subcommand = program.command("entities");

apps/cli/src/commands/tastings.ts

@@ -1,6 +1,6 @@
 import program from "@peated/cli/program";
 import { db } from "@peated/server/db";
-import { pushJob } from "@peated/server/jobs";
+import { pushJob } from "@peated/server/jobs/client";
 
 const subcommand = program.command("tastings");
 

apps/cli/src/index.ts

@@ -2,9 +2,13 @@
 
 import "./sentry";
 
+import { logError } from "@peated/server/lib/log";
 import "./commands";
 import program from "./program";
 
 export { program };
 
-program.parseAsync();
+program.parseAsync().catch((err) => {
+  logError(err);
+  process.exit(1);
+});

apps/cli/src/program.ts

@@ -1,5 +1,5 @@
+import { shutdownClient } from "@peated/server/jobs/client";
 import { Command } from "commander";
-import { shutdownClient } from "../../server/src/jobs";
 
 const program = new Command();
 

apps/cli/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "@peated/tsconfig/tsconfig.json",
   "compilerOptions": {
-    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json",
     "types": [],
     "baseUrl": "./",
     "paths": {

apps/server/.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}",
+      "name": "Debug Current Test File",
+      "autoAttachChildProcesses": true,
+      "skipFiles": ["<node_internals>/**", "**/node_modules/**"],
+      "runtimeExecutable": "npm",
+      "runtimeArgs": ["test", "--", "${relativeFile}"],
+      "smartStep": true,
+      "console": "integratedTerminal"
+    }
+  ]
+}

apps/server/package.json

@@ -3,67 +3,72 @@
   "name": "@peated/server",
   "scripts": {
     "build": "tsc",
-    "dev": "esno watch --require dotenv/config ./src/server.ts",
-    "start": "esno --require dotenv/config ./src/server.ts",
+    "dev:api": "tsx watch --require dotenv/config ./src/server.ts",
+    "dev:worker": "tsx watch --require dotenv/config ./src/worker.ts",
+    "start:api": "tsx --require dotenv/config ./src/server.ts",
+    "start:worker": "tsx --require dotenv/config ./src/worker.ts",
     "typecheck": "tsc --noEmit",
-    "test": "dotenv -e .env.test -- vitest watch --no-threads",
-    "test:watch": "dotenv -e .env.test -- vitest --no-threads",
-    "test:ci": "dotenv -e .env.test -- vitest --no-threads --coverage --reporter=junit --reporter=default --outputFile=junit.xml"
+    "test": "pnpm test:watch",
+    "test:watch": "dotenv -e .env.test -- vitest watch",
+    "test:ci": "dotenv -e .env.test -- vitest --coverage --reporter=junit --reporter=default --outputFile=junit.xml"
   },
   "dependencies": {
-    "@faker-js/faker": "^8.3.1",
+    "@faker-js/faker": "^8.4.1",
     "@fastify/cors": "^8.5.0",
     "@fastify/helmet": "^11.1.1",
-    "@fastify/multipart": "^8.0.0",
-    "@fastify/one-line-logger": "^1.2.0",
+    "@fastify/multipart": "^8.2.0",
+    "@fastify/one-line-logger": "^1.3.0",
     "@google-cloud/storage": "^6.12.0",
-    "@opentelemetry/api": "^1.7.0",
+    "@opentelemetry/api": "^1.8.0",
     "@paralleldrive/cuid2": "^2.2.2",
     "@peated/design": "workspace:*",
     "@peated/tsconfig": "workspace:*",
     "@sentry/node": "~8.0.0-alpha.7",
     "@sentry/profiling-node": "~8.0.0-alpha.7",
     "@sentry/utils": "~8.0.0-alpha.7",
-    "@trpc/client": "^10.45.0",
-    "@trpc/server": "^10.45.0",
+    "@trpc/client": "^10.45.2",
+    "@trpc/server": "^10.45.2",
     "@types/bcrypt": "^5.0.2",
-    "@types/jsonwebtoken": "^9.0.5",
+    "@types/error-cause": "^1.0.4",
+    "@types/jsonwebtoken": "^9.0.6",
     "@types/mime-types": "^2.1.4",
-    "@types/node": "^18.19.3",
+    "@types/node": "~20.12.5",
     "@types/nodemailer": "^6.4.14",
-    "@types/pg": "^8.10.9",
-    "@vitest/coverage-v8": "^0.34.6",
+    "@types/pg": "^8.11.4",
+    "@vitest/coverage-v8": "^1.4.0",
+    "axios": "^1.6.8",
+    "axios-mock-adapter": "^1.22.0",
     "bcrypt": "^5.1.1",
-    "commander": "^10.0.1",
-    "dotenv": "^16.3.1",
-    "dotenv-cli": "^7.3.0",
-    "drizzle-kit": "^0.20.9",
-    "drizzle-orm": "^0.29.2",
-    "esbuild": "^0.19.10",
-    "esbuild-node-tsc": "^2.0.5",
-    "esno": "^4.7.0",
+    "cheerio": "1.0.0-rc.12",
+    "commander": "^12.0.0",
+    "croner": "^7.0.7",
+    "dotenv": "^16.4.5",
+    "dotenv-cli": "^7.4.1",
+    "drizzle-kit": "^0.20.14",
+    "drizzle-orm": "^0.30.7",
+    "error-cause": "^1.0.8",
     "faktory-worker": "^4.5.1",
-    "fastify": "^4.25.2",
+    "fastify": "^4.26.2",
     "fastify-plugin": "^4.5.1",
     "form-data-encoder": "^3.0.1",
     "formdata-node": "^5.0.1",
     "google-auth-library": "^8.9.0",
     "jsonwebtoken": "^9.0.2",
-    "jwks-rsa": "^3.1.0",
     "mime-types": "^2.1.35",
     "nanoid": "^3.3.7",
-    "nodemailer": "^6.9.9",
-    "nodemon": "^3.0.2",
-    "pg": "^8.11.3",
+    "nodemailer": "^6.9.13",
+    "openai": "^4.33.0",
+    "pg": "^8.11.5",
     "pino-pretty": "^10.3.1",
-    "regenerator-runtime": "^0.14.1",
-    "sharp": "^0.32.6",
+    "sharp": "^0.33.3",
+    "toad-scheduler": "^3.0.1",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.3.3",
-    "vite-tsconfig-paths": "^4.2.3",
-    "vitest": "^0.34.6",
+    "tsx": "^4.7.2",
+    "typescript": "~5.4.4",
+    "vite-tsconfig-paths": "^4.3.2",
+    "vitest": "^1.4.0",
     "zod": "^3.22.4",
-    "zod-to-json-schema": "^3.22.3"
+    "zod-to-json-schema": "^3.22.5"
   },
   "volta": {
     "extends": "../../package.json"

apps/server/src/__mocks__/jobs.ts → apps/server/src/__mocks__/jobs/client.ts

@@ -1,6 +1,6 @@
+import * as jobs from "@peated/server/jobs/client";
 import { type JobFunction } from "faktory-worker";
 import { vi } from "vitest";
-import * as jobs from "../jobs";
 
 export const getClient: typeof jobs.getClient = vi.fn();
 

apps/server/src/app.ts

@@ -4,11 +4,11 @@ import "./sentry";
 import FastifyCors from "@fastify/cors";
 import FastifyHelmet from "@fastify/helmet";
 import FastifyMultipart from "@fastify/multipart";
+import { shutdownClient } from "@peated/server/jobs/client";
 import { fastifyTRPCPlugin } from "@trpc/server/adapters/fastify";
 import { fastify } from "fastify";
 import config from "./config";
 import { MAX_FILESIZE } from "./constants";
-import { shutdownClient } from "./jobs";
 import { injectAuth } from "./middleware/auth";
 import { router } from "./routes";
 import FastifySentry from "./sentryPlugin";

apps/server/src/config.ts

@@ -21,6 +21,7 @@ export default {
   VERSION: process.env.VERSION || "",
 
   SENTRY_DSN: process.env.SENTRY_DSN || "",
+  SENTRY_SERVICE: process.env.SENTRY_SERVICE || "@peated/server",
 
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,
   GOOGLE_CLIENT_SECRET: process.env.GOOGLE_CLIENT_SECRET,
@@ -36,4 +37,8 @@ export default {
     : null,
 
   OPENAI_API_KEY: process.env.OPENAI_API_KEY,
+  OPENAI_HOST: process.env.OPENAI_HOST || "https://api.openai.com/v1",
+  OPENAI_MODEL: process.env.OPENAI_MODEL || "gpt-4-turbo-preview",
+
+  DISCORD_WEBHOOK: process.env.DISCORD_WEBHOOK,
 };

apps/server/src/constants.ts

@@ -478,3 +478,17 @@ export const DEFAULT_TAGS = [
   "woody",
   "zesty",
 ] as const;
+
+// used for web scraping
+export const defaultHeaders = (url: string) => {
+  const urlParts = new URL(url);
+  return {
+    Authority: urlParts.hostname,
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
+    Accept:
+      "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7,",
+    "Accept-Language": "en-US,en:q=0.9",
+    Referer: urlParts.origin,
+  };
+};

apps/server/src/db/index.ts

@@ -3,6 +3,7 @@ import type { NodePgQueryResultHKT } from "drizzle-orm/node-postgres";
 import { drizzle } from "drizzle-orm/node-postgres";
 import type { PgTransaction } from "drizzle-orm/pg-core";
 import { Pool } from "pg";
+import config from "../config";
 import * as schema from "./schema";
 
 declare global {
@@ -28,7 +29,10 @@ export const pool = new Pool(
       },
 );
 
-export const db = drizzle(pool, { schema });
+export const db = drizzle(pool, {
+  schema,
+  logger: config.DEBUG,
+});
 
 export type DatabaseType = typeof db;