refactor: native postgres migrations

Dockerfile

@@ -24,6 +24,7 @@ RUN npm run build
 WORKDIR /dashboard
 COPY ./packages/dashboard /dashboard
 RUN npm run build
+RUN npm run bundle:preload
 
 FROM node:${NODE_VERSION}-buster-slim as app
 
@@ -38,6 +39,8 @@ COPY --from=builder /api/dist /api/dist
 
 WORKDIR /dashboard
 COPY --from=builder /dashboard/next.config.mjs ./
+COPY --from=builder /dashboard/migrations ./migrations
+COPY --from=builder /dashboard/server-preload.js ./
 COPY --from=builder /dashboard/public ./public
 COPY --from=builder /dashboard/package.json ./package.json
 COPY --from=builder --chown=node:node /dashboard/.next/standalone ./

Dockerfile.dev

@@ -21,4 +21,7 @@ RUN npm install
 COPY ./packages/system-api /api
 COPY ./packages/dashboard /dashboard
 
+WORKDIR /dashboard
+RUN npm run bundle:preload
+
 WORKDIR /

docker-compose.dev.yml

@@ -106,6 +106,7 @@ services:
       api:
         condition: service_started
     environment:
+      NODE_ENV: development
       INTERNAL_IP: ${INTERNAL_IP}
       TIPI_VERSION: ${TIPI_VERSION}
       JWT_SECRET: ${JWT_SECRET}
@@ -118,6 +119,7 @@ services:
       APPS_REPO_URL: ${APPS_REPO_URL}
       DOMAIN: ${DOMAIN}
       ARCHITECTURE: ${ARCHITECTURE}
+      REDIS_HOST: ${REDIS_HOST}
     networks:
       - tipi_main_network
     volumes:

docker-compose.rc.yml

@@ -95,7 +95,7 @@ services:
 
   dashboard:
     image: meienberger/runtipi:rc-${TIPI_VERSION}
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     container_name: dashboard
     networks:
       - tipi_main_network
@@ -118,6 +118,7 @@ services:
       APPS_REPO_URL: ${APPS_REPO_URL}
       DOMAIN: ${DOMAIN}
       ARCHITECTURE: ${ARCHITECTURE}
+      REDIS_HOST: ${REDIS_HOST}
     volumes:
       - ${PWD}/state:/runtipi/state
       - ${PWD}/logs:/app/logs

docker-compose.test.yml

@@ -99,7 +99,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     restart: unless-stopped
     container_name: dashboard
     networks:
@@ -123,6 +123,7 @@ services:
       APPS_REPO_URL: ${APPS_REPO_URL}
       DOMAIN: ${DOMAIN}
       ARCHITECTURE: ${ARCHITECTURE}
+      REDIS_HOST: ${REDIS_HOST}
     volumes:
       - ${PWD}/state:/runtipi/state
       - ${PWD}/logs:/app/logs

docker-compose.yml

@@ -95,7 +95,7 @@ services:
 
   dashboard:
     image: meienberger/runtipi:${TIPI_VERSION}
-    command: /bin/sh -c "cd /dashboard && node server.js"
+    command: /bin/sh -c "cd /dashboard && npm run start"
     restart: unless-stopped
     container_name: dashboard
     networks:
@@ -119,6 +119,7 @@ services:
       APPS_REPO_URL: ${APPS_REPO_URL}
       DOMAIN: ${DOMAIN}
       ARCHITECTURE: ${ARCHITECTURE}
+      REDIS_HOST: ${REDIS_HOST}
     volumes:
       - ${PWD}/state:/runtipi/state
       - ${PWD}/logs:/app/logs

package.json

@@ -9,7 +9,7 @@
     "start:dev": "./scripts/start-dev.sh",
     "start:rc": "docker-compose -f docker-compose.rc.yml --env-file .env up --build",
     "start:prod": "docker-compose -f docker-compose.test.yml --env-file .env up --build",
-    "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres",
+    "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres:14",
     "version": "echo $npm_package_version",
     "release:rc": "./scripts/deploy/release-rc.sh",
     "test:build": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t meienberger/runtipi:test .",

packages/dashboard/.env.test

@@ -3,5 +3,3 @@ POSTGRES_DBNAME=postgres
 POSTGRES_USERNAME=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_PORT=5433
-
-DATABASE_URL="postgresql://postgres:postgres@localhost:5433/postgres"

packages/dashboard/.gitignore

@@ -14,6 +14,8 @@
 
 # production
 /build
+/dist
+server-preload.js
 
 # misc
 .DS_Store
@@ -32,4 +34,4 @@ yarn-error.log*
 .vercel
 
 # typescript
-*.tsbuildinfo
+*.tsbuildinfo

packages/dashboard/esbuild.js

@@ -0,0 +1,15 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+/* eslint-disable import/no-extraneous-dependencies */
+const esbuild = require('esbuild');
+
+/* Bundle server preload */
+esbuild.build({
+  entryPoints: ['./server-preload.ts'],
+  bundle: true,
+  allowOverwrite: true,
+  external: ['pg-native'],
+  platform: 'node',
+  target: 'node18',
+  outfile: 'server-preload.js',
+  logLevel: 'info',
+});

packages/dashboard/migrations/00000-create-migrations-table.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS migrations (
+    id integer PRIMARY KEY,
+    name varchar(100) UNIQUE NOT NULL,
+    hash varchar(40) NOT NULL, -- sha1 hex encoded hash of the file name and contents, to ensure it hasn't been altered since applying the migration
+    executed_at timestamp DEFAULT CURRENT_TIMESTAMP
+);

packages/dashboard/migrations/00001-initial.sql

@@ -0,0 +1,69 @@
+DO $$
+BEGIN
+    -- check if enum update_status_enum exists
+    IF NOT EXISTS (
+        SELECT
+            1
+        FROM
+            pg_type
+        WHERE
+            typname = 'update_status_enum') THEN
+    -- create enum
+    CREATE TYPE "public"."update_status_enum" AS ENUM (
+        'FAILED',
+        'SUCCESS'
+);
+END IF;
+    -- check if enum app_status_enum exists
+    IF NOT EXISTS (
+        SELECT
+            1
+        FROM
+            pg_type
+        WHERE
+            typname = 'app_status_enum') THEN
+    -- create enum
+    CREATE TYPE "public"."app_status_enum" AS ENUM (
+        'running',
+        'stopped',
+        'installing',
+        'uninstalling',
+        'stopping',
+        'starting',
+        'missing'
+);
+END IF;
+END
+$$;
+
+CREATE TABLE IF NOT EXISTS "update" (
+    "id" serial NOT NULL,
+    "name" character varying NOT NULL,
+    "status" "public"."update_status_enum" NOT NULL,
+    "createdAt" timestamp NOT NULL DEFAULT now(),
+    "updatedAt" timestamp NOT NULL DEFAULT now(),
+    CONSTRAINT "UQ_6e7d7ecccdc972caa0ad33cb014" UNIQUE ("name"),
+    CONSTRAINT "PK_575f77a0576d6293bc1cb752847" PRIMARY KEY ("id")
+);
+
+CREATE TABLE IF NOT EXISTS "user" (
+    "id" serial NOT NULL,
+    "username" character varying NOT NULL,
+    "password" character varying NOT NULL,
+    "createdAt" timestamp NOT NULL DEFAULT now(),
+    "updatedAt" timestamp NOT NULL DEFAULT now(),
+    CONSTRAINT "UQ_78a916df40e02a9deb1c4b75edb" UNIQUE ("username"),
+    CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id")
+);
+
+CREATE TABLE IF NOT EXISTS "app" (
+    "id" character varying NOT NULL,
+    "status" "public"."app_status_enum" NOT NULL DEFAULT 'stopped',
+    "lastOpened" timestamp with time zone DEFAULT now(),
+    "numOpened" integer NOT NULL DEFAULT '0',
+    "config" jsonb NOT NULL,
+    "createdAt" timestamp NOT NULL DEFAULT now(),
+    "updatedAt" timestamp NOT NULL DEFAULT now(),
+    CONSTRAINT "UQ_9478629fc093d229df09e560aea" UNIQUE ("id"),
+    CONSTRAINT "PK_9478629fc093d229df09e560aea" PRIMARY KEY ("id")
+);

packages/dashboard/migrations/00002-add-app-version.sql

@@ -0,0 +1,31 @@
+-- Create version field if it doesn't exist
+ALTER TABLE "app"
+    ADD COLUMN IF NOT EXISTS "version" integer DEFAULT '0';
+
+-- Set version field to 1 for all apps that have no version
+UPDATE
+    "app"
+SET
+    "version" = '1'
+WHERE
+    "version" IS NULL;
+
+-- Set version field to not null
+ALTER TABLE "app"
+    ALTER COLUMN "version" SET NOT NULL;
+
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+            *
+        FROM
+            information_schema.table_constraints
+        WHERE
+            constraint_name = 'UQ_9478629fc093d229df09e560aea'
+            AND table_name = 'app') THEN
+    ALTER TABLE "app"
+        ADD CONSTRAINT "UQ_9478629fc093d229df09e560aea" UNIQUE ("id");
+END IF;
+END
+$$;

packages/dashboard/migrations/00003-add-status-updating.sql

@@ -0,0 +1,15 @@
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT
+            1
+        FROM
+            pg_enum
+        WHERE
+            enumlabel = 'updating'::text
+            AND enumtypid = 'public.app_status_enum'::regtype) THEN
+    ALTER TYPE "public"."app_status_enum"
+        ADD VALUE 'updating';
+END IF;
+END
+$$;

packages/dashboard/migrations/00004-add-exposed-domain-fields.sql

@@ -0,0 +1,23 @@
+-- Create exposed field if it doesn't exist
+ALTER TABLE "app"
+    ADD COLUMN IF NOT EXISTS "exposed" boolean DEFAULT FALSE;
+
+-- Select all apps that have not the exposed field and put it to false
+UPDATE
+    "app"
+SET
+    "exposed" = FALSE
+WHERE
+    "exposed" IS NULL;
+
+-- Set exposed column to not null constraint
+ALTER TABLE "app"
+    ALTER COLUMN "exposed" SET NOT NULL;
+
+-- Create domain column if it doesn't exist
+ALTER TABLE "app"
+    ADD COLUMN IF NOT EXISTS "domain" character varying;
+
+-- Set default version to 1
+ALTER TABLE "app"
+    ALTER COLUMN "version" SET DEFAULT '1';

packages/dashboard/package.json

@@ -3,22 +3,24 @@
   "version": "0.8.1",
   "private": true,
   "scripts": {
-    "migrate:postgres:test": "dotenv -e .env.test -- npx prisma migrate dev --name postgres-init",
-    "test": "npm run migrate:postgres:test && jest --colors",
+    "prisma:pull": "prisma db pull",
+    "test": "dotenv -e .env.test -- ts-node ./run-migration.ts && jest --colors",
     "test:client": "jest --colors --selectProjects client --",
     "test:server": "jest --colors --selectProjects server --",
     "postinstall": "prisma generate",
-    "dev": "next dev",
+    "dev": "node --require ./server-preload.js ./node_modules/.bin/next dev",
     "build": "next build",
-    "start": "next start",
+    "start": "NODE_ENV=production node --require ./server-preload.js server.js",
     "lint": "next lint",
     "lint:fix": "next lint --fix",
-    "gen": "graphql-codegen --config codegen.yml"
+    "gen": "graphql-codegen --config codegen.yml",
+    "bundle:preload": "node esbuild.js"
   },
   "dependencies": {
     "@apollo/client": "^3.6.8",
     "@hookform/resolvers": "^2.9.10",
     "@prisma/client": "^4.8.0",
+    "@runtipi/postgres-migrations": "^5.3.0",
     "@tabler/core": "1.0.0-beta16",
     "@tabler/icons": "^1.109.0",
     "@tanstack/react-query": "^4.20.4",
@@ -34,6 +36,7 @@
     "isomorphic-fetch": "^3.0.0",
     "jsonwebtoken": "^9.0.0",
     "next": "13.1.1",
+    "pg": "^8.7.3",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.38.0",
@@ -71,6 +74,7 @@
     "@types/jest": "^29.2.4",
     "@types/jsonwebtoken": "^9.0.0",
     "@types/node": "18.11.18",
+    "@types/pg": "^8.6.5",
     "@types/react": "18.0.8",
     "@types/react-dom": "18.0.3",
     "@types/semver": "^7.3.12",
@@ -80,6 +84,7 @@
     "@typescript-eslint/eslint-plugin": "^5.47.1",
     "@typescript-eslint/parser": "^5.47.1",
     "dotenv-cli": "^6.0.0",
+    "esbuild": "^0.16.6",
     "eslint": "8.30.0",
     "eslint-config-airbnb": "^19.0.4",
     "eslint-config-airbnb-typescript": "^17.0.0",