diff --git a/Dockerfile b/Dockerfile index 344eb5766e..a2834beb58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,7 @@ RUN npm run build WORKDIR /dashboard COPY ./packages/dashboard /dashboard RUN npm run build +RUN npm run bundle:preload FROM node:${NODE_VERSION}-buster-slim as app @@ -38,6 +39,8 @@ COPY --from=builder /api/dist /api/dist WORKDIR /dashboard COPY --from=builder /dashboard/next.config.mjs ./ +COPY --from=builder /dashboard/migrations ./migrations +COPY --from=builder /dashboard/server-preload.js ./ COPY --from=builder /dashboard/public ./public COPY --from=builder /dashboard/package.json ./package.json COPY --from=builder --chown=node:node /dashboard/.next/standalone ./ diff --git a/Dockerfile.dev b/Dockerfile.dev index 99bb53eba8..f2a5b659c2 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -21,4 +21,7 @@ RUN npm install COPY ./packages/system-api /api COPY ./packages/dashboard /dashboard +WORKDIR /dashboard +RUN npm run bundle:preload + WORKDIR / diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 2b58fdfd50..9cf5f2f724 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -106,6 +106,7 @@ services: api: condition: service_started environment: + NODE_ENV: development INTERNAL_IP: ${INTERNAL_IP} TIPI_VERSION: ${TIPI_VERSION} JWT_SECRET: ${JWT_SECRET} @@ -118,6 +119,7 @@ services: APPS_REPO_URL: ${APPS_REPO_URL} DOMAIN: ${DOMAIN} ARCHITECTURE: ${ARCHITECTURE} + REDIS_HOST: ${REDIS_HOST} networks: - tipi_main_network volumes: diff --git a/docker-compose.rc.yml b/docker-compose.rc.yml index 2b26b0573b..f2e2846fca 100644 --- a/docker-compose.rc.yml +++ b/docker-compose.rc.yml @@ -95,7 +95,7 @@ services: dashboard: image: meienberger/runtipi:rc-${TIPI_VERSION} - command: /bin/sh -c "cd /dashboard && node server.js" + command: /bin/sh -c "cd /dashboard && npm run start" container_name: dashboard networks: - tipi_main_network @@ -118,6 +118,7 @@ services: APPS_REPO_URL: ${APPS_REPO_URL} DOMAIN: ${DOMAIN} ARCHITECTURE: ${ARCHITECTURE} + REDIS_HOST: ${REDIS_HOST} volumes: - ${PWD}/state:/runtipi/state - ${PWD}/logs:/app/logs diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 455a07bde1..4cc8156bfd 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -99,7 +99,7 @@ services: build: context: . dockerfile: Dockerfile - command: /bin/sh -c "cd /dashboard && node server.js" + command: /bin/sh -c "cd /dashboard && npm run start" restart: unless-stopped container_name: dashboard networks: @@ -123,6 +123,7 @@ services: APPS_REPO_URL: ${APPS_REPO_URL} DOMAIN: ${DOMAIN} ARCHITECTURE: ${ARCHITECTURE} + REDIS_HOST: ${REDIS_HOST} volumes: - ${PWD}/state:/runtipi/state - ${PWD}/logs:/app/logs diff --git a/docker-compose.yml b/docker-compose.yml index 9bc2673d5e..a2468547d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -95,7 +95,7 @@ services: dashboard: image: meienberger/runtipi:${TIPI_VERSION} - command: /bin/sh -c "cd /dashboard && node server.js" + command: /bin/sh -c "cd /dashboard && npm run start" restart: unless-stopped container_name: dashboard networks: @@ -119,6 +119,7 @@ services: APPS_REPO_URL: ${APPS_REPO_URL} DOMAIN: ${DOMAIN} ARCHITECTURE: ${ARCHITECTURE} + REDIS_HOST: ${REDIS_HOST} volumes: - ${PWD}/state:/runtipi/state - ${PWD}/logs:/app/logs diff --git a/package.json b/package.json index f94c7fdd0c..613310e4a3 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "start:dev": "./scripts/start-dev.sh", "start:rc": "docker-compose -f docker-compose.rc.yml --env-file .env up --build", "start:prod": "docker-compose -f docker-compose.test.yml --env-file .env up --build", - "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres", + "start:pg": "docker run --name test-db -p 5433:5432 -d --rm -e POSTGRES_PASSWORD=postgres postgres:14", "version": "echo $npm_package_version", "release:rc": "./scripts/deploy/release-rc.sh", "test:build": "docker buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 -t meienberger/runtipi:test .", diff --git a/packages/dashboard/.env.test b/packages/dashboard/.env.test index cd2fee17f8..bdfed48684 100644 --- a/packages/dashboard/.env.test +++ b/packages/dashboard/.env.test @@ -3,5 +3,3 @@ POSTGRES_DBNAME=postgres POSTGRES_USERNAME=postgres POSTGRES_PASSWORD=postgres POSTGRES_PORT=5433 - -DATABASE_URL="postgresql://postgres:postgres@localhost:5433/postgres" \ No newline at end of file diff --git a/packages/dashboard/.gitignore b/packages/dashboard/.gitignore index 87606880eb..5ca4c43a26 100644 --- a/packages/dashboard/.gitignore +++ b/packages/dashboard/.gitignore @@ -14,6 +14,8 @@ # production /build +/dist +server-preload.js # misc .DS_Store @@ -32,4 +34,4 @@ yarn-error.log* .vercel # typescript -*.tsbuildinfo \ No newline at end of file +*.tsbuildinfo diff --git a/packages/dashboard/esbuild.js b/packages/dashboard/esbuild.js new file mode 100644 index 0000000000..65523f2a7a --- /dev/null +++ b/packages/dashboard/esbuild.js @@ -0,0 +1,15 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +/* eslint-disable import/no-extraneous-dependencies */ +const esbuild = require('esbuild'); + +/* Bundle server preload */ +esbuild.build({ + entryPoints: ['./server-preload.ts'], + bundle: true, + allowOverwrite: true, + external: ['pg-native'], + platform: 'node', + target: 'node18', + outfile: 'server-preload.js', + logLevel: 'info', +}); diff --git a/packages/dashboard/migrations/00000-create-migrations-table.sql b/packages/dashboard/migrations/00000-create-migrations-table.sql new file mode 100644 index 0000000000..9a9d1c1ce6 --- /dev/null +++ b/packages/dashboard/migrations/00000-create-migrations-table.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS migrations ( + id integer PRIMARY KEY, + name varchar(100) UNIQUE NOT NULL, + hash varchar(40) NOT NULL, -- sha1 hex encoded hash of the file name and contents, to ensure it hasn't been altered since applying the migration + executed_at timestamp DEFAULT CURRENT_TIMESTAMP +); diff --git a/packages/dashboard/migrations/00001-initial.sql b/packages/dashboard/migrations/00001-initial.sql new file mode 100644 index 0000000000..27f69b33a5 --- /dev/null +++ b/packages/dashboard/migrations/00001-initial.sql @@ -0,0 +1,69 @@ +DO $$ +BEGIN + -- check if enum update_status_enum exists + IF NOT EXISTS ( + SELECT + 1 + FROM + pg_type + WHERE + typname = 'update_status_enum') THEN + -- create enum + CREATE TYPE "public"."update_status_enum" AS ENUM ( + 'FAILED', + 'SUCCESS' +); +END IF; + -- check if enum app_status_enum exists + IF NOT EXISTS ( + SELECT + 1 + FROM + pg_type + WHERE + typname = 'app_status_enum') THEN + -- create enum + CREATE TYPE "public"."app_status_enum" AS ENUM ( + 'running', + 'stopped', + 'installing', + 'uninstalling', + 'stopping', + 'starting', + 'missing' +); +END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS "update" ( + "id" serial NOT NULL, + "name" character varying NOT NULL, + "status" "public"."update_status_enum" NOT NULL, + "createdAt" timestamp NOT NULL DEFAULT now(), + "updatedAt" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "UQ_6e7d7ecccdc972caa0ad33cb014" UNIQUE ("name"), + CONSTRAINT "PK_575f77a0576d6293bc1cb752847" PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "user" ( + "id" serial NOT NULL, + "username" character varying NOT NULL, + "password" character varying NOT NULL, + "createdAt" timestamp NOT NULL DEFAULT now(), + "updatedAt" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "UQ_78a916df40e02a9deb1c4b75edb" UNIQUE ("username"), + CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id") +); + +CREATE TABLE IF NOT EXISTS "app" ( + "id" character varying NOT NULL, + "status" "public"."app_status_enum" NOT NULL DEFAULT 'stopped', + "lastOpened" timestamp with time zone DEFAULT now(), + "numOpened" integer NOT NULL DEFAULT '0', + "config" jsonb NOT NULL, + "createdAt" timestamp NOT NULL DEFAULT now(), + "updatedAt" timestamp NOT NULL DEFAULT now(), + CONSTRAINT "UQ_9478629fc093d229df09e560aea" UNIQUE ("id"), + CONSTRAINT "PK_9478629fc093d229df09e560aea" PRIMARY KEY ("id") +); diff --git a/packages/dashboard/migrations/00002-add-app-version.sql b/packages/dashboard/migrations/00002-add-app-version.sql new file mode 100644 index 0000000000..2ec2bed06d --- /dev/null +++ b/packages/dashboard/migrations/00002-add-app-version.sql @@ -0,0 +1,31 @@ +-- Create version field if it doesn't exist +ALTER TABLE "app" + ADD COLUMN IF NOT EXISTS "version" integer DEFAULT '0'; + +-- Set version field to 1 for all apps that have no version +UPDATE + "app" +SET + "version" = '1' +WHERE + "version" IS NULL; + +-- Set version field to not null +ALTER TABLE "app" + ALTER COLUMN "version" SET NOT NULL; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT + * + FROM + information_schema.table_constraints + WHERE + constraint_name = 'UQ_9478629fc093d229df09e560aea' + AND table_name = 'app') THEN + ALTER TABLE "app" + ADD CONSTRAINT "UQ_9478629fc093d229df09e560aea" UNIQUE ("id"); +END IF; +END +$$; diff --git a/packages/dashboard/migrations/00003-add-status-updating.sql b/packages/dashboard/migrations/00003-add-status-updating.sql new file mode 100644 index 0000000000..02ed9cd817 --- /dev/null +++ b/packages/dashboard/migrations/00003-add-status-updating.sql @@ -0,0 +1,15 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT + 1 + FROM + pg_enum + WHERE + enumlabel = 'updating'::text + AND enumtypid = 'public.app_status_enum'::regtype) THEN + ALTER TYPE "public"."app_status_enum" + ADD VALUE 'updating'; +END IF; +END +$$; diff --git a/packages/dashboard/migrations/00004-add-exposed-domain-fields.sql b/packages/dashboard/migrations/00004-add-exposed-domain-fields.sql new file mode 100644 index 0000000000..a56cd26e7d --- /dev/null +++ b/packages/dashboard/migrations/00004-add-exposed-domain-fields.sql @@ -0,0 +1,23 @@ +-- Create exposed field if it doesn't exist +ALTER TABLE "app" + ADD COLUMN IF NOT EXISTS "exposed" boolean DEFAULT FALSE; + +-- Select all apps that have not the exposed field and put it to false +UPDATE + "app" +SET + "exposed" = FALSE +WHERE + "exposed" IS NULL; + +-- Set exposed column to not null constraint +ALTER TABLE "app" + ALTER COLUMN "exposed" SET NOT NULL; + +-- Create domain column if it doesn't exist +ALTER TABLE "app" + ADD COLUMN IF NOT EXISTS "domain" character varying; + +-- Set default version to 1 +ALTER TABLE "app" + ALTER COLUMN "version" SET DEFAULT '1'; diff --git a/packages/dashboard/package.json b/packages/dashboard/package.json index 22ca4b6291..f0223bd36f 100644 --- a/packages/dashboard/package.json +++ b/packages/dashboard/package.json @@ -3,22 +3,24 @@ "version": "0.8.1", "private": true, "scripts": { - "migrate:postgres:test": "dotenv -e .env.test -- npx prisma migrate dev --name postgres-init", - "test": "npm run migrate:postgres:test && jest --colors", + "prisma:pull": "prisma db pull", + "test": "dotenv -e .env.test -- ts-node ./run-migration.ts && jest --colors", "test:client": "jest --colors --selectProjects client --", "test:server": "jest --colors --selectProjects server --", "postinstall": "prisma generate", - "dev": "next dev", + "dev": "node --require ./server-preload.js ./node_modules/.bin/next dev", "build": "next build", - "start": "next start", + "start": "NODE_ENV=production node --require ./server-preload.js server.js", "lint": "next lint", "lint:fix": "next lint --fix", - "gen": "graphql-codegen --config codegen.yml" + "gen": "graphql-codegen --config codegen.yml", + "bundle:preload": "node esbuild.js" }, "dependencies": { "@apollo/client": "^3.6.8", "@hookform/resolvers": "^2.9.10", "@prisma/client": "^4.8.0", + "@runtipi/postgres-migrations": "^5.3.0", "@tabler/core": "1.0.0-beta16", "@tabler/icons": "^1.109.0", "@tanstack/react-query": "^4.20.4", @@ -34,6 +36,7 @@ "isomorphic-fetch": "^3.0.0", "jsonwebtoken": "^9.0.0", "next": "13.1.1", + "pg": "^8.7.3", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.38.0", @@ -71,6 +74,7 @@ "@types/jest": "^29.2.4", "@types/jsonwebtoken": "^9.0.0", "@types/node": "18.11.18", + "@types/pg": "^8.6.5", "@types/react": "18.0.8", "@types/react-dom": "18.0.3", "@types/semver": "^7.3.12", @@ -80,6 +84,7 @@ "@typescript-eslint/eslint-plugin": "^5.47.1", "@typescript-eslint/parser": "^5.47.1", "dotenv-cli": "^6.0.0", + "esbuild": "^0.16.6", "eslint": "8.30.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^17.0.0", diff --git a/packages/dashboard/prisma/migrations/20221226205928_postgres_init/migration.sql b/packages/dashboard/prisma/migrations/20221226205928_postgres_init/migration.sql deleted file mode 100644 index 3f41d10827..0000000000 --- a/packages/dashboard/prisma/migrations/20221226205928_postgres_init/migration.sql +++ /dev/null @@ -1,61 +0,0 @@ --- CreateEnum -CREATE TYPE "app_status_enum" AS ENUM ('running', 'stopped', 'installing', 'uninstalling', 'stopping', 'starting', 'missing', 'updating'); - --- CreateEnum -CREATE TYPE "update_status_enum" AS ENUM ('FAILED', 'SUCCESS'); - --- CreateTable -CREATE TABLE "app" ( - "id" VARCHAR NOT NULL, - "status" "app_status_enum" NOT NULL DEFAULT 'stopped', - "lastOpened" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP, - "numOpened" INTEGER NOT NULL DEFAULT 0, - "config" JSONB NOT NULL, - "createdAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "version" INTEGER NOT NULL DEFAULT 1, - "exposed" BOOLEAN NOT NULL DEFAULT false, - "domain" VARCHAR, - - CONSTRAINT "PK_9478629fc093d229df09e560aea" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "migrations" ( - "id" SERIAL NOT NULL, - "timestamp" BIGINT NOT NULL, - "name" VARCHAR NOT NULL, - - CONSTRAINT "PK_8c82d7f526340ab734260ea46be" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "update" ( - "id" SERIAL NOT NULL, - "name" VARCHAR NOT NULL, - "status" "update_status_enum" NOT NULL, - "createdAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "PK_575f77a0576d6293bc1cb752847" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "user" ( - "id" SERIAL NOT NULL, - "username" VARCHAR NOT NULL, - "password" VARCHAR NOT NULL, - "createdAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP, - - CONSTRAINT "PK_cace4a159ff9f2512dd42373760" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "UQ_9478629fc093d229df09e560aea" ON "app"("id"); - --- CreateIndex -CREATE UNIQUE INDEX "UQ_6e7d7ecccdc972caa0ad33cb014" ON "update"("name"); - --- CreateIndex -CREATE UNIQUE INDEX "UQ_78a916df40e02a9deb1c4b75edb" ON "user"("username"); diff --git a/packages/dashboard/prisma/migrations/migration_lock.toml b/packages/dashboard/prisma/migrations/migration_lock.toml deleted file mode 100644 index fbffa92c2b..0000000000 --- a/packages/dashboard/prisma/migrations/migration_lock.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Please do not edit this file manually -# It should be added in your version-control system (i.e. Git) -provider = "postgresql" \ No newline at end of file diff --git a/packages/dashboard/run-migration.ts b/packages/dashboard/run-migration.ts new file mode 100644 index 0000000000..b3fa49e491 --- /dev/null +++ b/packages/dashboard/run-migration.ts @@ -0,0 +1,48 @@ +/* eslint-disable no-console */ +import path from 'path'; +import pg from 'pg'; +import { migrate } from '@runtipi/postgres-migrations'; + +export const runPostgresMigrations = async () => { + console.log('Starting database migration'); + + const { POSTGRES_HOST, POSTGRES_DBNAME, POSTGRES_USERNAME, POSTGRES_PASSWORD, POSTGRES_PORT = 5432 } = process.env; + + console.log('Connecting to database', POSTGRES_DBNAME, 'on', POSTGRES_HOST, 'as', POSTGRES_USERNAME, 'on port', POSTGRES_PORT); + + const client = new pg.Client({ + user: POSTGRES_USERNAME, + host: POSTGRES_HOST, + database: POSTGRES_DBNAME, + password: POSTGRES_PASSWORD, + port: Number(POSTGRES_PORT), + }); + await client.connect(); + + console.log('Client connected'); + + try { + const { rows } = await client.query('SELECT * FROM migrations'); + // if rows contains a migration with name 'Initial1657299198975' (legacy typeorm) delete table migrations. As all migrations are idempotent we can safely delete the table and start over. + if (rows.find((row) => row.name === 'Initial1657299198975')) { + console.log('Found legacy migration. Deleting table migrations'); + await client.query('DROP TABLE migrations'); + } + } catch (e) { + console.log('Migrations table not found, creating it'); + } + + console.log('Running migrations'); + try { + await migrate({ client }, path.join(__dirname, 'migrations'), { skipCreateMigrationTable: true }); + } catch (e) { + console.log('Error running migrations. Dropping table migrations and trying again'); + await client.query('DROP TABLE migrations'); + await migrate({ client }, path.join(__dirname, 'migrations'), { skipCreateMigrationTable: true }); + } + + console.log('Migration complete'); + await client.end(); +}; + +runPostgresMigrations(); diff --git a/packages/dashboard/server-preload.ts b/packages/dashboard/server-preload.ts new file mode 100644 index 0000000000..add3c25528 --- /dev/null +++ b/packages/dashboard/server-preload.ts @@ -0,0 +1,21 @@ +import { runPostgresMigrations } from './run-migration'; +import { EventDispatcher, EventTypes } from './src/server/core/EventDispatcher'; +import { getConfig, setConfig } from './src/server/core/TipiConfig'; + +const main = async () => { + // Run database migrations + await runPostgresMigrations(); + + // Update app store repository + await EventDispatcher.dispatchEventAsync(EventTypes.CLONE_REPO, [getConfig().appsRepoUrl]); + await EventDispatcher.dispatchEventAsync(EventTypes.UPDATE_REPO, [getConfig().appsRepoUrl]); + + // startJobs(); + setConfig('status', 'RUNNING'); + + // Start apps + // appsService.startAllApps(); + console.info(`Config: ${JSON.stringify(getConfig(), null, 2)}`); +}; + +main(); diff --git a/packages/dashboard/src/server/core/TipiConfig/TipiConfig.ts b/packages/dashboard/src/server/core/TipiConfig/TipiConfig.ts index 7121fc2f30..85a3342e35 100644 --- a/packages/dashboard/src/server/core/TipiConfig/TipiConfig.ts +++ b/packages/dashboard/src/server/core/TipiConfig/TipiConfig.ts @@ -10,7 +10,6 @@ enum AppSupportedArchitecturesEnum { AMD64 = 'amd64', } -const { serverRuntimeConfig } = nextConfig(); const { NODE_ENV, JWT_SECRET, @@ -27,7 +26,8 @@ const { POSTGRES_USERNAME, POSTGRES_PASSWORD, POSTGRES_PORT = 5432, -} = serverRuntimeConfig; +} = nextConfig()?.serverRuntimeConfig || process.env; +// Use process.env if nextConfig is not available (e.g. in in server-preload.ts) const configSchema = z.object({ NODE_ENV: z.union([z.literal('development'), z.literal('production'), z.literal('test')]), diff --git a/packages/dashboard/src/server/db/client.ts b/packages/dashboard/src/server/db/client.ts index c5949ec70c..c7296ed243 100644 --- a/packages/dashboard/src/server/db/client.ts +++ b/packages/dashboard/src/server/db/client.ts @@ -11,11 +11,10 @@ declare global { export const prisma = global.prisma || new PrismaClient({ - log: getConfig().NODE_ENV === 'development' ? ['query', 'error', 'warn'] : ['error'], + log: getConfig().NODE_ENV === 'development' ? ['query', 'warn', 'error'] : ['error'], datasources: { db: { - url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}`, - // url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}`, + url: `postgresql://${getConfig().postgresUsername}:${getConfig().postgresPassword}@${getConfig().postgresHost}:${getConfig().postgresPort}/${getConfig().postgresDatabase}?connect_timeout=300`, }, }, }); diff --git a/packages/dashboard/tsconfig.json b/packages/dashboard/tsconfig.json index 8d0b6565c2..1a40550500 100644 --- a/packages/dashboard/tsconfig.json +++ b/packages/dashboard/tsconfig.json @@ -12,7 +12,7 @@ "forceConsistentCasingInFileNames": true, "noEmit": true, "esModuleInterop": true, - "module": "esnext", + "module": "CommonJS", "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, diff --git a/packages/system-api/esbuild.js b/packages/system-api/esbuild.js index cf08519dca..a1591626e6 100644 --- a/packages/system-api/esbuild.js +++ b/packages/system-api/esbuild.js @@ -68,18 +68,3 @@ esbuild.build({ }); const glob = require('glob'); - -/* Migrations */ -const migrationFiles = glob.sync('./src/config/migrations/*.ts'); - -esbuild.buildSync({ - entryPoints: migrationFiles, - platform: 'node', - target: 'node18', - minify: false, - outdir: 'dist/config/migrations', - logLevel: 'info', - format: 'cjs', - minifySyntax: true, - minifyWhitespace: true, -}); diff --git a/packages/system-api/src/server.ts b/packages/system-api/src/server.ts index 3ece0ccf63..19a500e81b 100644 --- a/packages/system-api/src/server.ts +++ b/packages/system-api/src/server.ts @@ -14,7 +14,6 @@ import { __prod__ } from './config/constants/constants'; import datasource from './config/datasource'; import appsService from './modules/apps/apps.service'; import { runUpdates } from './core/updates/run'; -import recover from './core/updates/recover-migrations'; import startJobs from './core/jobs/jobs'; import { applyJsonConfig, getConfig, setConfig } from './core/config/TipiConfig'; import { eventDispatcher, EventTypes } from './core/config/EventDispatcher'; @@ -71,14 +70,6 @@ const main = async () => { await apolloServer.start(); apolloServer.applyMiddleware({ app }); - try { - await datasource.runMigrations(); - } catch (e) { - logger.error(e); - await recover(datasource); - } - - // Run migrations await runUpdates(); httpServer.listen(port, async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 17c5d5ae8d..a6805fc417 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,7 @@ importers: '@graphql-codegen/typescript-react-apollo': ^3.2.16 '@hookform/resolvers': ^2.9.10 '@prisma/client': ^4.8.0 + '@runtipi/postgres-migrations': ^5.3.0 '@tabler/core': 1.0.0-beta16 '@tabler/icons': ^1.109.0 '@tanstack/react-query': ^4.20.4 @@ -43,6 +44,7 @@ importers: '@types/jest': ^29.2.4 '@types/jsonwebtoken': ^9.0.0 '@types/node': 18.11.18 + '@types/pg': ^8.6.5 '@types/react': 18.0.8 '@types/react-dom': 18.0.3 '@types/semver': ^7.3.12 @@ -54,6 +56,7 @@ importers: argon2: ^0.29.1 clsx: ^1.1.1 dotenv-cli: ^6.0.0 + esbuild: ^0.16.6 eslint: 8.30.0 eslint-config-airbnb: ^19.0.4 eslint-config-airbnb-typescript: ^17.0.0 @@ -73,6 +76,7 @@ importers: msw: ^0.49.2 next: 13.1.1 next-router-mock: ^0.8.0 + pg: ^8.7.3 prisma: ^4.8.0 react: 18.2.0 react-dom: 18.2.0 @@ -102,6 +106,7 @@ importers: '@apollo/client': 3.6.8_o264z5epwuajru7y4dsijkqr44 '@hookform/resolvers': 2.9.10_react-hook-form@7.40.0 '@prisma/client': 4.8.0_prisma@4.8.0 + '@runtipi/postgres-migrations': 5.3.0 '@tabler/core': 1.0.0-beta16_biqbaboplfbrettd7655fr4n2y '@tabler/icons': 1.116.1_biqbaboplfbrettd7655fr4n2y '@tanstack/react-query': 4.20.4_biqbaboplfbrettd7655fr4n2y @@ -117,6 +122,7 @@ importers: isomorphic-fetch: 3.0.0 jsonwebtoken: 9.0.0 next: 13.1.1_7nrowiyds4jpk2wpzkb7237oey + pg: 8.7.3 react: 18.2.0 react-dom: 18.2.0_react@18.2.0 react-hook-form: 7.40.0_react@18.2.0 @@ -153,6 +159,7 @@ importers: '@types/jest': 29.2.4 '@types/jsonwebtoken': 9.0.0 '@types/node': 18.11.18 + '@types/pg': 8.6.5 '@types/react': 18.0.8 '@types/react-dom': 18.0.3 '@types/semver': 7.3.12 @@ -162,6 +169,7 @@ importers: '@typescript-eslint/eslint-plugin': 5.47.1_txmweb6yn7coi7nfrp22gpyqmy '@typescript-eslint/parser': 5.47.1_lzzuuodtsqwxnvqeq4g4likcqa dotenv-cli: 6.0.0 + esbuild: 0.16.8 eslint: 8.30.0 eslint-config-airbnb: 19.0.4_j3uyvjk2vb2gkfzhvqukeu5rlq eslint-config-airbnb-typescript: 17.0.0_qipeoi3mvzxgzndpeo4r6kwevy @@ -176,7 +184,7 @@ importers: msw: 0.49.2_typescript@4.9.4 next-router-mock: 0.8.0_next@13.1.1+react@18.2.0 prisma: 4.8.0 - ts-jest: 29.0.3_fckurgq3dkz2wyxkuqw26iqkyi + ts-jest: 29.0.3_iyz3vhhlowkpp2xbqliblzwv3y ts-node: 10.9.1_awa2wsr5thmg3i7jqycphctjfq typescript: 4.9.4 whatwg-fetch: 3.6.2 @@ -3150,6 +3158,17 @@ packages: '@redis/client': 1.3.1 dev: false + /@runtipi/postgres-migrations/5.3.0: + resolution: {integrity: sha512-pTAx/8j843L4n9f4TOCRh6eGFQD827jY64EVy5luHZNOfaiX1KI6SaWpzMfNPdAwy1od0k5FZrDJjpyHXC0ppg==} + engines: {node: '>10.17.0'} + hasBin: true + dependencies: + pg: 8.7.3 + sql-template-strings: 2.2.2 + transitivePeerDependencies: + - pg-native + dev: false + /@rushstack/eslint-patch/1.2.0: resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==} dev: true @@ -3691,7 +3710,7 @@ packages: /@types/pg/8.6.5: resolution: {integrity: sha512-tOkGtAqRVkHa/PVZicq67zuujI4Oorfglsr2IbKofDwBSysnaqSx7W1mDqFqdkGE6Fbgh+PZAl0r/BWON/mozw==} dependencies: - '@types/node': 17.0.31 + '@types/node': 18.11.18 pg-protocol: 1.5.0 pg-types: 2.2.0 dev: true @@ -7554,7 +7573,7 @@ packages: dev: true /has-flag/3.0.0: - resolution: {integrity: sha1-tdRU3CGZriJWmfNGfloH87lVuv0=} + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} /has-flag/4.0.0: @@ -7708,7 +7727,7 @@ packages: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} /ignore-by-default/1.0.1: - resolution: {integrity: sha1-SMptcvbGo68Aqa1K5odr44ieKwk=} + resolution: {integrity: sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==} dev: true /ignore/5.2.0: @@ -10506,7 +10525,7 @@ packages: dev: true /nopt/1.0.10: - resolution: {integrity: sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=} + resolution: {integrity: sha512-NWmpvLSqUrgrAC9HCuxEvb+PSloHpqVu+FqcO4eeF2h5qYRhA7ev6KvelyQAKtegUbC6RypJnlEOhd8vloNKYg==} hasBin: true dependencies: abbrev: 1.1.1 @@ -12006,6 +12025,11 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true + /sql-template-strings/2.2.2: + resolution: {integrity: sha512-UXhXR2869FQaD+GMly8jAMCRZ94nU5KcrFetZfWEMd+LVVG6y0ExgHAhatEcKZ/wk8YcKPdi+hiD2wm75lq3/Q==} + engines: {node: '>=4.0.0'} + dev: false + /stack-trace/0.0.10: resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} dev: false @@ -12500,7 +12524,7 @@ packages: yargs-parser: 20.2.9 dev: true - /ts-jest/29.0.3_fckurgq3dkz2wyxkuqw26iqkyi: + /ts-jest/29.0.3_iyz3vhhlowkpp2xbqliblzwv3y: resolution: {integrity: sha512-Ibygvmuyq1qp/z3yTh9QTwVVAbFdDy/+4BtIQR2sp6baF2SJU/8CKK/hhnGIDY2L90Az2jIqTwZPnN2p+BweiQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} hasBin: true @@ -12523,6 +12547,7 @@ packages: dependencies: '@babel/core': 7.17.10 bs-logger: 0.2.6 + esbuild: 0.16.8 fast-json-stable-stringify: 2.1.0 jest: 29.3.1_zfha7dvnw4nti6zkbsmhmn6xo4 jest-util: 29.3.1 diff --git a/scripts/common.sh b/scripts/common.sh index f0b0f4b59f..84ba7b4b7a 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +ROOT_FOLDER="${PWD}" # Get field from json file function get_json_field() { local json_file="$1" diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 8aca3ef891..ce40878f80 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -8,6 +8,8 @@ fi source "${BASH_SOURCE%/*}/common.sh" +clean_logs + ### -------------------------------- ### General variables ### -------------------------------- @@ -26,6 +28,7 @@ POSTGRES_USERNAME=tipi POSTGRES_DBNAME=tipi POSTGRES_PORT=5432 POSTGRES_HOST=tipi-db +REDIS_HOST=tipi-redis TIPI_VERSION=$(get_json_field "${ROOT_FOLDER}/package.json" version) INTERNAL_IP=localhost storage_path="${ROOT_FOLDER}" @@ -109,6 +112,7 @@ for template in ${ENV_FILE}; do sed "${sed_args[@]}" "s//${POSTGRES_DBNAME}/g" "${template}" sed "${sed_args[@]}" "s//${POSTGRES_PORT}/g" "${template}" sed "${sed_args[@]}" "s//${POSTGRES_HOST}/g" "${template}" + sed "${sed_args[@]}" "s//${REDIS_HOST}/g" "${template}" done mv -f "$ENV_FILE" "$ROOT_FOLDER/.env.dev" diff --git a/scripts/start.sh b/scripts/start.sh index 1d91ce602a..be0d3d5179 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -54,6 +54,7 @@ storage_path="${ROOT_FOLDER}" STORAGE_PATH_ESCAPED="$(echo "${storage_path}" | sed 's/\//\\\//g')" NETWORK_INTERFACE="$(ip route | grep default | awk '{print $5}' | uniq)" NETWORK_INTERFACE_COUNT=$(echo "$NETWORK_INTERFACE" | wc -l) +REDIS_HOST=tipi-redis if [[ "$NETWORK_INTERFACE_COUNT" -eq 0 ]]; then echo "No network interface found!" @@ -233,6 +234,7 @@ for template in ${ENV_FILE}; do sed -i "s//${APPS_REPOSITORY_ESCAPED}/g" "${template}" sed -i "s//${DOMAIN}/g" "${template}" sed -i "s//${STORAGE_PATH_ESCAPED}/g" "${template}" + sed -i "s//${REDIS_HOST}/g" "${template}" done mv -f "$ENV_FILE" "$ROOT_FOLDER/.env" diff --git a/templates/env-sample b/templates/env-sample index 96b9588e64..58bfa69390 100644 --- a/templates/env-sample +++ b/templates/env-sample @@ -19,3 +19,4 @@ POSTGRES_DBNAME= POSTGRES_USERNAME= POSTGRES_PASSWORD= POSTGRES_PORT= +REDIS_HOST=