diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3b37d3a843..16317f7d0f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,6 +178,19 @@ jobs: --health-retries 5 ports: - 5432/tcp + postgres-seed: + image: postgres:${{ matrix.pg-version }} + env: + POSTGRES_DB: medplum_test + POSTGRES_USER: medplum + POSTGRES_PASSWORD: medplum + options: >- + --health-cmd pg_isready + --health-retries 5 + --health-interval 10s + --health-timeout 5s + ports: + - 5433/tcp:5432/tcp redis: image: redis:${{ matrix.redis-version }} options: >- @@ -225,6 +238,7 @@ jobs: env: POSTGRES_HOST: localhost POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_SEED_PORT: ${{ job.services.postgres.ports[5433] }} REDIS_PASSWORD_DISABLED_IN_TESTS: 1 - name: Upload code coverage if: ${{ matrix.node-version == 20 && matrix.pg-version == 14 && matrix.redis-version == 7 }} diff --git a/.gitignore b/.gitignore index 3e6907b917..7423c47f81 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ packages/react/build-storybook.log # Jest code coverage coverage/ +packages/server/coverage-seed/ # TypeScript incremental build tsconfig.tsbuildinfo diff --git a/docker-compose.seed.yml b/docker-compose.seed.yml new file mode 100644 index 0000000000..17ecc8af88 --- /dev/null +++ b/docker-compose.seed.yml @@ -0,0 +1,15 @@ +version: '3.7' +services: + postgres-seed: + image: postgres:12 + restart: always + environment: + - POSTGRES_USER=medplum + - POSTGRES_PASSWORD=medplum + + volumes: + - ./postgres/postgres.conf:/usr/local/etc/postgres/postgres.conf + - ./postgres/:/docker-entrypoint-initdb.d/ + command: postgres -c config_file=/usr/local/etc/postgres/postgres.conf + ports: + - '5433:5432' diff --git a/docker-compose.yml b/docker-compose.yml index 6b072b5578..7726992e47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,19 @@ services: command: postgres -c config_file=/usr/local/etc/postgres/postgres.conf ports: - '5432:5432' + postgres-seed: + image: postgres:12 + restart: always + environment: + - POSTGRES_USER=medplum + - POSTGRES_PASSWORD=medplum + + volumes: + - ./postgres/postgres.conf:/usr/local/etc/postgres/postgres.conf + - ./postgres/:/docker-entrypoint-initdb.d/ + command: postgres -c config_file=/usr/local/etc/postgres/postgres.conf + ports: + - '5433:5432' redis: image: redis:7 restart: always diff --git a/package-lock.json b/package-lock.json index b5e3cbf0ce..7ad4d653ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@types/node": "20.12.7", "babel-jest": "29.7.0", "babel-preset-vite": "1.1.3", + "concurrently": "8.2.2", "cross-env": "7.0.3", "danger": "11.3.1", "esbuild": "0.20.2", @@ -26164,6 +26165,92 @@ "typedarray": "^0.0.6" } }, + "node_modules/concurrently": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/concurrently/-/concurrently-8.2.2.tgz", + "integrity": "sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.2", + "date-fns": "^2.30.0", + "lodash": "^4.17.21", + "rxjs": "^7.8.1", + "shell-quote": "^1.8.1", + "spawn-command": "0.0.2", + "supports-color": "^8.1.1", + "tree-kill": "^1.2.2", + "yargs": "^17.7.2" + }, + "bin": { + "conc": "dist/bin/concurrently.js", + "concurrently": "dist/bin/concurrently.js" + }, + "engines": { + "node": "^14.13.0 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/open-cli-tools/concurrently?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/concurrently/node_modules/chalk/node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/concurrently/node_modules/date-fns": { + "version": "2.30.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", + "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.21.0" + }, + "engines": { + "node": ">=0.11" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/date-fns" + } + }, "node_modules/config-chain": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", @@ -50331,6 +50418,15 @@ "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", "dev": true }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dev": true, + "dependencies": { + "tslib": "^2.1.0" + } + }, "node_modules/sade": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/sade/-/sade-1.8.1.tgz", @@ -51752,6 +51848,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/spawn-command": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/spawn-command/-/spawn-command-0.0.2.tgz", + "integrity": "sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==", + "dev": true + }, "node_modules/spawn-please": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/spawn-please/-/spawn-please-2.0.2.tgz", diff --git a/package.json b/package.json index f186c49fcd..fc1b07a932 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@types/node": "20.12.7", "babel-jest": "29.7.0", "babel-preset-vite": "1.1.3", + "concurrently": "8.2.2", "cross-env": "7.0.3", "danger": "11.3.1", "esbuild": "0.20.2", diff --git a/packages/server/jest.config.json b/packages/server/jest.config.json deleted file mode 100644 index 8e2a0d7497..0000000000 --- a/packages/server/jest.config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "testEnvironment": "node", - "testTimeout": 600000, - "testSequencer": "/jest.sequencer.js", - "transform": { - "^.+\\.(js|jsx|ts|tsx)$": "babel-jest" - }, - "moduleFileExtensions": ["ts", "js", "json", "node"], - "testMatch": ["**/src/**/*.test.ts"], - "coverageDirectory": "coverage", - "coverageReporters": ["json", "text"], - "collectCoverageFrom": ["**/src/**/*", "!**/src/__mocks__/**/*.ts", "!**/src/migrations/**/*.ts"] -} diff --git a/packages/server/jest.config.ts b/packages/server/jest.config.ts new file mode 100644 index 0000000000..9c24d5533e --- /dev/null +++ b/packages/server/jest.config.ts @@ -0,0 +1,15 @@ +import type { Config } from 'jest'; + +export default { + testEnvironment: 'node', + testTimeout: 600000, + testSequencer: '/jest.sequencer.js', + transform: { + '^.+\\.(js|jsx|ts|tsx)$': 'babel-jest', + }, + testMatch: ['/src/**/*.test.ts'], + moduleFileExtensions: ['ts', 'js', 'json', 'node'], + coverageDirectory: 'coverage', + coverageReporters: ['json', 'text'], + collectCoverageFrom: ['**/src/**/*', '!**/src/__mocks__/**/*.ts', '!**/src/migrations/**/*.ts'], +} satisfies Config; diff --git a/packages/server/jest.seed.config.ts b/packages/server/jest.seed.config.ts new file mode 100644 index 0000000000..b428e8b3d5 --- /dev/null +++ b/packages/server/jest.seed.config.ts @@ -0,0 +1,8 @@ +import type { Config } from 'jest'; +import defaultConfig from './jest.config'; + +export default { + ...defaultConfig, + testMatch: ['/seed-tests/**/*.test.ts'], + collectCoverageFrom: ['/seed-tests/**/*'], +} satisfies Config; diff --git a/packages/server/package.json b/packages/server/package.json index cfdc3739d6..4abc16f616 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,9 @@ "clean": "rimraf dist", "dev": "ts-node-dev --poll --respawn --transpile-only --require ./src/otel/instrumentation.ts src/index.ts", "start": "node --require ./dist/otel/instrumentation.js dist/index.js", - "test": "jest" + "test:seed:serial": "jest seed-serial.test.ts --config jest.seed.config.ts --coverageDirectory \"/coverage-seed/serial\"", + "test:seed:parallel": "jest seed.test.ts --config jest.seed.config.ts --coverageDirectory \"/coverage-seed/parallel\"", + "test": "docker-compose -f ../../docker-compose.seed.yml up -d && npm run test:seed:parallel && jest" }, "dependencies": { "@aws-sdk/client-cloudwatch-logs": "3.554.0", diff --git a/packages/server/seed-tests/seed-serial.test.ts b/packages/server/seed-tests/seed-serial.test.ts new file mode 100644 index 0000000000..1e8500b6c2 --- /dev/null +++ b/packages/server/seed-tests/seed-serial.test.ts @@ -0,0 +1,42 @@ +import { Project } from '@medplum/fhirtypes'; +import { initAppServices, shutdownApp } from '../src/app'; +import { loadTestConfig } from '../src/config'; +import { getDatabasePool } from '../src/database'; +import { SelectQuery } from '../src/fhir/sql'; +import { seedDatabase } from '../src/seed'; +import { withTestContext } from '../src/test.setup'; + +describe('Seed', () => { + beforeAll(async () => { + console.log = jest.fn(); + + const config = await loadTestConfig(); + config.database.port = process.env['POSTGRES_SEED_PORT'] + ? Number.parseInt(process.env['POSTGRES_SEED_PORT'], 10) + : 5433; + return withTestContext(() => initAppServices(config)); + }); + + afterAll(async () => { + await shutdownApp(); + }); + + test('Seeder completes successfully -- serial version', async () => { + // First time, seeder should run + await seedDatabase({ parallel: false }); + + // Make sure the first project is a super admin + const rows = await new SelectQuery('Project') + .column('content') + .where('name', '=', 'Super Admin') + .execute(getDatabasePool()); + expect(rows.length).toBe(1); + + const project = JSON.parse(rows[0].content) as Project; + expect(project.superAdmin).toBe(true); + expect(project.strictMode).toBe(true); + + // Second time, seeder should silently ignore + await seedDatabase({ parallel: false }); + }, 240000); +}); diff --git a/packages/server/src/seed.test.ts b/packages/server/seed-tests/seed.test.ts similarity index 78% rename from packages/server/src/seed.test.ts rename to packages/server/seed-tests/seed.test.ts index 1f5d77ace9..129bd383d2 100644 --- a/packages/server/src/seed.test.ts +++ b/packages/server/seed-tests/seed.test.ts @@ -1,10 +1,10 @@ import { Project } from '@medplum/fhirtypes'; -import { initAppServices, shutdownApp } from './app'; -import { loadTestConfig } from './config'; -import { getDatabasePool } from './database'; -import { SelectQuery } from './fhir/sql'; -import { seedDatabase } from './seed'; -import { withTestContext } from './test.setup'; +import { initAppServices, shutdownApp } from '../src/app'; +import { loadTestConfig } from '../src/config'; +import { getDatabasePool } from '../src/database'; +import { SelectQuery } from '../src/fhir/sql'; +import { seedDatabase } from '../src/seed'; +import { withTestContext } from '../src/test.setup'; describe('Seed', () => { beforeAll(async () => { diff --git a/packages/server/src/config.ts b/packages/server/src/config.ts index dac56e7df0..76d5a43556 100644 --- a/packages/server/src/config.ts +++ b/packages/server/src/config.ts @@ -175,7 +175,7 @@ export async function loadTestConfig(): Promise { config.binaryStorage = 'file:' + mkdtempSync(join(tmpdir(), 'medplum-temp-storage')); config.allowedOrigins = undefined; config.database.host = process.env['POSTGRES_HOST'] ?? 'localhost'; - config.database.port = process.env['POSTGRES_PORT'] ? parseInt(process.env['POSTGRES_PORT'], 10) : 5432; + config.database.port = process.env['POSTGRES_PORT'] ? Number.parseInt(process.env['POSTGRES_PORT'], 10) : 5432; config.database.dbname = 'medplum_test'; config.redis.db = 7; // Select logical DB `7` so we don't collide with existing dev Redis cache. config.redis.password = process.env['REDIS_PASSWORD_DISABLED_IN_TESTS'] ? undefined : config.redis.password; diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index a5cb75c562..e53cd271e9 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -1,4 +1,4 @@ { "extends": "../../tsconfig.json", - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts", "seed-tests/seed-serial.test.ts", "seed-tests/seed.test.ts"] } diff --git a/packages/server/turbo.json b/packages/server/turbo.json new file mode 100644 index 0000000000..d00fb4125e --- /dev/null +++ b/packages/server/turbo.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://turborepo.org/schema.json", + "extends": ["//"], + "pipeline": { + "test:seed:serial": { + "dependsOn": ["build"], + "outputs": ["coverage/**"], + "inputs": ["src/**/*.tsx", "src/**/*.ts"] + }, + "test:seed:parallel": { + "dependsOn": ["build"], + "outputs": ["coverage/**"], + "inputs": ["src/**/*.tsx", "src/**/*.ts"] + } + } +} diff --git a/scripts/test.sh b/scripts/test.sh index ae8f0a7084..c338b04f6e 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -7,19 +7,29 @@ set -e set -x # Set node options -export NODE_OPTIONS='--max-old-space-size=5120' +export NODE_OPTIONS='--max-old-space-size=8192' # Clear old code coverage data rm -rf coverage +rm -rf coverage-seed mkdir -p coverage/packages mkdir -p coverage/combined +mkdir -p coverage-seed/serial +mkdir -p coverage-seed/parallel -# Seed the database +# Testing production path of seeding the database # This is a special "test" which runs all of the seed logic, such as setting up structure definitions # On a normal developer machine, this is run only rarely when setting up a new database -# This test must be run first, and cannot be run concurrently with other tests -time npx turbo run test --filter=./packages/server -- seed.test.ts --coverage -cp "packages/server/coverage/coverage-final.json" "coverage/packages/coverage-server-seed.json" +# We execute this in parallel with the main line of tests +{ + time npx turbo run test:seed:serial --filter=./packages/server -- --coverage + cp "packages/server/coverage-seed/serial/coverage-final.json" "coverage/packages/coverage-server-seed-serial.json" +} & + +# Seed the database before testing +# This is the parallel implementation so it's faster +time npx turbo run test:seed:parallel --filter=./packages/server -- --coverage +cp "packages/server/coverage-seed/parallel/coverage-final.json" "coverage/packages/coverage-server-seed-parallel.json" # Test # Run them separately because code coverage is resource intensive @@ -36,6 +46,7 @@ for dir in `ls examples`; do fi done +wait # Combine test coverage PACKAGES=(