diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 000000000..609350cf7 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,46 @@ +name: Test Coverage + +on: + pull_request_target: + types: + - closed + +jobs: + unit-tests: + if: github.event.pull_request.merged == true + timeout-minutes: 60 + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [20] + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v2 + name: Install pnpm + with: + version: 8 + run_install: false + - uses: actions/cache@v3 + name: restore/setup pnpm cache + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + + - name: Install dependencies + run: pnpm install + + - name: Test + run: pnpm test:coverage + + - name: Upload Coverage + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + fail_ci_if_error: false + files: ./coverage/lcov.info + flags: unittests + name: codecov-nectar + path_to_write_report: ./coverage/codecov_report.txt + verbose: true + diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 37569dfd3..8e0254964 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,19 +1,29 @@ name: Pull Request -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + branches: + - master + +# Cancel in-progress workflows when we update the branch +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: build-and-lint: timeout-minutes: 60 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - node-version: [ 18 ] + node-version: [20] steps: - uses: actions/checkout@v3 - uses: pnpm/action-setup@v2 name: Install pnpm - id: pnpm-install with: version: 8 run_install: false @@ -24,7 +34,6 @@ jobs: key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - name: Install dependencies - if: steps.pnpm-cache.outputs.cache-hit != 'true' run: pnpm install - name: Lint @@ -35,15 +44,14 @@ jobs: unit-tests: timeout-minutes: 60 - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - node-version: [ 18 ] + node-version: [20] steps: - uses: actions/checkout@v3 - uses: pnpm/action-setup@v2 name: Install pnpm - id: pnpm-install with: version: 8 run_install: false @@ -54,72 +62,82 @@ jobs: key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} - name: Install dependencies - if: steps.pnpm-cache.outputs.cache-hit != 'true' run: pnpm install - name: Test - run: pnpm test:coverage + run: pnpm test:no-coverage - - name: Upload Coverage - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - fail_ci_if_error: false - files: ./coverage/lcov.info - flags: unittests - name: codecov-nectar - path_to_write_report: ./coverage/codecov_report.txt - verbose: true - - e2e-tests: + integration-tests: + runs-on: ubuntu-22.04 timeout-minutes: 60 - runs-on: ubuntu-latest env: - CI: true - BASE_CANONICAL_URL: ${{ vars.BASE_CANONICAL_URL }} - API_HOST_CLIENT: ${{ vars.API_HOST_CLIENT }} - API_HOST_SERVER: ${{ vars.API_HOST_SERVER }} - COOKIE_SECRET: ${{ vars.COOKIE_SECRET }} - strategy: - matrix: - node-version: [ 18 ] + API_HOST_SERVER: https://devapi.adsabs.harvard.edu/v1 + API_HOST_CLIENT: https://devapi.adsabs.harvard.edu/v1 + SCIX_SESSION_COOKIE_NAME: test + COOKIE_SECRET: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + steps: - - uses: actions/checkout@v3 - - uses: pnpm/action-setup@v2 - name: Install pnpm - id: pnpm-install + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 with: - version: 8 - run_install: false - - uses: actions/cache@v3 - name: restore/setup pnpm cache + install: true + + - name: Cache Docker layers + uses: actions/cache@v3 with: - path: ~/.pnpm-store - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - - name: Install dependencies - if: steps.pnpm-cache.outputs.cache-hit != 'true' - run: pnpm install + - name: Create .env.local file + run: | + touch .env.local + echo "API_HOST_CLIENT=$API_HOST_CLIENT" >> .env.local + echo "API_HOST_SERVER=$API_HOST_SERVER" >> .env.local + echo "SCIX_SESSION_COOKIE_NAME=$SCIX_SESSION_COOKIE_NAME" >> .env.local + echo "COOKIE_SECRET=$COOKIE_SECRET" >> .env.local + + - name: Build e2e Service (from cache if possible) + run: | + USER_UID=$(id -u) USER_GID=$(id -g) DOCKER_BUILDKIT=1 docker buildx build \ + --cache-from type=local,src=/tmp/.buildx-cache \ + --cache-to type=local,dest=/tmp/.buildx-cache,mode=max \ + -t nectar-ci:e2e -f Dockerfile --target e2e . - - name: Install Playwright Browsers - run: pnpm exec playwright install --with-deps + - name: Create test-results directory + run: mkdir -p test-results && chmod 0777 test-results - - name: setup environment variables + - name: Run e2e tests run: | - touch .env.local - echo "CI=${{ env.CI }}" >> .env.local - echo "BASE_CANONICAL_URL=${{ env.BASE_CANONICAL_URL }}" >> .env.local - echo "API_HOST_CLIENT=${{ env.API_HOST_CLIENT }}" >> .env.local - echo "API_HOST_SERVER=${{ env.API_HOST_SERVER }}" >> .env.local - echo "COOKIE_SECRET=${{ env.COOKIE_SECRET }}" >> .env.local + USER_UID=$(id -u) USER_GID=$(id -g) docker compose \ + -f docker-compose.yml up --exit-code-from e2e e2e - - name: Run integration tests - run: pnpm integration + - name: Archive test results + if: failure() + run: | + zip -r test-results.zip test-results - - uses: actions/upload-artifact@v3 - if: always() + - name: Upload test results + if: failure() + uses: actions/upload-artifact@v2 with: - name: playwright-report - path: playwright-report/ - retention-days: 30 + name: test-results + path: test-results.zip + + - name: Get prod container logs + if: failure() + run: docker logs $(docker-compose ps -q prod) > prod-container-logs.txt + + - name: Upload prod container logs + if: failure() + uses: actions/upload-artifact@v2 + with: + name: prod-container-logs + path: prod-container-logs.txt + + - name: Shutdown Docker Compose + run: docker compose down diff --git a/.gitignore b/.gitignore index 681a8a03a..9fa7eb1c9 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,3 @@ storybook-static/ # Sentry Config File .sentryclirc - -# Sentry Config File -.sentryclirc diff --git a/Dockerfile b/Dockerfile index 452a5c2dc..64a6f290b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM node:20-bookworm AS base ARG USER_UID=1001 ARG USER_GID=1001 -ARG USERNAME=node +ARG USERNAME=nectar ENV PNPM_HOME=/pnpm ENV DIST_DIR="dist" ENV STANDALONE=1 @@ -22,26 +22,30 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && apt-get --no-install-recommends install -y libc6; + +RUN groupadd -g $USER_GID $USERNAME || true && \ + useradd -u $USER_UID -g $USER_GID -m -s /bin/bash $USERNAME || true && \ + usermod -u $USER_UID -g $USER_GID $USERNAME || true + WORKDIR /app -RUN groupmod --gid $USER_GID $USERNAME \ - && usermod --uid $USER_UID --gid $USER_GID $USERNAME \ - && chown -R $USER_UID:$USER_GID /app; +RUN chown -R $USER_UID:$USER_GID /app FROM base as dev COPY --link package.json pnpm-lock.yaml ./ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile --ignore-scripts --no-optional -USER $USERNAME COPY --link . ./ + +USER $USERNAME ENTRYPOINT ["pnpm", "run", "dev"] FROM base as unit USER root COPY --link package.json pnpm-lock.yaml ./ RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install vitest -USER $USERNAME COPY --link vitest-setup.ts vitest.config.js tsconfig.json ./ COPY --link src /app/src +USER $USERNAME ENTRYPOINT ["vitest"] FROM base AS build_prod @@ -56,13 +60,13 @@ RUN --mount=type=cache,id=nextjs,target=./dist/cache pnpm run build FROM base AS prod ENV NODE_ENV=production -USER $USERNAME COPY --link --from=build_prod /app/dist/standalone ./ COPY --link --from=build_prod /app/node_modules ./node_modules COPY --link --from=build_prod /app/dist/static ./dist/static COPY --link --from=build_prod /app/public ./public -COPY --link --from=build_prod --chown="$USERNAME":"$USERNAME" /app/dist/cache ./dist/cache +COPY --link --from=build_prod /app/dist/cache ./dist/cache +USER $USERNAME EXPOSE 8000 ENTRYPOINT ["node", "server.js"] @@ -73,11 +77,13 @@ RUN playwright install --with-deps FROM e2e-browsers as e2e ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 -USER $USERNAME COPY --link src ./src COPY --link e2e ./e2e -COPY --link playwright.config.ts ./ -COPY --link tsconfig.json ./ +COPY --link playwright.config.ts ./playwright.config.ts +COPY --link tsconfig.json ./tsconfig.json + +RUN chown -R $USER_UID:$USER_GID /app; +USER $USERNAME ENTRYPOINT ["playwright"] CMD ["test"] diff --git a/docker-compose.overrides.yml b/docker-compose.overrides.yml deleted file mode 100644 index 1755d5288..000000000 --- a/docker-compose.overrides.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - common: - build: - args: - USER_UID: 1000 - USER_GID: 1000 - user: 1000:1000 - env_file: .env.local diff --git a/docker-compose.yml b/docker-compose.yml index 751864176..63744c755 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,69 +2,89 @@ services: # Run build server dev: - extends: - file: docker-compose.overrides.yml - service: common env_file: .env.local ports: - '8000:8000' build: context: . target: dev + args: + USER_UID: ${USER_UID:-1000} + USER_GID: ${USER_GID:-1000} + user: ${USER_UID:-1000}:${USER_GID:-1000} volumes: - .:/app # Run production server prod: - extends: - file: docker-compose.overrides.yml - service: common + env_file: .env.local ports: - '8000:8000' tty: true build: context: . target: prod + args: + USER_UID: ${USER_UID:-1000} + USER_GID: ${USER_GID:-1000} + user: ${USER_UID:-1000}:${USER_GID:-1000} + networks: + - internal # Run unit tests unit: - extends: - file: docker-compose.overrides.yml - service: common + env_file: .env.local build: context: . target: unit + args: + USER_UID: ${USER_UID:-1000} + USER_GID: ${USER_GID:-1000} + user: ${USER_UID:-1000}:${USER_GID:-1000} tty: true volumes: - ./src:/app/src # Run e2e tests on production build e2e: - extends: - file: docker-compose.overrides.yml - service: common + env_file: .env.local + init: true + ipc: host build: context: . target: e2e + args: + USER_UID: ${USER_UID:-1000} + USER_GID: ${USER_GID:-1000} + user: ${USER_UID:-1000}:${USER_GID:-1000} tty: true - network_mode: host depends_on: - prod volumes: - ./playwright.config.ts:/app/playwright.config.ts - ./.playwright:/app/.playwright - ./e2e:/app/e2e + - ./test-results:/app/test-results + networks: + - internal # Runs UI web server e2e-ui: - extends: - file: docker-compose.overrides.yml - service: common + env_file: .env.local + init: true + ipc: host + cap_add: + - SYS_ADMIN build: context: . target: e2e + args: + USER_UID: ${USER_UID:-1000} + USER_GID: ${USER_GID:-1000} + user: ${USER_UID:-1000}:${USER_GID:-1000} tty: true - network_mode: host + ports: + - '3000:3000' depends_on: - prod volumes: @@ -72,3 +92,9 @@ services: - ./.playwright:/app/.playwright - ./playwright.config.ts:/app/playwright.config.ts command: [ "test", "--ui", "--ui-host=0.0.0.0", "--ui-port=3000" ] + networks: + - internal + +networks: + internal: + driver: bridge diff --git a/e2e/Navigation.e2e.ts b/e2e/Navigation.e2e.ts index efc991aab..ea893a9c9 100644 --- a/e2e/Navigation.e2e.ts +++ b/e2e/Navigation.e2e.ts @@ -1,8 +1,8 @@ import { expect, test } from '@playwright/test'; -import { changeAppMode, doSearch } from './helpers'; +import { changeAppMode } from './helpers'; test('Landing Pages', async ({ page }) => { - await page.goto('http://localhost:8000/'); + await page.goto('/'); await expect(page).toHaveURL(/\/$/); await expect(page).toHaveTitle(/NASA Science Explorer/); await expect(page).toHaveScreenshot('modern-form.png', { fullPage: true }); @@ -25,7 +25,7 @@ test('Landing Pages', async ({ page }) => { }); test('Feedback Pages', async ({ page }) => { - await page.goto('http://localhost:8000/'); + await page.goto('/'); // Missing/Incorrect Record await page.getByRole('button', { name: 'Feedback' }).click(); @@ -35,31 +35,17 @@ test('Feedback Pages', async ({ page }) => { await expect(page).toHaveScreenshot('feedback-missing-incorrect-record.png', { fullPage: true }); // Missing References - await page.goto('http://localhost:8000/feedback/missingreferences'); + await page.goto('/feedback/missingreferences'); await expect(page.getByText(/submit one or more citations currently missing/i)).toBeVisible(); await expect(page).toHaveScreenshot('feedback-missing-references.png', { fullPage: true }); // Associated Articles - await page.goto('http://localhost:8000/feedback/associatedarticles'); + await page.goto('/feedback/associatedarticles'); await expect(page.getByText(/associated references are connected with links/i)).toBeVisible(); await expect(page).toHaveScreenshot('feedback-associated-articles.png', { fullPage: true }); // General Feedback - await page.goto('http://localhost:8000/feedback/general'); + await page.goto('/feedback/general'); await expect(page.getByText(/you can also reach us at adshelp/i)).toBeVisible(); await expect(page).toHaveScreenshot('feedback-general.png', { fullPage: true }); }); - -test('Search Pages', async ({ page }) => { - // In order to prime the browser context, go to the homepage initially so we have a token - // TODO: this is a hack, we should be able to go directly to the search page - await page.goto('https://scixplorer.org'); - await page.goto('http://localhost:8000/'); - await doSearch(page, 'author:"Finkelstein, David"'); - await expect(page).toHaveURL(/\/search\?/); - await expect(page).toHaveTitle(/author:"Finkelstein, David"/); - await expect(page.getByText(/Your search returned \d{3,} results/)).toBeVisible(); - await expect(page).toHaveScreenshot('search-results.png', { fullPage: true }); - - // TODO: pagination is not working in the test -}); diff --git a/package.json b/package.json index f67fcc196..4374f4fd4 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "start:local": "NODE_ENV=production PORT=8000 DIST_DIR=./dist/ next start", "analyze": "NEXT_TELEMETRY_DISABLED=1 DIST_DIR=./dev/ ANALYZE=true pnpm build-bibstem-index && next build", "test": "NODE_ENV=test vitest", + "test:no-coverage": "NODE_ENV=test vitest --run", "test:coverage": "NODE_ENV=test vitest --run --coverage", "integration": "playwright test --config=./playwright.config.ts", "storybook": "storybook dev -p 8001 --no-open --disable-telemetry", @@ -22,11 +23,11 @@ "build-bibstem-index": "node ./scripts/gen-bibstem-index", "lint": "tsc --project ./tsconfig.json && eslint .", "prepare": "husky install", - "docker:unit": "docker compose -f docker-compose.yml run -it --rm unit", - "docker:integration": "docker compose -f docker-compose.yml up --build e2e", - "docker:integration:ui": "docker compose -f docker-compose.yml up --build e2e-ui", - "docker:dev": "docker compose -f docker-compose.yml up dev --build", - "docker:prod": "docker compose -f docker-compose.yml up prod --build" + "docker:unit": "USER_UID=$(id -u) USER_GID=$(id -g) docker compose -f docker-compose.yml run -it --rm unit", + "docker:integration": "USER_UID=$(id -u) USER_GID=$(id -g) docker compose -f docker-compose.yml up --build e2e", + "docker:integration:ui": "USER_UID=$(id -u) USER_GID=$(id -g) docker compose -f docker-compose.yml up --build e2e-ui", + "docker:dev": "USER_UID=$(id -u) USER_GID=$(id -g) docker compose -f docker-compose.yml up dev --build", + "docker:prod": "USER_UID=$(id -u) USER_GID=$(id -g) docker compose -f docker-compose.yml up prod --build" }, "dependencies": { "@chakra-ui/icons": "^2.1.0", diff --git a/playwright.config.ts b/playwright.config.ts index 9ab27a3a4..4e7066125 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -22,7 +22,7 @@ export default defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: 'http://localhost:8000', + baseURL: 'http://prod:8000', /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', @@ -58,23 +58,23 @@ export default defineConfig({ use: { ...devices['Desktop Safari'] }, }, ], - webServer: [ - { - env: { - BASE_CANONICAL_URL: process.env.BASE_CANONICAL_URL || 'https://ui.adsabs.harvard.edu', - API_HOST_CLIENT: process.env.API_HOST_CLIENT || 'https://devapi.adsabs.harvard.edu/v1', - API_HOST_SERVER: process.env.API_HOST_SERVER || 'https://devapi.adsabs.harvard.edu/v1', - COOKIE_SECRET: process.env.COOKIE_SECRET || 'secret_secret_secret_secret_secret', - ADS_SESSION_COOKIE_NAME: process.env.ADS_SESSION_COOKIE_NAME || 'ads_session', - SCIX_SESSION_COOKIE_NAME: process.env.SCIX_SESSION_COOKIE_NAME || 'scix_session', - }, - command: 'pnpm run dev:mocks', - // 5 minute timeout - timeout: 300000, - reuseExistingServer: !process.env.CI, - stdout: 'ignore', - stderr: 'pipe', - url: 'http://localhost:8000', - }, - ], + // webServer: [ + // { + // env: { + // BASE_CANONICAL_URL: process.env.BASE_CANONICAL_URL || 'https://ui.adsabs.harvard.edu', + // API_HOST_CLIENT: process.env.API_HOST_CLIENT || 'https://devapi.adsabs.harvard.edu/v1', + // API_HOST_SERVER: process.env.API_HOST_SERVER || 'https://devapi.adsabs.harvard.edu/v1', + // COOKIE_SECRET: process.env.COOKIE_SECRET || 'secret_secret_secret_secret_secret', + // ADS_SESSION_COOKIE_NAME: process.env.ADS_SESSION_COOKIE_NAME || 'ads_session', + // SCIX_SESSION_COOKIE_NAME: process.env.SCIX_SESSION_COOKIE_NAME || 'scix_session', + // }, + // command: 'pnpm run docker:prod', + // // 5 minute timeout + // timeout: 300000, + // reuseExistingServer: !process.env.CI, + // stdout: 'ignore', + // stderr: 'pipe', + // url: 'http://localhost:8000', + // }, + // ], });