diff --git a/.github/workflows/global-api-actions.yml b/.github/workflows/global-api-develop.yml similarity index 97% rename from .github/workflows/global-api-actions.yml rename to .github/workflows/global-api-develop.yml index 4bd7b49d3..e5aea87b6 100644 --- a/.github/workflows/global-api-actions.yml +++ b/.github/workflows/global-api-develop.yml @@ -1,4 +1,4 @@ -name: Push Global API to GHCR and deploy to EKS +name: Push Global API to latest and deploy to EKS on: workflow_dispatch: diff --git a/.github/workflows/global-api-test.yml b/.github/workflows/global-api-test.yml new file mode 100644 index 000000000..09369e351 --- /dev/null +++ b/.github/workflows/global-api-test.yml @@ -0,0 +1,59 @@ +name: Push Global API to GHCR and deploy to EKS + +on: + workflow_dispatch: + + push: + paths: + - global-api/** + - k8s/test/cc-test-global-api-deploy.yml + - k8s/test/cc-test-global-api.yml + - k8s/test/cc-test-global-api-migrate.yml + - .github/workflows/global-api-test.yml + branches: ["main"] + +jobs: + + pushToGHCR: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pushing Global API to GHCR + env: + VERSION: ${{ github.sha }} + IMAGE: ghcr.io/open-earth-foundation/citycatalyst-global-api + run: | + docker build -t $IMAGE:$VERSION global-api + docker tag $IMAGE:$VERSION $IMAGE:main + docker push $IMAGE:$VERSION + docker push $IMAGE:main + + deployToEKS: + needs: pushToGHCR + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }} + EKS_DEV_NAME: ${{ secrets.EKS_DEV_NAME }} + steps: + - uses: actions/checkout@v4 + + - name: Creating kubeconfig file + run: aws eks update-kubeconfig --name ${{secrets.EKS_DEV_NAME}} --region us-east-1 + + - name: Testing connection to EKS + run: kubectl get pods -n default + + - name: Deploying service + run: | + kubectl create -f k8s/cc-test-global-api-migrate.yml -n default + kubectl apply -f k8s/cc-test-global-api-deploy.yml -n default + kubectl rollout restart deployment cc-test-global-api-deploy -n default diff --git a/.github/workflows/web-actions.yml b/.github/workflows/web-develop.yml similarity index 100% rename from .github/workflows/web-actions.yml rename to .github/workflows/web-develop.yml diff --git a/.github/workflows/web-test.yml b/.github/workflows/web-test.yml new file mode 100644 index 000000000..70638bf7c --- /dev/null +++ b/.github/workflows/web-test.yml @@ -0,0 +1,139 @@ +name: Push Web app to GHCR and deploy to EKS + +on: + workflow_dispatch: + push: + paths: + - app/** + - k8s/test/cc-test-migrate.yml + - k8s/test/cc-test-web-deploy.yml + - k8s/test/cc-test/web.yml + - .github/workflows/web-test.yml + branches: ["develop"] + pull_request: + +jobs: + runTests: + runs-on: ubuntu-latest + env: + NODE_ENV: test + NEXTAUTH_SECRET: "diTMz/XLX4edSmmfzwJtmzKjCJGRt81Gf0PdjO3IPs8=" + NEXTAUTH_URL: "http://localhost:3000" + defaults: + run: + working-directory: ./app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Set up database + run: | + docker run --name github_action_postgresql -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_PASSWORD="" postgres + sleep 10 + createuser -w -h localhost -p 5432 -U postgres citycatalyst + createdb -w -h localhost -p 5432 -U postgres citycatalyst -O citycatalyst + cp env.example .env + npm run db:migrate + + - name: Run NextJS build + run: npm run build + + # - name: Run Cypress tests + # run: npm run cy:test + + # ... Generate LCOV files or download it from a different job + - name: Run tests and generate coverage file + run: npm run ci:test + + # - name: Setup LCOV + # uses: hrishikesh-kadam/setup-lcov@v1 + # - name: Report code coverage + # uses: zgosalvez/github-actions-report-lcov@v3 + # with: + # coverage-files: ./app/lcov*.info + # minimum-coverage: 40 + # artifact-name: code-coverage-report + # github-token: ${{ secrets.GITHUB_TOKEN }} + # working-directory: ./app + # update-comment: true + + - name: Upload coverage reports to Codecov + continue-on-error: true + uses: codecov/codecov-action@v4.0.1 + with: + fail_ci_if_error: false + flags: unittests + name: citycatalyst-backend + token: ${{ secrets.CODECOV_TOKEN }} + slug: Open-Earth-Foundation/CityCatalyst + + - name: Shut down database + run: docker stop github_action_postgresql + + pushToGHCR: + needs: runTests + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop' + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pushing CC Web to GHCR + env: + VERSION: ${{ github.sha }} + IMAGE: ghcr.io/open-earth-foundation/citycatalyst + run: | + docker build -t $IMAGE:$VERSION app + docker tag $IMAGE:$VERSION $IMAGE:main + docker push $IMAGE:$VERSION + docker push $IMAGE:main + + deployToEKS: + needs: pushToGHCR + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }} + EKS_DEV_NAME: ${{ secrets.EKS_DEV_NAME }} + steps: + - uses: actions/checkout@v4 + + - name: Creating kubeconfig file + run: aws eks update-kubeconfig --name ${{secrets.EKS_DEV_NAME}} --region us-east-1 + + - name: Testing connection to EKS + run: kubectl get pods -n default + + - name: Deploying service + run: | + kubectl create -f k8s/test/cc-test-migrate.yml -n default + kubectl create -f k8s/test/cc-test-seed.yml -n default + kubectl apply -f k8s/test/cc-test-sync-catalogue.yml -n default + kubectl apply -f k8s/test/cc-test-web-deploy.yml -n default + kubectl set env deployment/cc-test-web-deploy SMTP_USER=${{secrets.SMTP_USER}} + kubectl set env deployment/cc-test-web-deploy SMTP_PASSWORD=${{secrets.SMTP_PASSWORD}} + kubectl set env deployment/cc-test-web-deploy NEXTAUTH_SECRET=${{secrets.NEXTAUTH_SECRET}} + kubectl set env deployment/cc-test-web-deploy RESET_TOKEN_SECRET=${{secrets.RESET_TOKEN_SECRET}} + kubectl set env deployment/cc-test-web-deploy VERIFICATION_TOKEN_SECRET=${{secrets.VERIFICATION_TOKEN_SECRET}} + kubectl set env deployment/cc-test-web-deploy CHAT_PROVIDER=openai + kubectl set env deployment/cc-test-web-deploy OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}} + kubectl set env deployment/cc-test-web-deploy HUGGINGFACE_API_KEY=${{secrets.HUGGINGFACE_API_KEY}} + kubectl set env deployment/cc-test-web-deploy "DEFAULT_ADMIN_EMAIL=${{secrets.DEFAULT_ADMIN_EMAIL}}" + kubectl set env deployment/cc-test-web-deploy "DEFAULT_ADMIN_PASSWORD=${{secrets.DEFAULT_ADMIN_PASSWORD}}" + kubectl create -f k8s/test/cc-test-create-admin.yml -n default + kubectl rollout restart deployment cc-test-web-deploy -n default diff --git a/k8s/test/cc-test-create-admin.yml b/k8s/test/cc-test-create-admin.yml new file mode 100644 index 000000000..7763e898e --- /dev/null +++ b/k8s/test/cc-test-create-admin.yml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: cc-test-create-admin- +spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-create-admin + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npm", "run", "create-admin"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-global-api-deploy.yml b/k8s/test/cc-test-global-api-deploy.yml index 779672b4d..1bafd997c 100644 --- a/k8s/test/cc-test-global-api-deploy.yml +++ b/k8s/test/cc-test-global-api-deploy.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: cc-test-global-api - image: ghcr.io/open-earth-foundation/citycatalyst-global-api:latest + image: ghcr.io/open-earth-foundation/citycatalyst-global-api:main # Set to Never for local imagePullPolicy: Always ports: diff --git a/k8s/test/cc-test-global-api-migrate.yml b/k8s/test/cc-test-global-api-migrate.yml index 3aca58920..fdb899c35 100644 --- a/k8s/test/cc-test-global-api-migrate.yml +++ b/k8s/test/cc-test-global-api-migrate.yml @@ -9,7 +9,7 @@ spec: restartPolicy: OnFailure containers: - name: cc-test-global-api - image: ghcr.io/open-earth-foundation/citycatalyst-global-api:latest + image: ghcr.io/open-earth-foundation/citycatalyst-global-api:main env: - name: ALEMBIC_URL value: "postgresql://ccglobaltest:ccglobaltest@cc-global-api-db/ccglobaltest" diff --git a/k8s/test/cc-test-migrate.yml b/k8s/test/cc-test-migrate.yml index fe45d433a..2ecd41a4d 100644 --- a/k8s/test/cc-test-migrate.yml +++ b/k8s/test/cc-test-migrate.yml @@ -9,7 +9,7 @@ spec: restartPolicy: OnFailure containers: - name: cc-test-migrate - image: ghcr.io/open-earth-foundation/citycatalyst:latest + image: ghcr.io/open-earth-foundation/citycatalyst:main imagePullPolicy: Always env: - name: NODE_ENV diff --git a/k8s/test/cc-test-seed.yml b/k8s/test/cc-test-seed.yml new file mode 100644 index 000000000..1f1f35418 --- /dev/null +++ b/k8s/test/cc-test-seed.yml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: cc-test-seed- +spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-seed + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npm", "run", "db:seed"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-sync-catalogue.yml b/k8s/test/cc-test-sync-catalogue.yml new file mode 100644 index 000000000..5dc1e9186 --- /dev/null +++ b/k8s/test/cc-test-sync-catalogue.yml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cc-test-sync-catalogue +spec: + # run every monday + schedule: 0 0 * * 1 + jobTemplate: + spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-sync-catalogue + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npx", "tsx", "sync-catalogue"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-web-deploy.yml b/k8s/test/cc-test-web-deploy.yml index f8d38926c..8308685e5 100644 --- a/k8s/test/cc-test-web-deploy.yml +++ b/k8s/test/cc-test-web-deploy.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: cc-test-web - image: ghcr.io/open-earth-foundation/citycatalyst:latest + image: ghcr.io/open-earth-foundation/citycatalyst:main # Set to Never for local imagePullPolicy: Always ports: