Merge branch 'develop' into mendoza_EIE_integration

Open-Earth-Foundation · Mar 28, 2024 · b039d5e · b039d5e
2 parents 78c6571 + 2b37a5a
commit b039d5e
Show file tree

Hide file tree

Showing 99 changed files with 5,763 additions and 1,307 deletions.
diff --git a/.github/workflows/global-api-develop.yml b/.github/workflows/global-api-develop.yml
@@ -0,0 +1,107 @@
+name: Push Global API to latest and deploy to EKS
+
+on:
+  workflow_dispatch:
+
+  push:
+    paths:
+      - global-api/**
+      - k8s/cc-global-api-deploy.yml
+      - k8s/cc-global-api.yml
+      - k8s/cc-global-api-migrate.yml
+    branches: ["develop"]
+
+jobs:
+  runTests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./global-api
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest pytest-cov
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      
+      - name: Set up database
+        run: |
+          docker run --name github_action_postgresql -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_PASSWORD="" postgres
+          sleep 10
+          createuser -w -h localhost -p 5432 -U postgres citycatalyst
+          createdb -w -h localhost -p 5432 -U postgres citycatalyst -O citycatalyst
+          cp env.example .env
+
+      - name: Build coverage file
+        run: |
+          pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=app tests/ | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml
+
+      - name: Upload coverage reports to Codecov
+        continue-on-error: true
+        uses: codecov/codecov-action@v4.0.1
+        with:
+          fail_ci_if_error: false
+          flags: unittests
+          name: citycatalyst-global-api
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: Open-Earth-Foundation/CityCatalyst
+
+      - name: Shut down database
+        run: docker stop github_action_postgresql
+
+  pushToGHCR:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pushing Global API to GHCR
+        env:
+          VERSION: ${{ github.sha }}
+          IMAGE: ghcr.io/open-earth-foundation/citycatalyst-global-api
+        run: |
+          docker build -t $IMAGE:$VERSION global-api
+          docker tag $IMAGE:$VERSION $IMAGE:latest
+          docker push $IMAGE:$VERSION
+          docker push $IMAGE:latest
+
+  deployToEKS:
+    needs: pushToGHCR
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }}
+      EKS_DEV_NAME: ${{ secrets.EKS_DEV_NAME }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Creating kubeconfig file
+        run: aws eks update-kubeconfig --name ${{secrets.EKS_DEV_NAME}} --region us-east-1
+
+      - name: Testing connection to EKS
+        run: kubectl get pods -n default
+
+      - name: Deploying service
+        run: |
+          kubectl create -f k8s/cc-global-api-migrate.yml -n default
+          kubectl apply -f k8s/cc-global-api-deploy.yml -n default
+          kubectl rollout restart deployment cc-global-api-deploy -n default
diff --git a/.github/workflows/global-api-tag.yml b/.github/workflows/global-api-tag.yml
@@ -10,7 +10,7 @@ on:
       - k8s/cc-global-api.yml
       - k8s/cc-global-api-migrate.yml
     tags:
-      - 'v*'
+      - '^v\d+\.\d+\.\d+$'
 
 jobs:
 

diff --git a/.github/workflows/global-api-actions.yml → .github/workflows/global-api-test.yml b/.github/workflows/global-api-actions.yml → .github/workflows/global-api-test.yml
@@ -6,10 +6,11 @@ on:
   push:
     paths:
       - global-api/**
-      - k8s/cc-global-api-deploy.yml
-      - k8s/cc-global-api.yml
-      - k8s/cc-global-api-migrate.yml
-    branches: ["develop"]
+      - k8s/test/cc-test-global-api-deploy.yml
+      - k8s/test/cc-test-global-api.yml
+      - k8s/test/cc-test-global-api-migrate.yml
+      - .github/workflows/global-api-test.yml
+    branches: ["main"]
 
 jobs:
 
@@ -31,9 +32,9 @@ jobs:
           IMAGE: ghcr.io/open-earth-foundation/citycatalyst-global-api
         run: |
           docker build -t $IMAGE:$VERSION global-api
-          docker tag $IMAGE:$VERSION $IMAGE:latest
+          docker tag $IMAGE:$VERSION $IMAGE:main
           docker push $IMAGE:$VERSION
-          docker push $IMAGE:latest
+          docker push $IMAGE:main
 
   deployToEKS:
     needs: pushToGHCR
@@ -53,6 +54,6 @@ jobs:
 
       - name: Deploying service
         run: |
-          kubectl create -f k8s/cc-global-api-migrate.yml -n default
-          kubectl apply -f k8s/cc-global-api-deploy.yml -n default
-          kubectl rollout restart deployment cc-global-api-deploy -n default
+          kubectl create -f k8s/test/cc-test-global-api-migrate.yml -n default
+          kubectl apply -f k8s/test/cc-test-global-api-deploy.yml -n default
+          kubectl rollout restart deployment cc-test-global-api-deploy -n default
diff --git a/.github/workflows/web-actions.yml → .github/workflows/web-develop.yml b/.github/workflows/web-actions.yml → .github/workflows/web-develop.yml
@@ -53,21 +53,25 @@ jobs:
       - name: Run tests and generate coverage file
         run: npm run ci:test
 
-      - name: Setup LCOV
-        uses: hrishikesh-kadam/setup-lcov@v1
-      - name: Report code coverage
-        uses: zgosalvez/github-actions-report-lcov@v3
-        with:
-          coverage-files: ./app/lcov*.info
-          minimum-coverage: 40
-          artifact-name: code-coverage-report
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          working-directory: ./app
-          update-comment: true
+      # - name: Setup LCOV
+      #   uses: hrishikesh-kadam/setup-lcov@v1
+      # - name: Report code coverage
+      #   uses: zgosalvez/github-actions-report-lcov@v3
+      #   with:
+      #     coverage-files: ./app/lcov*.info
+      #     minimum-coverage: 40
+      #     artifact-name: code-coverage-report
+      #     github-token: ${{ secrets.GITHUB_TOKEN }}
+      #     working-directory: ./app
+      #     update-comment: true
 
       - name: Upload coverage reports to Codecov
+        continue-on-error: true
         uses: codecov/codecov-action@v4.0.1
         with:
+          fail_ci_if_error: false
+          flags: unittests
+          name: citycatalyst-backend
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: Open-Earth-Foundation/CityCatalyst
 
@@ -126,8 +130,11 @@ jobs:
           kubectl set env deployment/cc-web-deploy SMTP_PASSWORD=${{secrets.SMTP_PASSWORD}}
           kubectl set env deployment/cc-web-deploy NEXTAUTH_SECRET=${{secrets.NEXTAUTH_SECRET}}
           kubectl set env deployment/cc-web-deploy RESET_TOKEN_SECRET=${{secrets.RESET_TOKEN_SECRET}}
-          kubectl set env deployment/cc-web-deploy CHAT_PROVIDER=huggingface
+          kubectl set env deployment/cc-web-deploy VERIFICATION_TOKEN_SECRET=${{secrets.VERIFICATION_TOKEN_SECRET}}
+          kubectl set env deployment/cc-web-deploy CHAT_PROVIDER=openai
           kubectl set env deployment/cc-web-deploy OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}}
           kubectl set env deployment/cc-web-deploy HUGGINGFACE_API_KEY=${{secrets.HUGGINGFACE_API_KEY}}
+          kubectl set env deployment/cc-web-deploy "DEFAULT_ADMIN_EMAIL=${{secrets.DEFAULT_ADMIN_EMAIL}}"
+          kubectl set env deployment/cc-web-deploy "DEFAULT_ADMIN_PASSWORD=${{secrets.DEFAULT_ADMIN_PASSWORD}}"
+          kubectl create -f k8s/cc-create-admin.yml -n default
           kubectl rollout restart deployment cc-web-deploy -n default
-
diff --git a/.github/workflows/web-tag.yml b/.github/workflows/web-tag.yml
@@ -8,9 +8,9 @@ on:
       - k8s/cc-migrate.yml
       - k8s/cc-web-deploy.yml
       - k8s/cc-web.yml
-      - .github/workflows/web-actions.yml
+      - .github/workflows/web-tag.yml
     tags:
-    - 'v*'
+      - '^v\d+\.\d+\.\d+$'
 
 jobs:
   runTests:
@@ -28,7 +28,7 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-          cache: 'npm'
+          cache: "npm"
           cache-dependency-path: app/package-lock.json
 
       - name: Install dependencies
@@ -59,34 +59,33 @@ jobs:
     needs: runTests
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v4
 
-    - uses: actions/checkout@v4
-
-    - name: Log in to the Container registry
-      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-      with:
-        registry: ghcr.io
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Pushing citycatalyst to GHCR
-      env:
-        SHA: ${{ github.sha }}
-        REF: ${{ github.ref }}
-        IMAGE: ghcr.io/open-earth-foundation/citycatalyst
+      - name: Pushing citycatalyst to GHCR
+        env:
+          SHA: ${{ github.sha }}
+          REF: ${{ github.ref }}
+          IMAGE: ghcr.io/open-earth-foundation/citycatalyst
 
-      run: |
-        export VERSION=${REF#refs/tags/v}
-        export MAJOR=${VERSION%.*.*}
-        export MINOR=${VERSION%.*}
-        echo Version: ${VERSION} Major: ${MAJOR} Minor: ${MINOR}
-        docker build -t $IMAGE:$SHA app
-        docker tag $IMAGE:$SHA $IMAGE:$VERSION
-        docker tag $IMAGE:$SHA $IMAGE:$MAJOR
-        docker tag $IMAGE:$SHA $IMAGE:$MINOR
-        docker tag $IMAGE:$SHA $IMAGE:stable
-        docker push $IMAGE:$SHA
-        docker push $IMAGE:$VERSION
-        docker push $IMAGE:$MAJOR
-        docker push $IMAGE:$MINOR
-        docker push $IMAGE:stable
+        run: |
+          export VERSION=${REF#refs/tags/v}
+          export MAJOR=${VERSION%.*.*}
+          export MINOR=${VERSION%.*}
+          echo Version: ${VERSION} Major: ${MAJOR} Minor: ${MINOR}
+          docker build -t $IMAGE:$SHA app
+          docker tag $IMAGE:$SHA $IMAGE:$VERSION
+          docker tag $IMAGE:$SHA $IMAGE:$MAJOR
+          docker tag $IMAGE:$SHA $IMAGE:$MINOR
+          docker tag $IMAGE:$SHA $IMAGE:stable
+          docker push $IMAGE:$SHA
+          docker push $IMAGE:$VERSION
+          docker push $IMAGE:$MAJOR
+          docker push $IMAGE:$MINOR
+          docker push $IMAGE:stable