ci: Cache pip dependencies

edgarrmondragon · Dec 6, 2023 · 225f7b6 · 225f7b6
1 parent 74e7fc2
commit 225f7b6
Show file tree

Hide file tree

Showing 6 changed files with 84 additions and 55 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,15 +1,16 @@
 version: 2
 updates:
-  # - package-ecosystem: pip
-  #   directory: "/.github/workflows"
-  #   schedule:
-  #     interval: daily
-  #     time: "11:35"
-  #     timezone: "America/Mexico_City"
-  #   reviewers:
-  #     - "edgarrmondragon"
-  #   labels:
-  #     - "dependencies"
+  - package-ecosystem: pip
+    directory: "/.github/workflows"
+    schedule:
+      interval: weekly
+      time: "11:35"
+      timezone: "America/Mexico_City"
+    reviewers:
+      - "edgarrmondragon"
+    labels:
+      - "dependencies"
+      - "Test/ELTP"
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
@@ -21,7 +22,7 @@ updates:
   - package-ecosystem: "npm"
     directory: "/evidence"
     schedule:
-      interval: daily
+      interval: weekly
       time: "11:35"
       timezone: "America/Mexico_City"
     reviewers:

diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt
@@ -0,0 +1,2 @@
+pip==23.3.1
+meltano==3.2.0
diff --git a/.github/workflows/contraints.txt b/.github/workflows/contraints.txt
diff --git a/.github/workflows/melty.yaml b/.github/workflows/melty.yaml
@@ -34,18 +34,13 @@ jobs:
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'  || contains(github.event.pull_request.labels.*.name, 'Test/ELTP') }}
     name: "Meltano Job: ${{ matrix.job }}"
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/meltano/meltano:latest-python3.11
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_IO_TOKEN }}
-      env:
-        TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }}
-        TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }}
-        TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }}
-        TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }}
-        TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }}
-        TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }}
+    env:
+      TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }}
+      TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }}
+      TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }}
+      TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }}
+      TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }}
+      TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }}
     strategy:
       matrix:
         include:
@@ -63,8 +58,19 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
 
-      - name: Check Meltano
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            meltano.yml
+            .github/workflows/constraints.txt
+
+      - name: Install Meltano
+        env:
+          PIP_CONSTRAINT: .github/workflows/constraints.txt
         run: |
+          pipx install meltano[postgres]
           meltano --version
 
       - name: Install Plugins
@@ -91,20 +97,26 @@ jobs:
     name: Lint models
     runs-on: ubuntu-latest
     continue-on-error: true
-    container:
-      image: ghcr.io/meltano/meltano:latest-python3.11
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_IO_TOKEN }}
-      env:
-        DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }}
+    env:
+      DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }}
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
 
-      - name: Check Meltano
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            meltano.yml
+            .github/workflows/constraints.txt
+
+      - name: Install Meltano
+        env:
+          PIP_CONSTRAINT: .github/workflows/constraints.txt
         run: |
+          pipx install meltano[postgres]
           meltano --version
 
       - name: Install SQLFluff
@@ -121,18 +133,23 @@ jobs:
     name: Data Transformation with dbt-duckdb
     runs-on: ubuntu-latest
     needs: [meltano-run, lint]
-    container:
-      image: ghcr.io/meltano/meltano:latest-python3.11
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_IO_TOKEN }}
-
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
 
-      - name: Check Meltano
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            meltano.yml
+            .github/workflows/constraints.txt
+
+      - name: Install Meltano
+        env:
+          PIP_CONSTRAINT: .github/workflows/constraints.txt
         run: |
+          pipx install meltano[postgres]
           meltano --version
 
       - name: Install dbt-duckdb
@@ -147,17 +164,23 @@ jobs:
     name: Build Evidence site locally
     needs: [transform]
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/meltano/meltano:latest-python3.11
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_IO_TOKEN }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4.1.1
 
-      - name: Check Meltano
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            meltano.yml
+            .github/workflows/constraints.txt
+
+      - name: Install Meltano
+        env:
+          PIP_CONSTRAINT: .github/workflows/constraints.txt
         run: |
+          pipx install meltano[postgres]
           meltano --version
 
       - name: Install evidence

diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml
@@ -16,18 +16,23 @@ jobs:
   meltano-lock:
     name: Meltano Lock Update
     runs-on: ubuntu-latest
-    container:
-      image: ghcr.io/meltano/meltano:latest-python3.11
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GHCR_IO_TOKEN }}
-
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4.1.1
 
-    - name: Check Meltano
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: |
+          meltano.yml
+          .github/workflows/constraints.txt
+
+    - name: Install Meltano
+      env:
+        PIP_CONSTRAINT: .github/workflows/constraints.txt
       run: |
+        pipx install meltano[postgres]
         meltano --version
 
     - name: Meltano Lock Update

diff --git a/logging/ci.yaml b/logging/ci.yaml
@@ -10,7 +10,7 @@ formatters:
 handlers:
   console:
     class: logging.StreamHandler
-    level: ERROR
+    level: DEBUG
     formatter: structured
     stream: "ext://sys.stderr"
   file: