From 225f7b616e00b29d92c6cc3f4bee6ed120c514ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= Date: Tue, 5 Dec 2023 23:53:42 -0600 Subject: [PATCH] ci: Cache pip dependencies --- .github/dependabot.yml | 23 ++++---- .github/workflows/constraints.txt | 2 + .github/workflows/contraints.txt | 2 - .github/workflows/melty.yaml | 91 +++++++++++++++++++------------ .github/workflows/update.yml | 19 ++++--- logging/ci.yaml | 2 +- 6 files changed, 84 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/constraints.txt delete mode 100644 .github/workflows/contraints.txt diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0b4ce4c..46cb87b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,15 +1,16 @@ version: 2 updates: - # - package-ecosystem: pip - # directory: "/.github/workflows" - # schedule: - # interval: daily - # time: "11:35" - # timezone: "America/Mexico_City" - # reviewers: - # - "edgarrmondragon" - # labels: - # - "dependencies" + - package-ecosystem: pip + directory: "/.github/workflows" + schedule: + interval: weekly + time: "11:35" + timezone: "America/Mexico_City" + reviewers: + - "edgarrmondragon" + labels: + - "dependencies" + - "Test/ELTP" - package-ecosystem: github-actions directory: "/" schedule: @@ -21,7 +22,7 @@ updates: - package-ecosystem: "npm" directory: "/evidence" schedule: - interval: daily + interval: weekly time: "11:35" timezone: "America/Mexico_City" reviewers: diff --git a/.github/workflows/constraints.txt b/.github/workflows/constraints.txt new file mode 100644 index 0000000..7ce79ae --- /dev/null +++ b/.github/workflows/constraints.txt @@ -0,0 +1,2 @@ +pip==23.3.1 +meltano==3.2.0 diff --git a/.github/workflows/contraints.txt b/.github/workflows/contraints.txt deleted file mode 100644 index 61ebdc7..0000000 --- a/.github/workflows/contraints.txt +++ /dev/null @@ -1,2 +0,0 @@ -pip==22.2.2 -meltano==2.7.1 diff --git a/.github/workflows/melty.yaml b/.github/workflows/melty.yaml index 67ebb3f..12d939b 100644 --- a/.github/workflows/melty.yaml +++ b/.github/workflows/melty.yaml @@ -34,18 +34,13 @@ jobs: if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'Test/ELTP') }} name: "Meltano Job: ${{ matrix.job }}" runs-on: ubuntu-latest - container: - image: ghcr.io/meltano/meltano:latest-python3.11 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GHCR_IO_TOKEN }} - env: - TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }} - TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }} - TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }} - TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }} - TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }} - TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }} + env: + TAP_GETPOCKET_CONSUMER_KEY: ${{ secrets.TAP_GETPOCKET_CONSUMER_KEY }} + TAP_GETPOCKET_ACCESS_TOKEN: ${{ secrets.TAP_GETPOCKET_ACCESS_TOKEN }} + TAP_GITHUB_AUTH_TOKEN: ${{ secrets.TAP_GITHUB_AUTH_TOKEN }} + TAP_READTHEDOCS_TOKEN: ${{ secrets.TAP_READTHEDOCS_TOKEN }} + TAP_STACKEXCHANGE_KEY: ${{ secrets.TAP_STACKEXCHANGE_KEY }} + TARGET_DUCKDB_PATH: ${{ secrets.TARGET_DUCKDB_PATH }} strategy: matrix: include: @@ -63,8 +58,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4.1.1 - - name: Check Meltano + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + meltano.yml + .github/workflows/constraints.txt + + - name: Install Meltano + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | + pipx install meltano[postgres] meltano --version - name: Install Plugins @@ -91,20 +97,26 @@ jobs: name: Lint models runs-on: ubuntu-latest continue-on-error: true - container: - image: ghcr.io/meltano/meltano:latest-python3.11 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GHCR_IO_TOKEN }} - env: - DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} + env: + DUCKDB_PATH: ${{ secrets.DUCKDB_PATH }} steps: - name: Checkout code uses: actions/checkout@v4.1.1 - - name: Check Meltano + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + meltano.yml + .github/workflows/constraints.txt + + - name: Install Meltano + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | + pipx install meltano[postgres] meltano --version - name: Install SQLFluff @@ -121,18 +133,23 @@ jobs: name: Data Transformation with dbt-duckdb runs-on: ubuntu-latest needs: [meltano-run, lint] - container: - image: ghcr.io/meltano/meltano:latest-python3.11 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GHCR_IO_TOKEN }} - steps: - name: Checkout code uses: actions/checkout@v4.1.1 - - name: Check Meltano + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + meltano.yml + .github/workflows/constraints.txt + + - name: Install Meltano + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | + pipx install meltano[postgres] meltano --version - name: Install dbt-duckdb @@ -147,17 +164,23 @@ jobs: name: Build Evidence site locally needs: [transform] runs-on: ubuntu-latest - container: - image: ghcr.io/meltano/meltano:latest-python3.11 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GHCR_IO_TOKEN }} steps: - name: Checkout code uses: actions/checkout@v4.1.1 - - name: Check Meltano + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + meltano.yml + .github/workflows/constraints.txt + + - name: Install Meltano + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | + pipx install meltano[postgres] meltano --version - name: Install evidence diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 8d8ec2c..67cae43 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -16,18 +16,23 @@ jobs: meltano-lock: name: Meltano Lock Update runs-on: ubuntu-latest - container: - image: ghcr.io/meltano/meltano:latest-python3.11 - credentials: - username: ${{ github.actor }} - password: ${{ secrets.GHCR_IO_TOKEN }} - steps: - name: Checkout Code uses: actions/checkout@v4.1.1 - - name: Check Meltano + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + meltano.yml + .github/workflows/constraints.txt + + - name: Install Meltano + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt run: | + pipx install meltano[postgres] meltano --version - name: Meltano Lock Update diff --git a/logging/ci.yaml b/logging/ci.yaml index 5e33b9d..8e39c67 100644 --- a/logging/ci.yaml +++ b/logging/ci.yaml @@ -10,7 +10,7 @@ formatters: handlers: console: class: logging.StreamHandler - level: ERROR + level: DEBUG formatter: structured stream: "ext://sys.stderr" file: