Skip to content

Commit 444d3da

Browse files
seismanweiji14
andauthored
CI: Add a new workflow to cache dvc cache (#4184)
Co-authored-by: Wei Ji <23487320+weiji14@users.noreply.github.com>
1 parent c74e4af commit 444d3da

File tree

4 files changed

+82
-2
lines changed

4 files changed

+82
-2
lines changed

.github/workflows/cache_dvc.yaml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Cache baseline images tracked by DVC and upload as artifacts.
2+
#
3+
# This workflow downloads dvc cache needed by PyGMT tests and uploads them as workflow
4+
# artifacts, which can then be accessed by other GitHub Actions workflows.
5+
#
6+
# This workflow serves as a workaround for the DagsHub authentication issue that
7+
# prevents PRs from forks to access the DVC remote.
8+
# Related issue: https://github.com/GenericMappingTools/pygmt/issues/4147
9+
#
10+
# It is scheduled to run every Sunday at 12:00 (UTC) and on the main branch if there are
11+
# any changes to the baseline images.
12+
#
13+
name: Cache DVC
14+
15+
on:
16+
push:
17+
branches: [ main ]
18+
paths:
19+
- 'pygmt/tests/baseline/*.png.dvc'
20+
# pull_request: # For testing only.
21+
workflow_dispatch:
22+
# Schedule runs on 12 noon every Sunday
23+
schedule:
24+
- cron: '0 12 * * 0'
25+
26+
permissions: {}
27+
28+
jobs:
29+
dvc_cache:
30+
name: Cache DVC baseline images
31+
runs-on: ubuntu-latest
32+
defaults:
33+
run:
34+
shell: bash -l {0}
35+
36+
steps:
37+
- name: Checkout
38+
uses: actions/checkout@v5.0.0
39+
with:
40+
# fetch all history so that setuptools-scm works
41+
fetch-depth: 0
42+
persist-credentials: false
43+
44+
- name: Setup data version control (DVC)
45+
uses: iterative/setup-dvc@175771be1dc3d119268e00a896b52a4b77decb5e # v1.2.0
46+
47+
- name: Pull baseline image data from dvc remote
48+
env:
49+
DAGSHUB_TOKEN: ${{ secrets.DAGSHUB_TOKEN }}
50+
run: |
51+
dvc remote modify upstream url https://${DAGSHUB_TOKEN}@dagshub.com/GenericMappingTools/pygmt.dvc --local
52+
dvc pull --no-run-cache --verbose && ls -lhR pygmt/tests/baseline/
53+
54+
- name: Upload DVC cache as artifacts to GitHub
55+
uses: actions/upload-artifact@v5.0.0
56+
with:
57+
name: dvc-cache
58+
include-hidden-files: true
59+
path: .dvc/cache/

.github/workflows/ci_tests.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,22 @@ jobs:
163163
164164
# Pull baseline image data from dvc remote (DAGsHub)
165165
- name: Pull baseline image data from dvc remote
166+
id: dvc-pull
167+
continue-on-error: true
166168
run: |
167169
uv run dvc remote modify upstream url https://${DAGSHUB_TOKEN}@dagshub.com/GenericMappingTools/pygmt.dvc --local
168170
uv run dvc pull --no-run-cache --verbose && ls -lhR pygmt/tests/baseline/
169171
env:
170172
DAGSHUB_TOKEN: ${{ secrets.DAGSHUB_TOKEN }}
171173

174+
- name: Download DVC cache as artifacts from GitHub
175+
if: steps.dvc-pull.outcome == 'failure'
176+
run: |
177+
gh run download --name dvc-cache --dir .dvc/cache/
178+
uv run dvc checkout --verbose
179+
env:
180+
GH_TOKEN: ${{ github.token }}
181+
172182
# Install the package that we want to test
173183
- name: Install the package
174184
run: make install

.github/workflows/ci_tests_dev.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,12 +161,22 @@ jobs:
161161

162162
# Pull baseline image data from dvc remote (DAGsHub)
163163
- name: Pull baseline image data from dvc remote
164+
id: dvc-pull
165+
continue-on-error: true
164166
run: |
165167
dvc remote modify upstream url https://${DAGSHUB_TOKEN}@dagshub.com/GenericMappingTools/pygmt.dvc --local
166168
dvc pull --no-run-cache --verbose && ls -lhR pygmt/tests/baseline/
167169
env:
168170
DAGSHUB_TOKEN: ${{ secrets.DAGSHUB_TOKEN }}
169171

172+
- name: Download DVC cache as artifacts from GitHub
173+
if: steps.dvc-pull.outcome == 'failure'
174+
run: |
175+
gh run download --name dvc-cache --dir .dvc/cache/
176+
dvc checkout --verbose
177+
env:
178+
GH_TOKEN: ${{ github.token }}
179+
170180
# Download cached remote files (artifacts) from GitHub
171181
- name: Download remote data from GitHub
172182
run: |

doc/maintenance.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,9 @@ controlled by workflow files located in `.github/workflows`. Here we briefly
9595
summarize the functions of the workflows. Please refer to the comments in the
9696
workflow files for more details.
9797

98-
- `benchmarks.yml`: Benchmarks the execution speed of tests to track performance of PyGMT functions
99-
- `cache_data.yaml`: Cache GMT remote data files and uplodas as artifacts
98+
- `benchmarks.yml`: Benchmark the execution speed of tests to track performance of PyGMT functions
99+
- `cache_data.yaml`: Cache GMT remote data files and upload as artifacts
100+
- `cache_dvc.yaml`: Cache baseline images tracked by DVC and upload as artifacts
100101
- `check-links.yml`: Check links in the repository and documentation
101102
- `ci_docs.yml`: Build documentation on Linux/macOS/Windows and deploy to GitHub
102103
- `ci_doctest.yaml`: Run all doctests on Linux/macOS/Windows

0 commit comments

Comments
 (0)