From 7d86716a48f21b8e43ba018db58f29fdef9bd6de Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 12:52:09 -0400 Subject: [PATCH 1/6] ci: add scripts for compatibility checks --- Makefile | 4 ++++ script/ci-compat-check/.gitignore | 1 + script/ci-compat-check/Makefile | 20 ++++++++++++++++ script/ci-compat-check/dump_py_pins.py | 12 ++++++++++ script/ci-compat-check/dump_r_pins.R | 5 ++++ script/ci-compat-check/validate_py_to_r.R | 25 ++++++++++++++++++++ script/ci-compat-check/validate_r_to_py.py | 27 ++++++++++++++++++++++ 7 files changed, 94 insertions(+) create mode 100644 script/ci-compat-check/.gitignore create mode 100644 script/ci-compat-check/Makefile create mode 100644 script/ci-compat-check/dump_py_pins.py create mode 100644 script/ci-compat-check/dump_r_pins.R create mode 100644 script/ci-compat-check/validate_py_to_r.R create mode 100644 script/ci-compat-check/validate_r_to_py.py diff --git a/Makefile b/Makefile index b62e5fb7..8be38df1 100644 --- a/Makefile +++ b/Makefile @@ -36,3 +36,7 @@ requirements/dev.txt: setup.cfg binder/requirements.txt: requirements/dev.txt cp $< $@ + +ci-compat-check: + # TODO: mark as dummy + $(MAKE) -C script/$@ diff --git a/script/ci-compat-check/.gitignore b/script/ci-compat-check/.gitignore new file mode 100644 index 00000000..a9a5aecf --- /dev/null +++ b/script/ci-compat-check/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/script/ci-compat-check/Makefile b/script/ci-compat-check/Makefile new file mode 100644 index 00000000..741029df --- /dev/null +++ b/script/ci-compat-check/Makefile @@ -0,0 +1,20 @@ +BOARD_BASE_DIR=tmp +BOARD_PY=$(BOARD_BASE_DIR)/board-py +BOARD_R=$(BOARD_BASE_DIR)/board-r + +all: validate + +clean: + rm -r $(BOARD_PY) $(BOARD_R) + +validate: $(BOARD_PY) $(BOARD_R) + @echo "\n\nRUNNING R PINS ---\n" + Rscript validate_py_to_r.R $(BOARD_PY) $(BOARD_R) + @echo "\n\nRUNNING PYTHON PINS ---\n" + python validate_r_to_py.py $(BOARD_PY) $(BOARD_R) + +$(BOARD_PY): dump_py_pins.py + python dump_py_pins.py $@ + +$(BOARD_R): dump_r_pins.R + Rscript dump_r_pins.R $@ diff --git a/script/ci-compat-check/dump_py_pins.py b/script/ci-compat-check/dump_py_pins.py new file mode 100644 index 00000000..b9d22a13 --- /dev/null +++ b/script/ci-compat-check/dump_py_pins.py @@ -0,0 +1,12 @@ +import sys + +from pins.data import mtcars +from pins import board_folder + +if len(sys.argv) < 2: + raise ValueError("must pass board location as command-line argument") +else: + BOARD_PATH = sys.argv[1] + +board = board_folder(BOARD_PATH) +board.pin_write(mtcars, "mtcars", type="csv") diff --git a/script/ci-compat-check/dump_r_pins.R b/script/ci-compat-check/dump_r_pins.R new file mode 100644 index 00000000..7deff314 --- /dev/null +++ b/script/ci-compat-check/dump_r_pins.R @@ -0,0 +1,5 @@ +library(pins) +args <- commandArgs(trailingOnly=TRUE) + +board <- board_folder(args[1]) +board %>% pin_write(mtcars, "mtcars", type="csv") diff --git a/script/ci-compat-check/validate_py_to_r.R b/script/ci-compat-check/validate_py_to_r.R new file mode 100644 index 00000000..1106796b --- /dev/null +++ b/script/ci-compat-check/validate_py_to_r.R @@ -0,0 +1,25 @@ +library(pins) + +args <- commandArgs(trailingOnly=TRUE) + + +# create board ---- + +board_py <- board_folder(args[1]) +board_r <- board_folder(args[2]) + + +# check pins ---- + +cat("Checking mtcars pin\n") + +res_mtcars <- board_py %>% pin_read("mtcars") +stopifnot(all.equal(res_mtcars, datasets::mtcars, check.attributes=FALSE)) + +meta_mtcars_py <- board_py %>% pin_meta("mtcars") +cat("\nPython meta:\n\n") +print(meta_mtcars_py) + +meta_mtcars_r <- board_r %>% pin_meta("mtcars") +cat("\nR meta:\n\n") +print(meta_mtcars_r) diff --git a/script/ci-compat-check/validate_r_to_py.py b/script/ci-compat-check/validate_r_to_py.py new file mode 100644 index 00000000..593be711 --- /dev/null +++ b/script/ci-compat-check/validate_r_to_py.py @@ -0,0 +1,27 @@ +import sys + +from pins import board_folder +from pins import data + +path_py, path_r = sys.argv[1], sys.argv[2] + +# create board ---- + +board_py = board_folder(path_py) +board_r = board_folder(path_r) + + +# check pins ---- + +print("Checking mtcars pin") + +res_mtcars = board_r.pin_read("mtcars") +assert res_mtcars.equals(data.mtcars) + +meta_mtcars_py = board_py.pin_meta("mtcars") +print("\nPython meta:\n") +print(meta_mtcars_py) + +meta_mtcars_r = board_r.pin_meta("mtcars") +print("\nR meta:\n") +print(meta_mtcars_r) From 5cc2c970010239da3f03b09ac5907808e3093a02 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 12:54:18 -0400 Subject: [PATCH 2/6] ci: check cross lib compatibility --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9731ea19..3bb8d638 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,37 @@ jobs: run: | pytest pins -m 'fs_rsc and not skip_on_github' + check-cross-compatibility: + name: "Check cross lib compatibility" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + # r --- + + - uses: r-lib/actions/setup-r@v2 + with: + r-version: '3.5.3' + - name: Install R dependencies + run: "install.packages('pins')" + shell: Rscript {0} + + # python --- + + - uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install py dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements/dev.txt + python -m pip install -e . + + # write and test --- + + - name: Run script/ci-compat-check + run: make ci-compat-check + build-docs: name: "Build Docs" runs-on: ubuntu-latest From 08fcf8757f01cd620c7b1f1908b61ac1944db879 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 14:16:17 -0400 Subject: [PATCH 3/6] ci: try using setup-r-dependencies --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bb8d638..f4518e28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,9 +83,13 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: r-version: '3.5.3' - - name: Install R dependencies - run: "install.packages('pins')" - shell: Rscript {0} + - uses: r-lib/actions/setup-r-dependencies@v2 + extra-packages: | + any::pins + + #- name: Install R dependencies + # run: "install.packages('pins')" + # shell: Rscript {0} # python --- From 181a6cf851852665e0a3ceaaa6513599293dde37 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 14:17:09 -0400 Subject: [PATCH 4/6] ci: fix workflow typo --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4518e28..f660bbcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,8 +84,9 @@ jobs: with: r-version: '3.5.3' - uses: r-lib/actions/setup-r-dependencies@v2 - extra-packages: | - any::pins + with: + extra-packages: | + any::pins #- name: Install R dependencies # run: "install.packages('pins')" From b93150eb2a4a919a3edbf6e0bf16ec56a4280d86 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 14:30:29 -0400 Subject: [PATCH 5/6] ci: try R install again --- .github/workflows/ci.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f660bbcf..68f8888a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,19 +78,18 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install libcurl on Linux + run: sudo apt-get update -y && sudo apt-get install -y libcurl4-openssl-dev + # r --- - uses: r-lib/actions/setup-r@v2 with: r-version: '3.5.3' - - uses: r-lib/actions/setup-r-dependencies@v2 - with: - extra-packages: | - any::pins - #- name: Install R dependencies - # run: "install.packages('pins')" - # shell: Rscript {0} + - name: Install R dependencies + run: "install.packages('pins')" + shell: Rscript {0} # python --- From 8006734abd5ff204d29e28e5761d1758b32f10e3 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 27 Apr 2022 14:40:12 -0400 Subject: [PATCH 6/6] ci: make R not old and install faster --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68f8888a..a184015f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - uses: r-lib/actions/setup-r@v2 with: - r-version: '3.5.3' + use-public-rspm: true - name: Install R dependencies run: "install.packages('pins')"