From c4231f5f62ffaba99a285b479292a5cb7dee75b4 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 20 Jul 2022 13:27:45 -0400 Subject: [PATCH 1/7] ci: add windows to test matrix --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edc02dae..a3a1a379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,10 @@ jobs: # ignore doctests, as they involve calls to github, and all mac machines # use the same IP address pytest_opts: "-k pins/tests" + - os: "windows-latest" + python: "3.10" + # ignore doctests + pytest_opts: "-k pins/tests" steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 From be87780991682d0117a06e3ac6fd6979803d8c65 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 20 Jul 2022 13:41:56 -0400 Subject: [PATCH 2/7] ci: explicitly use bash shell --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3a1a379..05fa0914 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,7 @@ jobs: with: python-version: ${{ matrix.python }} - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip @@ -61,6 +62,7 @@ jobs: export_default_credentials: true - name: Run tests + shell: bash run: | pytest pins -m 'not fs_rsc and not skip_on_github' --workers 4 --tests-per-worker 1 $PYTEST_OPTS env: From 0b6d80acc71b624f1a1a8c36922f5f733804a904 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Wed, 20 Jul 2022 14:11:54 -0400 Subject: [PATCH 3/7] ci: do not use pytest-parallel on windows --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05fa0914..d72d36d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: matrix: python: ["3.7", "3.8", "3.9", "3.10"] os: ["ubuntu-latest"] - pytest_ops: [""] + pytest_opts: ["--workers 4 --tests-per-worker 1"] requirements: [""] include: - os: "ubuntu-latest" @@ -32,7 +32,7 @@ jobs: python: "3.10" # ignore doctests, as they involve calls to github, and all mac machines # use the same IP address - pytest_opts: "-k pins/tests" + pytest_opts: "--workers 4 --tests-per-worker 1 -k pins/tests" - os: "windows-latest" python: "3.10" # ignore doctests @@ -64,13 +64,14 @@ jobs: - name: Run tests shell: bash run: | - pytest pins -m 'not fs_rsc and not skip_on_github' --workers 4 --tests-per-worker 1 $PYTEST_OPTS + pytest pins -m 'not fs_rsc and not skip_on_github' $PYTEST_OPTS env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: "us-east-1" PYTEST_OPTS: ${{ matrix.pytest_opts }} REQUIREMENTS: ${{ matrix.requirements }} + ACTION_OS: ${{ matrix.os }} # fixes error on macosx virtual machine with pytest-parallel # https://github.com/browsertron/pytest-parallel/issues/93 no_proxy: "*" From 73d287c5f4f17e98a65e8f67b37740b898b6614c Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 30 Aug 2022 10:46:16 -0400 Subject: [PATCH 4/7] tests: path tests now windows compatible --- pins/tests/test_cache.py | 10 ++++++++-- pins/tests/test_constructors.py | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pins/tests/test_cache.py b/pins/tests/test_cache.py index 9bc95692..52ec8cda 100644 --- a/pins/tests/test_cache.py +++ b/pins/tests/test_cache.py @@ -10,6 +10,7 @@ ) from fsspec import filesystem +from pathlib import Path # Utilities =================================================================== @@ -55,9 +56,14 @@ def test_pins_cache_url_hash_name(): cache = PinsUrlCache(fs=filesystem("file")) hashed = cache.hash_name("http://example.com/a.txt", True) + p_hash = Path(hashed) + # should have form // - assert hashed.endswith("/a.txt") - assert hashed.count("/") == 2 + assert p_hash.name == "a.txt" + + # count parent dirs, excluding root (e.g. "." or "/") + n_parents = len(p_hash.parents) - 1 + assert n_parents == 2 @pytest.mark.skip("TODO") diff --git a/pins/tests/test_constructors.py b/pins/tests/test_constructors.py index 6b28ede0..805e8012 100644 --- a/pins/tests/test_constructors.py +++ b/pins/tests/test_constructors.py @@ -25,7 +25,14 @@ def check_dir_writable(p_dir): def check_cache_file_path(p_file, p_cache): - assert str(p_file.relative_to(p_cache)).count("/") == 2 + rel_path = p_file.relative_to(p_cache) + + # parents has every entry you'd get if you called .parents all the way to some root. + # for a relative path, the root is likely ".", so we subtract 1 to get the number + # of parent directories. + # note this essentially counts slashes, in a inter-OS friendly way. + n_parents = len(rel_path.parents) - 1 + assert n_parents == 2 def construct_from_board(board): From 3392b420497012d97fef1629fa8f55af71ffb8ce Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 30 Aug 2022 11:18:01 -0400 Subject: [PATCH 5/7] fix(rsconnect): possible race condition in bundle creation --- pins/rsconnect/api.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pins/rsconnect/api.py b/pins/rsconnect/api.py index b634b950..6ef4ce5b 100644 --- a/pins/rsconnect/api.py +++ b/pins/rsconnect/api.py @@ -371,15 +371,13 @@ def post_content_bundle(self, guid, fname, gzip=True) -> Bundle: if p.is_dir() and gzip: import tarfile - with tempfile.NamedTemporaryFile(mode="wb", suffix=".tar.gz") as tmp: - with tarfile.open(fileobj=tmp.file, mode="w:gz") as tar: - tar.add(str(p.absolute()), arcname="") + with tempfile.TemporaryDirectory() as tmp_dir: + p_archive = Path(tmp_dir) / "bundle.tar.gz" - # close the underlying file. note we don't call the top-level - # close method, since that would delete the temporary file - tmp.file.close() + with tarfile.open(p_archive, mode="w:gz") as tar: + tar.add(str(p.absolute()), arcname="") - with open(tmp.name, "rb") as f: + with open(p_archive, "rb") as f: result = f_request(data=f) else: with open(str(p.absolute()), "rb") as f: From a7471e4c671d316f7e4b6f855da3424dd7769723 Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 30 Aug 2022 11:18:36 -0400 Subject: [PATCH 6/7] tests: add time delay to cache tests for windows --- pins/tests/test_cache.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pins/tests/test_cache.py b/pins/tests/test_cache.py index 52ec8cda..a257a309 100644 --- a/pins/tests/test_cache.py +++ b/pins/tests/test_cache.py @@ -12,10 +12,18 @@ from fsspec import filesystem from pathlib import Path +# NOTE: windows time.time() implementation appears to have 16 millisecond precision, so +# we need to add a small delay, in order to avoid prune checks appearing to happen at the +# exact same moment something earlier was created / accessed. + # Utilities =================================================================== +def _sleep(): + time.sleep(0.2) + + @pytest.fixture def some_file(tmp_dir2): p = tmp_dir2 / "some_file.txt" @@ -35,7 +43,7 @@ def test_touch_access_time_manual(some_file): def test_touch_access_time_auto(some_file): orig_access = some_file.stat().st_atime - time.sleep(0.2) + _sleep() new_time = touch_access_time(some_file) assert some_file.stat().st_atime == new_time @@ -112,6 +120,8 @@ def pin2_v3(a_cache): def test_cache_pruner_old_versions_none(a_cache, pin1_v1): + _sleep() + pruner = CachePruner(a_cache) old = pruner.old_versions(days=1) @@ -120,6 +130,8 @@ def test_cache_pruner_old_versions_none(a_cache, pin1_v1): def test_cache_pruner_old_versions_days0(a_cache, pin1_v1): + _sleep() + pruner = CachePruner(a_cache) old = pruner.old_versions(days=0) @@ -128,6 +140,8 @@ def test_cache_pruner_old_versions_days0(a_cache, pin1_v1): def test_cache_pruner_old_versions_some(a_cache, pin1_v1, pin1_v2): + _sleep() + # create: tmp_dir/pin1/version1 pruner = CachePruner(a_cache) @@ -147,6 +161,8 @@ def test_cache_pruner_old_versions_multi_pins(a_cache, pin1_v2, pin2_v3): def test_cache_prune_prompt(a_cache, pin1_v1, pin2_v3, monkeypatch): + _sleep() + cache_prune(days=1, cache_root=a_cache.parent, prompt=False) versions = list(a_cache.glob("*/*")) From 1802128325116b5caad77353ad2285e1ef2b664d Mon Sep 17 00:00:00 2001 From: Michael Chow Date: Tue, 30 Aug 2022 11:39:09 -0400 Subject: [PATCH 7/7] tests: try increasing time delay --- pins/tests/test_cache.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pins/tests/test_cache.py b/pins/tests/test_cache.py index a257a309..d62257ea 100644 --- a/pins/tests/test_cache.py +++ b/pins/tests/test_cache.py @@ -15,13 +15,16 @@ # NOTE: windows time.time() implementation appears to have 16 millisecond precision, so # we need to add a small delay, in order to avoid prune checks appearing to happen at the # exact same moment something earlier was created / accessed. +# see: https://stackoverflow.com/a/1938096/1144523 # Utilities =================================================================== def _sleep(): - time.sleep(0.2) + # time-based issues keep arising erratically in windows checks, so try to shoot + # well past + time.sleep(0.3) @pytest.fixture @@ -153,6 +156,8 @@ def test_cache_pruner_old_versions_some(a_cache, pin1_v1, pin1_v2): def test_cache_pruner_old_versions_multi_pins(a_cache, pin1_v2, pin2_v3): + _sleep() + pruner = CachePruner(a_cache) old = pruner.old_versions(days=1)