From 316e0740af1aa69cfdfe0d1d6163ff7f34b2d56a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 17 Dec 2024 09:41:36 -0500 Subject: [PATCH 01/12] changelog --- docs/source/changelog.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index fea257b44..df4f83d6f 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,6 +1,31 @@ Changelog ========= +2024.12.0 +--------- + +Enhancements + +- "exclusive" mode for writing (#1762, 1756, 174+) +- "tree" text display of filesystem contents (#1750) +- async wrapper for sync FSs (#1745) +- new known implementation: tosfs (#1739) +- consilidate block fetch requests (#1733) + +Fixes + +- better webHDFS proxies (# +- syn FSs in referenceFS (#1755) +- don't serialize file caches (#1753) +- race condition in local ls() (#1744) +- missing/nan references in parquet (#1738) +- _un_chain kwargs (@1736) +- async _cat_file in referenceFS (#1734) + +Other + +- fallback implementation for _fetch_range (#1732) + 2024.10.0 --------- From fa7c6cecea833ea8f412f41492f1cf5b6a9decb4 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 17 Dec 2024 21:28:07 -0500 Subject: [PATCH 02/12] try s3fs branch --- ci/environment-friends.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index 6e6b79a14..bcd735b64 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -27,3 +27,5 @@ dependencies: - moto - flask - pyopenssl>=24.0.0 + - pip: + - git+https://github.com/martindurant/s3fs.git@test From 45805f65fb5ebf7aa3d320c7637f1d7a94a6f8be Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 17 Dec 2024 21:35:26 -0500 Subject: [PATCH 03/12] in the workflow --- .github/workflows/main.yaml | 7 +++++-- ci/environment-friends.yml | 2 -- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 23dc746bf..05617122a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -130,7 +130,7 @@ jobs: strategy: fail-fast: false matrix: - FRIEND: [gcsfs, s3fs] + FRIEND: [s3fs] env: CIRUN: true @@ -149,13 +149,16 @@ jobs: - name: Clone shell: bash -l {0} - run: git clone https://github.com/fsspec/${{ matrix.FRIEND }} + run: git clone https://github.com/martindurant/${{ matrix.FRIEND }} - name: Install shell: bash -l {0} run: | pip install -e . --no-deps + cd s3fs + git checkout test pip install -e ./${{ matrix.FRIEND }} --no-deps + cd .. pip list - name: Test diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index bcd735b64..6e6b79a14 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -27,5 +27,3 @@ dependencies: - moto - flask - pyopenssl>=24.0.0 - - pip: - - git+https://github.com/martindurant/s3fs.git@test From e5f2218b4784daaa437816c88995d61ee0af387a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Tue, 17 Dec 2024 22:02:52 -0500 Subject: [PATCH 04/12] order --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 05617122a..bb4b71944 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -157,8 +157,8 @@ jobs: pip install -e . --no-deps cd s3fs git checkout test - pip install -e ./${{ matrix.FRIEND }} --no-deps cd .. + pip install -e ./${{ matrix.FRIEND }} --no-deps pip list - name: Test From 7a9a3d0193635004716ccb6819e22304169d118b Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 10:07:16 -0500 Subject: [PATCH 05/12] (temp) test simplest env --- ci/environment-friends.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index 6e6b79a14..b7b89a291 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -2,28 +2,17 @@ name: test_env channels: - conda-forge dependencies: - - python=3.12 - pytest - - pytest-asyncio !=0.22.0 - - pytest-benchmark - - pytest-cov - - pytest-mock + - pytest-asyncio - pip - ujson - requests - decorator - pytest-timeout - - google-auth - - aiohttp - - google-auth-oauthlib - flake8 - black - - google-cloud-core - - google-cloud-storage - - google-api-core - - google-api-python-client - httpretty - aiobotocore - moto - flask - - pyopenssl>=24.0.0 + - fsspec From 063cd4914919990bfeb6d56a677331d748ef9fbb Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 10:48:26 -0500 Subject: [PATCH 06/12] add a bit? --- .github/workflows/main.yaml | 2 ++ ci/environment-friends.yml | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index bb4b71944..45fca1d1e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -158,6 +158,8 @@ jobs: cd s3fs git checkout test cd .. + pip install --upgrade "aiobotocore" + pip install --upgrade "botocore" --no-deps pip install -e ./${{ matrix.FRIEND }} --no-deps pip list diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index b7b89a291..6e6b79a14 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -2,17 +2,28 @@ name: test_env channels: - conda-forge dependencies: + - python=3.12 - pytest - - pytest-asyncio + - pytest-asyncio !=0.22.0 + - pytest-benchmark + - pytest-cov + - pytest-mock - pip - ujson - requests - decorator - pytest-timeout + - google-auth + - aiohttp + - google-auth-oauthlib - flake8 - black + - google-cloud-core + - google-cloud-storage + - google-api-core + - google-api-python-client - httpretty - aiobotocore - moto - flask - - fsspec + - pyopenssl>=24.0.0 From 2ce3a3b0b3a591b2b0ab00503c4e3344eb8812e6 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 14:19:55 -0500 Subject: [PATCH 07/12] add to base env --- ci/environment-friends.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index 6e6b79a14..c72b54fdd 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -27,3 +27,5 @@ dependencies: - moto - flask - pyopenssl>=24.0.0 + - s3fs + - gcsfs From a8e52a0612b9a128aef590e415eae2decabf11e4 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 14:21:14 -0500 Subject: [PATCH 08/12] reset build --- .github/workflows/main.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 45fca1d1e..b3d6e6261 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -130,7 +130,7 @@ jobs: strategy: fail-fast: false matrix: - FRIEND: [s3fs] + FRIEND: [s3fs, gcsfs] env: CIRUN: true @@ -155,11 +155,7 @@ jobs: shell: bash -l {0} run: | pip install -e . --no-deps - cd s3fs - git checkout test - cd .. pip install --upgrade "aiobotocore" - pip install --upgrade "botocore" --no-deps pip install -e ./${{ matrix.FRIEND }} --no-deps pip list From 163ad4665dc06c53daaac49923288f956359aff3 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 14:24:49 -0500 Subject: [PATCH 09/12] simplify --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b3d6e6261..42cfae341 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -149,13 +149,12 @@ jobs: - name: Clone shell: bash -l {0} - run: git clone https://github.com/martindurant/${{ matrix.FRIEND }} + run: git clone https://github.com/fsspec/${{ matrix.FRIEND }} - name: Install shell: bash -l {0} run: | pip install -e . --no-deps - pip install --upgrade "aiobotocore" pip install -e ./${{ matrix.FRIEND }} --no-deps pip list From 69d7476c3e7b5ee929fde51032b43460d27626cf Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Wed, 18 Dec 2024 14:49:21 -0500 Subject: [PATCH 10/12] alternate --- .github/workflows/main.yaml | 6 ++++-- ci/environment-friends.yml | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 42cfae341..2c6250e36 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -155,9 +155,11 @@ jobs: shell: bash -l {0} run: | pip install -e . --no-deps - pip install -e ./${{ matrix.FRIEND }} --no-deps pip list - name: Test shell: bash -l {0} - run: pytest -v ${{ matrix.FRIEND }} + run: | + cd ${{ matrix.FRIEND }} + pytest -v + cd .. diff --git a/ci/environment-friends.yml b/ci/environment-friends.yml index c72b54fdd..2126744af 100644 --- a/ci/environment-friends.yml +++ b/ci/environment-friends.yml @@ -26,6 +26,6 @@ dependencies: - aiobotocore - moto - flask - - pyopenssl>=24.0.0 - - s3fs - - gcsfs + - pip: + - git+https://github.com/fsspec/s3fs + - git+https://github.com/fsspec/gcsfs From 77909687a4a34b5e036d08089563695f49a7b196 Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 19 Dec 2024 14:37:11 -0500 Subject: [PATCH 11/12] try on ubuntu 24.04 --- .github/workflows/main.yaml | 206 ++++++++++++++++++------------------ 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2c6250e36..55f520daa 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,74 +7,74 @@ on: branches: [master] jobs: - linux: - name: ${{ matrix.PY }}-pytest - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - PY: - - "3.9" - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - env: - CIRUN: true - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: ci/environment-py38.yml - python-version: ${{ matrix.PY }} - - - name: Run Tests - shell: bash -l {0} - run: | - pip install -e .[test_full] - pytest -v - - win: - name: pytest-win - runs-on: windows-2019 - - env: - CIRUN: true - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: ci/environment-win.yml - - - name: Run Tests - shell: bash -l {0} - run: | - pip install -e .[test] - pytest -v - - lint: - name: lint - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@main - - uses: actions/setup-python@main - with: - python-version: "3.11" - - uses: pre-commit/action@main - +# linux: +# name: ${{ matrix.PY }}-pytest +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# PY: +# - "3.9" +# - "3.10" +# - "3.11" +# - "3.12" +# - "3.13" +# +# env: +# CIRUN: true +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# +# - name: Setup conda +# uses: conda-incubator/setup-miniconda@v3 +# with: +# environment-file: ci/environment-py38.yml +# python-version: ${{ matrix.PY }} +# +# - name: Run Tests +# shell: bash -l {0} +# run: | +# pip install -e .[test_full] +# pytest -v +# +# win: +# name: pytest-win +# runs-on: windows-2019 +# +# env: +# CIRUN: true +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# +# - name: Setup conda +# uses: conda-incubator/setup-miniconda@v3 +# with: +# environment-file: ci/environment-win.yml +# +# - name: Run Tests +# shell: bash -l {0} +# run: | +# pip install -e .[test] +# pytest -v +# +# lint: +# name: lint +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@main +# - uses: actions/setup-python@main +# with: +# python-version: "3.11" +# - uses: pre-commit/action@main +# # typecheck: # runs-on: ubuntu-latest # steps: @@ -91,46 +91,46 @@ jobs: # run: | # mypy fsspec # - downstream: - name: downstream - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup conda - uses: conda-incubator/setup-miniconda@v3 - with: - environment-file: ci/environment-downstream.yml - - - name: Local install - shell: bash -l {0} - run: | - sh install_s3fs.sh - pip install -e .[test,test_downstream] - pip list - - - name: Run fsspec tests - shell: bash -l {0} - run: | - pytest -v fsspec/tests/test_downstream.py - - - name: Run dask tests - shell: bash -l {0} - run: | - dask_test_path=$(python -c "import dask.bytes;print(dask.bytes.__path__[0])") - pytest -v $dask_test_path - +# downstream: +# name: downstream +# runs-on: ubuntu-latest +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# fetch-depth: 0 +# +# - name: Setup conda +# uses: conda-incubator/setup-miniconda@v3 +# with: +# environment-file: ci/environment-downstream.yml +# +# - name: Local install +# shell: bash -l {0} +# run: | +# sh install_s3fs.sh +# pip install -e .[test,test_downstream] +# pip list +# +# - name: Run fsspec tests +# shell: bash -l {0} +# run: | +# pytest -v fsspec/tests/test_downstream.py +# +# - name: Run dask tests +# shell: bash -l {0} +# run: | +# dask_test_path=$(python -c "import dask.bytes;print(dask.bytes.__path__[0])") +# pytest -v $dask_test_path +# fsspec_friends: name: ${{ matrix.FRIEND }}-pytest - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - FRIEND: [s3fs, gcsfs] + FRIEND: [s3fs] env: CIRUN: true From 2e6fb6dfe832bfcbc287cf1fb9a6c5056cb94b3b Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Thu, 19 Dec 2024 14:42:07 -0500 Subject: [PATCH 12/12] revert debug --- .github/workflows/main.yaml | 204 ++++++++++++++++++------------------ 1 file changed, 102 insertions(+), 102 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 55f520daa..ef29dd67c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,74 +7,74 @@ on: branches: [master] jobs: -# linux: -# name: ${{ matrix.PY }}-pytest -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# PY: -# - "3.9" -# - "3.10" -# - "3.11" -# - "3.12" -# - "3.13" -# -# env: -# CIRUN: true -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# - name: Setup conda -# uses: conda-incubator/setup-miniconda@v3 -# with: -# environment-file: ci/environment-py38.yml -# python-version: ${{ matrix.PY }} -# -# - name: Run Tests -# shell: bash -l {0} -# run: | -# pip install -e .[test_full] -# pytest -v -# -# win: -# name: pytest-win -# runs-on: windows-2019 -# -# env: -# CIRUN: true -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# - name: Setup conda -# uses: conda-incubator/setup-miniconda@v3 -# with: -# environment-file: ci/environment-win.yml -# -# - name: Run Tests -# shell: bash -l {0} -# run: | -# pip install -e .[test] -# pytest -v -# -# lint: -# name: lint -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@main -# - uses: actions/setup-python@main -# with: -# python-version: "3.11" -# - uses: pre-commit/action@main -# + linux: + name: ${{ matrix.PY }}-pytest + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + PY: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + + env: + CIRUN: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: ci/environment-py38.yml + python-version: ${{ matrix.PY }} + + - name: Run Tests + shell: bash -l {0} + run: | + pip install -e .[test_full] + pytest -v + + win: + name: pytest-win + runs-on: windows-2019 + + env: + CIRUN: true + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: ci/environment-win.yml + + - name: Run Tests + shell: bash -l {0} + run: | + pip install -e .[test] + pytest -v + + lint: + name: lint + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@main + - uses: actions/setup-python@main + with: + python-version: "3.11" + - uses: pre-commit/action@main + # typecheck: # runs-on: ubuntu-latest # steps: @@ -91,46 +91,46 @@ jobs: # run: | # mypy fsspec # -# downstream: -# name: downstream -# runs-on: ubuntu-latest -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# fetch-depth: 0 -# -# - name: Setup conda -# uses: conda-incubator/setup-miniconda@v3 -# with: -# environment-file: ci/environment-downstream.yml -# -# - name: Local install -# shell: bash -l {0} -# run: | -# sh install_s3fs.sh -# pip install -e .[test,test_downstream] -# pip list -# -# - name: Run fsspec tests -# shell: bash -l {0} -# run: | -# pytest -v fsspec/tests/test_downstream.py -# -# - name: Run dask tests -# shell: bash -l {0} -# run: | -# dask_test_path=$(python -c "import dask.bytes;print(dask.bytes.__path__[0])") -# pytest -v $dask_test_path -# + downstream: + name: downstream + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + environment-file: ci/environment-downstream.yml + + - name: Local install + shell: bash -l {0} + run: | + sh install_s3fs.sh + pip install -e .[test,test_downstream] + pip list + + - name: Run fsspec tests + shell: bash -l {0} + run: | + pytest -v fsspec/tests/test_downstream.py + + - name: Run dask tests + shell: bash -l {0} + run: | + dask_test_path=$(python -c "import dask.bytes;print(dask.bytes.__path__[0])") + pytest -v $dask_test_path + fsspec_friends: name: ${{ matrix.FRIEND }}-pytest runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: - FRIEND: [s3fs] + FRIEND: [s3fs, gcsfs] env: CIRUN: true