From a1a1bdfb9502b77c4631ea1c2f846914a7acd2c9 Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Thu, 23 Mar 2023 13:12:26 +0000 Subject: [PATCH 1/4] Abstract test harness --- s3fs/tests/derived/__init__.py | 0 s3fs/tests/derived/s3fs_fixtures.py | 116 ++++++++++++++++++++++++++++ s3fs/tests/derived/s3fs_test.py | 14 ++++ 3 files changed, 130 insertions(+) create mode 100644 s3fs/tests/derived/__init__.py create mode 100644 s3fs/tests/derived/s3fs_fixtures.py create mode 100644 s3fs/tests/derived/s3fs_test.py diff --git a/s3fs/tests/derived/__init__.py b/s3fs/tests/derived/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/s3fs/tests/derived/s3fs_fixtures.py b/s3fs/tests/derived/s3fs_fixtures.py new file mode 100644 index 00000000..5b565e31 --- /dev/null +++ b/s3fs/tests/derived/s3fs_fixtures.py @@ -0,0 +1,116 @@ +import json +import os +import pytest +import requests +import time + +from fsspec.tests.abstract import AbstractFixtures +from s3fs.core import S3FileSystem + + +test_bucket_name = "test" +secure_bucket_name = "test-secure" +versioned_bucket_name = "test-versioned" +port = 5555 +endpoint_uri = "http://127.0.0.1:%s/" % port + + +class S3fsFixtures(AbstractFixtures): + @staticmethod + @pytest.fixture + def fs(_s3_base, _get_boto3_client): + client = _get_boto3_client + client.create_bucket(Bucket=test_bucket_name, ACL="public-read") + + client.create_bucket(Bucket=versioned_bucket_name, ACL="public-read") + client.put_bucket_versioning( + Bucket=versioned_bucket_name, VersioningConfiguration={"Status": "Enabled"} + ) + + # initialize secure bucket + client.create_bucket(Bucket=secure_bucket_name, ACL="public-read") + policy = json.dumps( + { + "Version": "2012-10-17", + "Id": "PutObjPolicy", + "Statement": [ + { + "Sid": "DenyUnEncryptedObjectUploads", + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::{bucket_name}/*".format( + bucket_name=secure_bucket_name + ), + "Condition": { + "StringNotEquals": { + "s3:x-amz-server-side-encryption": "aws:kms" + } + }, + } + ], + } + ) + client.put_bucket_policy(Bucket=secure_bucket_name, Policy=policy) + + S3FileSystem.clear_instance_cache() + s3 = S3FileSystem(anon=False, client_kwargs={"endpoint_url": endpoint_uri}) + s3.invalidate_cache() + yield s3 + + @staticmethod + @pytest.fixture + def fs_path(): + return test_bucket_name + + @staticmethod + @pytest.fixture + def _get_boto3_client(): + from botocore.session import Session + + # NB: we use the sync botocore client for setup + session = Session() + return session.create_client("s3", endpoint_url=endpoint_uri) + + @staticmethod + @pytest.fixture + def _s3_base(): + # writable local S3 system + import shlex + import subprocess + + try: + # should fail since we didn't start server yet + r = requests.get(endpoint_uri) + except: + pass + else: + if r.ok: + raise RuntimeError("moto server already up") + if "AWS_SECRET_ACCESS_KEY" not in os.environ: + os.environ["AWS_SECRET_ACCESS_KEY"] = "foo" + if "AWS_ACCESS_KEY_ID" not in os.environ: + os.environ["AWS_ACCESS_KEY_ID"] = "foo" + proc = subprocess.Popen( + shlex.split("moto_server s3 -p %s" % port), + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + ) + + timeout = 5 + while timeout > 0: + try: + print("polling for moto server") + r = requests.get(endpoint_uri) + if r.ok: + break + except: + pass + timeout -= 0.1 + time.sleep(0.1) + print("server up") + yield + print("moto done") + proc.terminate() + proc.wait() diff --git a/s3fs/tests/derived/s3fs_test.py b/s3fs/tests/derived/s3fs_test.py new file mode 100644 index 00000000..2430e0a8 --- /dev/null +++ b/s3fs/tests/derived/s3fs_test.py @@ -0,0 +1,14 @@ +import fsspec.tests.abstract as abstract +from s3fs.tests.derived.s3fs_fixtures import S3fsFixtures + + +class TestS3fsCopy(abstract.AbstractCopyTests, S3fsFixtures): + pass + + +class TestS3fsGet(abstract.AbstractGetTests, S3fsFixtures): + pass + + +class TestS3fsPut(abstract.AbstractPutTests, S3fsFixtures): + pass From 10cfea09d925681d027255f9eb33a584d751a503 Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Thu, 23 Mar 2023 14:50:14 +0000 Subject: [PATCH 2/4] Debug --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e950985a..abe4ce83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,9 @@ jobs: - name: Install shell: bash -l {0} run: | - pip install git+https://github.com/fsspec/filesystem_spec + # Temporary fix so that fsspec.tests is available. + pip install -ve git+https://github.com/fsspec/filesystem_spec#egg=fsspec + #pip install git+https://github.com/fsspec/filesystem_spec pip install . --no-deps - name: Run Tests From 4a8ff5ccfab130c2ac82a132d2aa5a7de0e3d37d Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Mon, 3 Apr 2023 10:51:49 +0100 Subject: [PATCH 3/4] Add S3fsFixtures.supports_empty_directories --- s3fs/tests/derived/s3fs_fixtures.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/s3fs/tests/derived/s3fs_fixtures.py b/s3fs/tests/derived/s3fs_fixtures.py index 5b565e31..c22b78b0 100644 --- a/s3fs/tests/derived/s3fs_fixtures.py +++ b/s3fs/tests/derived/s3fs_fixtures.py @@ -63,6 +63,9 @@ def fs(_s3_base, _get_boto3_client): def fs_path(): return test_bucket_name + def supports_empty_directories(self): + return False + @staticmethod @pytest.fixture def _get_boto3_client(): From 0ec0be4b66bfbc2e03276f9a80856337490c94fa Mon Sep 17 00:00:00 2001 From: Ian Thomas Date: Mon, 15 May 2023 15:40:27 +0100 Subject: [PATCH 4/4] Update in line with latest fsspec --- .github/workflows/ci.yml | 4 +--- s3fs/tests/derived/s3fs_fixtures.py | 19 ++++++++----------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abe4ce83..e950985a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,7 @@ jobs: - name: Install shell: bash -l {0} run: | - # Temporary fix so that fsspec.tests is available. - pip install -ve git+https://github.com/fsspec/filesystem_spec#egg=fsspec - #pip install git+https://github.com/fsspec/filesystem_spec + pip install git+https://github.com/fsspec/filesystem_spec pip install . --no-deps - name: Run Tests diff --git a/s3fs/tests/derived/s3fs_fixtures.py b/s3fs/tests/derived/s3fs_fixtures.py index c22b78b0..8e56aa78 100644 --- a/s3fs/tests/derived/s3fs_fixtures.py +++ b/s3fs/tests/derived/s3fs_fixtures.py @@ -16,9 +16,9 @@ class S3fsFixtures(AbstractFixtures): - @staticmethod - @pytest.fixture - def fs(_s3_base, _get_boto3_client): + @pytest.fixture(scope="class") + def fs(self, _s3_base, _get_boto3_client): + print("FS") client = _get_boto3_client client.create_bucket(Bucket=test_bucket_name, ACL="public-read") @@ -58,26 +58,23 @@ def fs(_s3_base, _get_boto3_client): s3.invalidate_cache() yield s3 - @staticmethod @pytest.fixture - def fs_path(): + def fs_path(self): return test_bucket_name def supports_empty_directories(self): return False - @staticmethod - @pytest.fixture - def _get_boto3_client(): + @pytest.fixture(scope="class") + def _get_boto3_client(self): from botocore.session import Session # NB: we use the sync botocore client for setup session = Session() return session.create_client("s3", endpoint_url=endpoint_uri) - @staticmethod - @pytest.fixture - def _s3_base(): + @pytest.fixture(scope="class") + def _s3_base(self): # writable local S3 system import shlex import subprocess