-
Notifications
You must be signed in to change notification settings - Fork 267
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Test using new abstract test harness (#713)
* Abstract test harness * Add S3fsFixtures.supports_empty_directories * Update in line with latest fsspec
- Loading branch information
1 parent
676b7b6
commit ab460d5
Showing
3 changed files
with
130 additions
and
0 deletions.
There are no files selected for viewing
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,116 @@ | ||
import json | ||
import os | ||
import pytest | ||
import requests | ||
import time | ||
|
||
from fsspec.tests.abstract import AbstractFixtures | ||
from s3fs.core import S3FileSystem | ||
|
||
|
||
test_bucket_name = "test" | ||
secure_bucket_name = "test-secure" | ||
versioned_bucket_name = "test-versioned" | ||
port = 5555 | ||
endpoint_uri = "http://127.0.0.1:%s/" % port | ||
|
||
|
||
class S3fsFixtures(AbstractFixtures): | ||
@pytest.fixture(scope="class") | ||
def fs(self, _s3_base, _get_boto3_client): | ||
print("FS") | ||
client = _get_boto3_client | ||
client.create_bucket(Bucket=test_bucket_name, ACL="public-read") | ||
|
||
client.create_bucket(Bucket=versioned_bucket_name, ACL="public-read") | ||
client.put_bucket_versioning( | ||
Bucket=versioned_bucket_name, VersioningConfiguration={"Status": "Enabled"} | ||
) | ||
|
||
# initialize secure bucket | ||
client.create_bucket(Bucket=secure_bucket_name, ACL="public-read") | ||
policy = json.dumps( | ||
{ | ||
"Version": "2012-10-17", | ||
"Id": "PutObjPolicy", | ||
"Statement": [ | ||
{ | ||
"Sid": "DenyUnEncryptedObjectUploads", | ||
"Effect": "Deny", | ||
"Principal": "*", | ||
"Action": "s3:PutObject", | ||
"Resource": "arn:aws:s3:::{bucket_name}/*".format( | ||
bucket_name=secure_bucket_name | ||
), | ||
"Condition": { | ||
"StringNotEquals": { | ||
"s3:x-amz-server-side-encryption": "aws:kms" | ||
} | ||
}, | ||
} | ||
], | ||
} | ||
) | ||
client.put_bucket_policy(Bucket=secure_bucket_name, Policy=policy) | ||
|
||
S3FileSystem.clear_instance_cache() | ||
s3 = S3FileSystem(anon=False, client_kwargs={"endpoint_url": endpoint_uri}) | ||
s3.invalidate_cache() | ||
yield s3 | ||
|
||
@pytest.fixture | ||
def fs_path(self): | ||
return test_bucket_name | ||
|
||
def supports_empty_directories(self): | ||
return False | ||
|
||
@pytest.fixture(scope="class") | ||
def _get_boto3_client(self): | ||
from botocore.session import Session | ||
|
||
# NB: we use the sync botocore client for setup | ||
session = Session() | ||
return session.create_client("s3", endpoint_url=endpoint_uri) | ||
|
||
@pytest.fixture(scope="class") | ||
def _s3_base(self): | ||
# writable local S3 system | ||
import shlex | ||
import subprocess | ||
|
||
try: | ||
# should fail since we didn't start server yet | ||
r = requests.get(endpoint_uri) | ||
except: | ||
pass | ||
else: | ||
if r.ok: | ||
raise RuntimeError("moto server already up") | ||
if "AWS_SECRET_ACCESS_KEY" not in os.environ: | ||
os.environ["AWS_SECRET_ACCESS_KEY"] = "foo" | ||
if "AWS_ACCESS_KEY_ID" not in os.environ: | ||
os.environ["AWS_ACCESS_KEY_ID"] = "foo" | ||
proc = subprocess.Popen( | ||
shlex.split("moto_server s3 -p %s" % port), | ||
stderr=subprocess.DEVNULL, | ||
stdout=subprocess.DEVNULL, | ||
stdin=subprocess.DEVNULL, | ||
) | ||
|
||
timeout = 5 | ||
while timeout > 0: | ||
try: | ||
print("polling for moto server") | ||
r = requests.get(endpoint_uri) | ||
if r.ok: | ||
break | ||
except: | ||
pass | ||
timeout -= 0.1 | ||
time.sleep(0.1) | ||
print("server up") | ||
yield | ||
print("moto done") | ||
proc.terminate() | ||
proc.wait() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
import fsspec.tests.abstract as abstract | ||
from s3fs.tests.derived.s3fs_fixtures import S3fsFixtures | ||
|
||
|
||
class TestS3fsCopy(abstract.AbstractCopyTests, S3fsFixtures): | ||
pass | ||
|
||
|
||
class TestS3fsGet(abstract.AbstractGetTests, S3fsFixtures): | ||
pass | ||
|
||
|
||
class TestS3fsPut(abstract.AbstractPutTests, S3fsFixtures): | ||
pass |