Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move all use of s3path's configuration_map to public API #1724

Merged
merged 4 commits into from
May 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion requirements_s3.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ boto3==1.34.103
botocore==1.34.103
jmespath==1.0.1
python-dateutil==2.9.0.post0
s3path==0.5.6
s3path==0.5.7
s3transfer==0.10.1
six==1.16.0
smart-open==7.0.4
Expand Down
10 changes: 8 additions & 2 deletions src/bandersnatch/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,13 @@
from _pytest.capture import CaptureFixture
from _pytest.fixtures import FixtureRequest
from _pytest.monkeypatch import MonkeyPatch
from s3path import PureS3Path, S3Path, accessor, register_configuration_parameter
from s3path import (
PureS3Path,
S3Path,
accessor,
configuration_map,
register_configuration_parameter,
)

if TYPE_CHECKING:
from bandersnatch.master import Master
Expand Down Expand Up @@ -202,7 +208,7 @@ def s3_mock(reset_configuration_cache: None) -> S3Path:
new_bucket = S3Path("/test-bucket")
new_bucket.mkdir(exist_ok=True)
yield new_bucket
resource, _ = new_bucket._accessor.configuration_map.get_configuration(new_bucket)
resource, _ = configuration_map.get_configuration(new_bucket)
bucket = resource.Bucket(new_bucket.bucket)
for key in bucket.objects.all():
key.delete()
Expand Down
10 changes: 5 additions & 5 deletions src/bandersnatch_storage_plugins/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from botocore.client import Config
from s3path import PureS3Path
from s3path import S3Path as _S3Path
from s3path import register_configuration_parameter
from s3path import configuration_map, register_configuration_parameter

if TYPE_CHECKING:
from s3path.accessor import _S3DirEntry
Expand All @@ -37,7 +37,7 @@ def mkdir(

def glob(self, pattern: str) -> Iterator[S3Path]:
bucket_name = self.bucket
resource, _ = self._accessor.configuration_map.get_configuration(self)
resource, _ = configuration_map.get_configuration(self)
bucket = resource.Bucket(bucket_name)

kwargs = {
Expand Down Expand Up @@ -261,7 +261,7 @@ def copy_file(self, source: PATH_TYPES, dest: PATH_TYPES) -> None:
dest = self.PATH_BACKEND(dest)
if not self.exists(source):
raise FileNotFoundError(source)
resource, _ = source._accessor.configuration_map.get_configuration(source)
resource, _ = configuration_map.get_configuration(source)
client = resource.meta.client
client.copy_object(
Key=dest.key,
Expand Down Expand Up @@ -422,7 +422,7 @@ def get_file_size(self, path: PATH_TYPES) -> int:
def get_upload_time(self, path: PATH_TYPES) -> datetime.datetime:
if not isinstance(path, self.PATH_BACKEND):
path = self.create_path_backend(path)
resource, _ = path._accessor.configuration_map.get_configuration(path)
resource, _ = configuration_map.get_configuration(path)
s3object = resource.Object(path.bucket, str(path.key))
ts = s3object.metadata.get(self.UPLOAD_TIME_METADATA_KEY, 0)
if not isinstance(ts, int):
Expand All @@ -432,7 +432,7 @@ def get_upload_time(self, path: PATH_TYPES) -> datetime.datetime:
def set_upload_time(self, path: PATH_TYPES, time: datetime.datetime) -> None:
if not isinstance(path, self.PATH_BACKEND):
path = self.create_path_backend(path)
resource, _ = path._accessor.configuration_map.get_configuration(path)
resource, _ = configuration_map.get_configuration(path)
s3object = resource.Object(path.bucket, str(path.key))
s3object.metadata.update({self.UPLOAD_TIME_METADATA_KEY: str(time.timestamp())})
# s3 does not support editing metadata after upload, it can be done better.
Expand Down
Loading