diff --git a/pfio/v2/s3.py b/pfio/v2/s3.py index 92eaa48b..28c27382 100644 --- a/pfio/v2/s3.py +++ b/pfio/v2/s3.py @@ -30,6 +30,16 @@ def isdir(self): return False +class S3PrefixStat(FileStat): + def __init__(self, key): + self.filename = key + self.last_modified = 0 + self.size = -1 + + def isdir(self): + return True + + class _ObjectReader: def __init__(self, client, bucket, key, mode, kwargs): self.client = client @@ -398,29 +408,51 @@ def stat(self, path): return S3ObjectStat(key, res) except ClientError as e: if e.response['Error']['Code'] == '404': + if self.isdir(path): + return S3PrefixStat(key) raise FileNotFoundError() else: raise e def isdir(self, file_path: str): + '''Imitate isdir by handling common prefix ending with "/" as directory + + AWS S3 does not have concept of directory tree, but this class + imitates other file systems to increase compatibility. + ''' + self._checkfork() + key = _normalize_key(os.path.join(self.cwd, file_path)) + if key == '.': + key = '' + elif key.endswith('/'): + key = key[:-1] + if '/../' in key or key.startswith('..'): + raise ValueError('Invalid S3 key: {} as {}'.format(file_path, key)) + + if len(key) == 0: + return True + + res = self.client.list_objects_v2( + Bucket=self.bucket, + Prefix=key, + Delimiter="/", + MaxKeys=1, + ) + for common_prefix in res.get('CommonPrefixes', []): + if common_prefix['Prefix'] == key + "/": + return True + return False + + def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): '''Does nothing .. note:: AWS S3 does not have concept of directory tree; what - this function (and ``mkdir()`` and ``makedirs()`` should do + this function (and ``makedirs()``) should do and return? To be strict, it would be straightforward to raise ``io.UnsupportedOperation`` exception. But it just breaks users' applications that except quasi-compatible behaviour. Thus, imitating other file systems, like - returning boolean or ``None`` would be nicer. - - ''' - # raise io.UnsupportedOperation("S3 doesn't have directory") - pass - - def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): - '''Does nothing - - .. note:: see discussion in ``isdir()``. + returning ``None`` would be nicer. ''' # raise io.UnsupportedOperation("S3 doesn't have directory") pass @@ -428,7 +460,7 @@ def mkdir(self, file_path: str, mode=0o777, *args, dir_fd=None): def makedirs(self, file_path: str, mode=0o777, exist_ok=False): '''Does nothing - .. note:: see discussion in ``isdir()``. + .. note:: see discussion in ``mkdir()``. ''' # raise io.UnsupportedOperation("S3 doesn't have directory") pass @@ -447,6 +479,8 @@ def exists(self, file_path: str): return not res.get('DeleteMarker') except ClientError as e: if e.response['Error']['Code'] == '404': + if self.isdir(file_path): + return True return False else: raise e diff --git a/tests/v2_tests/test_s3.py b/tests/v2_tests/test_s3.py index 498518cd..00920fc2 100644 --- a/tests/v2_tests/test_s3.py +++ b/tests/v2_tests/test_s3.py @@ -57,6 +57,12 @@ def test_s3(): assert ['dir/', 'foo.txt'] == list(s3.list()) + assert not s3.isdir("foo.txt") + assert s3.isdir(".") + assert s3.isdir("/base/") + assert s3.isdir("/base") + assert not s3.isdir("/bas") + def f(s3): try: s3.open('foo.txt', 'r')