Downloading public files with no_sign_request throws error

[pjbull]

Our downloading code checks if an S3Path is a file or directory in order to know how to process the download. This calls _s3_file_query which checks for existing using Bucket.objects.filter.

This call requires list permissions to the bucket, so we need another way to check if an S3Path is a file to support downloading assets without S3 credentials configured.

Here's an example traceback:

----> 1 download_weights(filename="zamba_time_distributed.ckpt", destination_dir=Path.cwd())

~/zamba-algorithms/zamba_algorithms/models/utils.py in download_weights(filename, weight_region, destination_dir)
     38         client=S3Client(local_cache_dir=destination_dir, no_sign_request=True),
     39     )
---> 40     return s3p.fspath

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/cloudpath.py in fspath(self)
    304     @property
    305     def fspath(self) -> str:
--> 306         return self.__fspath__()
    307 
    308     def glob(self, pattern: str) -> Iterable["CloudPath"]:

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/cloudpath.py in __fspath__(self)
    208 
    209     def __fspath__(self):
--> 210         if self.is_file():
    211             self._refresh_cache(force_overwrite_from_cloud=False)
    212         return str(self._local)

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/s3/s3path.py in is_file(self)
     37 
     38     def is_file(self) -> bool:
---> 39         return self.client._is_file_or_dir(self) == "file"
     40 
     41     def mkdir(self, parents=False, exist_ok=False):

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/s3/s3client.py in _is_file_or_dir(self, cloud_path)
    117 
    118         # get first item by listing at least one key
--> 119         s3_obj = self._s3_file_query(cloud_path)
    120 
    121         if s3_obj is None:

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/s3/s3client.py in _s3_file_query(self, cloud_path)
    131     def _s3_file_query(self, cloud_path: S3Path):
    132         """Boto3 query used for quick checks of existence and if path is file/dir"""
--> 133         return next(
    134             (
    135                 obj

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/cloudpathlib/s3/s3client.py in <genexpr>(.0)
    132         """Boto3 query used for quick checks of existence and if path is file/dir"""
    133         return next(
--> 134             (
    135                 obj
    136                 for obj in (

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/boto3/resources/collection.py in __iter__(self)
     81 
     82         count = 0
---> 83         for page in self.pages():
     84             for item in page:
     85                 yield item

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/boto3/resources/collection.py in pages(self)
    164         # we start processing and yielding individual items.
    165         count = 0
--> 166         for page in pages:
    167             page_items = []
    168             for item in self._handler(self._parent, params, page):

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/botocore/paginate.py in __iter__(self)
    253         self._inject_starting_params(current_kwargs)
    254         while True:
--> 255             response = self._make_request(current_kwargs)
    256             parsed = self._extract_parsed_response(response)
    257             if first_request:

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/botocore/paginate.py in _make_request(self, current_kwargs)
    330 
    331     def _make_request(self, current_kwargs):
--> 332         return self._method(**current_kwargs)
    333 
    334     def _extract_parsed_response(self, response):

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
    384                     "%s() only accepts keyword arguments." % py_operation_name)
    385             # The "self" in this scope is referring to the BaseClient.
--> 386             return self._make_api_call(operation_name, kwargs)
    387 
    388         _api_call.__name__ = str(py_operation_name)

~/anaconda3/envs/zamba-algorithms/lib/python3.8/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
    703             error_code = parsed_response.get("Error", {}).get("Code")
    704             error_class = self.exceptions.from_code(error_code)
--> 705             raise error_class(parsed_response, operation_name)
    706         else:
    707             return parsed_response

ClientError: An error occurred (AccessDenied) when calling the ListObjects operation: Access Denied