Create a separate boto3 session for each file store thread

This prevents threading problems, see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html#multithreading-multiprocessing.
djmaze · Sep 13, 2018 · 05d9bf6 · 05d9bf6
1 parent 503fd7b
commit 05d9bf6
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/s3_storage_provider.py b/s3_storage_provider.py
@@ -54,7 +54,8 @@ def store_file(self, path, file_info):
         """See StorageProvider.store_file"""
 
         def _store_file():
-            boto3.resource('s3', **self.api_kwargs).Bucket(self.bucket).upload_file(
+            session = boto3.session.Session()
+            session.resource('s3', **self.api_kwargs).Bucket(self.bucket).upload_file(
                 Filename=os.path.join(self.cache_directory, path),
                 Key=path,
                 ExtraArgs={"StorageClass": self.storage_class},