avoid loading stale metadata in S3 persister

ottogroup · May 6, 2020 · ed6842d · ed6842d
1 parent a4efa8a
commit ed6842d
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/palladium/persistence.py b/palladium/persistence.py
@@ -712,6 +712,8 @@ def __init__(self, **kwargs):
         self.fs = s3fs.S3FileSystem(anon=False)
 
     def open(self, path, mode='r'):
+        # this is needed to avoid reading stale metadata JSONs
+        self.fs.invalidate_cache()
         return self.fs.open(path, mode=mode)
 
     def exists(self, path):