Skip to content
This repository
Browse code

fix StorageUri and file.Key file descriptor leaks

  • Loading branch information...
commit 21323833ae6d423832f6a3ff0049c41632070e50 1 parent bcb7199
Mike Schwartz authored
4  boto/connection.py
@@ -242,7 +242,9 @@ def size(self):
242 242
     def get_http_connection(self, host, is_secure):
243 243
         """
244 244
         Gets a connection from the pool for the named host.  Returns
245  
-        None if there is no connection that can be reused.
  245
+        None if there is no connection that can be reused. It's the caller's
  246
+        responsibility to call close() on the connection when it's no longer
  247
+        needed.
246 248
         """
247 249
         self.clean()
248 250
         with self.mutex:
20  boto/file/key.py
@@ -74,7 +74,10 @@ def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False):
74 74
             key_file = self.fp
75 75
         else:
76 76
             key_file = open(self.full_path, 'rb')
77  
-        shutil.copyfileobj(key_file, fp)
  77
+        try:
  78
+            shutil.copyfileobj(key_file, fp)
  79
+        finally:
  80
+            key_file.close()
78 81
 
79 82
     def set_contents_from_file(self, fp, headers=None, replace=True, cb=None,
80 83
                                num_cb=10, policy=None, md5=None):
@@ -119,8 +122,10 @@ def set_contents_from_file(self, fp, headers=None, replace=True, cb=None,
119 122
             if not replace and os.path.exists(self.full_path):
120 123
                 return
121 124
             key_file = open(self.full_path, 'wb')
122  
-        shutil.copyfileobj(fp, key_file)
123  
-        key_file.close()
  125
+        try:
  126
+            shutil.copyfileobj(fp, key_file)
  127
+        finally:
  128
+            key_file.close()
124 129
 
125 130
     def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
126 131
                                torrent=False):
@@ -152,3 +157,12 @@ def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
152 157
 
153 158
     def is_stream(self):
154 159
         return (self.key_type & self.KEY_STREAM)
  160
+
  161
+    def close(self):
  162
+        """
  163
+        Closes fp associated with underlying file.
  164
+        Caller should call this method when done with this class, to avoid
  165
+        using up OS resources (e.g., when iterating over a large number
  166
+        of files).
  167
+        """
  168
+        self.fp.close()
18  boto/storage_uri.py
@@ -41,6 +41,13 @@ class StorageUri(object):
41 41
     # https_connection_factory).
42 42
     connection_args = None
43 43
 
  44
+    # Map of provider scheme ('s3' or 'gs') to AWSAuthConnection object. We
  45
+    # maintain a pool here in addition to the connection pool implemented
  46
+    # in AWSAuthConnection because the latter re-creates its connection pool
  47
+    # every time that class is instantiated (so the current pool is used to
  48
+    # avoid re-instantiating AWSAuthConnection).
  49
+    provider_pool = {}
  50
+
44 51
     def __init__(self):
45 52
         """Uncallable constructor on abstract base StorageUri class.
46 53
         """
@@ -84,16 +91,20 @@ def connect(self, access_key_id=None, secret_access_key=None, **kwargs):
84 91
         connection_args['calling_format'] = OrdinaryCallingFormat()
85 92
         connection_args.update(kwargs)
86 93
         if not self.connection:
87  
-            if self.scheme == 's3':
  94
+            if self.scheme in self.provider_pool:
  95
+                self.connection = self.provider_pool[self.scheme]
  96
+            elif self.scheme == 's3':
88 97
                 from boto.s3.connection import S3Connection
89 98
                 self.connection = S3Connection(access_key_id,
90 99
                                                secret_access_key,
91 100
                                                **connection_args)
  101
+                self.provider_pool[self.scheme] = self.connection
92 102
             elif self.scheme == 'gs':
93 103
                 from boto.gs.connection import GSConnection
94 104
                 self.connection = GSConnection(access_key_id,
95 105
                                                secret_access_key,
96 106
                                                **connection_args)
  107
+                self.provider_pool[self.scheme] = self.connection
97 108
             elif self.scheme == 'file':
98 109
                 from boto.file.connection import FileConnection
99 110
                 self.connection = FileConnection(self)
@@ -493,3 +504,8 @@ def is_stream(self):
493 504
         """Retruns True if this URI represents input/output stream.
494 505
         """
495 506
         return self.stream
  507
+
  508
+    def close(self):
  509
+        """Closes the underlying file.
  510
+        """
  511
+        self.get_key().close()

0 notes on commit 2132383

Please sign in to comment.
Something went wrong with that request. Please try again.