Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

test_web.py: fix memory leak when run with --until-failure

The Fake*Node classes in test/common.py were accumulating share data in
a class-level dictionary, which persisted from one test run to the next.
As a result, running test_web.py over and over (with trial's
--until-failure feature) made this dictionary grow without bound,
eventually running out of memory.

This fix moves that dictionary into the FakeClient built fresh for each
test, so it doesn't build up. It does the same thing for "file_types",
which was much smaller but still lived at the class level.

Closes #1729
  • Loading branch information...
commit bfee999e20aa9fdc0b05c7d72cf11077cb319d78 1 parent bcdfb58
Brian Warner authored May 22, 2012
31  src/allmydata/test/common.py
@@ -6,7 +6,7 @@
6 6
 from twisted.application import service
7 7
 from twisted.web.error import Error as WebError
8 8
 from foolscap.api import flushEventualQueue, fireEventually
9  
-from allmydata import uri, dirnode, client
  9
+from allmydata import uri, client
10 10
 from allmydata.introducer.server import IntroducerNode
11 11
 from allmydata.interfaces import IMutableFileNode, IImmutableFileNode,\
12 12
                                  NotEnoughSharesError, ICheckable, \
@@ -43,10 +43,10 @@ class FakeCHKFileNode:
43 43
     """I provide IImmutableFileNode, but all of my data is stored in a
44 44
     class-level dictionary."""
45 45
     implements(IImmutableFileNode)
46  
-    all_contents = {}
47 46
 
48  
-    def __init__(self, filecap):
  47
+    def __init__(self, filecap, all_contents):
49 48
         precondition(isinstance(filecap, (uri.CHKFileURI, uri.LiteralFileURI)), filecap)
  49
+        self.all_contents = all_contents
50 50
         self.my_uri = filecap
51 51
         self.storage_index = self.my_uri.get_storage_index()
52 52
 
@@ -167,10 +167,10 @@ def make_chk_file_cap(size):
167 167
 def make_chk_file_uri(size):
168 168
     return make_chk_file_cap(size).to_string()
169 169
 
170  
-def create_chk_filenode(contents):
  170
+def create_chk_filenode(contents, all_contents):
171 171
     filecap = make_chk_file_cap(len(contents))
172  
-    n = FakeCHKFileNode(filecap)
173  
-    FakeCHKFileNode.all_contents[filecap.to_string()] = contents
  172
+    n = FakeCHKFileNode(filecap, all_contents)
  173
+    all_contents[filecap.to_string()] = contents
174 174
     return n
175 175
 
176 176
 
@@ -180,11 +180,11 @@ class FakeMutableFileNode:
180 180
 
181 181
     implements(IMutableFileNode, ICheckable)
182 182
     MUTABLE_SIZELIMIT = 10000
183  
-    all_contents = {}
184  
-    file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
185 183
 
186 184
     def __init__(self, storage_broker, secret_holder,
187  
-                 default_encoding_parameters, history):
  185
+                 default_encoding_parameters, history, all_contents):
  186
+        self.all_contents = all_contents
  187
+        self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
188 188
         self.init_from_cap(make_mutable_file_cap())
189 189
         self._k = default_encoding_parameters['k']
190 190
         self._segsize = default_encoding_parameters['max_segment_size']
@@ -402,7 +402,7 @@ def make_verifier_uri():
402 402
     return uri.SSKVerifierURI(storage_index=os.urandom(16),
403 403
                               fingerprint=os.urandom(32)).to_string()
404 404
 
405  
-def create_mutable_filenode(contents, mdmf=False):
  405
+def create_mutable_filenode(contents, mdmf=False, all_contents=None):
406 406
     # XXX: All of these arguments are kind of stupid. 
407 407
     if mdmf:
408 408
         cap = make_mdmf_mutable_file_cap()
@@ -413,7 +413,8 @@ def create_mutable_filenode(contents, mdmf=False):
413 413
     encoding_params['k'] = 3
414 414
     encoding_params['max_segment_size'] = 128*1024
415 415
 
416  
-    filenode = FakeMutableFileNode(None, None, encoding_params, None)
  416
+    filenode = FakeMutableFileNode(None, None, encoding_params, None,
  417
+                                   all_contents)
417 418
     filenode.init_from_cap(cap)
418 419
     if mdmf:
419 420
         filenode.create(MutableData(contents), version=MDMF_VERSION)
@@ -422,14 +423,6 @@ def create_mutable_filenode(contents, mdmf=False):
422 423
     return filenode
423 424
 
424 425
 
425  
-class FakeDirectoryNode(dirnode.DirectoryNode):
426  
-    """This offers IDirectoryNode, but uses a FakeMutableFileNode for the
427  
-    backing store, so it doesn't go to the grid. The child data is still
428  
-    encrypted and serialized, so this isn't useful for tests that want to
429  
-    look inside the dirnodes and check their contents.
430  
-    """
431  
-    filenode_class = FakeMutableFileNode
432  
-
433 426
 class LoggingServiceParent(service.MultiService):
434 427
     def log(self, *args, **kwargs):
435 428
         return log.msg(*args, **kwargs)
44  src/allmydata/test/test_web.py
@@ -64,16 +64,17 @@ class FakeNodeMaker(NodeMaker):
64 64
         'max_segment_size':128*1024 # 1024=KiB
65 65
     }
66 66
     def _create_lit(self, cap):
67  
-        return FakeCHKFileNode(cap)
  67
+        return FakeCHKFileNode(cap, self.all_contents)
68 68
     def _create_immutable(self, cap):
69  
-        return FakeCHKFileNode(cap)
  69
+        return FakeCHKFileNode(cap, self.all_contents)
70 70
     def _create_mutable(self, cap):
71  
-        return FakeMutableFileNode(None,
72  
-                                   None,
73  
-                                   self.encoding_params, None).init_from_cap(cap)
  71
+        return FakeMutableFileNode(None, None,
  72
+                                   self.encoding_params, None,
  73
+                                   self.all_contents).init_from_cap(cap)
74 74
     def create_mutable_file(self, contents="", keysize=None,
75 75
                             version=SDMF_VERSION):
76  
-        n = FakeMutableFileNode(None, None, self.encoding_params, None)
  76
+        n = FakeMutableFileNode(None, None, self.encoding_params, None,
  77
+                                self.all_contents)
77 78
         return n.create(contents, version=version)
78 79
 
79 80
 class FakeUploader(service.Service):
@@ -83,7 +84,7 @@ def upload(self, uploadable):
83 84
         d.addCallback(lambda size: uploadable.read(size))
84 85
         def _got_data(datav):
85 86
             data = "".join(datav)
86  
-            n = create_chk_filenode(data)
  87
+            n = create_chk_filenode(data, self.all_contents)
87 88
             ur = upload.UploadResults(file_size=len(data),
88 89
                                       ciphertext_fetched=0,
89 90
                                       preexisting_shares=0,
@@ -176,6 +177,7 @@ def __init__(self):
176 177
         # don't upcall to Client.__init__, since we only want to initialize a
177 178
         # minimal subset
178 179
         service.MultiService.__init__(self)
  180
+        self.all_contents = {}
179 181
         self.nodeid = "fake_nodeid"
180 182
         self.nickname = "fake_nickname"
181 183
         self.introducer_furl = "None"
@@ -187,11 +189,13 @@ def __init__(self):
187 189
         self.introducer_client = None
188 190
         self.history = FakeHistory()
189 191
         self.uploader = FakeUploader()
  192
+        self.uploader.all_contents = self.all_contents
190 193
         self.uploader.setServiceParent(self)
191 194
         self.blacklist = None
192 195
         self.nodemaker = FakeNodeMaker(None, self._secret_holder, None,
193 196
                                        self.uploader, None,
194 197
                                        None, None, None)
  198
+        self.nodemaker.all_contents = self.all_contents
195 199
         self.mutable_file_default = SDMF_VERSION
196 200
 
197 201
     def startService(self):
@@ -270,7 +274,7 @@ def _then(res):
270 274
 
271 275
             _ign, n, self._bad_file_uri = self.makefile(3)
272 276
             # this uri should not be downloadable
273  
-            del FakeCHKFileNode.all_contents[self._bad_file_uri]
  277
+            del self.s.all_contents[self._bad_file_uri]
274 278
 
275 279
             rodir = res[5][1]
276 280
             self.public_root.set_uri(u"reedownlee", rodir.get_readonly_uri(),
@@ -297,14 +301,17 @@ def _got_metadata(metadata):
297 301
         d.addCallback(_got_metadata)
298 302
         return d
299 303
 
  304
+    def get_all_contents(self):
  305
+        return self.s.all_contents
  306
+
300 307
     def makefile(self, number):
301 308
         contents = "contents of file %s\n" % number
302  
-        n = create_chk_filenode(contents)
  309
+        n = create_chk_filenode(contents, self.get_all_contents())
303 310
         return contents, n, n.get_uri()
304 311
 
305 312
     def makefile_mutable(self, number, mdmf=False):
306 313
         contents = "contents of mutable file %s\n" % number
307  
-        n = create_mutable_filenode(contents, mdmf)
  314
+        n = create_mutable_filenode(contents, mdmf, self.s.all_contents)
308 315
         return contents, n, n.get_uri(), n.get_readonly_uri()
309 316
 
310 317
     def tearDown(self):
@@ -1997,7 +2004,7 @@ def _check(child):
1997 2004
         return d
1998 2005
 
1999 2006
     def failUnlessCHKURIHasContents(self, got_uri, contents):
2000  
-        self.failUnless(FakeCHKFileNode.all_contents[got_uri] == contents)
  2007
+        self.failUnless(self.get_all_contents()[got_uri] == contents)
2001 2008
 
2002 2009
     def test_POST_upload(self):
2003 2010
         d = self.POST(self.public_url + "/foo", t="upload",
@@ -2100,7 +2107,7 @@ def _check(filecap):
2100 2107
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
2101 2108
             self.filecap = filecap
2102 2109
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
2103  
-            self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
  2110
+            self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
2104 2111
             n = self.s.create_node_from_uri(filecap)
2105 2112
             return n.download_best_version()
2106 2113
         d.addCallback(_check)
@@ -2929,7 +2936,8 @@ def _create_initial_children(self):
2929 2936
     def _create_immutable_children(self):
2930 2937
         contents, n, filecap1 = self.makefile(12)
2931 2938
         md1 = {"metakey1": "metavalue1"}
2932  
-        tnode = create_chk_filenode("immutable directory contents\n"*10)
  2939
+        tnode = create_chk_filenode("immutable directory contents\n"*10,
  2940
+                                    self.get_all_contents())
2933 2941
         dnode = DirectoryNode(tnode, None, None)
2934 2942
         assert not dnode.is_mutable()
2935 2943
         immdircap = dnode.get_uri()
@@ -3724,8 +3732,8 @@ def test_PUT_NEWFILE_URI(self):
3724 3732
         d = self.PUT("/uri", file_contents)
3725 3733
         def _check(uri):
3726 3734
             assert isinstance(uri, str), uri
3727  
-            self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3728  
-            self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
  3735
+            self.failUnlessIn(uri, self.get_all_contents())
  3736
+            self.failUnlessReallyEqual(self.get_all_contents()[uri],
3729 3737
                                        file_contents)
3730 3738
             return self.GET("/uri/%s" % uri)
3731 3739
         d.addCallback(_check)
@@ -3739,8 +3747,8 @@ def test_PUT_NEWFILE_URI_not_mutable(self):
3739 3747
         d = self.PUT("/uri?mutable=false", file_contents)
3740 3748
         def _check(uri):
3741 3749
             assert isinstance(uri, str), uri
3742  
-            self.failUnlessIn(uri, FakeCHKFileNode.all_contents)
3743  
-            self.failUnlessReallyEqual(FakeCHKFileNode.all_contents[uri],
  3750
+            self.failUnlessIn(uri, self.get_all_contents())
  3751
+            self.failUnlessReallyEqual(self.get_all_contents()[uri],
3744 3752
                                        file_contents)
3745 3753
             return self.GET("/uri/%s" % uri)
3746 3754
         d.addCallback(_check)
@@ -3765,7 +3773,7 @@ def _check1(filecap):
3765 3773
             self.failUnless(filecap.startswith("URI:SSK:"), filecap)
3766 3774
             self.filecap = filecap
3767 3775
             u = uri.WriteableSSKFileURI.init_from_string(filecap)
3768  
-            self.failUnlessIn(u.get_storage_index(), FakeMutableFileNode.all_contents)
  3776
+            self.failUnlessIn(u.get_storage_index(), self.get_all_contents())
3769 3777
             n = self.s.create_node_from_uri(filecap)
3770 3778
             return n.download_best_version()
3771 3779
         d.addCallback(_check1)

0 notes on commit bfee999

Please sign in to comment.
Something went wrong with that request. Please try again.