Fix high memory issue when calculating metadata open checksum

closes #7480 https://pulp.plan.io/issues/7480
pulp · Sep 9, 2020 · 5b546ca · 5b546ca
1 parent 97b3748
commit 5b546ca
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 5 deletions.
diff --git a/plugins/pulp_rpm/plugins/distributors/yum/metadata/repomd.py b/plugins/pulp_rpm/plugins/distributors/yum/metadata/repomd.py
@@ -129,10 +129,18 @@ def add_metadata_file_metadata(self, data_type, file_path, precalculated_checksu
 
             else:
                 try:
-                    content = file_handle.read()
-                    open_size_element.text = str(len(content))
-                    open_checksum_element.text = self.checksum_constructor(content).hexdigest()
-
+                    open_size = 0
+                    checksum_const = self.checksum_constructor()
+                    while True:
+                        # Read 1MB each time to save memory
+                        content = file_handle.read(1048576)
+                        if not content:
+                            break
+                        open_size += len(content)
+                        checksum_const.update(content)
+
+                    open_size_element.text = str(open_size)
+                    open_checksum_element.text = checksum_const.hexdigest()
                 finally:
                     file_handle.close()
 

diff --git a/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py b/plugins/test/unit/plugins/distributors/yum/metadata/test_metadata.py
@@ -484,7 +484,10 @@ def test_repomd_metadata_file_metadata(self, mock_getmtime):
             self.assertEqual(content.count('<checksum type="sha256">'), 1)
             self.assertEqual(
                 content.count('<open-size>%s</open-size>' % len(test_metadata_content)), 1)
-            self.assertEqual(content.count('<open-checksum type="sha256">'), 1)
+            self.assertEqual(content.count(
+                '<open-checksum type="sha256">'
+                'd7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592'
+                '</open-checksum>'), 1)
 
     @mock.patch("pulp_rpm.yum_plugin.util.Signer")
     def test_finalize_sign(self, _signer):