Merge pull request #35 from bochecha/updateinfo

Prefix the updateinfo file with its hash

bodhi/masher.py

@@ -14,6 +14,7 @@
 #
 # Authors: Luke Macken <lmacken@redhat.com>
 
+import glob
 import os
 import sha
 import time
@@ -949,9 +950,21 @@ def generate_updateinfo(self):
         self.genmd = True
         t0 = time.time()
         for repo, mashdir in self.mashed_repos.items():
-            olduinfo = join(config.get('mashed_dir'), '%s.repodata' % repo,
-                            'i386', 'updateinfo.xml.gz')
-            olduinfo = exists(olduinfo) and olduinfo or None
+            # File name is prefixed with a hash, use a glob to find it
+            olduinfos = glob.glob(os.path.join(config.get('mashed_dir'),
+                                               '%s.repodata' % repo, 'i386',
+                                               "*updateinfo.xml.gz"))
+
+            if len(olduinfos) > 1:
+                # TODO: Shouldn't that be an error?
+                olduinfo = olduinfos[0]
+
+            if len(olduinfos) == 1:
+                olduinfo = olduinfos[0]
+
+            else:
+                olduinfo = None
+
             repo = join(mashdir, repo)
             log.debug("Generating updateinfo.xml.gz for %s" % repo)
             uinfo = ExtendedMetadata(repo, olduinfo)

bodhi/metadata.py

@@ -53,6 +53,10 @@ def __init__(self, repo, cacheduinfo=None):
             log.debug("Loading cached updateinfo.xml.gz")
             umd = UpdateMetadata()
             umd.add(cacheduinfo)
+
+            # Drop the old cached updateinfo.xml.gz, it's unneeded now
+            os.unlink(cacheduinfo)
+
             existing_ids = set([up['update_id'] for up in umd.get_notices()])
             seen_ids = set()
             from_cache = set()

bodhi/modifyrepo.py

@@ -28,7 +28,7 @@
 import sys
 import gzip
 
-from hashlib import sha1 as sha
+from hashlib import sha256
 from xml.dom import minidom
 from kitchen.text.converters import to_bytes
 
@@ -83,13 +83,20 @@ def add(self, metadata):
         newmd = gzip.GzipFile(destmd, 'wb')
         newmd.write(to_bytes(md, errors='ignore', non_string='passthru'))
         newmd.close()
-        print "Wrote:", destmd
 
         ## Read the gzipped metadata
         f = file(destmd, 'r')
         newmd = f.read()
         f.close()
 
+        ## Prefix the file name with its hash
+        hashed_md = sha256(newmd).hexdigest()
+        hashed_mdname = "%s-%s" % (hashed_md, mdname)
+        hashed_destmd = os.path.join(self.repodir, hashed_mdname)
+        os.rename(destmd, hashed_destmd)
+
+        print "Wrote:", hashed_destmd
+
         ## Remove any stale metadata
         for elem in self.doc.getElementsByTagName('data'):
             if elem.attributes['type'].value == mdtype:
@@ -98,19 +105,13 @@ def add(self, metadata):
         root = self.doc.firstChild
         data = self._insert_element(root, 'data', attrs={'type' : mdtype})
         self._insert_element(data, 'location',
-                             attrs={'href' : 'repodata/' + mdname})
-        self._insert_element(data, 'checksum', attrs={'type' : 'sha'},
-                             text=sha(newmd).hexdigest())
+                             attrs={'href' : 'repodata/' + hashed_mdname})
+        self._insert_element(data, 'checksum', attrs={'type' : 'sha256'},
+                             text=hashed_md)
         self._insert_element(data, 'timestamp',
-                             text=str(os.stat(destmd).st_mtime))
-        self._insert_element(data, 'open-checksum', attrs={'type' : 'sha'},
-                             text=sha(to_bytes(md, errors='ignore', non_string='passthru')).hexdigest())
-
-        #print "           type =", mdtype
-        #print "       location =", 'repodata/' + mdname
-        #print "       checksum =", sha(newmd).hexdigest()
-        #print "      timestamp =", str(os.stat(destmd).st_mtime)
-        #print "  open-checksum =", sha(md).hexdigest()
+                             text=str(os.stat(hashed_destmd).st_mtime))
+        self._insert_element(data, 'open-checksum', attrs={'type' : 'sha256'},
+                             text=sha256(to_bytes(md, errors='ignore', non_string='passthru')).hexdigest())
 
         ## Write the updated repomd.xml
         outmd = file(self.repomdxml, 'w')

bodhi/tests/test_controllers.py

@@ -1487,7 +1487,7 @@ def test_admin_push(self):
         me.addGroup(testers)
         create_release()
         testutil.create_request('/updates/admin/push', headers=session)
-        assert '0 pending requests' in cherrypy.response.body[0]
+        assert '0 pending requests' in cherrypy.response.body[0], cherrypy.response.body[0]
         params = {
                 'builds'  : 'TurboGears-2.6.23.1-21.fc7',
                 'release' : 'Fedora 7',

bodhi/tests/test_metadata.py

@@ -5,11 +5,13 @@
 turbogears.update_config(configfile='bodhi.cfg', modulename='bodhi.config')
 database.set_db_uri("sqlite:///:memory:")
 
+import glob
 import shutil
 import tempfile
 
 from datetime import datetime
-from os.path import join, exists
+from hashlib import sha256
+from os.path import join, exists, basename
 from datetime import datetime
 from bodhi.util import mkmetadatadir, get_nvr
 from bodhi.model import (Release, Package, PackageUpdate, Bugzilla, CVE,
@@ -20,6 +22,21 @@
 
 
 class TestExtendedMetadata(testutil.DBTest):
+    def __verify_updateinfo(self, repodata):
+        """Verify the updateinfo file
+
+        This is not a test, just a helper function.
+        """
+        updateinfos = glob.glob(join(repodata, "*-updateinfo.xml.gz"))
+        assert len(updateinfos) == 1, "We generated %d updateinfo metadata" % len(updateinfos)
+
+        updateinfo = updateinfos[0]
+        hash = basename(updateinfo).split("-", 1)[0]
+        hashed = sha256(open(updateinfo).read()).hexdigest()
+
+        assert hash == hashed, "File: %s\nHash: %s" % (basename(updateinfo), hashed)
+
+        return updateinfo
 
     def test_extended_metadata(self):
         # grab the name of a build in updates-testing, and create it in our db
@@ -58,8 +75,7 @@ def test_extended_metadata(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -131,8 +147,7 @@ def test_extended_metadata_updating(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -167,8 +182,7 @@ def test_extended_metadata_updating(self):
         ## Test out updateinfo.xml updating via our ExtendedMetadata
         md = ExtendedMetadata(temprepo, updateinfo)
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -234,8 +248,7 @@ def test_extended_metadata_updating_with_edited_update_details(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -262,8 +275,7 @@ def test_extended_metadata_updating_with_edited_update_details(self):
         ## Test out updateinfo.xml updating via our ExtendedMetadata
         md = ExtendedMetadata(temprepo, updateinfo)
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -315,8 +327,7 @@ def test_extended_metadata_updating_with_edited_updates(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -349,8 +360,7 @@ def test_extended_metadata_updating_with_edited_updates(self):
         ## Test out updateinfo.xml updating via our ExtendedMetadata
         md = ExtendedMetadata(temprepo, updateinfo)
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -405,8 +415,7 @@ def test_extended_metadata_updating_with_old_testing_security(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         # Create a new non-security update for the same package
         newbuild = 'TurboGears-1.0.2.2-3.fc7'
@@ -425,7 +434,7 @@ def test_extended_metadata_updating_with_old_testing_security(self):
         ## Test out updateinfo.xml updating via our ExtendedMetadata
         md = ExtendedMetadata(temprepo, updateinfo)
         md.insert_updateinfo()
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()
@@ -483,8 +492,7 @@ def test_extended_metadata_updating_with_old_stable_security(self):
 
         ## Insert the updateinfo.xml into the repository
         md.insert_updateinfo()
-        updateinfo = join(repodata, 'updateinfo.xml.gz')
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         # Create a new non-security update for the same package
         newbuild = 'TurboGears-1.0.2.2-3.fc7'
@@ -503,7 +511,7 @@ def test_extended_metadata_updating_with_old_stable_security(self):
         ## Test out updateinfo.xml updating via our ExtendedMetadata
         md = ExtendedMetadata(temprepo, updateinfo)
         md.insert_updateinfo()
-        assert exists(updateinfo)
+        updateinfo = self.__verify_updateinfo(repodata)
 
         ## Read an verify the updateinfo.xml.gz
         uinfo = UpdateMetadata()