Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Prefix the updateinfo file with its hash #35

Merged
merged 7 commits into from Mar 12, 2014
Merged

Prefix the updateinfo file with its hash #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
146f35e
updateinfo: Prefix the file name with its hash
Mar 4, 2014
213179a
updateinfo: While we're at it, move to sha256
Mar 4, 2014
2a6fc20
updateinfo: Remove misleading comments
Mar 4, 2014
37ac331
updateinfo: Remove previous cached updateinfo
Mar 4, 2014
6a45da5
masher: Find the old updateinfo files
Mar 4, 2014
8193e7c
tests: Teach the metadata tests about the new updateinfo name
Mar 4, 2014
66e9f90
tests: Add a message on the assertion error
Mar 4, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 16 additions & 3 deletions bodhi/masher.py
View file
Open in desktop
Expand Up @@ -14,6 +14,7 @@
#
# Authors: Luke Macken <lmacken@redhat.com>

import glob
import os
import sha
import time
Expand Down Expand Up @@ -949,9 +950,21 @@ def generate_updateinfo(self):
self.genmd = True
t0 = time.time()
for repo, mashdir in self.mashed_repos.items():
olduinfo = join(config.get('mashed_dir'), '%s.repodata' % repo,
'i386', 'updateinfo.xml.gz')
olduinfo = exists(olduinfo) and olduinfo or None
# File name is prefixed with a hash, use a glob to find it
olduinfos = glob.glob(os.path.join(config.get('mashed_dir'),
'%s.repodata' % repo, 'i386',
"*updateinfo.xml.gz"))

if len(olduinfos) > 1:
# TODO: Shouldn't that be an error?
olduinfo = olduinfos[0]

if len(olduinfos) == 1:
olduinfo = olduinfos[0]

else:
olduinfo = None

repo = join(mashdir, repo)
log.debug("Generating updateinfo.xml.gz for %s" % repo)
uinfo = ExtendedMetadata(repo, olduinfo)
Expand Down
4 changes: 4 additions & 0 deletions bodhi/metadata.py
View file
Open in desktop
Expand Up @@ -53,6 +53,10 @@ def __init__(self, repo, cacheduinfo=None):
log.debug("Loading cached updateinfo.xml.gz")
umd = UpdateMetadata()
umd.add(cacheduinfo)

# Drop the old cached updateinfo.xml.gz, it's unneeded now
os.unlink(cacheduinfo)

existing_ids = set([up['update_id'] for up in umd.get_notices()])
seen_ids = set()
from_cache = set()
Expand Down
29 changes: 15 additions & 14 deletions bodhi/modifyrepo.py
View file
Open in desktop
Expand Up @@ -28,7 +28,7 @@
import sys
import gzip

from hashlib import sha1 as sha
from hashlib import sha256
from xml.dom import minidom
from kitchen.text.converters import to_bytes

Expand Down Expand Up @@ -83,13 +83,20 @@ def add(self, metadata):
newmd = gzip.GzipFile(destmd, 'wb')
newmd.write(to_bytes(md, errors='ignore', non_string='passthru'))
newmd.close()
print "Wrote:", destmd

## Read the gzipped metadata
f = file(destmd, 'r')
newmd = f.read()
f.close()

## Prefix the file name with its hash
hashed_md = sha256(newmd).hexdigest()
hashed_mdname = "%s-%s" % (hashed_md, mdname)
hashed_destmd = os.path.join(self.repodir, hashed_mdname)
os.rename(destmd, hashed_destmd)

print "Wrote:", hashed_destmd

## Remove any stale metadata
for elem in self.doc.getElementsByTagName('data'):
if elem.attributes['type'].value == mdtype:
Expand All @@ -98,19 +105,13 @@ def add(self, metadata):
root = self.doc.firstChild
data = self._insert_element(root, 'data', attrs={'type' : mdtype})
self._insert_element(data, 'location',
attrs={'href' : 'repodata/' + mdname})
self._insert_element(data, 'checksum', attrs={'type' : 'sha'},
text=sha(newmd).hexdigest())
attrs={'href' : 'repodata/' + hashed_mdname})
self._insert_element(data, 'checksum', attrs={'type' : 'sha256'},
text=hashed_md)
self._insert_element(data, 'timestamp',
text=str(os.stat(destmd).st_mtime))
self._insert_element(data, 'open-checksum', attrs={'type' : 'sha'},
text=sha(to_bytes(md, errors='ignore', non_string='passthru')).hexdigest())

#print " type =", mdtype
#print " location =", 'repodata/' + mdname
#print " checksum =", sha(newmd).hexdigest()
#print " timestamp =", str(os.stat(destmd).st_mtime)
#print " open-checksum =", sha(md).hexdigest()
text=str(os.stat(hashed_destmd).st_mtime))
self._insert_element(data, 'open-checksum', attrs={'type' : 'sha256'},
text=sha256(to_bytes(md, errors='ignore', non_string='passthru')).hexdigest())

## Write the updated repomd.xml
outmd = file(self.repomdxml, 'w')
Expand Down
2 changes: 1 addition & 1 deletion bodhi/tests/test_controllers.py
View file
Open in desktop
Expand Up @@ -1487,7 +1487,7 @@ def test_admin_push(self):
me.addGroup(testers)
create_release()
testutil.create_request('/updates/admin/push', headers=session)
assert '0 pending requests' in cherrypy.response.body[0]
assert '0 pending requests' in cherrypy.response.body[0], cherrypy.response.body[0]
params = {
'builds' : 'TurboGears-2.6.23.1-21.fc7',
'release' : 'Fedora 7',
Expand Down
50 changes: 29 additions & 21 deletions bodhi/tests/test_metadata.py
View file
Open in desktop
Expand Up @@ -5,11 +5,13 @@
turbogears.update_config(configfile='bodhi.cfg', modulename='bodhi.config')
database.set_db_uri("sqlite:///:memory:")

import glob
import shutil
import tempfile

from datetime import datetime
from os.path import join, exists
from hashlib import sha256
from os.path import join, exists, basename
from datetime import datetime
from bodhi.util import mkmetadatadir, get_nvr
from bodhi.model import (Release, Package, PackageUpdate, Bugzilla, CVE,
Expand All @@ -20,6 +22,21 @@


class TestExtendedMetadata(testutil.DBTest):
def __verify_updateinfo(self, repodata):
"""Verify the updateinfo file

This is not a test, just a helper function.
"""
updateinfos = glob.glob(join(repodata, "*-updateinfo.xml.gz"))
assert len(updateinfos) == 1, "We generated %d updateinfo metadata" % len(updateinfos)

updateinfo = updateinfos[0]
hash = basename(updateinfo).split("-", 1)[0]
hashed = sha256(open(updateinfo).read()).hexdigest()

assert hash == hashed, "File: %s\nHash: %s" % (basename(updateinfo), hashed)

return updateinfo

def test_extended_metadata(self):
# grab the name of a build in updates-testing, and create it in our db
Expand Down Expand Up @@ -58,8 +75,7 @@ def test_extended_metadata(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -131,8 +147,7 @@ def test_extended_metadata_updating(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -167,8 +182,7 @@ def test_extended_metadata_updating(self):
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -234,8 +248,7 @@ def test_extended_metadata_updating_with_edited_update_details(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand All @@ -262,8 +275,7 @@ def test_extended_metadata_updating_with_edited_update_details(self):
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -315,8 +327,7 @@ def test_extended_metadata_updating_with_edited_updates(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -349,8 +360,7 @@ def test_extended_metadata_updating_with_edited_updates(self):
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -405,8 +415,7 @@ def test_extended_metadata_updating_with_old_testing_security(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
Expand All @@ -425,7 +434,7 @@ def test_extended_metadata_updating_with_old_testing_security(self):
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down Expand Up @@ -483,8 +492,7 @@ def test_extended_metadata_updating_with_old_stable_security(self):

## Insert the updateinfo.xml into the repository
md.insert_updateinfo()
updateinfo = join(repodata, 'updateinfo.xml.gz')
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

# Create a new non-security update for the same package
newbuild = 'TurboGears-1.0.2.2-3.fc7'
Expand All @@ -503,7 +511,7 @@ def test_extended_metadata_updating_with_old_stable_security(self):
## Test out updateinfo.xml updating via our ExtendedMetadata
md = ExtendedMetadata(temprepo, updateinfo)
md.insert_updateinfo()
assert exists(updateinfo)
updateinfo = self.__verify_updateinfo(repodata)

## Read an verify the updateinfo.xml.gz
uinfo = UpdateMetadata()
Expand Down