Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Updates

  • Loading branch information...
commit c12201d695436012b4474f6cfef05483d185f937 1 parent 81b337c
@dagwieers authored
Showing with 123 additions and 87 deletions.
  1. +15 −19 README
  2. +3 −9 TODO
  3. +94 −51 sarahdb.py
  4. +2 −2 sarahdbo.py
  5. +9 −6 sarahinfo.py
View
34 README
@@ -10,23 +10,19 @@ Tim Rupp. Available from:
And issuing the following script:
cd aerrate
- for release in rh{el4,el3,21}{as,es,ws} rhel{4,3}-extras rh-desktop-{4,3} rhel3{cluster,devsuite} rh21aw rhshas; do
- for type in security; do
- echo "== $release / $type =="
- python ./aerrate.py -r --source=site --type=$type --release=$release
- done
+ for release in rh{el4,el3,21}{as,es,ws} rh-desktop-{4,3} rh21aw rhel{4,3}-extras rhel3{cluster,devsuite} rhshas; do
+ echo "== $release =="
+ python ./aerrate.py -r --source=site --type=all --release=$release
done
cd -
This will copy all errata as XML files into ./aerrata/advisories/
-Currently aerrate only implements security advisories, not bugs or enhancements.
-
USING SARAH
^^^^^^^^^^^
-sarah currently expects the advisories to be available from ./advisories/. So making
-a symlink from aerrate/advisories to ./advisories is probably easiest.
+sarah currently expects the advisories to be available from ./advisories/. So
+making a symlink from aerrate/advisories to ./advisories is probably easiest.
ln -sf aerrate/advisories .
@@ -39,12 +35,12 @@ To create some statistics from this database, use:
./sarahinfo.py
-The sarahinfo utility currently shows how to query the database. Not all information
-is currently available in the XML files. Red Hat will be releasing these XML files
-in the future with much more info.
+The sarahinfo utility currently shows how to query the database. Not all
+information is currently available in the XML files. Red Hat will be releasing
+these XML files in the future with much more info.
-I also added sarahsql to allow to query the database on the commandline, you can
-things in bash, like:
+I also added sarahsql to allow to query the database on the commandline, you
+can things in bash, like:
### Print filenames related to advisory
./sarahsql.py 'select distinct filename from rpm where advid == "RHSA-2005:039" order by filename'
@@ -52,15 +48,15 @@ things in bash, like:
### Show last 10 updated advisories
./sarahsql.py 'select advid, issuedate, updatedate, severity from adv order by updatedate' | tail
-Help is welcome to extend sarah much further. The TODO file is the first stop for
-interesting parties.
+Help is welcome to extend sarah much further. The TODO file is the first stop
+for interesting parties.
PROXY PROBLEMS ?
^^^^^^^^^^^^^^^^
-If you're behind a proxy that only allows SSL Proxy access (CONNECT method) for HTTPS sites,
-you're currently out of luck. The urllib2 implementation of python only has the HTTPS GET
-method implemented. More information at:
+If you're behind a proxy that only allows SSL Proxy access (CONNECT method) for
+HTTPS sites, you're currently out of luck. The urllib2 implementation of python
+only has the HTTPS GET method implemented. More information at:
https://devel.linux.duke.edu/bugzilla/show_bug.cgi?id=543
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/456195
View
12 TODO
@@ -1,15 +1,11 @@
-aerrate.py: scraps rhn advisory information
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-+ Add RHBA and RHEA support
+aerrate.py: scrapes rhn advisory information
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+ Allow to list a number of releases (eg. --release=rhel4as,rhel3as,rhel2.1as)
- and remember what advisories have already been scraped (recently)
-+ Allow to list all types of advisories (eg. --type=all or --type=critical,important)
-+ Make aerrate smart about advisories already downloaded
++ Make aerrate smart about advisories already scraped (recently)
+ Use timestamp and HTTP HEAD requests to check for updates
+ Parallelize the download process (use HTTP Pipelining and use
eg. 4 connections)
+ Fix SSL proxy support (CONNECT method) in urllib2
-+ Make aerrate.py executable :)
sarahdb.py: advisory database
@@ -89,8 +85,6 @@ Other interesting ideas
all affected machines.
+ What currently is in progress:
-
- - scrap RHBAs and RHEAs (aerrate)
- create default reports (sarahrep)
- verify a system's packages against this database
- send out security reports in different formats
View
145 sarahdb.py
@@ -7,6 +7,44 @@
sys.stdout = os.fdopen(1, 'w', 0)
+def elattr(ro, at, fail=True):
+ ret = ro.get(at)
+ if not ret and fail:
+ raise Exception, 'Attribute %s not found in element %s ' % (at, ro)
+ return ret
+
+def findel(ro, el, fail=True):
+ ret = ro.findtext(el)
+ if not ret and fail:
+ raise Exception, 'Element %s not found in root %s ' % (el, ro)
+ return ret
+
+def findelattr(ro, el, at, fail=True):
+ try:
+ ret = root.find(el).get(at)
+ except:
+ if fail:
+ raise Exception, 'Element %s not found in root %s ' % (el, ro)
+ return None
+ if not ret and fail:
+ raise Exception, 'Attribute %s not found in element %s ' % (at, el)
+ return ret
+
+def find(ro, el, fail=False):
+ ret = ro.find(el)
+ if not ret:
+ if fail:
+ raise Exception, 'Element %s not found in root %s ' % (el, ro)
+ return []
+ return ret
+def findall(ro, el, fail=False):
+ ret = ro.findall(el)
+ if not ret:
+ if fail:
+ raise Exception, 'Element list %s not found in root %s ' % (el, ro)
+ return []
+ return ret
+
con, cur = sarahlib.opendb()
sarahlib.createtb(cur, 'adv')
sarahlib.createtb(cur, 'ref')
@@ -24,86 +62,91 @@
root = tree.getroot()
advrec = {};
- advrec['sender'] = root.get('from')
- advrec['version'] = root.get('version')
- advrec['version'] = root.get('version')
-
- advrec['advid'] = root.findtext('id')
- advrec['pushcount'] = root.findtext('pushcount')
- advrec['type'] = root.find('type').get('short')
- advrec['keywords'] = root.findtext('keywords')
- advrec['obsoletes'] = root.findtext('obsoletes')
-
- if root.find('group'):
- advrec['rhgroup'] = root.find('group').get('name')
- else:
- advrec['rhgroup'] = None
- ### RHBAs and RHEAs do not have a severity level
+ ### FIXME: aerrate uses 'sender' instead of 'from' (unfixable)
+ try: advrec['sender'] = elattr(root, 'from')
+ except: advrec['sender'] = elattr(root, 'sender')
+
+ advrec['version'] = elattr(root, 'version')
+
+ advrec['advid'] = findel(root, 'id')
+ advrec['pushcount'] = findel(root, 'pushcount')
+
+ ### FIXME: aerrate does not (always) add type short info (use filename)
+ try: advrec['type'] = findelattr(root, 'type', 'short')
+ except: advrec['type'] = file[0:4]
+
+ try: advrec['keywords'] = ' '.join(findel(root, 'keywords'))
+ except: advrec['keywords'] = None
+ try: advrec['obsoletes'] = ' '.join(findel(root, 'obsoletes'))
+ except: advrec['obsoletes'] = None
+
+ advrec['rhgroup'] = findelattr(root, 'group', 'name', fail=False)
+
if advrec['type'] in ('RHBA', 'RHEA'):
advrec['severity'] = None
- elif root.find('severity') != None:
- advrec['severity'] = root.find('severity').get('level')
- elif root.findtext('severity'):
- advrec['severity'] = root.findtext('severity')
else:
-# raise Exception, 'severity not found.'
- advrec['severity'] = 'error'
+ ### FIXME: aerrate uses severity element text and not level attribute
+ try: advrec['severity'] = findelattr(root, 'severity', 'level')
+ except: advrec['severity'] = findel(root, 'severity')
- advrec['synopsis'] = root.findtext('synopsis')
- advrec['issued'] = root.find('issued').get('date')
- advrec['updated'] = root.find('updated').get('date')
- advrec['topic'] = root.findtext('topic')
- advrec['description'] = root.findtext('description')
+ advrec['synopsis'] = findel(root, 'synopsis')
+ advrec['issued'] = findelattr(root, 'issued', 'date')
+ advrec['updated'] = findelattr(root, 'updated', 'date')
+ advrec['topic'] = findel(root, 'topic')
+ ### FIXME: aerrate should replace <p> by \n\n for better formatting or not replace at all
+ advrec['description'] = findel(root, 'description')
+
+# print 'advrec:', advrec.keys()
# print 'advrec:', advrec
sarahlib.insertrec(cur, 'adv', advrec)
- for refnode in root.find('references'):
+ for refnode in find(root, 'references'):
refrec = { 'advid': advrec['advid'] }
- refrec['reftype'] = refnode.get('type')
- refrec['reference'] = refnode.get('href')
+ refrec['reftype'] = elattr(refnode, 'type')
+ ### FIXME: aerrate still implements the old format for reference information, skip
+ refrec['reference'] = elattr(refnode, 'href', fail=False)
if refrec['reftype'] == 'self':
refrec['refid'] = advrec['advid']
elif refnode.findtext('advisory'):
- refrec['refid'] = refnode.findtext('advisory')
+ refrec['refid'] = findel(refnode, 'advisory')
elif refnode.findtext('bugzilla'):
- refrec['refid'] = refnode.findtext('bugzilla')
+ refrec['refid'] = findel(refnode, 'bugzilla')
elif refnode.findtext('cve'):
- refrec['refid'] = refnode.findtext('cve')
+ refrec['refid'] = findel(refnode, 'cve')
elif refnode.findtext('self'):
- refrec['refid'] = refnode.findtext('self')
+ refrec['refid'] = findel(refnode, 'self')
else:
refrec['refid'] = 'error'
# raise Exception, 'refid not found.'
- if refnode.find('summary'):
- refrec['summary'] = refnode.findtext('summary')
- else:
- refrec['summary'] = None
+ refrec['summary'] = findel(refnode, 'summary', fail=False)
# print 'refrec:', refrec
sarahlib.insertrec(cur, 'ref', refrec)
- if not root.find('rpmlist'): continue
- for pronode in root.find('rpmlist'):
+ for pronode in find(root, 'rpmlist'):
prorec = { 'advid': advrec['advid'] }
- prorec['prodshort'] = pronode.get('short')
- prorec['product'] = pronode.findtext('name')
-# print prorec
- try: sarahlib.insertrec(cur, 'pro', prorec)
- except: pass
-
- if not pronode.find('file'): continue
- for rpmnode in pronode.findall('file'):
+ ### FIXME: aerrate does not (always) add product info, skip
+ try:
+ prorec['prodshort'] = elattr(pronode, 'short')
+ prorec['product'] = findel(pronode, 'name')
+# print prorec
+ try: sarahlib.insertrec(cur, 'pro', prorec)
+ except: pass
+ except:
+ pass
+
+ for rpmnode in findall(pronode, 'file'):
rpmrec = { 'advid': advrec['advid'], 'prodshort': prorec['prodshort'] }
- rpmrec['arch'] = rpmnode.get('arch')
- rpmrec['filename'] = rpmnode.findtext('filename')
+ rpmrec['arch'] = elattr(rpmnode, 'arch')
+ rpmrec['filename'] = findel(rpmnode, 'filename')
if rpmnode.find('sum').get('type') == 'md5':
- rpmrec['md5'] = rpmnode.findtext('sum')
+ rpmrec['md5'] = findel(rpmnode, 'sum')
rpmrec['channels'] = []
- for channel in root.findall('channel'):
+ for channel in findall(rpmnode, 'channel'):
rpmrec['channels'].append(channel.get('name'))
# print rpmrec
sarahlib.insertrec(cur, 'rpm', rpmrec)
View
4 sarahdbo.py
@@ -178,6 +178,6 @@ def handle_walker(walker):
except (xml.sax._exceptions.SAXParseException, AttributeError, KeyError), e:
print '\033[0;31m%s\033[0;0m' % os.path.basename(file),
print e
-# raise
- continue
+ raise
+# continue
con.commit()
View
15 sarahinfo.py
@@ -37,7 +37,7 @@
cur.execute('select distinct advid, prodshort from rpm')
prodlist = cur.fetchall()
-print 'Advisories per products'
+print 'Advisories per products:'
count = {}
for advid, prod in prodlist:
if not count.has_key(prod): count[prod] = 0
@@ -48,14 +48,16 @@
j = 2
print ' ',
for key in keys:
- i = int(key[0])
+ try: i = int(key[0])
+ except: i = 0
if i != j:
- print 'Other: %s\n ' % c[j],
+ print 'Other (%s): %s\n ' % (j, c[j]),
j = i
if key in ('2.1AS', '2.1ES', '2.1WS', '2.1AW'):
print '%s: %s ' % (key, count[key]),
continue
- elif key in ('3AS', '3ES', '3WS', '3Desktop'):
+ ### FIXME: aerrate should rename 3desktop to 3Desktop
+ elif key in ('3AS', '3ES', '3WS', '3Desktop', '3desktop'):
print '%s: %s ' % (key, count[key]),
continue
elif key in ('4AS', '4ES', '4WS', '4Desktop'):
@@ -63,14 +65,15 @@
continue
else:
c[i] += count[key]
-print 'Other: %s\n ' % c[j],
+print 'Other (%s): %s\n ' % (j, c[j]),
+print 'Other (unknown): %s\n ' % c[0],
print
print 'Advisories per year:'
print ' ',
for year in ('2002', '2003', '2004', '2005', '2006'):
cur.execute('select advid from adv where issued glob "*%s*"' % year)
- print '%s: %s \t' % (year, len(cur.fetchall())),
+ print '%s: %s ' % (year, len(cur.fetchall())),
print
print
Please sign in to comment.
Something went wrong with that request. Please try again.