Browse files

Important updates

  • Loading branch information...
1 parent 41d0377 commit 03f8ef96d3d2ee7c14e42c126f94605a8f375af7 @dagwieers committed Jan 10, 2006
Showing with 97 additions and 59 deletions.
  1. +37 −1 README
  2. +2 −2 TODO
  3. +16 −18 sarahdb.py
  4. +20 −14 sarahlib.py
  5. +22 −24 sarahprint.py
View
38 README
@@ -1,12 +1,48 @@
+DOWNLOADING ERRATA
+^^^^^^^^^^^^^^^^^^
Sarah currently requires a stuffed directory of advisories from RHN as XML files.
You can download these files automatically by using the aerrate.py script from
Tim Rupp. Available from:
- :pserver:anonymous@cdcvs.fnal.gov:/cvs/scientificlinux
+ cvs -d :pserver:anonymous@cdcvs.fnal.gov:/cvs/scientificlinux co aerrate
And issuing the following script:
+ cd aerrate
for release in rh{21,el3,el4}as rh{21,el3,el4}es rh{21,el3,el4}ws rhel3{cluster,devsuite} rhel{3,4}-extras rh-desktop-{3,4} rh21aw rhshas; do
./aerrate.py --source=site --type=security --release=$release
done
+ cd -
+
+This will copy all errata as XML files into ./advisories/
+
+
+USING SARAH
+^^^^^^^^^^^
+sarah currently expects the advisories to be available from ./advisories/ as well. So making
+a symlink from aerrate.advisories to sarah/advisories is probably easiest.
+
+ ln -sf aerrate/advisories .
+
+Then to create an sqlite database out of these XML files, run:
+
+ ./sarahdb.py
+
+You should now have a sarahdb.sqlite file in your current directory.
+To create some statistics from this database, use:
+
+ ./sarahprint.py
+
+The sarahprint utility currently shows how to query the database(s). Not all information
+is currently available from the XML files or from RHN. Help is welcome.
+
+
+PROXY PROBLEMS ?
+^^^^^^^^^^^^^^^^
+If you're behind a proxy that only allows SSL Proxy access (CONNECT method) for HTTPS sites,
+you're currently out of luck. The urllib2 implementation of python only has the HTTPS GET
+method implemented. More information at:
+
+ https://devel.linux.duke.edu/bugzilla/show_bug.cgi?id=543
+ http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/456195
View
4 TODO
@@ -4,9 +4,9 @@ aeratta.py
eg. 4 connections)
+ Allow to list a number of releases (eg. --release=rhel4as,rhel3as,rhel2.1as)
and remember what advisories have already been scraped (recently)
-+ Use timestamp and HTTP HEAD requests to check for updates)
+ Allow to list all types of advisories (eg. --type=all or --type=critical,important)
-+ Fix proxy support
++ Use timestamp and HTTP HEAD requests to check for updates)
++ Fix SSL proxy support (CONNECT method) in urllib2
+ Validate XML file after writing (some files are corrupted/unfinished)
+ Make aerrate.py executable :)
+ A lot of files do not include <severity>
View
34 sarahdb.py
@@ -7,15 +7,15 @@
import sarahlib
-advcon, advcur = sarahlib.opendb('adv', create=True)
-refcon, refcur = sarahlib.opendb('ref', create=True)
-rpmcon, rpmcur = sarahlib.opendb('rpm', create=True)
-procon, procur = sarahlib.opendb('pro', create=True)
-typcon, typcur = sarahlib.opendb('typ', create=True)
-
-### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
+con, cur = sarahlib.opendb()
+sarahlib.createtb(cur, 'adv')
+sarahlib.createtb(cur, 'ref')
+sarahlib.createtb(cur, 'rpm')
+sarahlib.createtb(cur, 'pro')
+sarahlib.createtb(cur, 'typ')
+
reader = Sax2.Reader()
filelist = glob.glob('advisories/RHSA-*.xml')
@@ -28,6 +28,7 @@
except xml.sax._exceptions.SAXParseException:
print '**%s**' % os.path.basename(file),
continue
+
print os.path.basename(file),
next = True
@@ -48,11 +49,11 @@
elif walker.currentNode.tagName == 'type':
typrec['type'] = walker.currentNode.firstChild.data
advrec['typeshort'] = typrec['typeshort'] = walker.currentNode.getAttribute('short')
- typcur.execute('select type from typ where typeshort = "%(typeshort)s"' % typrec)
- typelist = [type for type, in typcur.fetchall()]
+ cur.execute('select type from typ where typeshort = "%(typeshort)s"' % typrec)
+ typelist = [type for type, in cur.fetchall()]
if not typelist:
- sarahlib.insertdb(typcur, 'typ', typrec)
- typcon.commit()
+ sarahlib.insertrec(cur, 'typ', typrec)
+ con.commit()
elif typrec['type'] not in typelist:
print "ERROR: Wrong type exists (%s not in %s)" % (typrec['type'], typelist)
@@ -98,14 +99,11 @@
if product != prorec['product']:
print "ERROR: Wrong product exists (%s != %s)" % prorec['product'], product
else:
- sarahlib.insertdb(procur, 'pro', prorec)
- procon.commit()
+ sarahlib.insertrec(cur, 'pro', prorec)
+ con.commit()
next = walker.nextNode()
- if not advrec.has_key('severitylevel'):
- advrec['severitylevel'] = 'unknown'
-
# print advrec
- sarahlib.insertdb(advcur, 'adv', advrec)
- advcon.commit()
+ sarahlib.insertrec(cur, 'adv', advrec)
+ con.commit()
View
34 sarahlib.py
@@ -1,12 +1,6 @@
import os, sqlite, types
-dbase = {
- 'adv': 'sarahdb/Advisories.sqlite',
- 'ref': 'sarahdb/References.sqlite',
- 'rpm': 'sarahdb/RPMList.sqlite',
- 'pro': 'sarahdb/Products.sqlite',
- 'typ': 'sarahdb/Types.sqlite',
-}
+dbase = 'sarahdb.sqlite'
headers = {
# 'adv': ('advid', 'pushcount', 'severitylevel', 'issuedate', 'updatedate', 'typeshort', 'synopsis', 'description', 'topic', 'sender', 'version'),
@@ -17,10 +11,19 @@
'typ': ('typeshort', 'type'),
}
+dataset = {
+ 'spec': { 'name': 'varchar(10) unique primary key', },
+ 'info': { 'name': 'varchar(10) unique primary key', },
+}
+
def sqlcreate(name):
'Return a database create SQL statement'
str = 'create table %s ( ' % name
- for key in headers[name]: str += '%s varchar(10), ' % key
+ for key in headers[name]:
+ if dataset.has_key(name) and dataset[name].has_key(key):
+ str += '%s %s,' % (key, dataset[name][key])
+ else:
+ str += '%s varchar(10), ' % key
return str.rstrip(', ') + ' )'
def sqlinsert(name):
@@ -31,15 +34,18 @@ def sqlinsert(name):
for key in headers[name]: str += '"%%(%s)s", ' % key
return str.rstrip(', ') + ' )'
-def opendb(name, create=False):
+def opendb():
'Open a database and return references'
- con = sqlite.connect(dbase[name])
+ con = sqlite.connect(dbase)
cur = con.cursor()
- if create:
- cur.execute(sqlcreate(name))
- return (con, cur)
+ return con, cur
+
+def createtb(cur, name, create=False):
+ try: cur.execute('drop table "%s"' % name)
+ except: pass
+ cur.execute(sqlcreate(name))
-def insertdb(cur, name, rec):
+def insertrec(cur, name, rec):
'Insert a record in a database'
### Convert unicode to UTF-8
for key in rec.keys():
View
46 sarahprint.py
@@ -1,39 +1,37 @@
#!/usr/bin/python
import glob, sqlite, sys, re, os, string, shutil
+import sarahlib
+
from xml.dom.ext.reader import Sax2
from xml.dom.NodeFilter import NodeFilter
-import sarahlib
-
-advcon, advcur = sarahlib.opendb('adv')
-refcon, refcur = sarahlib.opendb('ref')
-rpmcon, rpmcur = sarahlib.opendb('rpm')
-procon, procur = sarahlib.opendb('pro')
-typcon, typcur = sarahlib.opendb('typ')
+con, cur = sarahlib.opendb()
### Unbuffered sys.stdout
sys.stdout = os.fdopen(1, 'w', 0)
-advcur.execute('select distinct advid from adv order by advid')
-print 'Number of advisories:', len(advcur.fetchall())
+cur.execute('select distinct advid from adv order by advid')
+print 'Number of advisories:', len(cur.fetchall())
-advcur.execute('select distinct advid from adv where severitylevel = "critical"')
-print '\tcritical:', len(advcur.fetchall()),
-advcur.execute('select distinct advid from adv where severitylevel = "important"')
-print 'important:', len(advcur.fetchall()),
-advcur.execute('select distinct advid from adv where severitylevel = "moderate"')
-print 'moderate:', len(advcur.fetchall()),
-advcur.execute('select distinct advid from adv where severitylevel = "low"')
-print 'low:', len(advcur.fetchall()),
-advcur.execute('select distinct advid from adv where severitylevel = "unknown"')
-print 'unknown:', len(advcur.fetchall()),
-print
+cur.execute('select distinct advid from adv where severitylevel = "critical"')
+print '\tcritical:', len(cur.fetchall()), '/',
+cur.execute('select distinct advid from adv where severitylevel = "important"')
+print 'important:', len(cur.fetchall()), '/',
+cur.execute('select distinct advid from adv where severitylevel = "moderate"')
+print 'moderate:', len(cur.fetchall()), '/',
+cur.execute('select distinct advid from adv where severitylevel = "low"')
+print 'low:', len(cur.fetchall()), '/',
+cur.execute('select distinct advid from adv where severitylevel = "unknown"')
+print 'unknown:', len(cur.fetchall())
-typcur.execute('select typeshort from typ order by typeshort')
-print 'Number of types:', len(typcur.fetchall())
+cur.execute('select typeshort from typ order by typeshort')
+print 'Number of types:', len(cur.fetchall())
### Debug database
-#advcur.execute('select * from adv order by advid')
-#for all in advcur.fetchall():
+#cur.execute('select * from adv order by advid')
+#for all in cur.fetchall():
# print all
+
+#cur.execute('select distinct advid from adv where severitylevel = "unknown" order by advid')
+#print cur.fetchall()

0 comments on commit 03f8ef9

Please sign in to comment.