update redhat-oval->yvc converter to handle input as a stream

CHANGES

@@ -1,5 +1,9 @@
 Changelog:
 
+Version 3.3.0 (2011-12-07)
+ * update redhat-oval->yvc converter to handle input as a stream
+ * update sample Makefile to include RH6 lists
+
 Version 3.2.0 (2011-04-20)
  * add support for an optional 'severity' field in a vlist
 

Makefile

@@ -1,21 +1,27 @@
-# Copyright (c) 2008,2010  Yahoo! Inc.
+# Copyright (c) 2008,2010,2011 Yahoo! Inc.
 #
 # This example Makefile can be used to maintain vulnerability list.
 # See 'make help' for more information.
 
 # Location to which to upload the vlists.
 LOCATION="<hostname>:~/public_html/yvc/"
+
+RHEL_URL=http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2
+RHEL_XML=com.redhat.rhsa-all.xml.bz2
+RH_VERSIONS=4 5 6
 FBVLIST=fbvlist
 RH4VLIST=rh4vlist
 RH5VLIST=rh5vlist
-LISTS= ${RH5VLIST} ${RH4VLIST} ${FBVLIST}
+RH6VLIST=rh6vlist
+YVLIST=yvlist
+LISTS= ${YVLIST} ${RH6VLIST} ${RH5VLIST} ${RH4VLIST} ${FBVLIST}
 
-GONERS= ${RH5VLIST}.in ${RH4VLIST}.in ${FBVLIST}.in \
-	com.redhat.rhsa-all.xml.bz2
+GONERS= ${RH6VLIST}.in ${RH5VLIST}.in ${RH4VLIST}.in ${FBVLIST}.in \
+	${RHEL_XML}
 
 date!=date
 
-all: sign upload
+all: fetch sign upload
 
 help:
 	@echo "The following targets are available:"
@@ -25,32 +31,34 @@ help:
 	@echo "sign	sign the vulnerability list"
 	@echo "upload	upload the vulnerability list"
 
+fetch: ${RHEL_XML}
+
+${RHEL_XML}:
+	wget -q ${RHEL_URL}
+
 sign: ${LISTS}
 
+${YVLIST}: ${YVLIST}.in
+	gpg -o ${YVLIST} --clearsign ${YVLIST}.in
+	chmod a+r ${YVLIST}
+
 ${FBVLIST}: ${FBVLIST}.in
 	gpg -o ${FBVLIST} --clearsign ${FBVLIST}.in
 	chmod a+r ${FBVLIST}
 
 ${FBVLIST}.in:
-	@echo "# Generated on ${date}" > ${FBVLIST}.in
+	echo "# Generated on ${date}" > ${FBVLIST}.in
 	perl ./misc/harvest_freebsd_yvc.pl >> ${FBVLIST}.in
 
+.for n in ${RH_VERSIONS}
 
-${RH4VLIST}: ${RH4VLIST}.in
-	gpg -o ${RH4VLIST} --clearsign ${RH4VLIST}.in
-	chmod a+r ${RH4VLIST}
-
-${RH4VLIST}.in:
-	python ./misc/redhat_oval_to_yvc.py 4 > ${RH4VLIST}.in
-
-
-${RH5VLIST}: ${RH5VLIST}.in
-	gpg -o ${RH5VLIST} --clearsign ${RH5VLIST}.in
-	chmod a+r ${RH5VLIST}
-
-${RH5VLIST}.in:
-	python ./misc/redhat_oval_to_yvc.py 5 > ${RH5VLIST}.in
+rh${n}vlist.in:
+	python ./misc/redhat_oval_to_yvc.py ${n} | sort -u > rh${n}vlist.in
 
+rh${n}vlist: rh${n}vlist.in
+	gpg -o $@ --clearsign $>
+	chmod a+r $>
+.endfor
 
 upload: sign
 	scp ${LISTS} ${LOCATION}

misc/redhat_oval_to_yvc.py

@@ -1,27 +1,25 @@
 #!/usr/local/bin/python
 
 #
-# Copyright (c) 2009,2010 Yahoo! Inc.
+# $Id: redhat_oval_to_yvc.py 71 2011-12-08 01:28:41Z jans $
+# $URL: svn+ssh://svn.corp.yahoo.com/yahoo/tools/yvc/branches/outgoing/misc/redhat_oval_to_yvc.py $
 #
-# Originally written by Joshua Moss <jmos@yahoo-inc.com> in October 2009.
+# Copyright (c) 2009,2011 Yahoo! Inc.
+#
+# Originally written by Joshua Moskovitz <jmos@yahoo-inc.com> in October 2009.
+# Rewritten as a streaming parser bi Jan Schaumann <jans@yahoo-inc.com> in
+# December 2011.
 #
 # This program reads the Open Vulnerability and Assessment Language (OVAL)
 # file, available from http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2
 # and generates a yvc(1) compatible vlist.
-#
 
 import re
 import sys
-import socket
 import os.path
-import xml.dom.minidom
 import bz2
-import urllib
 import time
 
-# Source of the oval xml.bz2 file
-oval_url = 'http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2'
-
 # Destination of the oval xml.bz2 file
 oval_bz2 = './com.redhat.rhsa-all.xml.bz2'
 
@@ -36,58 +34,83 @@
 ###
 
 
-# function : download_redhat_oval_bz2
-# purpose  : fetches the xml.bz2 file from redhat and write to local dir on disk
-# inputs   : none
-# returns  : void
-
-def download_redhat_oval_bz2():
-    if not os.path.isfile(oval_bz2): # XXX remove this if you want to download every time, rather than handle via make clean
-        socket.setdefaulttimeout(45)
-        urllib.urlretrieve(oval_url, oval_bz2) # in, out
-        urllib.urlcleanup()
-
-
-
 # function : print_redhat_yvc
-# purpose  : traverses the xml dom for vulnerability information and prints out in the desired
-#            format
-# inputs   : numeric string, appends to regex search for 'el'
+# purpose  : turn oval XML gunk into yvc goodness
+# inputs   : numeric string to restrict output by version
 # returns  : void, output is printed to stdout
 
-def print_redhat_yvc(version = '[45]'):
+def print_redhat_yvc(version):
 
     print '# Generated on ' + time.strftime("%a %b %e %H:%M:%S %Z %Y")  # Wed Oct 28 12:05:05 PDT 2009
-
     arch = bz2.BZ2File(oval_bz2, 'r')
-    document = xml.dom.minidom.parse(arch)
-
-    apps = {} # they seem to keep their tests in chronological order, we only want the latest
-    definitions = document.getElementsByTagName('definition')
-    for d in definitions:
-        #description = d.getElementsByTagName('metadata')[0].getElementsByTagName('description')[0].lastChild.nodeValue # TODO parse descriptions for keywords
-        title = d.getElementsByTagName('metadata')[0].getElementsByTagName('title')[0].lastChild.nodeValue.split(': ')[1].replace('\n', '')
-        title = re.sub('[^a-zA-Z0-9]+', '-', title).rstrip('-').lower()
-        ref_url = d.getElementsByTagName('reference')[0].getAttribute('ref_url')
-
-        criterions = d.getElementsByTagName('criterion')
-        for c in criterions:
-            criteria = c.getAttribute('comment').replace('\t', ' ')
-
-            if re.search('el'+version, criteria, re.I) == None:
-                continue # skipping lines that don't match version
-
-            criteria = re.sub('\d+:', '', criteria)
-            criteria = criteria.replace(' is earlier than ','<')
-
-            app = criteria.split('<')[0]
-
-            apps[app] = "%s\t%s\t%s" % (criteria, title, ref_url)
-
-    for app in apps:
-        print apps[app]
 
+    def_p = re.compile('.*<definition id="(?P<id>[^"]*)"', re.I)
+    title_p = re.compile('.*<title>RHSA-[0-9:]*: (?P<title>.*)</title>', re.I)
+    platform_p = re.compile('.*<platform>.* Linux (?P<num>[0-9])</platform>', re.I)
+    reference_p = re.compile('.*<reference source.*ref_url="(?P<url>.*)"', re.I)
+    criterion_p = re.compile('.*<criterion .*comment="(?P<pkg>[^ ]*) is earlier than [0-9]+:(?P<version>[^"]*)"', re.I)
+    end_p = re.compile('.*</definition>', re.I)
+
+    found = 0
+    id = 0
+    title = ""
+    platform = 0
+    reference = ""
+    criteria = []
+
+    lineno = 0
+
+    for line in arch.readlines():
+        lineno += 1
+        m = def_p.match(line)
+        if m:
+            found = 1
+            id = m.group('id')
+            title = ""
+            platform = 0
+            reference = ""
+            criteria = []
+            continue
+
+        if not found:
+            continue
+
+        m = title_p.match(line)
+        if m:
+            title = m.group('title')
+            continue
+
+        m = platform_p.match(line)
+        if m:
+            platform = m.group('num')
+            continue
+
+        m = reference_p.match(line)
+        if m:
+            reference = m.group('url')
+            continue
+
+        m = criterion_p.match(line)
+        if m:
+            criteria.append(m.group('pkg') + '<' + m.group('version'))
+            continue
+
+        m = end_p.match(line)
+        if m:
+            if title and platform and reference and len(criteria):
+                if platform == version:
+                    title = re.sub('[^a-zA-Z0-9]+', '-', title).rstrip('-').lower()
+		    for c in criteria:
+                        try:
+                            c.index("el%s" % version)
+                            print "%s\t%s\t%s" % (c, title, reference)
+                        except ValueError:
+                             pass
+            else:
+                sys.stderr.write("Incomplete vulnerability encountered in line %d\n" % lineno)
+                sys.stderr.write("%s\n%d\n%s\n%s\n" % (title, platform, reference, criteria))
+            found = 0
+            continue
 
-download_redhat_oval_bz2()
 print_redhat_yvc(sys.argv[1])
 sys.exit(0)