Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

update redhat-oval->yvc converter to handle input as a stream

  • Loading branch information...
commit 434919559b5c3e51d935ba88649c1c8db7b253c7 1 parent 0b38185
@jschauma authored
Showing with 109 additions and 74 deletions.
  1. +4 −0 CHANGES
  2. +28 −20 Makefile
  3. +77 −54 misc/redhat_oval_to_yvc.py
View
4 CHANGES
@@ -1,5 +1,9 @@
Changelog:
+Version 3.3.0 (2011-12-07)
+ * update redhat-oval->yvc converter to handle input as a stream
+ * update sample Makefile to include RH6 lists
+
Version 3.2.0 (2011-04-20)
* add support for an optional 'severity' field in a vlist
View
48 Makefile
@@ -1,21 +1,27 @@
-# Copyright (c) 2008,2010 Yahoo! Inc.
+# Copyright (c) 2008,2010,2011 Yahoo! Inc.
#
# This example Makefile can be used to maintain vulnerability list.
# See 'make help' for more information.
# Location to which to upload the vlists.
LOCATION="<hostname>:~/public_html/yvc/"
+
+RHEL_URL=http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2
+RHEL_XML=com.redhat.rhsa-all.xml.bz2
+RH_VERSIONS=4 5 6
FBVLIST=fbvlist
RH4VLIST=rh4vlist
RH5VLIST=rh5vlist
-LISTS= ${RH5VLIST} ${RH4VLIST} ${FBVLIST}
+RH6VLIST=rh6vlist
+YVLIST=yvlist
+LISTS= ${YVLIST} ${RH6VLIST} ${RH5VLIST} ${RH4VLIST} ${FBVLIST}
-GONERS= ${RH5VLIST}.in ${RH4VLIST}.in ${FBVLIST}.in \
- com.redhat.rhsa-all.xml.bz2
+GONERS= ${RH6VLIST}.in ${RH5VLIST}.in ${RH4VLIST}.in ${FBVLIST}.in \
+ ${RHEL_XML}
date!=date
-all: sign upload
+all: fetch sign upload
help:
@echo "The following targets are available:"
@@ -25,32 +31,34 @@ help:
@echo "sign sign the vulnerability list"
@echo "upload upload the vulnerability list"
+fetch: ${RHEL_XML}
+
+${RHEL_XML}:
+ wget -q ${RHEL_URL}
+
sign: ${LISTS}
+${YVLIST}: ${YVLIST}.in
+ gpg -o ${YVLIST} --clearsign ${YVLIST}.in
+ chmod a+r ${YVLIST}
+
${FBVLIST}: ${FBVLIST}.in
gpg -o ${FBVLIST} --clearsign ${FBVLIST}.in
chmod a+r ${FBVLIST}
${FBVLIST}.in:
- @echo "# Generated on ${date}" > ${FBVLIST}.in
+ echo "# Generated on ${date}" > ${FBVLIST}.in
perl ./misc/harvest_freebsd_yvc.pl >> ${FBVLIST}.in
+.for n in ${RH_VERSIONS}
-${RH4VLIST}: ${RH4VLIST}.in
- gpg -o ${RH4VLIST} --clearsign ${RH4VLIST}.in
- chmod a+r ${RH4VLIST}
-
-${RH4VLIST}.in:
- python ./misc/redhat_oval_to_yvc.py 4 > ${RH4VLIST}.in
-
-
-${RH5VLIST}: ${RH5VLIST}.in
- gpg -o ${RH5VLIST} --clearsign ${RH5VLIST}.in
- chmod a+r ${RH5VLIST}
-
-${RH5VLIST}.in:
- python ./misc/redhat_oval_to_yvc.py 5 > ${RH5VLIST}.in
+rh${n}vlist.in:
+ python ./misc/redhat_oval_to_yvc.py ${n} | sort -u > rh${n}vlist.in
+rh${n}vlist: rh${n}vlist.in
+ gpg -o $@ --clearsign $>
+ chmod a+r $>
+.endfor
upload: sign
scp ${LISTS} ${LOCATION}
View
131 misc/redhat_oval_to_yvc.py
@@ -1,27 +1,25 @@
#!/usr/local/bin/python
#
-# Copyright (c) 2009,2010 Yahoo! Inc.
+# $Id: redhat_oval_to_yvc.py 71 2011-12-08 01:28:41Z jans $
+# $URL: svn+ssh://svn.corp.yahoo.com/yahoo/tools/yvc/branches/outgoing/misc/redhat_oval_to_yvc.py $
#
-# Originally written by Joshua Moss <jmos@yahoo-inc.com> in October 2009.
+# Copyright (c) 2009,2011 Yahoo! Inc.
+#
+# Originally written by Joshua Moskovitz <jmos@yahoo-inc.com> in October 2009.
+# Rewritten as a streaming parser bi Jan Schaumann <jans@yahoo-inc.com> in
+# December 2011.
#
# This program reads the Open Vulnerability and Assessment Language (OVAL)
# file, available from http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2
# and generates a yvc(1) compatible vlist.
-#
import re
import sys
-import socket
import os.path
-import xml.dom.minidom
import bz2
-import urllib
import time
-# Source of the oval xml.bz2 file
-oval_url = 'http://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2'
-
# Destination of the oval xml.bz2 file
oval_bz2 = './com.redhat.rhsa-all.xml.bz2'
@@ -36,58 +34,83 @@
###
-# function : download_redhat_oval_bz2
-# purpose : fetches the xml.bz2 file from redhat and write to local dir on disk
-# inputs : none
-# returns : void
-
-def download_redhat_oval_bz2():
- if not os.path.isfile(oval_bz2): # XXX remove this if you want to download every time, rather than handle via make clean
- socket.setdefaulttimeout(45)
- urllib.urlretrieve(oval_url, oval_bz2) # in, out
- urllib.urlcleanup()
-
-
-
# function : print_redhat_yvc
-# purpose : traverses the xml dom for vulnerability information and prints out in the desired
-# format
-# inputs : numeric string, appends to regex search for 'el'
+# purpose : turn oval XML gunk into yvc goodness
+# inputs : numeric string to restrict output by version
# returns : void, output is printed to stdout
-def print_redhat_yvc(version = '[45]'):
+def print_redhat_yvc(version):
print '# Generated on ' + time.strftime("%a %b %e %H:%M:%S %Z %Y") # Wed Oct 28 12:05:05 PDT 2009
-
arch = bz2.BZ2File(oval_bz2, 'r')
- document = xml.dom.minidom.parse(arch)
-
- apps = {} # they seem to keep their tests in chronological order, we only want the latest
- definitions = document.getElementsByTagName('definition')
- for d in definitions:
- #description = d.getElementsByTagName('metadata')[0].getElementsByTagName('description')[0].lastChild.nodeValue # TODO parse descriptions for keywords
- title = d.getElementsByTagName('metadata')[0].getElementsByTagName('title')[0].lastChild.nodeValue.split(': ')[1].replace('\n', '')
- title = re.sub('[^a-zA-Z0-9]+', '-', title).rstrip('-').lower()
- ref_url = d.getElementsByTagName('reference')[0].getAttribute('ref_url')
-
- criterions = d.getElementsByTagName('criterion')
- for c in criterions:
- criteria = c.getAttribute('comment').replace('\t', ' ')
-
- if re.search('el'+version, criteria, re.I) == None:
- continue # skipping lines that don't match version
-
- criteria = re.sub('\d+:', '', criteria)
- criteria = criteria.replace(' is earlier than ','<')
-
- app = criteria.split('<')[0]
-
- apps[app] = "%s\t%s\t%s" % (criteria, title, ref_url)
-
- for app in apps:
- print apps[app]
+ def_p = re.compile('.*<definition id="(?P<id>[^"]*)"', re.I)
+ title_p = re.compile('.*<title>RHSA-[0-9:]*: (?P<title>.*)</title>', re.I)
+ platform_p = re.compile('.*<platform>.* Linux (?P<num>[0-9])</platform>', re.I)
+ reference_p = re.compile('.*<reference source.*ref_url="(?P<url>.*)"', re.I)
+ criterion_p = re.compile('.*<criterion .*comment="(?P<pkg>[^ ]*) is earlier than [0-9]+:(?P<version>[^"]*)"', re.I)
+ end_p = re.compile('.*</definition>', re.I)
+
+ found = 0
+ id = 0
+ title = ""
+ platform = 0
+ reference = ""
+ criteria = []
+
+ lineno = 0
+
+ for line in arch.readlines():
+ lineno += 1
+ m = def_p.match(line)
+ if m:
+ found = 1
+ id = m.group('id')
+ title = ""
+ platform = 0
+ reference = ""
+ criteria = []
+ continue
+
+ if not found:
+ continue
+
+ m = title_p.match(line)
+ if m:
+ title = m.group('title')
+ continue
+
+ m = platform_p.match(line)
+ if m:
+ platform = m.group('num')
+ continue
+
+ m = reference_p.match(line)
+ if m:
+ reference = m.group('url')
+ continue
+
+ m = criterion_p.match(line)
+ if m:
+ criteria.append(m.group('pkg') + '<' + m.group('version'))
+ continue
+
+ m = end_p.match(line)
+ if m:
+ if title and platform and reference and len(criteria):
+ if platform == version:
+ title = re.sub('[^a-zA-Z0-9]+', '-', title).rstrip('-').lower()
+ for c in criteria:
+ try:
+ c.index("el%s" % version)
+ print "%s\t%s\t%s" % (c, title, reference)
+ except ValueError:
+ pass
+ else:
+ sys.stderr.write("Incomplete vulnerability encountered in line %d\n" % lineno)
+ sys.stderr.write("%s\n%d\n%s\n%s\n" % (title, platform, reference, criteria))
+ found = 0
+ continue
-download_redhat_oval_bz2()
print_redhat_yvc(sys.argv[1])
sys.exit(0)
Please sign in to comment.
Something went wrong with that request. Please try again.