merge from Twitter

Makefile

@@ -0,0 +1,27 @@
+NAME=nvdsync
+VERSION=$(shell awk '/define version/ { print $$NF }' rpm/${NAME}.spec)
+MANPAGES=${NAME}.1
+BUILDHOST="opsnest1"
+
+help:
+	@echo "The following targets are available:"
+	@echo "build      build the RPM ${NAME}-${VERSION} on ${BUILDHOST}"
+	@echo "clean      remove any interim files"
+	@echo "help       print this help"
+	@echo "rpm        build an RPM"
+
+build:
+	@rsync -e ssh -avz . ${BUILDHOST}:${NAME}/.
+	@ssh ${BUILDHOST} "cd ${NAME} && make rpm"
+	@scp ${BUILDHOST}:redhat/RPMS/noarch/${NAME}-${VERSION}*rpm /tmp/
+
+rpm: man-compress
+	cd rpm && sh mkrpm.sh ${NAME}.spec
+
+man-compress:
+	@for f in ${MANPAGES}; do			\
+		gzip -9 doc/$${f} -c > doc/$${f}.gz;	\
+	done;
+
+clean:
+	rm -fr doc/*.gz

doc/nvdsync.1

@@ -1,23 +1,20 @@
-.Dd May 6, 2013
+.Dd March 11, 2013
 .Dt NVDSYNC 1
 .Os
 .Sh NAME
 .Nm nvdsync
-.Nd fetch NIST's NVD, sync into a local database and cross-reference with Jira
+.Nd fetch NIST's NVD data and cross-reference it with Jira
 .Sh SYNOPSIS
 .Nm
 .Op Fl hv
 .Sh DESCRIPTION
 The
 .Nm
 utility wraps the
-.Xr nvd2sqlite3 1
-and
 .Xr nvdXjira 1
-tools to provide a simple script suitable to be run from a cronjob in
-order to fetch NIST's National Vulnerability Database, sync it into a
-local sqlite3 database and then cross-reference that database with a Jira
-instance.
+tool to provide a simple script suitable to be run from a cronjob in order
+to fetch NIST's National Vulnerability Database and then cross-reference
+that database with a Jira instance.
 .Sh OPTIONS
 .Nm
 supports the following options:
@@ -31,14 +28,10 @@ Can be specified multiple times.
 .Sh ENVIRONMENT
 .Nm
 honors the following environment variables:
-.Bl -tag -width NVD2SQLITE_FLAGS_
-.It CURL_FLAGS
+.Bl -tag -width NVDXJIRA_FLAGS_
+.It WGET_FLAGS
 Flags to pass to
-.Xr curl 1 ,
-if any.
-.It NVD2SQLITE3_FLAGS
-Flags to pass to
-.Xr nvd2sqlite3 1 ,
+.Xr wget 1 ,
 if any.
 .It NVDXJIRA_FLAGS
 Flags to pass to
@@ -48,13 +41,11 @@ if any.
 .Sh EXIT STATUS
 .Ex -std
 .Sh SEE ALSO
-.Xr nvd2sqlite3 1 ,
 .Xr nvdXjira 1 ,
-.Xr sqlite3 1 ,
 .Xr yvc 1
 .Sh HISTORY
 .Nm
 was originally written by
 .An Jan Schaumann
-.Aq jschauma@netmeister.org
+.Aq jschauma@twitter.com
 in May 2013.

doc/nvdsync.1.txt

@@ -1,17 +1,15 @@
 NVDSYNC(1)		NetBSD General Commands Manual		    NVDSYNC(1)
 
 NAME
-     nvdsync -- fetch NIST's NVD, sync into a local database and cross-refer-
-     ence with Jira
+     nvdsync -- fetch NIST's NVD data and cross-reference it with Jira
 
 SYNOPSIS
      nvdsync [-hv]
 
 DESCRIPTION
-     The nvdsync utility wraps the nvd2sqlite3(1) and nvdXjira(1) tools to
-     provide a simple script suitable to be run from a cronjob in order to
-     fetch NIST's National Vulnerability Database, sync it into a local
-     sqlite3 database and then cross-reference that database with a Jira
+     The nvdsync utility wraps the nvdXjira(1) tool to provide a simple script
+     suitable to be run from a cronjob in order to fetch NIST's National Vul-
+     nerability Database and then cross-reference that database with a Jira
      instance.
 
 OPTIONS
@@ -24,20 +22,18 @@ OPTIONS
 ENVIRONMENT
      nvdsync honors the following environment variables:
 
-     CURL_FLAGS		Flags to pass to curl(1), if any.
+     WGET_FLAGS	      Flags to pass to wget(1), if any.
 
-     NVD2SQLITE3_FLAGS	Flags to pass to nvd2sqlite3(1), if any.
-
-     NVDXJIRA_FLAGS	Flags to pass to nvdXjira(1), if any.
+     NVDXJIRA_FLAGS   Flags to pass to nvdXjira(1), if any.
 
 EXIT STATUS
      The nvdsync utility exits 0 on success, and >0 if an error occurs.
 
 SEE ALSO
-     nvd2sqlite3(1), nvdXjira(1), sqlite3(1), yvc(1)
+     nvdXjira(1), yvc(1)
 
 HISTORY
-     nvdsync was originally written by Jan Schaumann <jschauma@netmeister.org>
-     in May 2013.
+     nvdsync was originally written by Jan Schaumann <jschauma@twitter.com> in
+     May 2013.
 
-NetBSD 5.0			  May 6, 2013			    NetBSD 5.0
+NetBSD 5.0			March 11, 2013			    NetBSD 5.0

rpm/nvdsync.spec

@@ -1,12 +1,12 @@
 %define name		nvdsync
 %define release		1
-%define version 	0.2
+%define version 	0.7
 %define mybuilddir	${HOME}/redhat/BUILD/%{name}-%{version}-root
 
-Requires:		bash, curl, nvd2sqlite3, nvdXjira
+Requires:		bash, nvdXjira, wget
 BuildRoot:		%{mybuilddir}
 BuildArch:		noarch
-Summary:		fetch NIST's NVD, sync into a local database and cross-reference with Jira
+Summary:		fetch NIST's NVD and cross-reference with Jira
 License: 		BSD
 Name: 			%{name}
 Version: 		%{version}
@@ -16,10 +16,10 @@ Prefix: 		/usr
 Group: 			Development/Tools
 
 %description
-The nvdsync utility wraps the nvd2sqlite3 and nvdXjira tools to provide a
-simple script suitable to be run from a cronjob in order to fetch NIST's
-National Vulnerability Database, sync it into a local sqlite3 database and
-then cross-reference that database with a Jira instance.
+The nvdsync utility wraps the nvdXjira tool to provide a simple script
+suitable to be run from a cronjob in order to fetch NIST's National
+Vulnerability Database, and then cross-reference that database with a Jira
+instance.
 
 %prep
 %setup -q
@@ -38,11 +38,42 @@ install -c -m 444 doc/nvdsync.1 %{mybuilddir}/usr/share/man/man1/nvdsync.1
 %doc /usr/share/man/man1/nvdsync.1.gz
 
 %changelog
-* Mon May 06 2013 - jschauma@netmeister.org
+* Fri Mar 14 2014 - jschauma@twitter.com
+- 0.7
+-  NIST changed the location of the feeds
+
+
+* Wed Mar 12 2014 - jschauma@twitter.com
+- 0.6.1
+-  minor fix for [SECURITY-11696]: eval wget invocation to properly expand
+   all flags
+
+* Tue Mar 11 2014 - jschauma@twitter.com
+- 0.6
+-  [SECURITY-11696]: use wget(1) instead of curl(1) to work around a bug
+   in some versions of curl(1) where it can't validate an SSL certificate
+   signed using SHA-256
+
+* Mon Feb 10 2014 - jschauma@twitter.com
+- 0.5
+-  use https for recent XML list
+-  properly report URL in error messages
+
+* Mon Oct 28 2013 - jschauma@twitter.com
+- 0.4
+-  [SECURITY-10345]: drop support for nvd2sqlite3
+
+* Tue Sep 17 2013 - jschauma@twitter.com
+- 0.3
+-  [SECURITY-9860]: update location of NIST feed
+-  [SECURITY-9860]: detect and bail out on HTTP errors when fetching NIST
+   feed
+
+* Mon May 06 2013 - jschauma@twitter.com
 - 0.2
 -  accept CURL_FLAGS
 -  fix usage
 
-* Fri May 03 2013 - jschauma@netmeister.org
+* Fri May 03 2013 - jschauma@twitter.com
 - 0.1:
 -  initial version

src/nvdsync

@@ -4,20 +4,22 @@
 #
 # Originally written by Jan Schaumann <jschauma@netmeister.org> in May 2013.
 #
-# This script fetches NIST's NVD, syncs it into a local database and
-# cross-references CVEs with Jira tickets.
+# This script fetches NIST's NVD data and cross-references the CVEs with
+# Jira tickets.
 
-set -e
+#set -e
 
 ###
 ### Globals
 ###
 
 umask 077
 
-NVD_URL="https://nvd.nist.gov/static/feeds/xml/cve/nvdcve-2.0-recent.xml"
+NVD_RECENT_URL="https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-recent.xml"
+NVD_MODIFIED_URL="https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-modified.xml"
 
 PROGNAME="${0##*/}"
+TDIR="$(mktemp -d "${TMPDIR:-/tmp}/${PROGNAME}.XXXX")"
 VERBOSITY=0
 VFLAGS=""
 
@@ -31,17 +33,38 @@ cleanup() {
 
 crossReferenceWithJira() {
 	verbose "Cross-referencing with Jira..."
-	nvdXjira ${VFLAGS} -n ${NVDXJIRA_FLAGS}
+	(
+		# XXX: This is far from ideal: wget(1) requires
+		# the proxy, but nvdXjira does not.  nvdXjira uses
+		# python's urllib2, which will use a proxy if the
+		# https_proxy environment variable is set.  So we nuke it
+		# here.  If you ever want to use the proxy from the
+		# environment in nvdXjira in this context, you're SOL.
+		unset https_proxy
+		cd ${TDIR}
+		for f in *; do
+			verbose "Processing ${f}..." 2
+			nvdXjira ${VFLAGS} -n ${NVDXJIRA_FLAGS} <${f}
+		done
+	)
 }
 
-updateLocalDatabase() {
-	verbose "Updating local database..."
-	verbose "Fetching ${NVD_URL}..." 2
+fetchNVDData() {
+	local code file url
+
+	verbose "Fetching NVD Data..."
+
+	for url in ${NVD_RECENT_URL} ${NVD_MODIFIED_URL}; do
+		verbose "Fetching ${url}..." 2
+		file=$(basename ${url})
 
-	eval curl -s ${CURL_FLAGS} "${NVD_URL}" | {
-		verbose "Processing XML and updating sqlite3 database..." 2
-		nvd2sqlite3 ${VFLAGS} ${NVD2SQLITE_FLAGS}
-	}
+		eval wget --quiet ${WGET_FLAGS} -O "${TDIR}/${file}" "${url}"
+		if [ $? -gt 0 ]; then
+			echo "Unable to fetch '${url}'." >&2
+			exit 1
+			# NOTREACHED
+		fi
+	done
 }
 
 usage() {
@@ -112,7 +135,7 @@ if [ $# -ne 0 ]; then
 fi
 
 varCheck
-updateLocalDatabase
+fetchNVDData
 crossReferenceWithJira
 
 exit 0