use yaml for remapping; remove json transpose code (#177)

* use yaml for remapping; remove json transpose code

* temporarily revert cpe change on win2k3

cpe-remap.yaml

@@ -0,0 +1,125 @@
+mappings:
+  apache:
+    vendor: apache
+    products:
+      httpd: http_server
+  alt-n:
+    vendor: altn
+  bea:
+    vendor: bea
+    products:
+      weblogic: weblogic_server
+  blue_coat:
+    vendor: bluecoat
+  centos:
+    vendor: centos
+    products:
+      linux: centos
+  check_point:
+    vendor: checkpoint
+  cisco:
+    vendor: cisco
+    products:
+      adaptive_security_appliance: adaptive_security_appliance_software
+      pix: pix_firewall_software
+      telepresence: telepresence_video_communication_server_software
+  debian:
+    vendor: debian
+    products:
+      linux: debian_linux
+  f5:
+    vendor: f5
+    products:
+      big-ip: big-ip_local_traffic_manager
+      big-ip_ltm: big-ip_local_traffic_manager
+  hp:
+    vendor: hp
+    products:
+      ilo: integrated_lights_out
+      lotus_domino: lotus_domino_server
+      tru64_unix: tru64
+  ibm:
+    vendor: ibm
+    products:
+      lotus_domino: lotus_domino_server
+  juniper:
+    vendor: juniper
+    products:
+      junos_os: junos
+  linux:
+    vendor: linux
+    products:
+      linux: linux_kernel
+  microsoft:
+    vendor: microsoft
+    products:
+      active_directory_controller: active_directory
+      exchange_server_5.5: exchange_server
+      exchange_2000_server: exchange_server
+      exchange_2003_server: exchange_server
+      exchange_2007_server: exchange_server
+      lightweight_directory_server: active_directory_lightweight_directory_service
+      windows_server_2003_datacenter_edition: windows_server_2003
+      windows_server_2003_r2: windows_server_2003
+      windows_2008_r2: windows_server_2008
+      windows_server_2008_datacenter_edition: windows_server_2008
+      windows_server_2008_r2: windows_server_2008
+      windows_server_2008_r2_datacenter_edition: windows_server_2008
+      windows_server_2012_r2: windows_server_2012
+      nt: windows_nt
+      windows_nt_desktop: windows_nt
+      windows_nt_server: windows_nt
+      windows_server_2000: windows_2000
+      windows_2000_server: windows_2000
+      windows_2000_datacenter_server: windows_2000
+      pws: personal_web_server
+  mod_ssl:
+    vendor: modssl
+  mod_wsgi:
+    vendor: modwsgi
+  mort_bay:
+    vendor: mortbay
+  net-snmp:
+    vendor: net-snmp
+    products:
+      snmp_agent: net-snmp
+  palo_alto_networks:
+    vendor: paloaltonetworks
+    products:
+      pa_firewall: pan-os
+  proftpd_project:
+    vendor: proftpd
+  realvnc_ltd.:
+    vendor: realvnc
+  red_hat:
+    vendor: redhat
+    products:
+      cygwin_x_server_project: cygwin
+      fedora_core_linux: fedora_core
+      jboss_as: jboss_wildfly_application_server
+      jboss_eap: jboss_enterprise_application_platform
+      jbossweb: jboss_web_framework_kit
+      red_hat_directory_server: directory_server
+  sun:
+    vendor: sun
+    products:
+      solaris: sunos
+  ubuntu:
+    vendor: canonical
+    products:
+      linux: ubuntu_linux
+  vandyke_software:
+    vendor: vandyke
+  vmware:
+    vendor: vmware
+    products:
+      photon_linux: photon_os
+      zimbra: zimbra_desktop
+      vmware_esx_server: esx
+      vmware_esxi_server: esxi
+  wind_river:
+    vendor: windriver
+  x.org:
+    vendor: x.org
+    products:
+      x.org_x11: x11

update_cpes.py

@@ -1,31 +1,15 @@
 #!/usr/bin/env python
 
-import json
+import yaml
 import logging
 import re
 import sys
 
 from lxml import etree
 
 def parse_r7_remapping(file):
-    remap = {} # r7_vendor => { 'cpe_vendor' => <cpe_vendor>, 'products': { r7_product1 => cpe_product1 }}
-    remappings = None
     with open(file) as remap_file:
-        remappings = json.load(remap_file)["remappings"]
-
-    for remap_json in remappings:
-        r7_vendor = remap_json['r7_vendor']
-        cpe_vendor = remap_json['cpe_vendor']
-        if r7_vendor in remap:
-            raise ValueError("R7 vendor {} duplicated in {}".format(r7_vendor, file))
-
-        product_map = {}
-        if 'products' in remap_json:
-            product_map = remap_json['products']
-        remap[r7_vendor] = {'cpe_vendor': cpe_vendor, 'products': product_map}
-
-    return remap
-
+        return yaml.load(remap_file)["mappings"]
 
 def parse_cpe_vp_map(file):
     vp_map = {} # cpe_type -> vendor -> products
@@ -63,7 +47,7 @@ def main():
     update_cpes(sys.argv[1], cpe_vp_map, r7_vp_map)
 
 def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
-    parser = etree.XMLParser(remove_comments=False)
+    parser = etree.XMLParser(remove_comments=False, remove_blank_text=True)
     doc = etree.parse(xml_file, parser)
 
     for fingerprint in doc.xpath('//fingerprint'):
@@ -140,7 +124,7 @@ def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
                 og_vendor = vendor
                 if not vendor in cpe_vp_map[cpe_type]:
                     if vendor in r7_vp_map:
-                        vendor = r7_vp_map[vendor]['cpe_vendor']
+                        vendor = r7_vp_map[vendor]['vendor']
                         remapped_vendor = True
                         if not vendor in cpe_vp_map[cpe_type]:
                             logging.error("Remapped vendor %s (remapped from %s) invalid for CPE %s (product %s)", vendor, og_vendor, cpe_type, product)
@@ -153,7 +137,7 @@ def update_cpes(xml_file, cpe_vp_map, r7_vp_map):
                 # if the product as specified is not found in the CPE dictionary for this vendor
                 if not product in cpe_vp_map[cpe_type][vendor]:
                     # if this vendor has a remapping from R7
-                    if og_vendor in r7_vp_map:
+                    if og_vendor in r7_vp_map and 'products' in r7_vp_map[og_vendor]:
                         # if this product has a remapping for this vendor from R7
                         if product in r7_vp_map[og_vendor]['products']:
                             og_product = product