Modification in the New Plugin for Microsoft Bulletin

lib/Config.py

@@ -22,8 +22,6 @@
 import zipfile
 from io import BytesIO
 
-import requests
-
 class Configuration():
   ConfigParser = configparser.ConfigParser()
   ConfigParser.read(os.path.join(runPath, "../etc/configuration.ini"))
@@ -84,19 +82,3 @@ def getFile(cls, getfile, unpack=True):
           if len(fzip.namelist())>0:
             data=BytesIO(fzip.read(fzip.namelist()[0]))
     return (data, response)
-
-  # Get Microsoft Bulletin
-  def get_msbulletin(url):
-    h1={"Accept": "application/json, text/plain, */*",
-    "Content-Type": "application/json;charset=utf-8",
-    "Referer": "https://portal.msrc.microsoft.com/en-us/security-guidance"
-    }
-    data1 ='{"familyIds":[],"productIds":[],"severityIds":[],"impactIds":[],"pageNumber":1,"pageSize":100,"includeCveNumber":true,"includeSeverity":true,"includeImpact":true,"orderBy":"publishedDate","orderByMonthly":"releaseDate","isDescending":true,"isDescendingMonthly":true,"queryText":"","isSearch":false,"filterText":"","fromPublishedDate":"04/12/2017","toPublishedDate":"05/17/2017"}'
-    post = requests.post(url,  headers=h1, data=data1)
-    data=b''
-    for chunk in post.iter_content(chunk_size=128):
-        data=data+chunk
-    if len(data)>0:
-      return data
-    else:
-      return None

sources/MSBulletin_V2.py

@@ -11,65 +11,73 @@
 import copy
 import datetime
 import time
-import xlrd
-import zipfile
-from io import BytesIO
+import json
+import requests
 
 from collections import defaultdict
 
 from lib.Config  import Configuration as conf
 from lib.Source  import Source
 
 
-SOURCE_NAME = "msbulletin.xlsx"
-SOURCE_FILE = "https://portal.msrc.microsoft.com/api/security-guidance/en-us/excel"
+SOURCE_NAME = "msbulletin"
+SOURCE_FILE = "https://portal.msrc.microsoft.com/api/security-guidance/en-us/"
 
-workbook_format = wf = {'date':             0,  'impact':       6,
-                        'bulletin_id':      1,  'title':        2,
-                        'knowledgebase_id': 2,  'cves':         4,
-                        'severity':     5}
 
-# Convert date "10-jun-17" to datetime "10-01-2017 00:00:00"
-def xldate_as_datetime(xldate, datemode):
-    date = time.strftime("%d/%m/%Y", time.strptime(xldate,"%d-%b-%y"))
-    return datetime.datetime.strptime(date, "%d/%m/%Y")
-
+# Get Ms Bulletins in Json Format
+def get_msbulletin(url, from_date= "01/01/1900" , to_Date= None ):
+    headers={"Accept": "application/json, text/plain, */*",
+    "Content-Type": "application/json;charset=utf-8",
+    "Referer": "https://portal.msrc.microsoft.com/en-us/security-guidance"
+    }
+    query = {
+          'familyIds': [], 'productIds': [], 'severityIds': [], 'impactIds': [],
+          'pageNumber': 1, 'pageSize': 50000,
+          'includeCveNumber':True, 'includeSeverity': True, 'includeImpact': True,
+          'orderBy': 'publishedDate', 'orderByMonthly': 'releaseDate',
+          'isDescending': True, 'isDescendingMonthly': True,
+          'queryText': '', 'isSearch': False, 'filterText': '',
+          'fromPublishedDate': from_date
+    }
+
+    if to_Date:
+      query['toPublishedDate'] = to_date
+
+    post = requests.post(url,  headers=headers, data=json.dumps(query) )
+    if post:
+      return post.json()
+    else:
+      return None
+
 class MSBulletin_V2(Source):
   def __init__(self):
     self.name = SOURCE_NAME
-    _file =   conf.get_msbulletin(SOURCE_FILE) # get file xlsx
-    workbook  = xlrd.open_workbook(file_contents = _file)
-    worksheet = workbook.sheet_by_index(0)
+    data_json =   get_msbulletin(SOURCE_FILE) # Get MS Bulletins in Json Format
     mskb      = defaultdict(dict)
     self.cves = defaultdict(list)
 
-    for rownum in range(worksheet.nrows-1): # -1 because we skip the header
-      row = worksheet.row_values(rownum+1)  # +1 because we skip the header
-      # Convert date to date object
-      date = xldate_as_datetime(row[wf['date']], 0).isoformat()
-      #date = row[wf['date']]
-      mskb[row[wf['bulletin_id']]]['date']             = date
-      mskb[row[wf['bulletin_id']]]['bulletin_id']      = row[wf['bulletin_id']]
-      mskb[row[wf['bulletin_id']]]['knowledgebase_id'] = row[wf['knowledgebase_id']]
-      mskb[row[wf['bulletin_id']]]['severity']         = row[wf['severity']]
-      mskb[row[wf['bulletin_id']]]['impact']           = row[wf['impact']]
-      mskb[row[wf['bulletin_id']]]['title']            = row[wf['title']]
-      mskb[row[wf['bulletin_id']]]['cves']             = row[wf['cves']].split(",")
+    for items in data_json["details"]:  
+      mskb[items["knowledgeBaseId"]]['date']             = items["publishedDate"] 
+      mskb[items["knowledgeBaseId"]]['kb_id']            = items['knowledgeBaseId'] # KB id
+      mskb[items["knowledgeBaseId"]]['knowledgebase_id'] = items['knowledgeBaseId'] # KB id
+      mskb[items["knowledgeBaseId"]]['severity']         = items['severity']
+      mskb[items["knowledgeBaseId"]]['impact']           = items['impact']
+      mskb[items["knowledgeBaseId"]]['title']            = items['name'] # Name of product
+      mskb[items["knowledgeBaseId"]]['cves']             = items['cveNumber'] # CVE  id
+      mskb[items["knowledgeBaseId"]]['cves_url']         = items['cveUrl']  # CVE url
 
-      bulletin_SOURCE_FILE      = worksheet.hyperlink_map.get((rownum, wf['bulletin_id']))
-      knowledgebase_SOURCE_FILE = worksheet.hyperlink_map.get((rownum, wf['knowledgebase_id']))
-      mskb[row[wf['bulletin_id']]]['bulletin_SOURCE_FILE']      = bulletin_SOURCE_FILE
-      mskb[row[wf['bulletin_id']]]['knowledgebase_SOURCE_FILE'] = knowledgebase_SOURCE_FILE
+      mskb[items["knowledgeBaseId"]]['bulletin_SOURCE_FILE']      = SOURCE_FILE
+      mskb[items["knowledgeBaseId"]]['knowledgebase_SOURCE_FILE'] = items["knowledgeBaseUrl"]
 
     for _id, data in mskb.items():
       to_store = copy.copy(data)
       to_store.pop("cves")
-      for cve in data['cves']:
-        if cve: self.cves[cve].append(to_store)
+      if data['cves']:
+          self.cves[data['cves']].append(to_store)
 
   def cleanUp(self, cveID, cveData):
     if cveData.get('refmap', {}).get('ms'):
       del cveData['refmap']['ms']
 
   def getSearchables(self):
-    return ['bulletin_id', 'knowledgebase_id']
+    return ['kb_id', 'knowledgebase_id']