Merge pull request #736 from BAngelique/fix-data-in-key

Nmap/XML: fix structured output for ms-sql-info and smb-enum-shares scripts

ivre/db/__init__.py

@@ -608,6 +608,7 @@ def __init__(self):
                 9: (10, self.__migrate_schema_hosts_9_10),
                 10: (11, self.__migrate_schema_hosts_10_11),
                 11: (12, self.__migrate_schema_hosts_11_12),
+                12: (13, self.__migrate_schema_hosts_12_13),
             },
         }
         self.argparser.add_argument(
@@ -984,6 +985,32 @@ def __migrate_schema_hosts_11_12(doc):
                         )
         return doc
 
+    @staticmethod
+    def __migrate_schema_hosts_12_13(doc):
+        """Converts a record from version 12 to version 13. Version 13 changes
+the structured output for ms-sql-info and smb-enum-shares scripts.
+
+        """
+        assert doc["schema_version"] == 12
+        doc["schema_version"] = 13
+        for port in doc.get('ports', []):
+            for script in port.get('scripts', []):
+                if script['id'] == "ms-sql-info":
+                    if "ms-sql-info" in script:
+                        script[
+                            "ms-sql-info"
+                        ] = xmlnmap.change_ms_sql_info(
+                            script["ms-sql-info"]
+                        )
+                elif script['id'] == "smb-enum-shares":
+                    if "smb-enum-shares" in script:
+                        script[
+                            "smb-enum-shares"
+                        ] = xmlnmap.change_smb_enum_shares(
+                            script["smb-enum-shares"]
+                        )
+        return doc
+
     @staticmethod
     def json2dbrec(host):
         return host

ivre/db/mongo.py

@@ -793,6 +793,7 @@ class MongoDBActive(MongoDB, DBActive):
         "ports.scripts.ike-info.vendor_ids",
         "ports.scripts.ls.volumes",
         "ports.scripts.ls.volumes.files",
+        "ports.scripts.ms-sql-info",
         "ports.scripts.mongodb-databases.databases",
         "ports.scripts.mongodb-databases.databases.shards",
         "ports.scripts.rpcinfo",
@@ -958,6 +959,7 @@ def __init__(self, url):
                 9: (10, self.migrate_schema_hosts_9_10),
                 10: (11, self.migrate_schema_hosts_10_11),
                 11: (12, self.migrate_schema_hosts_11_12),
+                12: (13, self.migrate_schema_hosts_12_13),
             },
         ]
 
@@ -1351,6 +1353,37 @@ def migrate_schema_hosts_11_12(doc):
             update["$set"]["ports"] = doc['ports']
         return update
 
+    @staticmethod
+    def migrate_schema_hosts_12_13(doc):
+        """Converts a record from version 12 to version 13. Version 13 changes
+        the structured output for ms-sql-info and smb-enum-shares scripts.
+
+        """
+        assert doc["schema_version"] == 12
+        update = {"$set": {"schema_version": 13}}
+        updated = False
+        for port in doc.get('ports', []):
+            for script in port.get('scripts', []):
+                if script['id'] == "ms-sql-info":
+                    if "ms-sql-info" in script:
+                        script[
+                            "ms-sql-info"
+                        ] = xmlnmap.change_ms_sql_info(
+                            script["ms-sql-info"]
+                        )
+                        updated = True
+                elif script['id'] == "smb-enum-shares":
+                    if "smb-enum-shares" in script:
+                        script[
+                            "smb-enum-shares"
+                        ] = xmlnmap.change_smb_enum_shares(
+                            script["smb-enum-shares"]
+                        )
+                        updated = True
+        if updated:
+            update["$set"]["ports"] = doc['ports']
+        return update
+
     def _get(self, flt, **kargs):
         """Like .get(), but returns a MongoDB cursor (suitable for use with
 e.g.  .explain()).

ivre/db/sql/__init__.py

@@ -735,6 +735,8 @@ def migrate_schema(self, version):
             failed += self._migrate_schema_10_11()
         if (version or 0) < 12:
             failed += self._migrate_schema_11_12()
+        if (version or 0) < 13:
+            failed += self._migrate_schema_12_13()
         return failed
 
     def _migrate_schema_8_9(self):
@@ -869,6 +871,50 @@ def _migrate_schema_11_12(self):
         )
         return len(failed)
 
+    def _migrate_schema_12_13(self):
+        """Converts a record from version 12 to version 13. Version 13 changes
+the structured output for ms-sql-info and smq-enum-shares scripts.
+
+        """
+        failed = set()
+        req = (select([self.tables.scan.id,
+                       self.tables.script.name,
+                       self.tables.script.port,
+                       self.tables.script.output,
+                       self.tables.script.data])
+               .select_from(join(join(self.tables.scan, self.tables.port),
+                                 self.tables.script))
+               .where(and_(self.tables.scan.schema_version == 12,
+                           self.tables.script.name.in_(["ms-sql-info",
+                                                        "smb-enum-shares"]))))
+        for rec in self.db.execute(req):
+            if rec.name in rec.data:
+                migr_func = {
+                    'ms-sql-info': xmlnmap.change_ms_sql_info,
+                    'smb-enum-shares': xmlnmap.change_smb_enum_shares,
+                }[rec.name]
+                try:
+                    data = migr_func(rec.data[rec.name])
+                except Exception:
+                    utils.LOGGER.warning("Cannot migrate host %r", rec.id,
+                                         exc_info=True)
+                    failed.add(rec.id)
+                else:
+                    if data:
+                        self.db.execute(
+                            update(self.tables.script)
+                            .where(and_(self.tables.script.port == rec.port,
+                                        self.tables.script.name == rec.name))
+                            .values(data={rec.name: data})
+                        )
+        self.db.execute(
+            update(self.tables.scan)
+            .where(and_(self.tables.scan.schema_version == 12,
+                        self.tables.scan.id.notin_(failed)))
+            .values(schema_version=13)
+        )
+        return len(failed)
+
     def count(self, flt, **_):
         return self.db.execute(
             flt.query(select([func.count()]))

ivre/xmlnmap.py

@@ -43,7 +43,7 @@
 from ivre.analyzer import ike
 
 
-SCHEMA_VERSION = 12
+SCHEMA_VERSION = 13
 
 # Scripts that mix elem/table tags with and without key attributes,
 # which is not supported for now
@@ -583,6 +583,14 @@ def change_s7_info_keys(table):
     return table
 
 
+def _smb_enum_shares_fix_share_name(name):
+    if not (name.startswith('\\\\') and '\\' in name[2:]):
+        utils.LOGGER.warning("Incorrect share name [%r]", name)
+        return name
+    server, share = name[2:].split('\\', 1)
+    return '\\\\%s\\%s' % (server.replace('_', '.'), share)
+
+
 def change_smb_enum_shares(table):
     """Adapt structured data from script smb-enum-shares so that it is
     easy to query when inserted in DB.
@@ -591,12 +599,37 @@ def change_smb_enum_shares(table):
     if not table:
         return table
     result = {}
-    for field in ["account_used", "note"]:
-        if field in table:
+    for field in list(table):
+        if field == 'shares':
+            continue
+        if not (field.startswith('\\\\')
+                and isinstance(table[field], dict)):
             result[field] = table.pop(field)
-    result["shares"] = [
-        dict(value, Share=key) for key, value in viewitems(table)
-    ]
+    if 'shares' in table:
+        # We just need to fix the share names
+        result["shares"] = sorted(
+            [
+                dict(elt, Share=_smb_enum_shares_fix_share_name(elt["Share"]))
+                for elt in table["shares"]
+            ],
+            key=lambda elt: elt["Share"],
+        )
+    else:
+        # We need to update the structured output to avoid data in field names:
+        #
+        # Old:
+        # {"\\ServerName\ShareName": {XXX}, ...}
+        # New:
+        # [{"Share": "\\ServerName\ShareName", XXX}, ...]
+        result["shares"] = sorted(
+            [
+                dict(
+                    value,
+                    Share=_smb_enum_shares_fix_share_name(key)
+                ) for key, value in viewitems(table)
+            ],
+            key=lambda elt: elt["Share"],
+        )
     return result
 
 
@@ -667,13 +700,38 @@ def change_rpcinfo(table):
             for proto, result in viewitems(protores)]
 
 
+def change_ms_sql_info(table):
+    """Adapt structured output generated by the "ms-sql-info" Nmap script. The
+structured output uses instances (hence, data) as keys, which is
+undesirable in the databases.
+
+New in SCHEMA_VERSION == 13.
+
+Use this function when migrating existing records.
+
+In previous schema versions, shares were used has keys; in keys,
+dots are replaced by underscores; this function reverts this change by
+replacing underscores by dots.
+
+    """
+    newlist = []
+    for key in list(table):
+        value = table[key]
+        if not isinstance(value, dict):
+            continue
+        newlist.append(dict(value, Instance=key.replace('_', '.')))
+        del table[key]
+    return newlist
+
+
 CHANGE_TABLE_ELEMS = {
     'smb-enum-shares': change_smb_enum_shares,
     "s7-info": change_s7_info_keys,
     'ls': change_ls,
     'vulns': change_vulns,
     'fcrdns': change_fcrdns,
     'rpcinfo': change_rpcinfo,
+    'ms-sql-info': change_ms_sql_info,
 }
 
 IGNORE_SCRIPTS = {

tests/tests.py

@@ -68,6 +68,7 @@
 import ivre.utils
 import ivre.web.utils
 import ivre.flow
+import ivre.xmlnmap
 
 HTTPD_PORT = 18080
 HTTPD_HOSTNAME = socket.gethostname()
@@ -750,6 +751,89 @@ def _json_loads(data, deflt=None):
         self.assertTrue('IVRE' in screenwords)
         shutil.rmtree('output')
 
+        # Test various structured results for smb-enum-shares
+        # before commit 79d25c5c6e4e2c3298f4f0771a183e82d06bfabe
+        json_12_smb_enum_shares_old = {
+            'note': 'ERROR: Enumerating shares failed, guessing at common ones'
+            ' (NT_STATUS_ACCESS_DENIED)',
+            'account_used': '<blank>',
+            '\\\\10_0_0_1\\C$': {
+                'warning': "Couldn't get details for share: "
+                "NT_STATUS_ACCESS_DENIED",
+                'Anonymous access': '<none>',
+            },
+            '\\\\10_0_0_1\\IPC$': {
+                'warning': "Couldn't get details for share: "
+                "NT_STATUS_ACCESS_DENIED",
+                'Anonymous access': 'READ',
+            },
+            '\\\\10_0_0_1\\ADMIN$': {
+                'warning': "Couldn't get details for share: "
+                "NT_STATUS_ACCESS_DENIED",
+                'Anonymous access': '<none>',
+            },
+            '\\\\10_0_0_1\\ADMIN_SOFTWARE': {
+                'warning': "Couldn't get details for share: "
+                "NT_STATUS_ACCESS_DENIED",
+                'Anonymous access': '<none>',
+            },
+        }
+        # after commit 79d25c5c6e4e2c3298f4f0771a183e82d06bfabe
+        json_12_smb_enum_shares_new = {
+            'note': 'ERROR: Enumerating shares failed, guessing at common ones'
+            ' (NT_STATUS_ACCESS_DENIED)',
+            'account_used': '<blank>',
+            'shares': [
+                {'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED",
+                 'Share': '\\\\10_0_0_1\\C$',
+                 'Anonymous access': '<none>'},
+                {'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED",
+                 'Share': '\\\\10_0_0_1\\IPC$',
+                 'Anonymous access': 'READ'},
+                {'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED",
+                 'Share': '\\\\10_0_0_1\\ADMIN$',
+                 'Anonymous access': '<none>'},
+                {'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED",
+                 'Share': '\\\\10_0_0_1\\ADMIN_SOFTWARE',
+                 'Anonymous access': '<none>'},
+            ]
+        }
+        json_13_expect_output = {
+            'account_used': '<blank>',
+            'note': 'ERROR: Enumerating shares failed, guessing at common ones'
+            ' (NT_STATUS_ACCESS_DENIED)',
+            'shares': [
+                {'Anonymous access': '<none>',
+                 'Share': '\\\\10.0.0.1\\ADMIN$',
+                 'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED"},
+                {'Anonymous access': '<none>',
+                 'Share': '\\\\10.0.0.1\\ADMIN_SOFTWARE',
+                 'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED"},
+                {'Anonymous access': '<none>',
+                 'Share': '\\\\10.0.0.1\\C$',
+                 'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED"},
+                {'Anonymous access': 'READ',
+                 'Share': '\\\\10.0.0.1\\IPC$',
+                 'warning': "Couldn't get details for share: "
+                 "NT_STATUS_ACCESS_DENIED"},
+            ]
+        }
+        self.assertEqual(
+            json_13_expect_output,
+            ivre.xmlnmap.change_smb_enum_shares(json_12_smb_enum_shares_old),
+        )
+        self.assertEqual(
+            json_13_expect_output,
+            ivre.xmlnmap.change_smb_enum_shares(json_12_smb_enum_shares_new),
+        )
+
         RUN(["ivre", "scancli", "--update-schema"])
 
         self.assertEqual(host_counter, host_counter_test)