Merge pull request #755 from p-l-/fix-schema

Active schemas: avoid different types on same fields

ivre/db/__init__.py

@@ -609,6 +609,7 @@ def __init__(self):
                 10: (11, self.__migrate_schema_hosts_10_11),
                 11: (12, self.__migrate_schema_hosts_11_12),
                 12: (13, self.__migrate_schema_hosts_12_13),
+                13: (14, self.__migrate_schema_hosts_13_14),
             },
         }
         self.argparser.add_argument(
@@ -1011,6 +1012,30 @@ def __migrate_schema_hosts_12_13(doc):
                         )
         return doc
 
+    @staticmethod
+    def __migrate_schema_hosts_13_14(doc):
+        """Converts a record from version 13 to version 14. Version 14 changes
+the structured output for ssh-hostkey and ls scripts to prevent a same
+field from having different data types.
+
+        """
+        assert doc["schema_version"] == 13
+        doc["schema_version"] = 14
+        for port in doc.get('ports', []):
+            for script in port.get('scripts', []):
+                if script['id'] == "ssh-hostkey" and 'ssh-hostkey' in script:
+                    script['ssh-hostkey'] = xmlnmap.change_ssh_hostkey(
+                        script["ssh-hostkey"]
+                    )
+                elif (xmlnmap.ALIASES_TABLE_ELEMS.get(script['id']) == 'ls' and
+                      "ls" in script):
+                    script[
+                        "ls"
+                    ] = xmlnmap.change_ls_migrate(
+                        script["ls"]
+                    )
+        return doc
+
     @staticmethod
     def json2dbrec(host):
         return host

ivre/db/mongo.py

@@ -960,6 +960,7 @@ def __init__(self, url):
                 10: (11, self.migrate_schema_hosts_10_11),
                 11: (12, self.migrate_schema_hosts_11_12),
                 12: (13, self.migrate_schema_hosts_12_13),
+                13: (14, self.migrate_schema_hosts_13_14),
             },
         ]
 
@@ -1384,6 +1385,35 @@ def migrate_schema_hosts_12_13(doc):
             update["$set"]["ports"] = doc['ports']
         return update
 
+    @staticmethod
+    def migrate_schema_hosts_13_14(doc):
+        """Converts a record from version 13 to version 14. Version 14 changes
+the structured output for ssh-hostkey and ls scripts to prevent a same
+field from having different data types.
+
+        """
+        assert doc["schema_version"] == 13
+        update = {"$set": {"schema_version": 13}}
+        updated = False
+        for port in doc.get('ports', []):
+            for script in port.get('scripts', []):
+                if script['id'] == "ssh-hostkey" and 'ssh-hostkey' in script:
+                    script['ssh-hostkey'] = xmlnmap.change_ssh_hostkey(
+                        script["ssh-hostkey"]
+                    )
+                    updated = True
+                elif (xmlnmap.ALIASES_TABLE_ELEMS.get(script['id']) == 'ls' and
+                      "ls" in script):
+                    script[
+                        "ls"
+                    ] = xmlnmap.change_ls_migrate(
+                        script["ls"]
+                    )
+                    updated = True
+        if updated:
+            update["$set"]["ports"] = doc['ports']
+        return update
+
     def _get(self, flt, **kargs):
         """Like .get(), but returns a MongoDB cursor (suitable for use with
 e.g.  .explain()).

ivre/db/sql/__init__.py

@@ -737,6 +737,8 @@ def migrate_schema(self, version):
             failed += self._migrate_schema_11_12()
         if (version or 0) < 13:
             failed += self._migrate_schema_12_13()
+        if (version or 0) < 14:
+            failed += self._migrate_schema_13_14()
         return failed
 
     def _migrate_schema_8_9(self):
@@ -915,6 +917,57 @@ def _migrate_schema_12_13(self):
         )
         return len(failed)
 
+    def _migrate_schema_13_14(self):
+        """Converts a record from version 13 to version 14. Version 14 changes
+the structured output for ssh-hostkey and ls scripts to prevent a same
+field from having different data types.
+
+        """
+        failed = set()
+        scripts = [
+            script_name
+            for script_name, alias in viewitems(xmlnmap.ALIASES_TABLE_ELEMS)
+            if alias == 'ls'
+        ]
+        scripts.append('ssh-hostkey')
+        req = (select([self.tables.scan.id,
+                       self.tables.script.name,
+                       self.tables.script.port,
+                       self.tables.script.output,
+                       self.tables.script.data])
+               .select_from(join(join(self.tables.scan, self.tables.port),
+                                 self.tables.script))
+               .where(and_(self.tables.scan.schema_version == 13,
+                           self.tables.script.name.in_(scripts))))
+        for rec in self.db.execute(req):
+            if rec.name in rec.data:
+                migr_func = (
+                    xmlnmap.change_ssh_hostkey
+                    if rec.name == 'ssh-hostkey' else
+                    xmlnmap.change_ls_migrate
+                )
+                try:
+                    data = migr_func(rec.data[rec.name])
+                except Exception:
+                    utils.LOGGER.warning("Cannot migrate host %r", rec.id,
+                                         exc_info=True)
+                    failed.add(rec.id)
+                else:
+                    if data:
+                        self.db.execute(
+                            update(self.tables.script)
+                            .where(and_(self.tables.script.port == rec.port,
+                                        self.tables.script.name == rec.name))
+                            .values(data={rec.name: data})
+                        )
+        self.db.execute(
+            update(self.tables.scan)
+            .where(and_(self.tables.scan.schema_version == 13,
+                        self.tables.scan.id.notin_(failed)))
+            .values(schema_version=14)
+        )
+        return len(failed)
+
     def count(self, flt, **_):
         return self.db.execute(
             flt.query(select([func.count()]))

ivre/xmlnmap.py

@@ -43,7 +43,7 @@
 from ivre.analyzer import ike
 
 
-SCHEMA_VERSION = 13
+SCHEMA_VERSION = 14
 
 # Scripts that mix elem/table tags with and without key attributes,
 # which is not supported for now
@@ -602,8 +602,8 @@ def change_smb_enum_shares(table):
     for field in list(table):
         if field == 'shares':
             continue
-        if not (field.startswith('\\\\')
-                and isinstance(table[field], dict)):
+        if not (field.startswith('\\\\') and
+                isinstance(table[field], dict)):
             result[field] = table.pop(field)
     if 'shares' in table:
         # We just need to fix the share names
@@ -637,15 +637,36 @@ def change_ls(table):
     """Adapt structured data from "ls" NSE module to convert some
     fields to integers.
 
+New in SCHEMA_VERSION == 14: special file size '<DIR>' is removed from
+the document.
+
     """
     if 'total' in table:
         for field in ['files', 'bytes']:
             if field in table['total'] and table['total'][field].isdigit():
                 table['total'][field] = int(table['total'][field])
     for volume in table.get('volumes', []):
         for fileentry in volume.get('files', []):
-            if 'size' in fileentry and fileentry['size'].isdigit():
-                fileentry['size'] = int(fileentry['size'])
+            if 'size' in fileentry:
+                if fileentry['size'].isdigit():
+                    fileentry['size'] = int(fileentry['size'])
+                elif fileentry['size'] == '<DIR>':
+                    del fileentry['size']
+    return table
+
+
+def change_ls_migrate(table):
+    """Adapt structured data from "ls" NSE module to convert some
+    fields to integers.
+
+New in SCHEMA_VERSION == 14: special file size '<DIR>' is removed from
+the document.
+
+    """
+    for volume in table.get('volumes', []):
+        for fileentry in volume.get('files', []):
+            if 'size' in fileentry and fileentry['size'] == '<DIR>':
+                del fileentry['size']
     return table
 
 
@@ -724,6 +745,19 @@ def change_ms_sql_info(table):
     return newlist
 
 
+def change_ssh_hostkey(table):
+    """Adapt structured output generated by the "ssh-hostkey" Nmap script.
+
+New in SCHEMA_VERSION == 14.
+
+    """
+    for key in table:
+        if 'bits' in key:
+            # int(float()): a (now fixed) bug in Nmap reports 2048.0.
+            key['bits'] = int(float(key['bits']))
+    return table
+
+
 CHANGE_TABLE_ELEMS = {
     'smb-enum-shares': change_smb_enum_shares,
     "s7-info": change_s7_info_keys,
@@ -732,6 +766,7 @@ def change_ms_sql_info(table):
     'fcrdns': change_fcrdns,
     'rpcinfo': change_rpcinfo,
     'ms-sql-info': change_ms_sql_info,
+    'ssh-hostkey': change_ssh_hostkey,
 }
 
 IGNORE_SCRIPTS = {