Merge pull request #1520 from p-l-/enh-govcloud

Tags: add GovClouds (AWS & Azure) IP addresses

ivre/active/data.py

@@ -40,6 +40,7 @@
     Optional,
     Set,
     Tuple,
+    TypeVar,
     Union,
     cast,
 )
@@ -383,6 +384,7 @@ def has_toomany_hostnames(host: NmapHost) -> bool:
 
 
 TAG_CDN: Tag = {"value": "CDN", "type": "info"}
+TAG_GOVCLOUD: Tag = {"value": "GovCloud", "type": "info"}
 TAG_DEFAULT_PASSWORD: Tag = {"value": "Default password", "type": "danger"}
 TAG_HONEYPOT: Tag = {"value": "Honeypot", "type": "warning"}
 TAG_MALWARE: Tag = {"value": "Malware", "type": "danger"}
@@ -416,11 +418,12 @@ def has_toomany_hostnames(host: NmapHost) -> bool:
 
 _TOR_NODES: Optional[Set[str]] = None
 _CDN_TABLE: Optional[Tuple[List[int], List[Optional[str]]]] = None
+_GOVCLOUD_TABLE: Optional[Tuple[List[int], List[Optional[List[str]]]]] = None
 _SCANNERS_TABLE: Optional[Tuple[List[int], List[Optional[str]]]] = None
 
 
 def _get_data() -> None:
-    global _TOR_NODES, _SCANNERS_TABLE, _CDN_TABLE
+    global _TOR_NODES, _SCANNERS_TABLE, _CDN_TABLE, _GOVCLOUD_TABLE
     assert DATA_PATH is not None
     if _TOR_NODES is None:
         try:
@@ -483,9 +486,22 @@ def _get_data() -> None:
                 "Cannot find file [cdn_nuclei.py]. Try running `ivre getwebdata`"
             )
             _CDN_TABLE = ([], [])
+    if _GOVCLOUD_TABLE is None:
+        try:
+            with open(os.path.join(DATA_PATH, "govcloud.py"), encoding="utf8") as fdesc:
+                # pylint: disable=eval-used
+                _GOVCLOUD_TABLE = eval(compile(fdesc.read(), "gov_cloud", "eval"))
+        except FileNotFoundError:
+            LOGGER.warning(
+                "Cannot find file [govcloud.py]. Try running `ivre getwebdata`"
+            )
+            _GOVCLOUD_TABLE = ([], [])
+
 
+T = TypeVar("T")
 
-def _get_name(table: Tuple[List[int], List[Optional[str]]], addr: str) -> Optional[str]:
+
+def _get_name(table: Tuple[List[int], List[Optional[T]]], addr: str) -> Optional[T]:
     """Devs: please make sure _get_data() has been called before calling me!"""
     addr_i = ip2int(addr) if ":" in addr else ip2int(f"::ffff:{addr}")
     try:
@@ -515,6 +531,7 @@ def gen_auto_tags(
     assert _TOR_NODES is not None
     assert _SCANNERS_TABLE is not None
     assert _CDN_TABLE is not None
+    assert _GOVCLOUD_TABLE is not None
     addr = host.get("addr")
     if isinstance(addr, str):
         if addr in _TOR_NODES:
@@ -536,6 +553,15 @@ def gen_auto_tags(
                     info=[f"{cdn_name} as listed at <https://cdn.nuclei.sh/>"],
                 ),
             )
+        govcloud_data = _get_name(_GOVCLOUD_TABLE, addr)
+        if govcloud_data is not None:
+            yield cast(
+                Tag,
+                dict(
+                    TAG_GOVCLOUD,
+                    info=govcloud_data,
+                ),
+            )
         scanner_name = _get_name(_SCANNERS_TABLE, addr)
         if scanner_name is not None:
             yield cast(

ivre/data/govcloud.py

@@ -0,0 +1,122 @@
+#! /usr/bin/env python
+
+# This file is part of IVRE.
+# Copyright 2011 - 2023 Pierre LALET <pierre@droids-corp.org>
+#
+# IVRE is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# IVRE is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+# License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with IVRE. If not, see <http://www.gnu.org/licenses/>.
+
+"""Handles data from (US) GovCloud, based on
+<https://github.com/daehee/govcloud/> (but we do not use code or data
+from this repository).
+
+"""
+
+
+import json
+import os
+import re
+from typing import List, Tuple
+
+from ivre import VERSION, config
+from ivre.utils import build_opener, download_if_newer, make_range_tables, net2range
+
+AZURE_URL = re.compile(b'<a href="(https://download.microsoft.com/download/[^"]*)"')
+
+
+def get_azure_url() -> str:
+    opener = build_opener()
+    opener.addheaders = [("User-Agent", "IVRE/%s +https://ivre.rocks/" % VERSION)]
+    with opener.open(
+        "https://www.microsoft.com/en-us/download/confirmation.aspx?id=57063"
+    ) as udesc:
+        for line in udesc:
+            match = AZURE_URL.search(line)
+            if match is None:
+                continue
+            return match.group(1).decode()
+    raise ValueError("URL for Azure US Government Cloud not found")
+
+
+def get_all_files() -> float:
+    assert config.DATA_PATH is not None
+    most_recent = 0.0
+    fname = os.path.join(config.DATA_PATH, "govcloud_azure.json")
+    download_if_newer(get_azure_url(), fname)
+    most_recent = max(most_recent, os.stat(fname).st_mtime)
+    fname = os.path.join(config.DATA_PATH, "govcloud_aws.json")
+    download_if_newer("https://ip-ranges.amazonaws.com/ip-ranges.json", fname)
+    most_recent = max(most_recent, os.stat(fname).st_mtime)
+    return most_recent
+
+
+def build_table() -> List[Tuple[str, str, List[str]]]:
+    assert config.DATA_PATH is not None
+    all_ranges = []
+
+    # Azure
+    with open(
+        os.path.join(config.DATA_PATH, "govcloud_azure.json"), encoding="utf8"
+    ) as fdesc:
+        all_entries = json.load(fdesc)["values"]
+    for entry in all_entries:
+        data = []
+        properties = entry.get("properties", {})
+        for fld, name in [
+            ("platform", "Platform"),
+            ("region", "Region"),
+            ("systemService", "Service"),
+        ]:
+            if properties.get(fld):
+                data.append(f"{name}: {properties[fld]}")
+        data = sorted(data)
+        for net in properties.get("addressPrefixes", []):
+            start, stop = net2range(net)
+            all_ranges.append((start, stop, data))
+
+    # AWS
+    with open(
+        os.path.join(config.DATA_PATH, "govcloud_aws.json"), encoding="utf8"
+    ) as fdesc:
+        all_entries = json.load(fdesc)["prefixes"]
+    for entry in all_entries:
+        if not entry.get("region", "").startswith("us-gov-"):
+            continue
+        start, stop = net2range(entry["ip_prefix"])
+        data = ["Platform: AWS"]
+        for fld, name in [("region", "Region"), ("service", "Service")]:
+            if entry.get(fld):
+                data.append(f"{name}: {entry[fld]}")
+        data = sorted(data)
+        all_ranges.append((start, stop, data))
+
+    return all_ranges
+
+
+def fetch_and_build() -> None:
+    assert config.DATA_PATH is not None
+    most_recent = get_all_files()
+    fname = os.path.join(config.DATA_PATH, "govcloud.py")
+    try:
+        current = os.stat(fname).st_mtime
+    except FileNotFoundError:
+        current = 0.0
+    if current > most_recent:
+        return
+    table = make_range_tables(build_table())
+    with open(fname, "w", encoding="utf8") as fdesc:
+        fdesc.write("[\n    (\n")
+        fdesc.writelines(f"        {elt[0]!r},\n" for elt in table)
+        fdesc.write("    ),\n    (\n")
+        fdesc.writelines(f"        {elt[1]!r},\n" for elt in table)
+        fdesc.write("    ),\n]\n")

ivre/tools/getwebdata.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 
 # This file is part of IVRE.
-# Copyright 2011 - 2022 Pierre LALET <pierre@droids-corp.org>
+# Copyright 2011 - 2023 Pierre LALET <pierre@droids-corp.org>
 #
 # IVRE is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by
@@ -22,6 +22,8 @@
 
   - CDN providers, from <https://cdn.nuclei.sh/>
 
+  - (US) GovCloud IP ranges, from <https://github.com/daehee/govcloud>
+
   - Tor Exit nodes, from
     <https://check.torproject.org/torbulkexitlist>
 
@@ -46,6 +48,7 @@
 from typing import BinaryIO, Callable, Generator, List, Tuple, cast
 
 from ivre import config
+from ivre.data import govcloud
 from ivre.utils import (
     IPADDR,
     download_if_newer,
@@ -130,3 +133,4 @@ def main() -> None:
             f"{addr}\n"
             for addr in dns_get_names("scanner.scanning.service.ncsc.gov.uk")
         )
+    govcloud.fetch_and_build()

ivre/utils.py

@@ -62,6 +62,7 @@
     Set,
     Tuple,
     Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -2519,7 +2520,10 @@ def generic_processor(
     )
 
 
-def _fix_range(start: str, stop: str, label: str) -> Tuple[int, int, str]:
+T = TypeVar("T")
+
+
+def _fix_range(start: str, stop: str, label: T) -> Tuple[int, int, T]:
     return (
         ip2int(start) if ":" in start else ip2int(f"::ffff:{start}"),
         ip2int(stop) if ":" in stop else ip2int(f"::ffff:{stop}"),
@@ -2528,12 +2532,12 @@ def _fix_range(start: str, stop: str, label: str) -> Tuple[int, int, str]:
 
 
 def make_range_tables(
-    ranges: Iterable[Tuple[str, str, str]]
-) -> List[Tuple[int, Optional[str]]]:
-    ranges_sorted: List[Tuple[int, int, str]] = sorted(
+    ranges: Iterable[Tuple[str, str, T]]
+) -> List[Tuple[int, Optional[T]]]:
+    ranges_sorted: List[Tuple[int, int, T]] = sorted(
         (_fix_range(start, stop, label) for start, stop, label in ranges), reverse=True
     )
-    result: List[Tuple[int, Optional[str]]] = []
+    result: List[Tuple[int, Optional[T]]] = []
     prev = 0
     while ranges_sorted:
         start, stop, label = ranges_sorted.pop()