From da1ffebaa9c13ebfc4a420310c3f09fa98293a98 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 11 Dec 2023 15:44:07 +0000 Subject: [PATCH 1/6] asmap: add builder Lifted from Sipa's ASMap builder: https://github.com/sipa/asmap/tree/nextgen Don't ruff format to pull upstream changes more easily. --- pyproject.toml | 2 +- src/external/__init__.py | 0 src/external/buildmap.py | 362 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 src/external/__init__.py create mode 100644 src/external/buildmap.py diff --git a/pyproject.toml b/pyproject.toml index 5d425f62c..99fa066e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,6 @@ exclude = ''' ''' [tool.ruff] -extend-exclude = ["src/test_framework/*.py"] +extend-exclude = ["src/test_framework/*.py", "src/external/buildmap.py"] line-length = 100 indent-width = 4 diff --git a/src/external/__init__.py b/src/external/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/external/buildmap.py b/src/external/buildmap.py new file mode 100644 index 000000000..cff3aa554 --- /dev/null +++ b/src/external/buildmap.py @@ -0,0 +1,362 @@ +""" +TAKEN FROM https://github.com/sipa/asmap/tree/nextgen +""" + +""" +Intake a map of IP prefixes -> AS numbers and output instructions that will +allow a decoder to match an IP address to an ASN by following a sequence +of instructions. +The instructions describe a prefix tree that can be navigated using the bits of +an IP address (i.e. 0 for left child, 1 for right child, leaf nodes +corresponding to a given ASN). The types of instructions are denoted by the +*Type() functions defined below. Once an IP address specifies a bit for which +there is no path in the tree (i.e. the part of its address more specific than +any known network prefix), the tree returns a "default" ASN value that has been +set based on the last valid location in the tree. +See `testmap.py:Interpret` for an illustration of how the decoding process +works. +Before the prefix tree is encoded into instructions using bits, it is compacted +(e.g. duplicate subtrees removed) and annotated with which default ASN values +should be set for particular regions of the tree. +""" +import sys +import ipaddress +from collections import namedtuple +from typing import Counter + + +def Parse(entries: list): + """ + Read in a file of the format + 1.0.0.0/24 AS13335 # ipv4.dump:4856343 + 1.0.4.0/22 AS56203 # ipv4.dump:2759291 + ... + Ignoring comments following '#'. Creates an Entry object for each line. + Maps IPv4 networks into IPv6 space. + Args: + entries: modified in place with the new Entrys. + """ + for line in sys.stdin: + line = line.split("#")[0].lstrip(" ").rstrip(" \r\n") + prefix, asn = line.split(" ") + assert len(asn) > 2 and asn[:2] == "AS" + network = ipaddress.ip_network(prefix) + + prefix_len = network.prefixlen + net_addr = int.from_bytes(network.network_address.packed, "big") + + # Map an IPv4 prefix into IPv6 space. + if isinstance(network, ipaddress.IPv4Network): + prefix_len += 96 + net_addr += 0xFFFF00000000 + + entries.append(Entry(prefix_len, net_addr, int(asn[2:]))) + + +Entry = namedtuple( + "Entry", + ( + # The length of the network prefix in bits. E.g. '26' for 255.255.0.0/26. + "prefix_len", + # An int containing the bits of the network address. + "net_addr", + # An int for the autonomous system (AS) number. + "asn", + ), +) + + +def UpdateTree(gtree, addrlen: int, entries: [Entry]): + """ + Returns a prefix tree such that following a path down through the + tree based on the bits of a network prefix (in order of most significant + bit) leads to an ASN. + Args: + gtree: tree structure to encode the mappings into. Modified in-place. + addrlen: The maximum number of bits in a network address. + This is 128 for IPv6 (16 bytes). + entries: The network prefix -> ASN mappings to encode. + """ + for prefix, val, asn in sorted(entries): + tree = gtree + default = None + + # Iterate through each bit in the network prefix, starting with the + # most significant bit. + for i in range(prefix): + bit = (val >> (addrlen - 1 - i)) & 1 + + # If we have passed the end of the network prefix, all entries + # under subsequent bits will be associated with the same ASN. + needs_inner = i < prefix - 1 + if tree[bit] is None: + if needs_inner: + tree[bit] = [default, default] + tree = tree[bit] + continue + else: + tree[bit] = asn + break + if isinstance(tree[bit], list): + assert needs_inner + tree = tree[bit] + continue + assert isinstance(tree[bit], int) + if tree[bit] == asn: + break + if not needs_inner: + tree[bit] = asn + break + default = tree[bit] + tree[bit] = [default, default] + tree = tree[bit] + return gtree + + +def CompactTree(tree, approx=True) -> (list, set): + """ + Remove redundancy from a tree. + E.g. if all nodes in a subtree point to the same ASN, compact the subtree + into a single int. + Returns: + (the compacted tree, a set of all ASNs in the tree) + Args: + approx: if True, unassigned ranges may get reassigned to arbitrary ASNs. + """ + num = 0 + if tree is None: + return (tree, set()) + if isinstance(tree, int): + return (tree, set([tree])) + tree[0], leftas = CompactTree(tree[0], approx) + tree[1], rightas = CompactTree(tree[1], approx) + allas = leftas | rightas + if len(allas) == 0: + return (None, allas) + if approx and len(allas) == 1: + return (list(allas)[0], allas) + if isinstance(tree[0], int) and isinstance(tree[1], int) and tree[0] == tree[1]: + return tree[0], set([tree[0]]) + return (tree, allas) + + +def PropTree(tree, approx=True) -> (list, Counter, bool): + """ + Annotate internal nodes in the tree with the most common leafs below it. + The binary serialization later uses this. + This changes the shape of the `tree` datastructure from + `[left_child, right_child]` to `[lc, rc, max_ASN_in_tree]`. + Returns: + (tree, Counter of ASNs in tree, whether or not tree is empty) + """ + if tree is None: + return (tree, Counter(), True) + if isinstance(tree, int): + return (tree, Counter({tree: 1}), False) + tree[0], leftcnt, leftnone = PropTree(tree[0], approx) + tree[1], rightcnt, rightnone = PropTree(tree[1], approx) + allcnt = leftcnt + rightcnt + allnone = leftnone | rightnone + maxasn, maxcount = allcnt.most_common(1)[0] + if maxcount is not None and maxcount >= 2 and (approx or not allnone): + return ([tree[0], tree[1], maxasn], Counter({maxasn: 1}), allnone) + return (tree, allcnt, allnone) + + +def EncodeBits(val, minval, bit_sizes) -> [int]: + """ + Perform a variable-length encoding of a value to bits, least significant + bit first. + For each `bit_sizes` passed, attempt to encode the value with that number + of bits + 1. Normalize the encoded value by `minval` to potentially save + bits - the value will be corrected during decoding. + Returns: + a list of bits representing the value to encode. + """ + val -= minval + ret = [] + for pos in range(len(bit_sizes)): + bit_size = bit_sizes[pos] + + # If the value will not fit in `bit_size` bits, absorb the largest + # value for this bitsize and continue to the next smallest size. + if val >= (1 << bit_size): + val -= 1 << bit_size + ret += [1] + else: + # If we aren't encoding the largest possible value per the largest + # bitsize... + if pos + 1 < len(bit_sizes): + ret += [0] + + # Use remaining bits to encode the rest of val. + for b in range(bit_size): + ret += [(val >> (bit_size - 1 - b)) & 1] + return ret + + # Couldn't fit val into any of the bit_sizes + assert False + + +def MatchType() -> [int]: + """ + The match instruction descends into the tree based on a bit path. If at any + point the match fails to hit a valid path through the tree, it will fail + and return the current default ASN (which changes as we move through the + tree). + """ + return EncodeType(2) + + +def JumpType() -> [int]: + """ + The jump instruction allows us to quickly seek to one side of the tree + or the other. By encoding the length of the left child, we can skip over + it to the right child if need be. + """ + return EncodeType(1) + + +def LeafType() -> [int]: + """The leaf instruction encodes an ASN at the end of a bit path.""" + return EncodeType(0) + + +def SetNewDefaultType() -> [int]: + """ + This instruction establishes a new default ASN to return should we fail + while traversing this path. + """ + return EncodeType(3) + + +def EncodeType(v) -> [int]: + return EncodeBits(v, 0, [0, 0, 1]) + + +def EncodeASN(v) -> [int]: + # It's reasonable to ask why "15" (indicating 16 bits) is the minimum size + # we might try to pack an ASN into, given there are many ASNs below 2**16. + # + # The reason that we start at 15 here is because we want the first bitsize + # we specify to contain ~50% of the values we are trying to encode - this + # is because each separate bitsize we try will add a digit to our encoded + # values, so we simultaneously want to minimize the number of bitsizes we + # allow while also minimizing the bit length of the encoded data, which + # is a trade-off. + return EncodeBits(v, 1, [15, 16, 17, 18, 19, 20, 21, 22, 23, 24]) + + +def EncodeMatch(v) -> [int]: + return EncodeBits(v, 2, [1, 2, 3, 4, 5, 6, 7, 8]) + + +def EncodeJump(v) -> [int]: + return EncodeBits( + v, + 17, + [ + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + ], + ) + + +def EncodeBytes(bits) -> [int]: + """Encode a sequence of bits as a sequence of bytes.""" + val = 0 + nbits = 0 + bytes = [] + for bit in bits: + val += bit << nbits + nbits += 1 + if nbits == 8: + bytes += [val] + val = 0 + nbits = 0 + if nbits: + bytes += [val] + return bytes + + +def TreeSer(tree, default): + match = 1 + assert tree is not None + assert not (isinstance(tree, int) and tree == default) + + # If one side of the tree is empty (i.e. represents a path without + # choices), encode a match instruction up to 8 bits. + while isinstance(tree, list) and match <= 0xFF: + if tree[0] is None or tree[0] == default: + match = (match << 1) + 1 + tree = tree[1] + elif tree[1] is None or tree[1] == default: + match = (match << 1) + 0 + tree = tree[0] + else: + break + if match >= 2: + return MatchType() + EncodeMatch(match) + TreeSer(tree, default) + + # Leaf node: return the ASN. + if isinstance(tree, int): + return LeafType() + EncodeASN(tree) + + # Return the tree along with a new "default" ASN value should we fail to + # match while along this path. + if len(tree) > 2 and tree[2] != default: + return SetNewDefaultType() + EncodeASN(tree[2]) + TreeSer(tree, tree[2]) + + left = TreeSer(tree[0], default) + right = TreeSer(tree[1], default) + + # Start the program by specifying a possible jump to either child of the + # first node. + return JumpType() + EncodeJump(len(left)) + left + right + + +def BuildTree(entries, approx=True): + tree = [None, None] + tree = UpdateTree(tree, 128, entries) + return tree + + +if __name__ == "__main__": + entries: [Entry] = [] + print("[INFO] Loading", file=sys.stderr) + Parse(entries) + print("[INFO] Read %i prefixes" % len(entries), file=sys.stderr) + print("[INFO] Constructing trie", file=sys.stderr) + tree = BuildTree(entries) + print("[INFO] Compacting tree", file=sys.stderr) + tree, _ = CompactTree(tree, True) + print("[INFO] Computing inner prefixes", file=sys.stderr) + tree, _, _ = PropTree(tree, True) + + ser = TreeSer(tree, None) + print("[INFO] Total bits: %i" % (len(ser)), file=sys.stderr) + sys.stdout.buffer.write(bytes(EncodeBytes(ser))) From a071818c61cec1205379bcd60afc01ccefe88a76 Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 11 Dec 2023 15:45:07 +0000 Subject: [PATCH 2/6] asmap: add random as generator --- src/warnet/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/warnet/utils.py b/src/warnet/utils.py index 908076b55..344964608 100644 --- a/src/warnet/utils.py +++ b/src/warnet/utils.py @@ -515,3 +515,9 @@ def convert_unsupported_attributes(graph): continue else: edge_data[key] = str(value) + + +def generate_as(): + while True: + as_number = random.randint(1, 64496) # I think these are not "reserved" + return as_number From 333c37afc9f0e826f84794c2c8f73eff5811731a Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 11 Dec 2023 15:45:41 +0000 Subject: [PATCH 3/6] asmap: add autonomous_system property to tank --- src/warnet/tank.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/warnet/tank.py b/src/warnet/tank.py index ced2c5187..7241368e9 100644 --- a/src/warnet/tank.py +++ b/src/warnet/tank.py @@ -8,6 +8,7 @@ from warnet.lnnode import LNNode from warnet.utils import ( exponential_backoff, + generate_as, generate_ipv4_addr, sanitize_tc_netem_command, SUPPORTED_TAGS, @@ -41,6 +42,7 @@ def __init__(self, index: int, config_dir: Path, warnet): self.rpc_password = "2themoon" self._suffix = None self._ipv4 = None + self._a_system = None self._exporter_name = None self.extra_build_args = "" self.lnnode = None @@ -112,6 +114,12 @@ def exporter_name(self): def status(self) -> RunningStatus: return self.warnet.container_interface.get_status(self.index, ServiceType.BITCOIN) + @property + def autonomous_system(self) -> int: + if self._a_system is None: + self._a_system = generate_as() + return self._a_system + @exponential_backoff() def exec(self, cmd: str, user: str = "root"): return self.warnet.container_interface.exec_run( From 8e7f0f672d1efcc0f98f70124a2ca3654f0daf6b Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Mon, 11 Dec 2023 15:48:20 +0000 Subject: [PATCH 4/6] asmap: add asmap generator to Warnet --- src/warnet/warnet.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/warnet/warnet.py b/src/warnet/warnet.py index e7defec86..da3c12a7a 100644 --- a/src/warnet/warnet.py +++ b/src/warnet/warnet.py @@ -3,21 +3,26 @@ """ import base64 +import io import json import logging import networkx import os import shutil +import sys from pathlib import Path from templates import TEMPLATES from typing import List, Optional +import external.buildmap as buildmap from backends import ComposeBackend, KubernetesBackend from warnet.tank import Tank from warnet.utils import gen_config_dir, bubble_exception_str, version_cmp_ge logger = logging.getLogger("warnet") FO_CONF_NAME = "fork_observer_config.toml" +ASMAP_TXT_PATH = "asmap.txt" +ASMAP_DAT_PATH = "asmap.dat" class Warnet: @@ -37,6 +42,8 @@ def __init__(self, config_dir, backend, network_name: str): self.tanks: List[Tank] = [] self.deployment_file: Optional[Path] = None self.backend = backend + self.as_map_txt_path = config_dir / ASMAP_TXT_PATH + self.as_map_dat_path = config_dir / ASMAP_DAT_PATH def __str__(self) -> str: # TODO: bitcoin_conf and tc_netem can be added back in to this table @@ -238,3 +245,35 @@ def export(self, subdir): config_path = os.path.join(subdir, "sim.json") with open(config_path, "a") as f: json.dump(config, f) + + def generate_as_map(self): + # Write AS mappings to file + with open(self.as_map_txt_path, "w") as f: + for tank in self.tanks: + f.write(f"{tank.ipv4}/32 AS{tank.autonomous_system}\n") + + # Yes, read back into a string... + with open(self.as_map_txt_path, "r") as f: + file_content = f.read() + + buffer = io.StringIO(file_content) + sys.stdin = buffer + + entries = [] + logger.info("AS map: Loading") + buildmap.Parse(entries) + logger.info(f"AS map: Read {len(entries)} prefixes") + logger.info("AS map: Constructing trie") + tree = buildmap.BuildTree(entries) + logger.info("AS map: Compacting tree") + tree, _ = buildmap.CompactTree(tree, True) + logger.info("AS map: Computing inner prefixes") + tree, _, _ = buildmap.PropTree(tree, True) + + ser = buildmap.TreeSer(tree, None) + logger.info(f"AS map: Total bits: {len(ser)}") + with open(self.as_map_dat_path, "wb") as f: + f.write(bytes(buildmap.EncodeBytes(ser))) + + # Reset sys.stdin to its original state + sys.stdin = sys.__stdin__ From 0bb8239895d090fb3d7501cc531ab32237a3539d Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 12 Dec 2023 10:14:09 +0000 Subject: [PATCH 5/6] asmap: mount asmap.dat in docker containers --- src/backends/compose_backend.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/backends/compose_backend.py b/src/backends/compose_backend.py index ff02ecc78..677b4a88c 100644 --- a/src/backends/compose_backend.py +++ b/src/backends/compose_backend.py @@ -255,12 +255,10 @@ def write_prometheus_config(self, warnet): "job_name": tank.exporter_name, "scrape_interval": "5s", "static_configs": [{"targets": [f"{tank.exporter_name}:9332"]}], - }) + } + ) - config = { - "global": {"scrape_interval": "15s"}, - "scrape_configs": scrape_configs - } + config = {"global": {"scrape_interval": "15s"}, "scrape_configs": scrape_configs} prometheus_path = self.config_dir / "prometheus.yml" try: @@ -381,6 +379,10 @@ def add_services(self, tank: Tank, services): "start_period": "5s", # Start checking after 5 seconds "retries": 3, }, + "volumes": [ + # Mount asmap.dat in rw mode (as entrypoint.sh will attempt to take ownership of it) + f"{self.config_dir / 'asmap.dat'}:/home/bitcoin/.bitcoin/regtest/asmap.dat:rw" + ], } ) @@ -398,7 +400,7 @@ def add_services(self, tank: Tank, services): "BITCOIN_RPC_USER": tank.rpc_user, "BITCOIN_RPC_PASSWORD": tank.rpc_password, }, - "networks": [tank.network_name] + "networks": [tank.network_name], } def add_lnd_service(self, tank, services): From b564d48c2a3c903c10b412e84efa5c3f3c33071e Mon Sep 17 00:00:00 2001 From: willcl-ark Date: Tue, 12 Dec 2023 10:20:34 +0000 Subject: [PATCH 6/6] asmap: load asmap --- src/templates/bitcoin.conf | 1 + src/warnet/warnet.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/templates/bitcoin.conf b/src/templates/bitcoin.conf index 8b52d5af8..b76139c6b 100644 --- a/src/templates/bitcoin.conf +++ b/src/templates/bitcoin.conf @@ -16,3 +16,4 @@ rest=1 fallbackfee=0.00001000 listen=1 +asmap=asmap.dat diff --git a/src/warnet/warnet.py b/src/warnet/warnet.py index da3c12a7a..f2ca4a96a 100644 --- a/src/warnet/warnet.py +++ b/src/warnet/warnet.py @@ -129,6 +129,7 @@ def from_graph_file( self.network_name = network self.graph = networkx.parse_graphml(graph_file.decode("utf-8"), node_type=int) self.tanks_from_graph() + self.generate_as_map() logger.info(f"Created Warnet using directory {self.config_dir}") return self @@ -152,6 +153,7 @@ def from_network(cls, network_name, backend="compose"): self.graph = networkx.read_graphml(Path(self.config_dir / self.graph_name), node_type=int) if self.tanks == []: self.tanks_from_graph() + # TODO: read the AS Map file to re-learn each tank AS mapping on restarts. return self @property