Skip to content

Commit

Permalink
trying out pytricia tree on all related modules, removing prefixtree …
Browse files Browse the repository at this point in the history 
…module
  • Loading branch information
@vkotronis
vkotronis committed Sep 16, 2019
1 parent 56b8b72 commit 53694e0
Show file tree
Hide file tree
Showing 16 changed files with 169 additions and 368 deletions.
2 changes: 1 addition & 1 deletion .flake8
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[flake8]
ignore = E203, E266, E501, W503
ignore = E203, E266, E501, W503, W601
max-line-length = 88
max-complexity = 40
select = B,C,E,F,W,T4,B9
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
- TBD (removed a feature)

### Deprecated
- TBD (soon-to-be removed feature)
- py-radix, substituted with pytricia

### Security
- TBD (addressing vulnerability)
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ in a Kubernetes environment, please contact the [ARTEMIS team](#development-team
## Minimum Technical Requirements

* CPU: 4 cores (note that needed CPU cores depend on the number of separate processes, e.g., detectors, you spawn)
* RAM: 4 GB (note that needed memory depends on the number of configured prefixes and load of incoming BGP updates)
* RAM: 4 GB (note that needed memory depends on the number of configured prefixes and load of incoming BGP updates; for example, we recommend +4 GB for each +1 million configured prefixes)
* HDD: 100 GB (less may suffice, depending on the use case for storing BGP updates and hijack alerts)
* NETWORK: 1 public-facing network interface
* OS: Ubuntu Linux 16.04+
Expand Down
63 changes: 38 additions & 25 deletions backend/core/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from xmlrpc.client import ServerProxy

import psycopg2.extras
import radix
import pytricia
import redis
import yaml
from kombu import Connection
Expand All @@ -29,6 +29,7 @@
from utils import REDIS_HOST
from utils import redis_key
from utils import REDIS_PORT
from utils import search_worst_prefix
from utils import translate_asn_range
from utils import translate_rfc2622

Expand Down Expand Up @@ -72,7 +73,7 @@ def __init__(self, connection, ro_conn, wo_conn):
self.connection = connection
self.prefix_tree = None
self.monitored_prefixes = set()
self.configured_prefixes = set()
self.configured_prefix_count = 0
self.monitor_peers = 0
self.rules = None
self.timestamp = -1
Expand Down Expand Up @@ -638,8 +639,10 @@ def handle_handled_bgp_update(self, message):
except Exception:
log.exception("{}".format(message))

def build_radix_tree(self):
self.prefix_tree = radix.Radix()
def build_prefix_tree(self):
log.info("Starting building database prefix tree...")
self.prefix_tree = pytricia.PyTricia(128)
raw_prefix_count = 0
for rule in self.rules:
try:
rule_translated_origin_asn_set = set()
Expand All @@ -662,40 +665,50 @@ def build_radix_tree(self):
}
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
node = self.prefix_tree.search_exact(translated_prefix)
if not node:
node = self.prefix_tree.add(translated_prefix)
node.data["confs"] = []
node.data["confs"].append(conf_obj)
if self.prefix_tree.has_key(translated_prefix):
node = self.prefix_tree[translated_prefix]
else:
node = {
"prefix": translated_prefix,
"data": {"confs": []},
}
self.prefix_tree.insert(translated_prefix, node)
node["data"]["confs"].append(conf_obj)
raw_prefix_count += 1
if raw_prefix_count % 100000 == 0:
log.info(
"{} prefixes integrated in database prefix tree till now".format(
raw_prefix_count
)
)
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in database prefix tree in total".format(
raw_prefix_count
)
)
# calculate the monitored and configured prefixes
self.monitored_prefixes = set()
self.configured_prefixes = set()
for prefix in self.prefix_tree.prefixes():
self.configured_prefixes.add(prefix)
monitored_prefix = self.find_worst_prefix_match(prefix)
self.configured_prefix_count = 0
for prefix in self.prefix_tree:
self.configured_prefix_count += 1
monitored_prefix = search_worst_prefix(prefix, self.prefix_tree)
if monitored_prefix:
self.monitored_prefixes.add(monitored_prefix)
try:
with get_wo_cursor(self.wo_conn) as db_cur:
db_cur.execute(
"UPDATE stats SET monitored_prefixes=%s, configured_prefixes=%s;",
(len(self.monitored_prefixes), len(self.configured_prefixes)),
(len(self.monitored_prefixes), self.configured_prefix_count),
)
except Exception:
log.exception("exception")
log.info("Finished building database prefix tree.")

def find_best_prefix_match(self, prefix):
prefix_node = self.prefix_tree.search_best(prefix)
if prefix_node:
return prefix_node.prefix
return None

def find_worst_prefix_match(self, prefix):
prefix_node = self.prefix_tree.search_worst(prefix)
if prefix_node:
return prefix_node.prefix
if prefix in self.prefix_tree:
return self.prefix_tree.get_key(prefix)
return None

def handle_config_notify(self, message):
Expand All @@ -705,7 +718,7 @@ def handle_config_notify(self, message):
if config["timestamp"] > self.timestamp:
self.timestamp = config["timestamp"]
self.rules = config.get("rules", [])
self.build_radix_tree()
self.build_prefix_tree()
if "timestamp" in config:
del config["timestamp"]
raw_config = ""
Expand All @@ -730,7 +743,7 @@ def handle_config_request_reply(self, message):
if config["timestamp"] > self.timestamp:
self.timestamp = config["timestamp"]
self.rules = config.get("rules", [])
self.build_radix_tree()
self.build_prefix_tree()
if "timestamp" in config:
del config["timestamp"]
raw_config = ""
Expand Down
80 changes: 51 additions & 29 deletions backend/core/detection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from typing import NoReturn
from typing import Tuple

import radix
import pytricia
import redis
import yaml
from kombu import Connection
Expand Down Expand Up @@ -341,7 +341,9 @@ def init_detection(self) -> NoReturn:
"""
Updates rules everytime it receives a new configuration.
"""
self.prefix_tree = radix.Radix()
log.info("Starting building detection prefix tree...")
self.prefix_tree = pytricia.PyTricia(128)
raw_prefix_count = 0
for rule in self.rules:
try:
rule_translated_origin_asn_set = set()
Expand All @@ -367,13 +369,30 @@ def init_detection(self) -> NoReturn:
}
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
node = self.prefix_tree.search_exact(translated_prefix)
if not node:
node = self.prefix_tree.add(translated_prefix)
node.data["confs"] = []
node.data["confs"].append(conf_obj)
if self.prefix_tree.has_key(translated_prefix):
node = self.prefix_tree[translated_prefix]
else:
node = {
"prefix": translated_prefix,
"data": {"confs": []},
}
self.prefix_tree.insert(translated_prefix, node)
node["data"]["confs"].append(conf_obj)
raw_prefix_count += 1
if raw_prefix_count % 100000 == 0:
log.info(
"{} prefixes integrated in detection prefix tree till now".format(
raw_prefix_count
)
)
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in detection prefix tree in total".format(
raw_prefix_count
)
)
log.info("Finished building detection prefix tree.")

def handle_ongoing_hijacks(self, message: Dict) -> NoReturn:
"""
Expand Down Expand Up @@ -431,10 +450,10 @@ def handle_bgp_update(self, message: Dict) -> NoReturn:
monitor_event["path"] = Detection.Worker.__clean_as_path(
monitor_event["path"]
)
prefix_node = self.prefix_tree.search_best(monitor_event["prefix"])

if prefix_node:
monitor_event["matched_prefix"] = prefix_node.prefix
if monitor_event["prefix"] in self.prefix_tree:
prefix_node = self.prefix_tree[monitor_event["prefix"]]
monitor_event["matched_prefix"] = prefix_node["prefix"]

try:
path_hijacker = -1
Expand Down Expand Up @@ -616,52 +635,55 @@ def __hijack_pol_checker_gen(self, path_len: int) -> Callable:

@exception_handler(log)
def detect_prefix_squatting_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> str:
"""
Squatting hijack detection.
"""
for item in prefix_node.data["confs"]:
for item in prefix_node["data"]["confs"]:
# check if there are origin_asns defined (even wildcards)
if item["origin_asns"]:
return "-"
return "Q"

@exception_handler(log)
def detect_prefix_subprefix_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> str:
"""
Subprefix or exact prefix hijack detection.
"""
mon_prefix = ipaddress.ip_network(monitor_event["prefix"])
if prefix_node.prefixlen < mon_prefix.prefixlen:
if (
ipaddress.ip_network(prefix_node["prefix"]).prefixlen
< mon_prefix.prefixlen
):
return "S"
return "E"

@exception_handler(log)
def detect_path_type_0_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
"""
Origin hijack detection.
"""
origin_asn = monitor_event["path"][-1]
for item in prefix_node.data["confs"]:
for item in prefix_node["data"]["confs"]:
if origin_asn in item["origin_asns"] or item["origin_asns"] == [-1]:
return (-1, "-")
return (origin_asn, "0")

@exception_handler(log)
def detect_path_type_1_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
"""
Type-1 hijack detection.
"""
origin_asn = monitor_event["path"][-1]
first_neighbor_asn = monitor_event["path"][-2]
for item in prefix_node.data["confs"]:
for item in prefix_node["data"]["confs"]:
# [] or [-1] neighbors means "allow everything"
if (
origin_asn in item["origin_asns"] or item["origin_asns"] == [-1]
Expand All @@ -675,54 +697,54 @@ def detect_path_type_1_hijack(

@exception_handler(log)
def detect_path_type_N_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
# Placeholder for type-N detection (not supported)
return (-1, "-")

@exception_handler(log)
def detect_path_type_U_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
# Placeholder for type-U detection (not supported)
return (-1, "-")

@exception_handler(log)
def detect_dplane_blackholing_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> str:
# Placeholder for blackholing detection (not supported)
return "-"

@exception_handler(log)
def detect_dplane_imposture_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> str:
# Placeholder for imposture detection (not supported)
return "-"

@exception_handler(log)
def detect_dplane_mitm_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> str:
# Placeholder for mitm detection (not supported)
return "-"

@exception_handler(log)
def detect_pol_leak_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
"""
Route leak hijack detection
"""
for item in prefix_node.data["confs"]:
for item in prefix_node["data"]["confs"]:
if "no-export" in item["policies"]:
return (monitor_event["path"][-2], "L")
return (-1, "-")

@exception_handler(log)
def detect_pol_other_hijack(
self, monitor_event: Dict, prefix_node: radix.Radix, *args, **kwargs
self, monitor_event: Dict, prefix_node: Dict, *args, **kwargs
) -> Tuple[int, str]:
# Placeholder for policy violation detection (not supported)
return (-1, "-")
Expand Down Expand Up @@ -949,9 +971,9 @@ def comm_annotate_hijack(self, monitor_event: Dict, hijack: Dict) -> NoReturn:
community = "{}:{}".format(comm_as_value[0], comm_as_value[1])
bgp_update_communities.add(community)

prefix_node = self.prefix_tree.search_best(monitor_event["prefix"])
if prefix_node:
for item in prefix_node.data["confs"]:
if monitor_event["prefix"] in self.prefix_tree:
prefix_node = self.prefix_tree[monitor_event["prefix"]]
for item in prefix_node["data"]["confs"]:
annotations = []
for annotation_element in item.get("community_annotations", []):
for annotation in annotation_element:
Expand Down

0 comments on commit 53694e0

Please sign in to comment.