Merge branch 'master' into withdrawal_logging

.env

@@ -1,7 +1,7 @@
 # Docker specific configs
 # use only letters and numbers for the project name
 COMPOSE_PROJECT_NAME=artemis
-DB_VERSION=17
+DB_VERSION=18
 GUI_ENABLED=true
 SYSTEM_VERSION=latest
 HISTORIC=false

CHANGELOG.md

@@ -4,17 +4,19 @@
 ### Added
 - IPv6 tests (backend testing)
 - PR labeler (GitHub actions)
-- env variable and formatter for selecting hijack log fields
-- env variable for ARTEMIS web host for backend logging
-- logging also withdrawn and outdated hijacks in mail and hijack loggers
-- extra end_tag and hijack_url in hijacks for logging
+- Env variable and formatter for selecting hijack log fields
+- Env variable for ARTEMIS web host for backend logging
+- Logging also withdrawn and outdated hijacks in mail and hijack loggers
+- Extra end_tag and hijack_url in hijacks for logging
+- Reinstating intended modules on ARTEMIS startup
 
 ### Changed
 - py-radix, substituted with pytricia tree
 - dep-licenses updated (with currently used modules)
 
 ### Fixed
 - Support for millions of prefixes in configuration file
+- Ensure ARTEMIS config file is not clobbered on pod restart
 
 ### Removed
 - ipaddress requirement from frontend (not needed)

artemis-chart/templates/backend-deployment.yaml

@@ -26,7 +26,7 @@ spec:
         - mountPath: /pvc
           name: backend-pvc
           subPath: configs
-        command: ['sh', '-c', 'cp -u /configmaps/config.yaml /configmaps/logging.yaml /configmaps/services.conf /pvc/']
+        command: ['sh', '-c', 'cp -n -u /configmaps/config.yaml /configmaps/logging.yaml /configmaps/services.conf /pvc/']
       - name: wait-for-rmq
         image: busybox
         command: ['sh', '-c', 'until nc -z {{ .Values.rabbitmqHost }} {{ .Values.rabbitmqPort}}; do echo waiting for services; sleep 10; done;']

artemis-chart/templates/configmap.yaml

@@ -19,7 +19,7 @@ data:
   risId: {{ .Values.risId | default "8522" | quote }}
   dbHost: {{ .Values.dbHost | default "postgres" | quote }}
   dbPort: {{ .Values.dbPort | default "5432" | quote }}
-  dbVersion: {{ .Values.dbVersion | default "17" | quote }}
+  dbVersion: {{ .Values.dbVersion | default "18" | quote }}
   dbName: {{ .Values.dbName | default "artemis_db" | quote }}
   dbUser: {{ .Values.dbUser | default "artemis_user" | quote }}
   dbSchema: {{ .Values.dbSchema | default "public" | quote }}

artemis-chart/templates/monitor-deployment.yaml

@@ -26,7 +26,7 @@ spec:
         - mountPath: /pvc
           name: monitor-pvc
           subPath: configs
-        command: ['sh', '-c', 'cp -u /configmaps/logging.yaml /configmaps/mon-services.conf /pvc/']
+        command: ['sh', '-c', 'cp -n -u /configmaps/logging.yaml /configmaps/mon-services.conf /pvc/']
       - name: wait-for-service
         image: busybox
         command: ['sh', '-c', 'until nc -z {{ .Values.rabbitmqHost }} {{ .Values.rabbitmqPort}}; do echo waiting for services; sleep 10; done;']

artemis-chart/values.yaml

@@ -20,7 +20,7 @@ risId: 8522
 # database
 dbHost: postgres
 dbPort: 5432
-dbVersion: 17
+dbVersion: 18
 dbName: artemis_db
 dbUser: artemis_user
 dbPass: Art3m1s

backend/core/database.py

@@ -26,6 +26,7 @@
 from utils import get_wo_cursor
 from utils import hijack_log_field_formatter
 from utils import HISTORIC
+from utils import ModulesState
 from utils import MON_SUPERVISOR_URI
 from utils import ping_redis
 from utils import purge_redis_eph_pers_keys
@@ -138,6 +139,26 @@ def __init__(self, connection, ro_conn, wo_conn):
             except Exception:
                 log.exception("exception")
 
+            try:
+                query = (
+                    "INSERT INTO intended_process_states (name, running) "
+                    "VALUES (%s, %s) ON CONFLICT(name) DO NOTHING"
+                )
+
+                for ctx in {BACKEND_SUPERVISOR_URI, MON_SUPERVISOR_URI}:
+                    server = ServerProxy(ctx)
+                    processes = [
+                        (x["group"], False)
+                        for x in server.supervisor.getAllProcessInfo()
+                        if x["group"] in ["monitor", "detection", "mitigation"]
+                    ]
+
+                    with get_wo_cursor(self.wo_conn) as db_cur:
+                        psycopg2.extras.execute_batch(db_cur, query, processes)
+
+            except Exception:
+                log.exception("exception")
+
             # redis db
             self.redis = redis.Redis(host=REDIS_HOST, port=REDIS_PORT)
             ping_redis(self.redis)
@@ -420,6 +441,26 @@ def get_consumers(self, Consumer, channel):
                 ),
             ]
 
+        def set_modules_to_intended_state(self):
+            try:
+                query = "SELECT name, running FROM intended_process_states"
+
+                with get_ro_cursor(self.ro_conn) as db_cur:
+                    db_cur.execute(query)
+                    entries = db_cur.fetchall()
+                modules_state = ModulesState()
+                for entry in entries:
+                    # entry[0] --> module name, entry[1] --> intended state
+                    # start only intended modules (after making sure they are stopped
+                    # to avoid stale entries)
+                    if entry[1]:
+                        log.info("Setting {} to start state.".format(entry[0]))
+                        modules_state.call(entry[0], "stop")
+                        time.sleep(1)
+                        modules_state.call(entry[0], "start")
+            except Exception:
+                log.exception("exception")
+
         def config_request_rpc(self):
             self.correlation_id = uuid()
             callback_queue = Queue(
@@ -745,7 +786,7 @@ def find_best_prefix_match(self, prefix):
             return None
 
         def handle_config_notify(self, message):
-            log.info("Reconfiguring database...")
+            log.info("Reconfiguring database due to conf update...")
 
             log.debug("Message: {}\npayload: {}".format(message, message.payload))
             config = message.payload
@@ -773,7 +814,7 @@ def handle_config_notify(self, message):
             log.info("Database initiated, configured and running.")
 
         def handle_config_request_reply(self, message):
-            log.info("Reconfiguring database...")
+            log.info("Configuring database for the first time...")
 
             log.debug("Message: {}\npayload: {}".format(message, message.payload))
             config = message.payload
@@ -803,6 +844,7 @@ def handle_config_request_reply(self, message):
                             log.debug("database config is up-to-date")
             except Exception:
                 log.exception("{}".format(config))
+            self.set_modules_to_intended_state()
 
             log.info("Database initiated, configured and running.")
 

backend/core/utils/__init__.py

@@ -8,6 +8,7 @@
 from contextlib import contextmanager
 from ipaddress import ip_network as str2ip
 from logging.handlers import SMTPHandler
+from xmlrpc.client import ServerProxy
 
 import psycopg2
 import requests
@@ -160,6 +161,60 @@ def get_logger(path="/etc/artemis/logging.yaml"):
 log = get_logger()
 
 
+class ModulesState:
+    def __init__(self):
+        self.backend_server = ServerProxy(BACKEND_SUPERVISOR_URI)
+        self.mon_server = ServerProxy(MON_SUPERVISOR_URI)
+
+    def call(self, module, action):
+        try:
+            if module == "all":
+                if action == "start":
+                    for ctx in {self.backend_server, self.mon_server}:
+                        ctx.supervisor.startAllProcesses()
+                elif action == "stop":
+                    for ctx in {self.backend_server, self.mon_server}:
+                        ctx.supervisor.stopAllProcesses()
+            else:
+                ctx = self.backend_server
+                if module == "monitor":
+                    ctx = self.mon_server
+
+                if action == "start":
+                    modules = self.is_any_up_or_running(module, up=False)
+                    for mod in modules:
+                        ctx.supervisor.startProcess(mod)
+
+                elif action == "stop":
+                    modules = self.is_any_up_or_running(module)
+                    for mod in modules:
+                        ctx.supervisor.stopProcess(mod)
+
+        except Exception:
+            log.exception("exception")
+
+    def is_any_up_or_running(self, module, up=True):
+        ctx = self.backend_server
+        if module == "monitor":
+            ctx = self.mon_server
+
+        try:
+            if up:
+                return [
+                    "{}:{}".format(x["group"], x["name"])
+                    for x in ctx.supervisor.getAllProcessInfo()
+                    if x["group"] == module and (x["state"] == 20 or x["state"] == 10)
+                ]
+            return [
+                "{}:{}".format(x["group"], x["name"])
+                for x in ctx.supervisor.getAllProcessInfo()
+                if x["group"] == module and (x["state"] != 20 and x["state"] != 10)
+            ]
+        except Exception:
+            log.exception("exception")
+            return False
+
+
 @contextmanager
 def get_ro_cursor(conn):
     with conn.cursor() as curr:

backend/hasura_init.json

@@ -54,6 +54,59 @@
         "delete_permissions": [],
         "event_triggers": []
       },
+      {
+        "table": "view_intended_process_states",
+        "object_relationships": [],
+        "array_relationships": [],
+        "insert_permissions": [],
+        "select_permissions": [
+          {
+            "role": "user",
+            "comment": null,
+            "permission": {
+              "allow_aggregations": true,
+              "columns": [
+                "name",
+                "running"
+              ],
+              "filter": {}
+            }
+          }
+        ],
+        "update_permissions": [
+          {
+            "role": "user",
+            "comment": null,
+            "permission": {
+              "columns": [
+                "name",
+                "running"
+              ],
+              "filter": {
+                "$or": [
+                  {
+                    "name": {
+                      "$eq": "monitor"
+                    }
+                  },
+                  {
+                    "name": {
+                      "$eq": "detection"
+                    }
+                  },
+                  {
+                    "name": {
+                      "$eq": "mitigation"
+                    }
+                  }
+                ]
+              }
+            }
+          }
+        ],
+        "delete_permissions": [],
+        "event_triggers": []
+      },
       {
         "table": "view_bgpupdates",
         "object_relationships": [],

backend/migrate/migrations/scripts/migration_18.sql

@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS intended_process_states (
+    name VARCHAR (32) UNIQUE,
+    running BOOLEAN DEFAULT FALSE
+);
+
+CREATE OR REPLACE VIEW view_intended_process_states AS SELECT * FROM intended_process_states;

backend/migrate/migrations/target_steps.json

@@ -101,6 +101,12 @@
             "db_version": "17",
             "description": "Added community_annotation column in hijacks table",
             "file": "migration_17.sql"
+        },
+        "18": {
+            "id": "18",
+            "db_version": "18",
+            "description": "Added intended process states table in db",
+            "file": "migration_18.sql"
         }
     }
 }

backend/testing/configs/config3.yaml

@@ -27,6 +27,12 @@ prefixes:
         - 10.0.20.0/24
     test_as_set_24: &test_as_set_24
         - 10.0.30.0/24
+    test_ipv6_benign: &test_ipv6_benign
+        - 2001:db8:abcd:10::/64
+    test_ipv6_exact: &test_ipv6_exact
+        - 2001:db8:abcd:11::/64
+    test_ipv6_sub: &test_ipv6_sub
+        - 2001:db8:abcd:12::/64
 asns:
     8_origin: &8_origin
         1
@@ -48,6 +54,10 @@ asns:
         444
     comm_test_origin: &comm_test_origin
         555
+    v6_origins: &v6_origins
+        - 777
+        - 888
+        - 999
 rules:
 - prefixes:
   - *8_prefix
@@ -144,3 +154,13 @@ rules:
   - *test_as_set_24
   origin_asns:
   - 6777
+- prefixes:
+  - *test_ipv6_benign
+  - *test_ipv6_exact
+  - *test_ipv6_sub
+  origin_asns:
+  - *v6_origins
+  neighbors:
+    '*'
+  mitigation:
+    manual

backend/testing/db/data/tables.sql

@@ -22,7 +22,7 @@ CREATE TRIGGER db_details_no_delete
 BEFORE DELETE ON db_details
 FOR EACH ROW EXECUTE PROCEDURE db_version_no_delete();
 
-INSERT INTO db_details (version, upgraded_on) VALUES (17, now());
+INSERT INTO db_details (version, upgraded_on) VALUES (18, now());
 
 CREATE TABLE IF NOT EXISTS bgp_updates (
     key VARCHAR ( 32 ) NOT NULL,
@@ -201,6 +201,11 @@ CREATE TABLE IF NOT EXISTS process_states (
     timestamp TIMESTAMP default current_timestamp
 );
 
+CREATE TABLE IF NOT EXISTS intended_process_states (
+    name VARCHAR (32) UNIQUE,
+    running BOOLEAN DEFAULT FALSE
+);
+
 CREATE OR REPLACE FUNCTION update_timestamp()
 RETURNS TRIGGER AS $$
 BEGIN
@@ -215,6 +220,8 @@ FOR EACH ROW EXECUTE PROCEDURE update_timestamp();
 
 CREATE OR REPLACE VIEW view_processes AS SELECT * FROM process_states;
 
+CREATE OR REPLACE VIEW view_intended_process_states AS SELECT * FROM intended_process_states;
+
 CREATE OR REPLACE VIEW view_db_details AS SELECT version, upgraded_on FROM db_details;
 
 CREATE FUNCTION search_bgpupdates_as_path(as_paths BIGINT[])