Translate local RC feeds (exabgp/hist) into ARTEMIS configuration primitives

.travis.yml

@@ -50,16 +50,29 @@ jobs:
     - docker tag inspiregroup/artemis-monitor:${DOCKER_TAG} artemis_monitor:latest
     script:
     - docker-compose -f docker-compose.testcafe.yaml up --abort-on-container-exit
-  - name: BackendTest
+  - name: TestDetection
     install:
     - pip install codecov
     before_script:
     - docker pull inspiregroup/artemis-backend:${DOCKER_TAG}
     - docker tag inspiregroup/artemis-backend:${DOCKER_TAG} artemis_backend:latest
-    - docker build -t artemis_testing tester
-    - docker-compose -f docker-compose.testbackend.yaml up -d
+    - docker build -t artemis_detection_testing tester/detection
+    - docker-compose -f docker-compose.testdetection.yaml up -d
     script:
-    - docker run --rm -ti --env-file .env --network artemis_artemis artemis_testing
+    - docker run --rm -ti --env-file .env --network artemis_artemis artemis_detection_testing
+    after_success:
+    - sh convert_coverage.sh
+    - codecov
+  - name: TestAutoconf
+    install:
+    - pip install codecov
+    before_script:
+    - docker pull inspiregroup/artemis-backend:${DOCKER_TAG}
+    - docker tag inspiregroup/artemis-backend:${DOCKER_TAG} artemis_backend:latest
+    - docker build -t artemis_autoconf_testing tester/autoconf
+    - docker-compose -f docker-compose.testautoconf.yaml up -d
+    script:
+    - docker run --rm -ti --env-file .env --network artemis_artemis artemis_autoconf_testing
     after_success:
     - sh convert_coverage.sh
     - codecov

CHANGELOG.md

@@ -7,6 +7,9 @@
   - Artemis::BGP Updates per prefix
   - Artemis::BGP Updates per service
   - Artemis::Offending ASes
+- Get prefix, origin and neighbor info (communities) from exaBGP or hist 'A'/'W' updates
+- autoconf flag in configuration for exaBGP and hist monitors
+- autoconf tester
 - ENV variable WITHDRAWN_HIJACK_THRESHOLD to consider hijacks withdrawn based on a threshold
 - TravisCI and CodeCov support
 
@@ -22,7 +25,7 @@
 - TBD (removed a feature)
 
 ### Deprecated
-- TBD (soon-to-be removed feature)
+- SemaphoreCI and coveralls
 
 ### Security
 - TBD (addressing vulnerability)

backend/configs/autoconf-config.yaml

@@ -0,0 +1,14 @@
+prefixes: {}
+monitors:
+  riperis: [''] # by default this uses all available monitors
+  bgpstreamlive:
+    - routeviews
+    - ris
+  betabmp:
+    - betabmp
+  exabgp:
+    - ip: exabgp # this will automatically be resolved to the exabgp container's IP
+      port: 5000 # default port
+      autoconf: "true"
+asns: {}
+rules: []

backend/core/database.py

@@ -498,7 +498,7 @@ def config_request_rpc(self):
                 queues=[callback_queue],
                 no_ack=True,
             ):
-                while not self.rules:
+                while self.rules is None:
                     self.connection.drain_events()
 
         def handle_bgp_update(self, message):

backend/core/detection.py

@@ -322,7 +322,7 @@ def config_request_rpc(self) -> NoReturn:
                 queues=[callback_queue],
                 no_ack=True,
             ):
-                while not self.rules:
+                while self.rules is None:
                     self.connection.drain_events()
             log.debug("{}".format(self.rules))
 

backend/core/mitigation.py

@@ -142,7 +142,7 @@ def config_request_rpc(self):
                 queues=[callback_queue],
                 no_ack=True,
             ):
-                while not self.rules:
+                while self.rules is None:
                     self.connection.drain_events()
 
         def handle_config_request_reply(self, message):

docker-compose.testbackend.yaml → docker-compose.testautoconf.yaml

@@ -36,8 +36,8 @@ services:
             HIJACK_LOG_FIELDS: ${HIJACK_LOG_FIELDS}
             WITHDRAWN_HIJACK_THRESHOLD: ${WITHDRAWN_HIJACK_THRESHOLD}
         volumes:
-            - ./tester/configs/:/etc/artemis/
-            - ./tester/supervisor.d/:/etc/supervisor/conf.d/
+            - ./tester/autoconf/configs/:/etc/artemis/
+            - ./tester/autoconf/supervisor.d/:/etc/supervisor/conf.d/
             - ./tester/entrypoint.test:/root/entrypoint
             - ./tester/.coveragerc:/root/core/.coveragerc
     rabbitmq:
@@ -49,7 +49,7 @@ services:
         expose:
             - ${RABBITMQ_PORT}
         volumes:
-            - ./tester/rabbitmq/enabled-plugins:/etc/rabbitmq/enabled_plugins
+            - ./other/rabbitmq/enabled-plugins:/etc/rabbitmq/enabled_plugins
     postgres:
         image: timescale/timescaledb:1.1.0-pg10
         container_name: postgres
@@ -63,9 +63,9 @@ services:
             POSTGRES_USER: ${DB_USER}
             POSTGRES_PASSWORD: ${DB_PASS}
         volumes:
-            - ./tester/db/init.sql:/docker-entrypoint-initdb.d/zinit.sql
-            - ./tester/db/data/:/docker-entrypoint-initdb.d/data/
-            - ./tester/db/libs/:/docker-entrypoint-initdb.d/libs/
+            - ./other/db/init.sql:/docker-entrypoint-initdb.d/zinit.sql
+            - ./other/db/data/:/docker-entrypoint-initdb.d/data/
+            - ./other/db/libs/:/docker-entrypoint-initdb.d/libs/
     pg-amqp-bridge:
         image: subzerocloud/pg-amqp-bridge:0.0.6
         container_name: pg-amqp-bridge
@@ -80,7 +80,7 @@ services:
             - AMQP_URI=amqp://${RABBITMQ_HOST}//
             - BRIDGE_CHANNELS=events:amq.direct
         volumes:
-            - ./tester/wait-for:/root/wait-for
+            - ./backend/wait-for:/root/wait-for
         command: ["/root/wait-for", "${RABBITMQ_HOST}:${RABBITMQ_PORT}", "-t", "0", "--", "pg-amqp-bridge"]
 
 networks:

docker-compose.testdetection.yaml

@@ -0,0 +1,87 @@
+version: '3'
+services:
+    backend:
+        image: artemis_backend
+        build: ./backend/
+        container_name: backend
+        depends_on:
+            - rabbitmq
+            - postgres
+        networks:
+            - artemis
+        expose:
+            - ${BACKEND_SUPERVISOR_PORT}
+            - ${REDIS_PORT}
+        environment:
+            ARTEMIS_WEB_HOST: ${ARTEMIS_WEB_HOST}
+            RABBITMQ_USER: ${RABBITMQ_USER}
+            RABBITMQ_PASS: ${RABBITMQ_PASS}
+            RABBITMQ_HOST: ${RABBITMQ_HOST}
+            RABBITMQ_PORT: ${RABBITMQ_PORT}
+            DB_USER: ${DB_USER}
+            DB_PASS: ${DB_PASS}
+            DB_HOST: ${DB_HOST}
+            DB_PORT: ${DB_PORT}
+            DB_NAME: ${DB_NAME}
+            DB_VERSION: ${DB_VERSION}
+            REDIS_HOST: ${REDIS_HOST}
+            REDIS_PORT: ${REDIS_PORT}
+            BACKEND_SUPERVISOR_HOST: ${BACKEND_SUPERVISOR_HOST}
+            BACKEND_SUPERVISOR_PORT: ${BACKEND_SUPERVISOR_PORT}
+            GUI_ENABLED: "false"
+            BULK_TIMER: 0.1
+            PULL_REQUEST_NUMBER: ${PULL_REQUEST_NUMBER}
+            HIJACK_LOG_FILTER: ${HIJACK_LOG_FILTER}
+            MON_TIMEOUT_LAST_BGP_UPDATE: ${MON_TIMEOUT_LAST_BGP_UPDATE}
+            HIJACK_LOG_FIELDS: ${HIJACK_LOG_FIELDS}
+            WITHDRAWN_HIJACK_THRESHOLD: ${WITHDRAWN_HIJACK_THRESHOLD}
+        volumes:
+            - ./tester/detection/configs/:/etc/artemis/
+            - ./tester/detection/supervisor.d/:/etc/supervisor/conf.d/
+            - ./tester/entrypoint.test:/root/entrypoint
+            - ./tester/.coveragerc:/root/core/.coveragerc
+    rabbitmq:
+        image: rabbitmq:3.7.8-management-alpine
+        container_name: rabbitmq
+        restart: always
+        networks:
+            - artemis
+        expose:
+            - ${RABBITMQ_PORT}
+        volumes:
+            - ./tester/detection/rabbitmq/enabled-plugins:/etc/rabbitmq/enabled_plugins
+    postgres:
+        image: timescale/timescaledb:1.1.0-pg10
+        container_name: postgres
+        restart: always
+        networks:
+            - artemis
+        expose:
+            - ${DB_PORT}
+        environment:
+            POSTGRES_DB: ${DB_NAME}
+            POSTGRES_USER: ${DB_USER}
+            POSTGRES_PASSWORD: ${DB_PASS}
+        volumes:
+            - ./tester/detection/db/init.sql:/docker-entrypoint-initdb.d/zinit.sql
+            - ./tester/detection/db/data/:/docker-entrypoint-initdb.d/data/
+            - ./tester/detection/db/libs/:/docker-entrypoint-initdb.d/libs/
+    pg-amqp-bridge:
+        image: subzerocloud/pg-amqp-bridge:0.0.6
+        container_name: pg-amqp-bridge
+        depends_on:
+            - postgres
+            - backend
+            - rabbitmq
+        networks:
+            - artemis
+        environment:
+            - POSTGRESQL_URI=postgres://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}
+            - AMQP_URI=amqp://${RABBITMQ_HOST}//
+            - BRIDGE_CHANNELS=events:amq.direct
+        volumes:
+            - ./backend/wait-for:/root/wait-for
+        command: ["/root/wait-for", "${RABBITMQ_HOST}:${RABBITMQ_PORT}", "-t", "0", "--", "pg-amqp-bridge"]
+
+networks:
+    artemis:

frontend/webapp/core/fetch_config.py

@@ -44,15 +44,15 @@ def get_prefixes_list(self):
             return []
 
         prefixes_list = []
-        for rule in self.config_yaml["rules"]:
+        for rule in self.config_yaml.get("rules", []):
             rule["prefixes"] = flatten(rule["prefixes"])
             for prefix in rule["prefixes"]:
                 if prefix not in prefixes_list:
                     prefixes_list.append(prefix)
         return prefixes_list
 
     def get_rules_list(self):
-        return self.config_yaml["rules"]
+        return self.config_yaml.get("rules", [])
 
     def get_raw_response(self):
         return self.raw_json

monitor/core/monitor.py

@@ -330,17 +330,17 @@ def init_exabgp_instance(self):
                     exabgp_monitor_str = "{}:{}".format(
                         exabgp_monitor["ip"], exabgp_monitor["port"]
                     )
-                    p = Popen(
-                        [
-                            "/usr/local/bin/python3",
-                            "taps/exabgp_client.py",
-                            "--prefixes",
-                            self.prefix_file,
-                            "--host",
-                            exabgp_monitor_str,
-                        ],
-                        shell=False,
-                    )
+                    exabgp_cmd = [
+                        "/usr/local/bin/python3",
+                        "taps/exabgp_client.py",
+                        "--prefixes",
+                        self.prefix_file,
+                        "--host",
+                        exabgp_monitor_str,
+                    ]
+                    if "autoconf" in exabgp_monitor:
+                        exabgp_cmd.append("-a")
+                    p = Popen(exabgp_cmd, shell=False)
                     self.process_ids.append(
                         (
                             "[exabgp] {} {}".format(
@@ -367,17 +367,19 @@ def init_bgpstreamhist_instance(self):
                     )
                 )
                 bgpstreamhist_dir = self.monitors["bgpstreamhist"]
-                p = Popen(
-                    [
-                        "/usr/local/bin/python3",
-                        "taps/bgpstreamhist.py",
-                        "--prefixes",
-                        self.prefix_file,
-                        "--dir",
-                        bgpstreamhist_dir,
-                    ],
-                    shell=False,
-                )
+                if "dir" in self.monitors["bgpstreamhist"]:
+                    bgpstreamhist_dir = self.monitors["bgpstreamhist"]["dir"]
+                bgpstreamhist_cmd = [
+                    "/usr/local/bin/python3",
+                    "taps/bgpstreamhist.py",
+                    "--prefixes",
+                    self.prefix_file,
+                    "--dir",
+                    bgpstreamhist_dir,
+                ]
+                if "autoconf" in self.monitors["bgpstreamhist"]:
+                    bgpstreamhist_cmd.append("-a")
+                p = Popen(bgpstreamhist_cmd, shell=False)
                 self.process_ids.append(
                     (
                         "[bgpstreamhist] {} {}".format(

monitor/core/taps/betabmp.py

@@ -140,6 +140,7 @@ def run_bgpstream_beta_bmp(prefixes_file=None):
                                     )
                             else:
                                 log.warning("Invalid format message: {}".format(msg))
+                            break
                 try:
                     elem = rec.get_next_elem()
                 except BaseException: