Merge branch 'master' into monitored_routes

.env

@@ -4,17 +4,20 @@ COMPOSE_PROJECT_NAME=artemis
 DB_VERSION=13
 GUI_ENABLED=true
 SYSTEM_VERSION=latest
+HISTORIC=false
 
-# Global configs
-POSTGREST_HOST=postgrest
-POSTGREST_PORT=3000
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_PORT=5672
-POSTGRES_HOST=postgres
-WEBAPP_HOST=frontend
+# Redis config
+REDIS_HOST=backend
+REDIS_PORT=6379
+
+# Supervisor config
 SUPERVISOR_HOST=backend
 SUPERVISOR_PORT=9001
 
+# API config
+API_HOST=postgrest
+API_PORT=3000
+
 # Monitor-specific configs
 RIS_ID=8522
 
@@ -29,19 +32,24 @@ DB_AUTOCLEAN=false
 DB_BACKUP=true
 DB_HIJACK_DORMANT=false
 
-# Frontend details
-MACHINE_IP=0.0.0.0
-FLASK_PORT=8000
+# Frontend config
+BIND_IP=0.0.0.0
+WEBAPP_HOST=frontend
+WEBAPP_PORT=8000
 ADMIN_USER=admin
 ADMIN_PASS=admin123
 ADMIN_EMAIL=admin@admin
 JS_VERSION=0.1.2
 
-# rabbitmq
+# rabbitmq config
+RABBITMQ_HOST=rabbitmq
+RABBITMQ_PORT=5672
 RABBITMQ_DEFAULT_USER=guest
 RABBITMQ_DEFAULT_PASS=guest
 
-# hasura
+# hasura config
+HASURA_HOST=graphql
+HASURA_PORT=8080
 HASURA_CONSOLE=false
 
 # secret keys

.flake8

@@ -1,5 +1,5 @@
 [flake8]
 ignore = E203, E266, E501, W503
 max-line-length = 88
-max-complexity = 30
+max-complexity = 40
 select = B,C,E,F,W,T4,B9

.gitignore

@@ -15,3 +15,4 @@ postgres-data-backup/*
 *.rdb
 *_history
 local_configs/*
+prod-*.js

.pre-commit-config.yaml

@@ -16,6 +16,7 @@
     - id: check-docstring-first
     - id: check-merge-conflict
     - id: check-yaml
+      exclude: '^artemis-chart/'
     - id: check-json
     - id: debug-statements
     - id: end-of-file-fixer

CHANGELOG.md

@@ -5,10 +5,14 @@
 - Support for dormant flags in hijacks
 - Storing hijack update (origin, neighbor) combinations in redis
 - Configured/matched prefix field and search in frontend hijack and update tables
-- Stats table initialized with monitored_prefixes counts
+- Monitored prefixes count in stats table (overview)
+- Initial kubernetes/helm (helm-charts) support
+- Reject old updates from taps and have a "HISTORIC" variable to enable/disable
+- Initial support for LDAP authentication
 
 ### Changed
-- TBD (Changed existing functionality)
+- Refactored environment variables
+- Use of RIPE RIS firehose stream instead of the websocket clientui8
 
 ### Fixed
 - Correct RFC2622 translation when needed in frontend and backend
@@ -21,7 +25,7 @@
 - TBD (soon-to-be removed feature)
 
 ### Security
-- TBD (addressing vulnerability)
+- Using yaml dump and safe_load instead of pickling/unpickling
 
 ## [1.1.1] (Atlas) - 2019-02-28
 ### Added

artemis-chart/Chart.yaml

@@ -0,0 +1,14 @@
+name: artemis-chart
+description: ARTEMIS helm chart for deploying on kubernetes.
+version: 0.0.1
+keywords:
+  - helm
+  - artemis
+  - bgp
+  - hijack
+home: https://github.com/FORTH-ICS-INSPIRE/artemis
+maintainers:
+  - name: Dimitrios Mavrommatis
+    email: jim.mavrommatis@gmail.com
+    url: https://slowr.github.io
+appVersion: latest

artemis-chart/README.md

@@ -0,0 +1 @@
+This chart was created by Kompose

artemis-chart/files/configmaps/config.yaml

@@ -0,0 +1 @@
+../../../backend/configs/config.yaml

artemis-chart/files/configmaps/enabled_plugins

@@ -0,0 +1 @@
+../../../other/rabbitmq/enabled-plugins

artemis-chart/files/configmaps/exabgp.conf

@@ -0,0 +1 @@
+../../../backend/configs/exabgp.conf

artemis-chart/files/configmaps/init.sql

@@ -0,0 +1 @@
+../../../other/db/init.sql

artemis-chart/files/configmaps/logging.yaml

@@ -0,0 +1 @@
+../../../backend/configs/logging.yaml

artemis-chart/files/configmaps/nginx.conf

@@ -0,0 +1 @@
+../../../frontend/webapp/configs/nginx.conf

artemis-chart/files/configmaps/postgres-entrypoint.sh

@@ -0,0 +1 @@
+../../../other/postgres-entrypoint.sh

artemis-chart/files/configmaps/restore.sql

@@ -0,0 +1 @@
+../../../other/db/data/restore.sql

artemis-chart/files/configmaps/schema.sql

@@ -0,0 +1 @@
+../../../other/db/libs/rabbitmq/schema.sql

artemis-chart/files/configmaps/services.conf

@@ -0,0 +1 @@
+../../../backend/supervisor.d/services.conf

artemis-chart/files/configmaps/tables.sql

@@ -0,0 +1 @@
+../../../other/db/data/tables.sql

artemis-chart/files/configmaps/wait-for

@@ -0,0 +1 @@
+../../../backend/wait-for

artemis-chart/files/configmaps/webapp.cfg

@@ -0,0 +1 @@
+../../../frontend/webapp/configs/webapp.cfg

artemis-chart/templates/backend-deployment.yaml

@@ -0,0 +1,151 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    io.kompose.service: backend
+    app: {{ .Release.Name }}
+  name: backend
+spec:
+  replicas: {{ .Values.replicaCount | default 1 }}
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        io.kompose.service: backend
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+    spec:
+      initContainers:
+      - name: populate-pvc
+        image: busybox
+        volumeMounts:
+        - mountPath: /configmaps
+          name: backend-configmap
+        - mountPath: /pvc
+          name: backend-pvc
+          subPath: configs
+        command: ['sh', '-c', 'cp -u /configmaps/config.yaml /configmaps/logging.yaml /configmaps/services.conf /pvc/']
+      - name: wait-for-service
+        image: busybox
+        command: ['sh', '-c', 'until nc -z {{ .Values.rabbitmqHost }} {{ .Values.rabbitmqPort}}; do echo waiting for services; sleep 10; done;']
+      containers:
+      - env:
+        - name: DB_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: dbHost
+        - name: DB_NAME
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: dbName
+        - name: DB_PASS
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-secret
+              key: dbPass
+        - name: DB_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: dbPort
+        - name: DB_USER
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: dbUser
+        - name: DB_VERSION
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: dbVersion
+        - name: GUI_ENABLED
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: guiEnabled
+        - name: HASURA_GRAPHQL_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-secret
+              key: hasuraSecret
+        - name: HASURA_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: hasuraHost
+        - name: HASURA_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: hasuraPort
+        - name: HISTORIC
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: historic
+        - name: RABBITMQ_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: rabbitmqHost
+        - name: RABBITMQ_PASS
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-secret
+              key: rabbitmqPass
+        - name: RABBITMQ_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: rabbitmqPort
+        - name: RABBITMQ_USER
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: rabbitmqUser
+        - name: REDIS_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: redisHost
+        - name: REDIS_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: redisPort
+        - name: SUPERVISOR_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: supervisorHost
+        - name: SUPERVISOR_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: {{ .Release.Name }}-configmap
+              key: supervisorPort
+        image: inspiregroup/artemis-backend:{{ .Values.systemVersion }}
+        name: backend
+        resources: {}
+        volumeMounts:
+        - mountPath: /etc/artemis/logging.yaml
+          name: backend-pvc
+          subPath: configs/logging.yaml
+        - mountPath: /etc/artemis/config.yaml
+          name: backend-pvc
+          subPath: configs/config.yaml
+        - mountPath: /etc/supervisor/conf.d/services.conf
+          name: backend-pvc
+          subPath: configs/services.conf
+      restartPolicy: Always
+      volumes:
+      - configMap:
+          name: {{ .Release.Name }}-volumes
+        name: backend-configmap
+      - persistentVolumeClaim:
+          claimName: {{ .Release.Name }}-pvc
+        name: backend-pvc
+status: {}

artemis-chart/templates/configmap.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-configmap
+  labels:
+    app: {{ .Release.Name }}
+data:
+  guiEnabled: {{ .Values.guiEnable | default "true" | quote }}
+  systemVersion: {{ .Values.systemVersion | default "latest" | quote }}
+  historic: {{ .Values.historic | default "false" | quote }}
+  redisHost: {{ .Values.redisHost | default "backend" | quote }}
+  redisPort: {{ .Values.redisPort | default "6379" | quote }}
+  supervisorHost: {{ .Values.supervisorHost | default "backend" | quote }}
+  supervisorPort: {{ .Values.supervisorPort | default "9001" | quote }}
+  apiHost: {{ .Values.apiHost | default "postgrest" | quote }}
+  apiPort: {{ .Values.apiPort | default "3000" | quote }}
+  dbHost: {{ .Values.dbHost | default "postgres" | quote }}
+  dbPort: {{ .Values.dbPort | default "5432" | quote }}
+  dbVersion: {{ .Values.dbVersion | default "12" | quote }}
+  dbName: {{ .Values.dbName | default "artemis_db" | quote }}
+  dbUser: {{ .Values.dbUser | default "artemis_user" | quote }}
+  dbSchema: {{ .Values.dbSchema | default "public" | quote }}
+  dbAutoClean: {{ .Values.dbAutoClean | default "false" | quote }}
+  dbBackup: {{ .Values.dbBackup | default "true" | quote }}
+  dbHijackDormant: {{ .Values.dbHijackDormant | default "false" | quote }}
+  dbUri: postgres://{{ .Values.dbUser | default "artemis_user" }}:{{ .Values.dbPass | default "Art3m1s" }}@{{ .Values.dbHost | default "postgres"}}:{{ .Values.dbPort | default 5432 }}/{{ .Values.dbName | default "artemis_db" }}
+  bindIp: {{ .Values.bindIp | default "0.0.0.0" | quote }}
+  webappHost: {{ .Values.webappHost | default "frontend" | quote }}
+  webappPort: {{ .Values.webappPort | default "8000" | quote }}
+  adminUser: {{ .Values.adminUser | default "admin" | quote }}
+  adminEmail: {{ .Values.adminEmail | default "admin@admin.com" | quote }}
+  jsVersion: {{ .Values.jsVersion | default "0.1.2" | quote }}
+  rabbitmqHost: {{ .Values.rabbimtqHost | default "rabbitmq" | quote }}
+  rabbitmqPort: {{ .Values.rabbimtqPort | default "5672" | quote }}
+  rabbitmqUser: {{ .Values.rabbitmqUser | default "guest" | quote }}
+  rabbitmqUri: amqp://{{ .Values.rabbitmqUser | default "guest" }}:{{ .Values.rabbitmqPass | default "guest" }}@{{ .Values.rabbitmqHost | default "rabbitmq"}}:{{ .Values.rabbitmqPort | default 5672 }}//
+  hasuraHost: {{ .Values.hasuraHost | default "graphql" | quote }}
+  hasuraPort: {{ .Values.hasuraPort | default "8080" | quote }}
+  hasuraGui: {{ .Values.hasuraGui | default "false" | quote }}