cmd: accept an !include tag for YAML files

.github/workflows/ci.yml

@@ -201,7 +201,7 @@ jobs:
           tar zcvf docker-compose-quickstart.tar.gz \
             docker-compose.yml docker-compose-demo.yml .env \
             orchestrator/clickhouse/data/docker-entrypoint.sh \
-            akvorado.yaml
+            akvorado*.yaml
 
       # Publish release
       - name: Publish release

akvorado-console.yaml

@@ -0,0 +1,16 @@
+---
+http:
+  cache:
+    type: redis
+    server: redis:6379
+database:
+  saved-filters:
+    # These are prepopulated filters you can select in a drop-down
+    # menu. Users can add more filters interactively.
+    - description: "From Netflix"
+      content: >-
+        InIfBoundary = external AND SrcAS = AS2906
+    - description: "From GAFAM"
+      content: >-
+        InIfBoundary = external AND
+        SrcAS IN (AS15169, AS16509, AS32934, AS6185, AS8075)

akvorado-demo.yaml

@@ -0,0 +1,347 @@
+---
+.demo-exporter-flows:
+  - &http-src
+    src-port: [80, 443]
+    dst-port: 0
+    protocol: tcp
+    size: 1300
+  - &http-dst
+    src-port: 0
+    dst-port: [80, 443]
+    protocol: tcp
+    size: 1300
+  - &quic-src
+    src-port: 443
+    dst-port: 0
+    protocol: udp
+    size: 1200
+  - &ssh-src
+    src-port: 22
+    dst-port: 0
+    protocol: tcp
+    size: 200
+  - &ssh-dst
+    src-port: 0
+    dst-port: 22
+    protocol: tcp
+    size: 300
+  - &to-v4-customers
+    dst-net: 192.0.2.0/24
+    dst-as: 64501
+  - &to-v6-customers
+    dst-net: 2a01:db8:cafe:1::/64
+    dst-as: 64501
+  - &to-v4-servers
+    dst-net: 203.0.113.0/24
+    dst-as: 64501
+  - &to-v6-servers
+    dst-net: 2a01:db8:cafe:2::/64
+    dst-as: 64501
+  - &from-v4-google
+    src-net: 216.58.206.0/24
+    src-as: 15169
+  - &from-v6-google
+    src-net: 2a00:1450:4007:807::2000/124
+    src-as: 15169
+  - &from-v4-facebook
+    src-net: 179.60.192.0/24
+    src-as: 32934
+  - &from-v6-facebook
+    src-net: 2a03:2880:f130:83:face:b00c:0::/112
+    src-as: 32934
+  - &from-v4-netflix
+    src-net: 198.38.120.0/23
+    src-as: 2906
+  - &from-v6-netflix
+    src-net: 2a00:86c0:115:115::/112
+    src-as: 2906
+  - &from-v4-akamai
+    src-net: 23.33.27.0/24
+    src-as: 20940
+  - &from-v6-akamai
+    src-net: 2a02:26f0:9100:28:0:17c0::/112
+    src-as: 20940
+  - &from-v4-amazon
+    src-net: 52.84.175.0/24
+    src-as: 16509
+  - &from-v6-amazon
+    src-net: 2600:9000:218d:4a00:15:74db::/112
+    src-as: 16509
+  - &from-v4-fastly
+    src-net: 199.232.178.0/29
+    src-as: 54113
+  - &from-v6-fastly
+    src-net: 2a04:4e42:1d::/126
+    src-as: 54113
+  - &from-v4-twitch
+    src-net: 52.223.202.128/27
+    src-as: 46489
+  - &from-v4-renater
+    src-net: 138.231.0.0/16
+    src-as: 2269
+  - &from-v4-random
+    src-net: 92.0.0.0/8
+    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
+  - &from-v6-random
+    src-net: 2a01:cb00::/32
+    src-as: [12322, 3215, 3303, 15557, 3320, 13335, 6185, 202818, 60068, 16276, 8075, 32590]
+
+"":
+  - snmp:
+      name: th2-edge1.example.com
+      interfaces:
+        10: "Transit: Telia"
+        11: "IX: AMSIX"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp: &bmp
+      target: akvorado-inlet:10179
+      routes:
+        - prefixes: 192.0.2.0/24,2a01:db8:cafe:1::/64
+          aspath: 64501
+          communities: 65401:10,65401:12
+          large-communities: 65401:100:200,65401:100:201
+        - prefixes: 203.0.113.0/24,2a01:db8:cafe:2::/64
+          aspath: 65401
+          communities: 65401:10,65401:13
+          large-communities: 65401:100:200,65401:100:213
+        - prefixes: 216.58.206.0/24,2a00:1450:4007:807::2000/124
+          aspath: 174,1299,15169
+          communities: 174:22004,174:21100
+        - prefixes: 179.60.192.0/24,2a03:2880:f130:83:face:b00c:0::/112
+          aspath: 1299,1299,32934
+          communities: 1299:30000,1299:30220
+        - prefixes: 198.38.120.0/23,2a00:86c0:115:115::/112
+          aspath: 5511,1299,1299,32934
+          communities: 1299:30000,1299:30310
+        - prefixes: 23.33.27.0/24,2a02:26f0:9100:28:0:17c0::/112
+          aspath: 174,174,174,20940
+          communities: 174:22002,174:21200
+        - prefixes: 52.84.175.0/24,2600:9000:218d:4a00:15:74db::/112
+          aspath: 16509
+        - prefixes: 199.232.178.0/29,2a04:4e42:1d::/126
+          aspath: 1299,54113
+          communities: 1299:35000,1299:35200
+        - prefixes: 52.223.202.128/27
+          aspath: 16509,46489
+        - prefixes: 138.231.0.0/16
+          aspath: 1299,174,2269,2269
+          communities: 1299:30000,1299:30400
+        - prefixes: 0.0.0.0/0
+          aspath: 174
+        - prefixes: ::/0
+          aspath: 1299
+    flows: &flows1
+      samplingrate: 50000
+      target: akvorado-inlet:2055
+      flows:
+        # Google
+        - per-second: 1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-google, *to-v4-customers, *http-src]
+        - per-second: 0.5
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 5
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-google, *to-v4-customers, *quic-src]
+        - per-second: 1.4
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-google, *to-v6-customers, *http-src]
+        - per-second: 0.8
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 5
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-google, *to-v6-customers, *quic-src]
+        # Facebook
+        - per-second: 1.1
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-facebook, *to-v4-customers, *http-src]
+        - per-second: 0.2
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 16h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-facebook, *to-v4-customers, *quic-src]
+        - per-second: 1.8
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v6-facebook, *to-v6-customers, *http-src]
+        - per-second: 0.2
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 20h
+          multiplier: 3
+          reverse-direction-ratio: 0.2
+          <<: [*from-v6-facebook, *to-v6-customers, *quic-src]
+        # Netflix
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 22h
+          multiplier: 20
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-netflix, *to-v4-customers, *http-src]
+        - per-second: 0.7
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 22h
+          multiplier: 20
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-netflix, *to-v6-customers, *http-src]
+        # Twitch
+        - per-second: 0.12
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 21h
+          multiplier: 17
+          reverse-direction-ratio: 0.4
+          <<: [*from-v4-twitch, *to-v4-customers, *http-src]
+        # Akamai
+        - per-second: 0.14
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-akamai, *to-v4-customers, *http-src]
+        - per-second: 0.8
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-akamai, *to-v6-customers, *http-src]
+        # Fastly
+        - per-second: 0.4
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v4-fastly, *to-v4-customers, *http-src]
+        - per-second: 0.7
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 14h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-fastly, *to-v6-customers, *http-src]
+        # Amazon
+        - per-second: 0.3
+          in-if-index: [10, 11]
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.15
+          <<: [*from-v4-amazon, *to-v4-customers, *http-src]
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 18h
+          multiplier: 1.3
+          reverse-direction-ratio: 0.15
+          <<: [*from-v6-amazon, *to-v6-customers, *http-src]
+
+        # Random SSH
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.5
+          <<: [*from-v4-renater, *to-v4-customers, *ssh-src]
+        # Servers
+        - per-second: 0.1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.2
+          <<: [*from-v4-renater, *to-v4-servers, *ssh-dst]
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.15
+          <<: [*from-v4-random, *to-v4-servers, *http-dst]
+        - per-second: 0.2
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 15h
+          multiplier: 1.2
+          reverse-direction-ratio: 0.1
+          <<: [*from-v6-random, *to-v6-servers, *http-dst]
+
+        # Noise
+        - &random-flow
+          per-second: 1
+          in-if-index: 10
+          out-if-index: [20, 21]
+          peak-hour: 20h
+          multiplier: 1
+          protocol: [tcp, udp]
+          srcport: [80, 443, 22, 25461, 8080, 4500, 993, 8801]
+          reverse-direction-ratio: 0.25
+          <<: [*from-v4-random, *to-v4-customers]
+        - <<: [*from-v6-random, *to-v6-customers, *random-flow]
+  - snmp:
+      name: th2-edge2.example.com
+      interfaces:
+        10: "Transit: Cogent"
+        11: "IX: DECIX"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 100
+  - snmp:
+      name: dc3-edge1.example.com
+      interfaces:
+        10: "Transit: Tata"
+        11: "Transit: Lumen"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 200
+  - snmp:
+      name: dc5-edge2.example.com
+      interfaces:
+        10: "IX: FranceIX"
+        11: "Transit: Cogent"
+        20: "core"
+        21: "core"
+      listen: 0.0.0.0:161
+    bmp:
+      <<: *bmp
+    flows:
+      <<: *flows1
+      seed: 300