[CI/CD][orc8r] Add deploy scripts for Kubernetes and Magma on bare me…

…tal (#3089) * Add magma CI bare metal lab ansible manifests Add scripts for deploying magma helm charts Signed-off-by: Matthew Mosesohn <matthew.mosesohn@gmail.com> * add note about orc8r repo Signed-off-by: Matthew Mosesohn <matthew.mosesohn@gmail.com>
magma · Nov 23, 2020 · 897af42 · 897af42
1 parent 4c4a863
commit 897af42
Show file tree

Hide file tree

Showing 21 changed files with 735 additions and 0 deletions.
diff --git a/orc8r/cloud/deploy/bare-metal/README.md b/orc8r/cloud/deploy/bare-metal/README.md
@@ -0,0 +1,22 @@
+Deploying Magma on bare metal
+
+The following files need to be edited before deployment can start:
+* deploy_ansible_vars.yaml
+* orc8r_settings (shell env file)
+* deploy_charts.sh
+
+The values that need to be customized are the IP settings for your network and
+the passwords which need to be generated. Additionally, you need to host a
+docker repo and helm chart repo for orc8r. Further details can be found at
+https://magma.github.io/magma/docs/orc8r/deploy_build#build-and-publish-helm-charts
+
+Once the config is set, just run the following commands:
+
+./deploy.sh
+./deploy_charts.sh
+
+If the deployment succeeds, you will see information on how to log into Magma
+web UI.
+
+Note: external-dns configuration is not automatic here because it is intended
+for on-premise. You are expected to update DNS or /etc/hosts on your own.
diff --git a/orc8r/cloud/deploy/bare-metal/charts/elasticsearch-curator.yaml b/orc8r/cloud/deploy/bare-metal/charts/elasticsearch-curator.yaml
@@ -0,0 +1,29 @@
+configMaps:
+  action_file_yml: |-
+    ---
+    actions:
+      1:
+        action: delete_indices
+        description: "Clean up ES by deleting old indices"
+        options:
+          timeout_override:
+          continue_if_exception: False
+          disable_action: False
+          ignore_empty_list: True
+        filters:
+        - filtertype: age
+          source: name
+          direction: older
+          timestring: '%Y.%m.%d'
+          unit: days
+          unit_count: 7
+          field:
+          stats_result:
+          epoch:
+          exclude: False
+  config_yml: |-
+    ---
+    client:
+      hosts:
+        - elasticsearch-client:9200
+      port: 9200
diff --git a/orc8r/cloud/deploy/bare-metal/charts/elasticsearch.yaml.tpl b/orc8r/cloud/deploy/bare-metal/charts/elasticsearch.yaml.tpl
@@ -0,0 +1,13 @@
+clusterName: "es-magma"
+replicas: 1
+minimumMasterNodes: 1
+rbac:
+  create: true
+antiAffinity: "soft"
+service:
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: elasticsearch.${dns_domain}
+data:
+  storageClass: ${storage_class}
+master:
+  storageClass: ${storage_class}
diff --git a/orc8r/cloud/deploy/bare-metal/charts/fluentd.yaml.tpl b/orc8r/cloud/deploy/bare-metal/charts/fluentd.yaml.tpl
@@ -0,0 +1,66 @@
+configMaps:
+  forward-input.conf: |-
+    <source>
+      @type forward
+      port 24224
+      bind 0.0.0.0
+      <transport tls>
+        ca_path /certs/certifier.pem
+        cert_path /certs/fluentd.pem
+        private_key_path /certs/fluentd.key
+        client_cert_auth true
+      </transport>
+    </source>
+  output.conf: |-
+    <match **>
+      @id elasticsearch
+      @type elasticsearch
+      @log_level info
+      include_tag_key true
+      host "#{ENV['OUTPUT_HOST']}"
+      port "#{ENV['OUTPUT_PORT']}"
+      scheme "#{ENV['OUTPUT_SCHEME']}"
+      ssl_version "#{ENV['OUTPUT_SSL_VERSION']}"
+      logstash_format true
+      logstash_prefix "magma"
+      reconnect_on_error true
+      reload_on_failure true
+      reload_connections false
+      <buffer>
+        @type file
+        path /var/log/fluentd-buffers/kubernetes.system.buffer
+        flush_mode interval
+        retry_type exponential_backoff
+        flush_thread_count 2
+        flush_interval 5s
+        retry_forever
+        retry_max_interval 30
+        chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
+        queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
+        overflow_action block
+      </buffer>
+    </match>
+extraVolumeMounts:
+- mountPath: /certs
+  name: certs
+  readOnly: true
+extraVolumes:
+- name: certs
+  secret:
+    defaultMode: 420
+    secretName: fluentd-certs
+output:
+  host: elasticsearch-client
+  port: 9200
+  scheme: http
+rbac:
+  create: false
+replicaCount: 2
+service:
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: fluentd.${dns_domain}
+  ports:
+  - containerPort: 24224
+    name: forward
+    protocol: TCP
+  type: LoadBalancer
diff --git a/orc8r/cloud/deploy/bare-metal/charts/kibana.yaml b/orc8r/cloud/deploy/bare-metal/charts/kibana.yaml
@@ -0,0 +1,21 @@
+image:
+  repository: "docker.elastic.co/kibana/kibana-oss"
+  tag: "6.8.2"
+  pullPolicy: "IfNotPresent"
+
+env:
+  LOGGING_VERBOSE: "false"
+
+files:
+  kibana.yml:
+    ## Default Kibana configuration from kibana-docker.
+    server.name: kibana
+    server.host: "0"
+    ## For kibana < 6.6, use elasticsearch.url instead
+    elasticsearch.hosts: http://elasticsearch-client:9200
+
+dashboardImport:
+  enabled: true
+  timeout: 60
+  dashboards:
+    k8s: https://raw.githubusercontent.com/monotek/kibana-dashboards/master/k8s-fluentd-elasticsearch.json
diff --git a/orc8r/cloud/deploy/bare-metal/charts/mariadb-galera.yaml.tpl b/orc8r/cloud/deploy/bare-metal/charts/mariadb-galera.yaml.tpl
@@ -0,0 +1,7 @@
+global:
+  storageClass: nfs
+rootUser:
+  password: ${db_admin_password}
+extraFlags: "--sql-mode=ANSI_QUOTES"
+image:
+  tag: 10.5.6-debian-10-r7
diff --git a/orc8r/cloud/deploy/bare-metal/charts/metallb.yaml.tpl b/orc8r/cloud/deploy/bare-metal/charts/metallb.yaml.tpl
@@ -0,0 +1,10 @@
+configInline:
+  address-pools:
+  - name: default
+    protocol: layer2
+    addresses:
+      - ${metallb_addresses}
+prometheus:
+  scrapeAnnotations: true
+psp:
+  create: false
diff --git a/orc8r/cloud/deploy/bare-metal/charts/nfs-server-provisioner.yaml b/orc8r/cloud/deploy/bare-metal/charts/nfs-server-provisioner.yaml
@@ -0,0 +1,24 @@
+image:
+  pullPolicy: IfNotPresent
+  repository: quay.io/kubernetes_incubator/nfs-provisioner
+  tag: v2.3.0
+
+nodeSelector:
+  kubernetes.io/hostname: compute1
+
+persistence:
+  enabled: true
+  size: 200Gi
+  storageClass: nfs-provisioner
+replicaCount: 1
+resources:
+  requests:
+    cpu: 500m
+    memory: 512Mi
+storageClass:
+  create: true
+  defaultClass: true
+  mountOptions:
+  - noatime
+  name: nfs
+  provisionerName: nfs
diff --git a/orc8r/cloud/deploy/bare-metal/charts/orc8r.yaml.tpl b/orc8r/cloud/deploy/bare-metal/charts/orc8r.yaml.tpl
@@ -0,0 +1,144 @@
+controller:
+  image:
+    repository: ${img_repo}/controller
+    tag: ${controller_tag}
+  podDisruptionBudget:
+    enabled: true
+  replicas: 2
+  spec:
+    database:
+      driver: mysql
+      sql_dialect: maria
+      db: ${orc8r_db_user}
+      host: ${orc8r_db_host}
+      pass: ${orc8r_db_pass}
+      port: 3306
+      user: ${orc8r_db_user}
+
+imagePullSecrets:
+- name: artifactory
+logging:
+  enabled: false
+metrics:
+  alertmanager:
+    create: true
+  alertmanagerConfigurer:
+    alertmanagerURL: orc8r-alertmanager:9093
+    create: true
+    image:
+      repository: docker.io/facebookincubator/alertmanager-configurer
+      tag: 1.0.0
+  grafana:
+    create: false
+  imagePullSecrets:
+  - name: artifactory
+  metrics:
+    volumes:
+      prometheusConfig:
+        volumeSpec:
+          persistentVolumeClaim:
+            claimName: promcfg
+      prometheusData:
+        volumeSpec:
+          persistentVolumeClaim:
+            claimName: promdata
+  prometheus:
+    create: true
+    includeOrc8rAlerts: true
+  prometheusCache:
+    create: true
+    image:
+      repository: docker.io/facebookincubator/prometheus-edge-hub
+      tag: 1.0.0
+    limit: 500000
+  prometheusConfigurer:
+    create: true
+    image:
+      repository: docker.io/facebookincubator/prometheus-configurer
+      tag: 1.0.0
+    prometheusURL: orc8r-prometheus:9090
+  userGrafana:
+    create: true
+    image:
+      repository: docker.io/grafana/grafana
+      tag: 6.6.2
+    volumes:
+      dashboardproviders:
+        persistentVolumeClaim:
+          claimName: grafanaproviders
+      dashboards:
+        persistentVolumeClaim:
+          claimName: grafanadashboards
+      datasources:
+        persistentVolumeClaim:
+          claimName: grafanadatasources
+      grafanaData:
+        persistentVolumeClaim:
+          claimName: grafanadata
+
+nms:
+  enabled: true
+  imagePullSecrets:
+  - name: artifactory
+  magmalte:
+    env:
+      api_host: api.nuclab.maxwellswireless.com
+      grafana_address: orc8r-user-grafana:3000
+      mysql_db: ${nms_db_user}
+      mysql_host: ${nms_db_host}
+      mysql_pass: ${nms_db_pass}
+      mysql_port: 3306
+      mysql_user: ${nms_db_user}
+      mysql_dialect: mariadb
+    image:
+      repository: ${img_repo}/magmalte
+      tag: ${nms_tag}
+    manifests:
+      deployment: true
+      rbac: false
+      secrets: true
+      service: true
+  nginx:
+    deployment:
+      spec:
+        ssl_cert_key_name: controller.key
+        ssl_cert_name: controller.crt
+    manifests:
+      configmap: true
+      deployment: true
+      rbac: false
+      secrets: true
+      service: true
+    service:
+      annotations:
+        external-dns.alpha.kubernetes.io/hostname: '*.nms.${dns_domain}'
+      type: LoadBalancer
+  secret:
+    certs: nms-certs
+nginx:
+  image:
+    repository: ${img_repo}/nginx
+    tag: ${nginx_tag}
+  podDisruptionBudget:
+    enabled: true
+  replicas: 2
+  service:
+    enabled: true
+    extraAnnotations:
+      bootstrapLagacy:
+        external-dns.alpha.kubernetes.io/hostname: bootstrapper-controller.${dns_domain}
+      clientcertLegacy:
+        external-dns.alpha.kubernetes.io/hostname: controller.${dns_domain},api.${dns_domain}
+    legacyEnabled: true
+    name: orc8r-bootstrap-legacy
+    type: LoadBalancer
+  spec:
+    hostname: controller.${dns_domain}
+
+secret:
+  certs: orc8r-certs
+  configs:
+    orc8r: orc8r-configs
+  envdir: orc8r-envdir
+secrets:
+  create: false
diff --git a/orc8r/cloud/deploy/bare-metal/db_setup.sql.tpl b/orc8r/cloud/deploy/bare-metal/db_setup.sql.tpl
@@ -0,0 +1,11 @@
+create database if not exists ${nms_db_user};
+create user if not exists '${nms_db_user}'@'localhost' identified by '${nms_db_pass}';
+create user if not exists '${nms_db_user}'@'%' identified by '${nms_db_pass}';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES ON ${nms_db_user}.* TO '${nms_db_user}'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES ON ${nms_db_user}.* TO '${nms_db_user}'@'%';
+
+create database if not exists ${orc8r_db_user};
+create user if not exists '${orc8r_db_user}'@'localhost' identified by '${orc8r_db_pass}';
+create user if not exists '${orc8r_db_user}'@'%' identified by '${orc8r_db_pass}';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES ON ${orc8r_db_user}.* TO '${orc8r_db_user}'@'localhost';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES ON ${orc8r_db_user}.* TO '${orc8r_db_user}'@'%';