BYO networking

examples/terraform-do/cluster_custom_network.yml

@@ -0,0 +1,9 @@
+network:
+  provider: custom
+  pod_network_cidr: 172.31.0.0/16
+  service_cidr: 172.32.0.0/16
+  custom:
+    manifest_path: ./flannel/
+addons:
+  ingress-nginx:
+    enabled: true

examples/terraform-do/flannel/01-cluster-role.yml

@@ -0,0 +1,25 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch

examples/terraform-do/flannel/02-cluster-role-binding.yml

@@ -0,0 +1,13 @@
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system

examples/terraform-do/flannel/03-service-account.yml

@@ -0,0 +1,6 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system

examples/terraform-do/flannel/04-config-map.yml.erb

@@ -0,0 +1,36 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "<%= cluster_config.network.pod_network_cidr %>",
+      "Backend": {
+        "Type": "vxlan"
+      }
+    }

examples/terraform-do/flannel/10-daemonset.yml

@@ -0,0 +1,76 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      tolerations:
+      - operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      initContainers:
+      - name: install-cni
+        image: quay.io/coreos/flannel:v0.10.0-amd64
+        command:
+        - cp
+        args:
+        - -f
+        - /etc/kube-flannel/cni-conf.json
+        - /etc/cni/net.d/10-flannel.conflist
+        volumeMounts:
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      containers:
+      - name: kube-flannel
+        image: quay.io/coreos/flannel:v0.10.0-amd64
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq
+        - --kube-subnet-mgr
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg

lib/pharos/cluster_manager.rb

@@ -102,6 +102,7 @@ def apply_phases
       apply_phase(Phases::ConfigureDNS, [master_hosts.first], master: master_hosts.first)
       apply_phase(Phases::ConfigureWeave, [master_hosts.first], master: master_hosts.first) if config.network.provider == 'weave'
       apply_phase(Phases::ConfigureCalico, [master_hosts.first], master: master_hosts.first) if config.network.provider == 'calico'
+      apply_phase(Phases::ConfigureCustomNetwork, [master_hosts.first], master: master_hosts.first) if config.network.provider == 'custom'
 
       apply_phase(Phases::ConfigureBootstrap, [master_hosts.first]) # using `kubeadm token`, not the kube API
 

lib/pharos/config_schema.rb

@@ -94,7 +94,7 @@ def self.messages
           optional(:endpoint).filled(:str?)
         end
         optional(:network).schema do
-          optional(:provider).filled(included_in?: %(weave calico))
+          optional(:provider).filled(included_in?: %(weave calico custom))
           optional(:dns_replicas).filled(:int?, gt?: 0)
           optional(:service_cidr).filled(:str?)
           optional(:pod_network_cidr).filled(:str?)
@@ -106,6 +106,10 @@ def self.messages
           optional(:calico).schema do
             optional(:ipip_mode).filled(included_in?: %(Always, CrossSubnet, Never))
           end
+          optional(:custom).schema do
+            required(:manifest_path).filled(:str?)
+            optional(:options).filled(:hash?)
+          end
         end
         optional(:etcd).schema do
           required(:endpoints).each(type?: String)

lib/pharos/configuration/network.rb

@@ -29,12 +29,25 @@ def self.filter_host_routes(routes)
         end
       end
 
+      class Custom < Pharos::Configuration::Struct
+        attribute :manifest_path, Pharos::Types::String
+        attribute :options, Pharos::Types::Hash
+
+        # @param _routes [Array<Pharos::Configuration::Host::Routes>]
+        # @return [Array<Pharos::Configuration::Host::Routes>]
+        def self.filter_host_routes(_routes)
+          # There's no way to validate routes for a custom CNI setup
+          []
+        end
+      end
+
       attribute :provider, Pharos::Types::String.default('weave')
       attribute :dns_replicas, Pharos::Types::Integer
       attribute :service_cidr, Pharos::Types::String.default('10.96.0.0/12')
       attribute :pod_network_cidr, Pharos::Types::String.default('10.32.0.0/12')
       attribute :weave, Weave
       attribute :calico, Calico
+      attribute :custom, Custom
 
       # @return [String] 10.96.0.10
       def dns_service_ip
@@ -49,6 +62,8 @@ def filter_host_routes(routes)
           Weave.filter_host_routes(routes)
         when 'calico'
           Calico.filter_host_routes(routes)
+        when 'custom'
+          Custom.filter_host_routes(routes)
         else
           fail
         end

lib/pharos/phases/configure_custom_network.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Pharos
+  module Phases
+    class ConfigureCustomNetwork < Pharos::Phase
+      title "Configure Custom network"
+
+      def call
+        logger.info { "Configuring custom network ..." }
+        # TODO Set options more granularly?
+        stack = Pharos::Kube.stack('custom-network', @config.network.custom.manifest_path, name: 'custom_network', cluster_config: @config)
+        stack.apply(kube_client)
+      end
+    end
+  end
+end