From 160ecb28abc349f03324f3a55a5fa4811fd292a0 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 15:30:25 +0100 Subject: [PATCH 01/12] generic deployment work --- Documentation/generic.md | 29 +++ ...erouter-all-features-advertise-routes.yaml | 191 ++++++++++++++++++ .../generic-kuberouter-all-features.yaml | 187 +++++++++++++++++ daemonset/generic-kuberouter.yaml | 187 +++++++++++++++++ 4 files changed, 594 insertions(+) create mode 100644 Documentation/generic.md create mode 100644 daemonset/generic-kuberouter-all-features-advertise-routes.yaml create mode 100644 daemonset/generic-kuberouter-all-features.yaml create mode 100644 daemonset/generic-kuberouter.yaml diff --git a/Documentation/generic.md b/Documentation/generic.md new file mode 100644 index 0000000000..915fb08311 --- /dev/null +++ b/Documentation/generic.md @@ -0,0 +1,29 @@ +# Deploying kube-router on generic cluster + +Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes. + +Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options. + +## kube-router providing pod networking and network policy + +```sh +KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter.yaml +``` + +## kube-router providing service proxy, firewall and pod networking. + +For the step #3 **Installing a pod network** install a kube-router pod network and network policy add-on with the following command: + +```sh +KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter-all-features.yaml +``` + +Now since kube-router provides service proxy as well. Run below commands to remove kube-proxy and cleanup any iptables configuration it may have done. + +```sh +KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system delete ds kube-proxy +docker run --privileged --net=host gcr.io/google_containers/kube-proxy-amd64:v1.7.3 kube-proxy --cleanup-iptables +``` + + + diff --git a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml new file mode 100644 index 0000000000..9d78633a61 --- /dev/null +++ b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml @@ -0,0 +1,191 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "ipam": { + "type":"host-local" + } + } + kubeconfig: | + apiVersion: v1 + kind: Config + clusterCIDR: "%CLUSTERCIDR%" + clusters: + - name: cluster + cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://%APISERVER% + users: + - name: kube-router + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + contexts: + - context: + cluster: cluster + user: kube-router + name: kube-router-context + current-context: kube-router-context + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: cloudnativelabs/kube-router + imagePullPolicy: Always + args: + - "--run-router=true" + - "--run-firewall=true" + - "--run-service-proxy=true" + - "--kubeconfig=/var/lib/kube-router/kubeconfig" + - "--peer-router-ips=10.1.201.254" + - "--peer-router-asns=64512" + - "--cluster-asn=64512" + - "--advertise-cluster-ip=true" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conf; + fi; + if [ ! -f /var/lib/kube-router/kubeconfig ]; then + TMP=/var/lib/kube-router/.tmp-kubeconfig; + cp /etc/kube-router/kubeconfig ${TMP}; + mv ${TMP} /var/lib/kube-router/kubeconfig; + fi + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-conf-dir + - mountPath: /etc/kube-router + name: kube-router-cfg + - name: kubeconfig + mountPath: /var/lib/kube-router + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + hostPath: + path: /var/lib/kube-router/kubeconfig +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/daemonset/generic-kuberouter-all-features.yaml b/daemonset/generic-kuberouter-all-features.yaml new file mode 100644 index 0000000000..ee4b54a522 --- /dev/null +++ b/daemonset/generic-kuberouter-all-features.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "ipam": { + "type":"host-local" + } + } + kubeconfig: | + apiVersion: v1 + kind: Config + clusterCIDR: "%CLUSTERCIDR%" + clusters: + - name: cluster + cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://%APISERVER% + users: + - name: kube-router + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + contexts: + - context: + cluster: cluster + user: kube-router + name: kube-router-context + current-context: kube-router-context + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: cloudnativelabs/kube-router + imagePullPolicy: Always + args: + - "--run-router=true" + - "--run-firewall=true" + - "--run-service-proxy=true" + - "--kubeconfig=/var/lib/kube-router/kubeconfig" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conf; + fi; + if [ ! -f /var/lib/kube-router/kubeconfig ]; then + TMP=/var/lib/kube-router/.tmp-kubeconfig; + cp /etc/kube-router/kubeconfig ${TMP}; + mv ${TMP} /var/lib/kube-router/kubeconfig; + fi + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-conf-dir + - mountPath: /etc/kube-router + name: kube-router-cfg + - name: kubeconfig + mountPath: /var/lib/kube-router + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + hostPath: + path: /var/lib/kube-router/kubeconfig +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system diff --git a/daemonset/generic-kuberouter.yaml b/daemonset/generic-kuberouter.yaml new file mode 100644 index 0000000000..dfc61714b5 --- /dev/null +++ b/daemonset/generic-kuberouter.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-router-cfg + namespace: kube-system + labels: + tier: node + k8s-app: kube-router +data: + cni-conf.json: | + { + "name":"kubernetes", + "type":"bridge", + "bridge":"kube-bridge", + "isDefaultGateway":true, + "ipam": { + "type":"host-local" + } + } + kubeconfig: | + apiVersion: v1 + kind: Config + clusterCIDR: %CLUSTERCIDR% + clusters: + - name: cluster + cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: %APISERVER% + users: + - name: kube-router + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + contexts: + - context: + cluster: cluster + user: kube-router + name: kube-router-context + current-context: kube-router-context + +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + labels: + k8s-app: kube-router + tier: node + name: kube-router + namespace: kube-system +spec: + template: + metadata: + labels: + k8s-app: kube-router + tier: node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: kube-router + serviceAccount: kube-router + containers: + - name: kube-router + image: cloudnativelabs/kube-router + imagePullPolicy: Always + args: + - "--run-router=true" + - "--run-firewall=true" + - "--run-service-proxy=false" + - "--kubeconfig=/var/lib/kube-router/kubeconfig" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + resources: + requests: + cpu: 250m + memory: 250Mi + securityContext: + privileged: true + volumeMounts: + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: cni-conf-dir + mountPath: /etc/cni/net.d + - name: kubeconfig + mountPath: /var/lib/kube-router + readOnly: true + initContainers: + - name: install-cni + image: busybox + imagePullPolicy: Always + command: + - /bin/sh + - -c + - set -e -x; + if [ ! -f /etc/cni/net.d/10-kuberouter.conf ]; then + TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; + cp /etc/kube-router/cni-conf.json ${TMP}; + mv ${TMP} /etc/cni/net.d/10-kuberouter.conf; + fi; + if [ ! -f /var/lib/kube-router/kubeconfig ]; then + TMP=/var/lib/kube-router/.tmp-kubeconfig; + cp /etc/kube-router/kubeconfig ${TMP}; + mv ${TMP} /var/lib/kube-router/kubeconfig; + fi + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni-conf-dir + - mountPath: /etc/kube-router + name: kube-router-cfg + - name: kubeconfig + mountPath: /var/lib/kube-router + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: cni-conf-dir + hostPath: + path: /etc/cni/net.d + - name: kube-router-cfg + configMap: + name: kube-router-cfg + - name: kubeconfig + hostPath: + path: /var/lib/kube-router/kubeconfig +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-router + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - namespaces + - pods + - services + - nodes + - endpoints + verbs: + - list + - get + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - networkpolicies + verbs: + - list + - get + - watch + - apiGroups: + - extensions + resources: + - networkpolicies + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kube-router +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-router +subjects: +- kind: ServiceAccount + name: kube-router + namespace: kube-system From 56e57997882d886766782803f465b9272b15fd73 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 15:59:33 +0100 Subject: [PATCH 02/12] better docs --- Documentation/generic.md | 43 ++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/Documentation/generic.md b/Documentation/generic.md index 915fb08311..8e5bb06b2e 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -4,26 +4,49 @@ Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options. +## Prerequisites + +### Kubelet + +kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni` +Startup options: + +- --cni-conf-dir=/etc/cni/net.d +- --network-plugin=cni + +### Kube controller-manager + +The following options needs to be set on the controller-manager: + +```text +--cluster-cidr=${POD_NETWORK} # for example 10.32.0.0/12 +--service-cluster-ip-range=${SERVICE_IP_RANGE} # for example 10.50.0.0/22 +``` + ## kube-router providing pod networking and network policy ```sh -KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter.yaml +CLUSTERCIDR=10.32.0.0/12 \ +APISERVER=https://cluster01.int.domain.com:6443 \ +sh -c 'curl https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter.yaml -o - | \ +sed -e "s;%APISERVER%;$APISERVER;g" -e "s;%CLUSTERCIDR%;$CLUSTERCIDR;g"' | \ +kubectl apply -f - ``` -## kube-router providing service proxy, firewall and pod networking. - -For the step #3 **Installing a pod network** install a kube-router pod network and network policy add-on with the following command: +## kube-router providing service proxy, firewall and pod networking ```sh -KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter-all-features.yaml +CLUSTERCIDR=10.32.0.0/12 \ +APISERVER=https://cluster01.int.domain.com:6443 \ +sh -c 'curl https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/generic-kuberouter-all-features.yaml -o - | \ +sed -e "s;%APISERVER%;$APISERVER;g" -e "s;%CLUSTERCIDR%;$CLUSTERCIDR;g"' | \ +kubectl apply -f - ``` Now since kube-router provides service proxy as well. Run below commands to remove kube-proxy and cleanup any iptables configuration it may have done. +Depending on if or how you installed kube-proxy these instructions will differ and have to be ran on every node where kube-proxy has run. ```sh -KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system delete ds kube-proxy +kubectl -n kube-system delete ds kube-proxy docker run --privileged --net=host gcr.io/google_containers/kube-proxy-amd64:v1.7.3 kube-proxy --cleanup-iptables -``` - - - +``` \ No newline at end of file From b343382dc967ad883bd04c2d76574210303acd64 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 16:05:04 +0100 Subject: [PATCH 03/12] more docs --- Documentation/generic.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Documentation/generic.md b/Documentation/generic.md index 8e5bb06b2e..ab5c4d9c06 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -2,14 +2,13 @@ Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes. -Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options. +Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options listed below the prerequisites. ## Prerequisites ### Kubelet -kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni` -Startup options: +kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni`. - --cni-conf-dir=/etc/cni/net.d - --network-plugin=cni From a6bf5bb0705b29a2f8809ed8929d887d0d054e20 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 16:13:06 +0100 Subject: [PATCH 04/12] prettify --- Documentation/generic.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Documentation/generic.md b/Documentation/generic.md index ab5c4d9c06..ed8b71c65d 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -1,4 +1,4 @@ -# Deploying kube-router on generic cluster +# Kube-router on generic cluster Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes. @@ -6,7 +6,11 @@ Kube-router provides pod networking, network policy and high perfoming IPVS/LVS ## Prerequisites -### Kubelet +kube-router can work as your whole network stack in Kubernetes on-prem & bare metall and works without any cloudproviders. + +below is the needed configuration to run kube-router in such environments + +### Kubelet on each node kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni`. @@ -22,7 +26,7 @@ The following options needs to be set on the controller-manager: --service-cluster-ip-range=${SERVICE_IP_RANGE} # for example 10.50.0.0/22 ``` -## kube-router providing pod networking and network policy +## Kube-router providing pod networking and network policy ```sh CLUSTERCIDR=10.32.0.0/12 \ @@ -32,7 +36,7 @@ sed -e "s;%APISERVER%;$APISERVER;g" -e "s;%CLUSTERCIDR%;$CLUSTERCIDR;g"' | \ kubectl apply -f - ``` -## kube-router providing service proxy, firewall and pod networking +## Kube-router providing service proxy, firewall and pod networking ```sh CLUSTERCIDR=10.32.0.0/12 \ From 9039ecebab321841036b3203a2cd358928c65855 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 16:17:13 +0100 Subject: [PATCH 05/12] more docs --- Documentation/generic.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Documentation/generic.md b/Documentation/generic.md index ed8b71c65d..37331ffce0 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -17,6 +17,10 @@ kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & net - --cni-conf-dir=/etc/cni/net.d - --network-plugin=cni +If you have been using other CNI providers such as weave-net, calico or flannel you will have to remove old configurations from this directory on each node. + +## __Switching CNI provider on a running cluster requires you to delete all the running pods and let them recreate and get new adresses assigned from the kubenet IPAM__ + ### Kube controller-manager The following options needs to be set on the controller-manager: From 89a93afe9e60a08678102be9f613ed55de81a13a Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:18:03 +0100 Subject: [PATCH 06/12] more docs --- Documentation/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Documentation/README.md b/Documentation/README.md index ba0d397ae8..90ded03ea4 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -87,6 +87,9 @@ Please see the [steps](https://github.com/cloudnativelabs/kube-router/tree/maste #### kubeadm Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/kubeadm.md) to deploy Kubernetes cluster with Kube-router using [Kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) +#### Generic platforms +Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/generic.md) to deploy kube-router on generic clusters like on-prem & bare metall + ### deployment Depending on what functionality of kube-router you want to use, multiple deployment options are possible. You can use the flags `--run-firewall`, `--run-router`, `--run-service-proxy` to selectively enable only required functionality of kube-router. From acc62b6d9046b4490f91a470f875ec064a64b701 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:24:11 +0100 Subject: [PATCH 07/12] more docs --- Documentation/README.md | 2 +- Documentation/generic.md | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Documentation/README.md b/Documentation/README.md index 90ded03ea4..cbedf8ffb2 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -87,7 +87,7 @@ Please see the [steps](https://github.com/cloudnativelabs/kube-router/tree/maste #### kubeadm Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/kubeadm.md) to deploy Kubernetes cluster with Kube-router using [Kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) -#### Generic platforms +#### generic Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/generic.md) to deploy kube-router on generic clusters like on-prem & bare metall ### deployment diff --git a/Documentation/generic.md b/Documentation/generic.md index 37331ffce0..3c902f22af 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -1,5 +1,7 @@ # Kube-router on generic cluster +This guide assumes you already have bootstrapped a Kubernets cluster from scratch yourself or used some other deployment tool. + Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes. Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options listed below the prerequisites. @@ -17,9 +19,9 @@ kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & net - --cni-conf-dir=/etc/cni/net.d - --network-plugin=cni -If you have been using other CNI providers such as weave-net, calico or flannel you will have to remove old configurations from this directory on each node. +If you have been using a other CNI providerssuch as weave-net, calico or flannel you will have to remove old configurations from /etc/cni/net.d on each node. -## __Switching CNI provider on a running cluster requires you to delete all the running pods and let them recreate and get new adresses assigned from the kubenet IPAM__ +## __Switching CNI provider on a running cluster will require you to delete all the running pods and let them recreate and get new adresses assigned from the Kubenet IPAM__ ### Kube controller-manager @@ -32,6 +34,8 @@ The following options needs to be set on the controller-manager: ## Kube-router providing pod networking and network policy +Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking). + ```sh CLUSTERCIDR=10.32.0.0/12 \ APISERVER=https://cluster01.int.domain.com:6443 \ @@ -42,6 +46,8 @@ kubectl apply -f - ## Kube-router providing service proxy, firewall and pod networking +Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking). + ```sh CLUSTERCIDR=10.32.0.0/12 \ APISERVER=https://cluster01.int.domain.com:6443 \ @@ -51,7 +57,8 @@ kubectl apply -f - ``` Now since kube-router provides service proxy as well. Run below commands to remove kube-proxy and cleanup any iptables configuration it may have done. -Depending on if or how you installed kube-proxy these instructions will differ and have to be ran on every node where kube-proxy has run. + +Depending on if or how you installed kube-proxy previously these instructions will differ and have to be ran on every node where kube-proxy has run. ```sh kubectl -n kube-system delete ds kube-proxy From c7f7c7da19e2acdf927fb285424642a2ad6da3dc Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:25:38 +0100 Subject: [PATCH 08/12] more docs --- Documentation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/README.md b/Documentation/README.md index cbedf8ffb2..885fba0bf5 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -88,7 +88,7 @@ Please see the [steps](https://github.com/cloudnativelabs/kube-router/tree/maste Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/kubeadm.md) to deploy Kubernetes cluster with Kube-router using [Kubeadm](https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/) #### generic -Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/generic.md) to deploy kube-router on generic clusters like on-prem & bare metall +Please see the [steps](https://github.com/cloudnativelabs/kube-router/blob/master/Documentation/generic.md) to deploy kube-router on generic installed clusters ### deployment From 3a40307438c0302d2250870e6528e415436ea26c Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:27:15 +0100 Subject: [PATCH 09/12] tidy up --- Documentation/generic.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Documentation/generic.md b/Documentation/generic.md index 3c902f22af..71b3ac14aa 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -1,31 +1,31 @@ # Kube-router on generic cluster -This guide assumes you already have bootstrapped a Kubernets cluster from scratch yourself or used some other deployment tool. +This guide assumes you already have bootstrapped a Kubernets cluster from scratch yourself or used some other deployment tool -Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes. +Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes -Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options listed below the prerequisites. +Kube-router provides pod networking, network policy and high perfoming IPVS/LVS based service proxy. Depending on you choose to use kube-router for service proxy you have two options listed below the prerequisites ## Prerequisites -kube-router can work as your whole network stack in Kubernetes on-prem & bare metall and works without any cloudproviders. +kube-router can work as your whole network stack in Kubernetes on-prem & bare metall and works without any cloudproviders below is the needed configuration to run kube-router in such environments ### Kubelet on each node -kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni`. +kube-router assumes each Kubelet is using `/etc/cni/net.d` as cni conf dir & network plugin `cni` - --cni-conf-dir=/etc/cni/net.d - --network-plugin=cni -If you have been using a other CNI providerssuch as weave-net, calico or flannel you will have to remove old configurations from /etc/cni/net.d on each node. +If you have been using a other CNI providerssuch as weave-net, calico or flannel you will have to remove old configurations from /etc/cni/net.d on each node ## __Switching CNI provider on a running cluster will require you to delete all the running pods and let them recreate and get new adresses assigned from the Kubenet IPAM__ ### Kube controller-manager -The following options needs to be set on the controller-manager: +The following options needs to be set on the controller-manager ```text --cluster-cidr=${POD_NETWORK} # for example 10.32.0.0/12 @@ -34,7 +34,7 @@ The following options needs to be set on the controller-manager: ## Kube-router providing pod networking and network policy -Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking). +Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking) ```sh CLUSTERCIDR=10.32.0.0/12 \ @@ -46,7 +46,7 @@ kubectl apply -f - ## Kube-router providing service proxy, firewall and pod networking -Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking). +Don't forgett to adjust values for Cluster CIDR (pod range) & apiserver adress (must be reachable directly from host networking) ```sh CLUSTERCIDR=10.32.0.0/12 \ @@ -56,9 +56,9 @@ sed -e "s;%APISERVER%;$APISERVER;g" -e "s;%CLUSTERCIDR%;$CLUSTERCIDR;g"' | \ kubectl apply -f - ``` -Now since kube-router provides service proxy as well. Run below commands to remove kube-proxy and cleanup any iptables configuration it may have done. +Now since kube-router provides service proxy as well. Run below commands to remove kube-proxy and cleanup any iptables configuration it may have done -Depending on if or how you installed kube-proxy previously these instructions will differ and have to be ran on every node where kube-proxy has run. +Depending on if or how you installed kube-proxy previously these instructions will differ and have to be ran on every node where kube-proxy has run ```sh kubectl -n kube-system delete ds kube-proxy From 90f13bb7d8d80d7d1fa9bcc57ce510849f6c48ad Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:31:11 +0100 Subject: [PATCH 10/12] fixed mount paths --- .../generic-kuberouter-all-features-advertise-routes.yaml | 3 +-- daemonset/generic-kuberouter-all-features.yaml | 3 +-- daemonset/generic-kuberouter.yaml | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml index 9d78633a61..ac8dd63796 100644 --- a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml +++ b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml @@ -134,8 +134,7 @@ spec: name: kube-router-cfg - name: kubeconfig hostPath: - path: /var/lib/kube-router/kubeconfig ---- + path: /var/lib/kube-router apiVersion: v1 kind: ServiceAccount metadata: diff --git a/daemonset/generic-kuberouter-all-features.yaml b/daemonset/generic-kuberouter-all-features.yaml index ee4b54a522..43a56814ed 100644 --- a/daemonset/generic-kuberouter-all-features.yaml +++ b/daemonset/generic-kuberouter-all-features.yaml @@ -130,8 +130,7 @@ spec: name: kube-router-cfg - name: kubeconfig hostPath: - path: /var/lib/kube-router/kubeconfig ---- + path: /var/lib/kube-router apiVersion: v1 kind: ServiceAccount metadata: diff --git a/daemonset/generic-kuberouter.yaml b/daemonset/generic-kuberouter.yaml index dfc61714b5..d28939dedf 100644 --- a/daemonset/generic-kuberouter.yaml +++ b/daemonset/generic-kuberouter.yaml @@ -130,7 +130,7 @@ spec: name: kube-router-cfg - name: kubeconfig hostPath: - path: /var/lib/kube-router/kubeconfig + path: /var/lib/kube-router --- apiVersion: v1 kind: ServiceAccount From 4277dc834b10a2d1738ba1df4d44de2553994ab5 Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:36:13 +0100 Subject: [PATCH 11/12] fixed manifests --- .../generic-kuberouter-all-features-advertise-routes.yaml | 5 ++++- daemonset/generic-kuberouter-all-features.yaml | 4 +++- daemonset/generic-kuberouter.yaml | 4 +++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml index ac8dd63796..57de528086 100644 --- a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml +++ b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml @@ -56,7 +56,6 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: kube-router - serviceAccount: kube-router containers: - name: kube-router image: cloudnativelabs/kube-router @@ -135,11 +134,14 @@ spec: - name: kubeconfig hostPath: path: /var/lib/kube-router + +--- apiVersion: v1 kind: ServiceAccount metadata: name: kube-router namespace: kube-system + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -175,6 +177,7 @@ rules: - get - list - watch + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/daemonset/generic-kuberouter-all-features.yaml b/daemonset/generic-kuberouter-all-features.yaml index 43a56814ed..58d3e37b4f 100644 --- a/daemonset/generic-kuberouter-all-features.yaml +++ b/daemonset/generic-kuberouter-all-features.yaml @@ -56,7 +56,6 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: kube-router - serviceAccount: kube-router containers: - name: kube-router image: cloudnativelabs/kube-router @@ -131,11 +130,14 @@ spec: - name: kubeconfig hostPath: path: /var/lib/kube-router + +--- apiVersion: v1 kind: ServiceAccount metadata: name: kube-router namespace: kube-system + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/daemonset/generic-kuberouter.yaml b/daemonset/generic-kuberouter.yaml index d28939dedf..904622223e 100644 --- a/daemonset/generic-kuberouter.yaml +++ b/daemonset/generic-kuberouter.yaml @@ -56,7 +56,6 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: kube-router - serviceAccount: kube-router containers: - name: kube-router image: cloudnativelabs/kube-router @@ -131,12 +130,14 @@ spec: - name: kubeconfig hostPath: path: /var/lib/kube-router + --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-router namespace: kube-system + --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -172,6 +173,7 @@ rules: - get - list - watch + --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 From 73357a4d4a9e29119e5f4da560f3118d9459d49f Mon Sep 17 00:00:00 2001 From: Joakim Karlsson Date: Thu, 11 Jan 2018 23:39:03 +0100 Subject: [PATCH 12/12] better clarification of scope --- Documentation/generic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/generic.md b/Documentation/generic.md index 71b3ac14aa..5491dd56e8 100644 --- a/Documentation/generic.md +++ b/Documentation/generic.md @@ -1,6 +1,6 @@ # Kube-router on generic cluster -This guide assumes you already have bootstrapped a Kubernets cluster from scratch yourself or used some other deployment tool +This guide assumes you already have bootstrapped the initial pieces for a Kubernets cluster and is about to switch or setup service & container networking provider Kube-router relies on kube-controller-manager to allocate pod CIDR for the nodes