diff --git a/README.md b/README.md index d93b82ec505..581570c199c 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.15.1 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.15.2 - [etcd](https://github.com/coreos/etcd) v3.3.10 - [docker](https://www.docker.com/) v18.06 (see note) - [cri-o](http://cri-o.io/) v1.11.5 (experimental: see [CRI-O Note](docs/cri-o.md). Only on centos based OS) @@ -117,7 +117,7 @@ Supported Components - [cni-plugins](https://github.com/containernetworking/plugins) v0.8.1 - [calico](https://github.com/projectcalico/calico) v3.7.3 - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - - [cilium](https://github.com/cilium/cilium) v1.3.0 + - [cilium](https://github.com/cilium/cilium) v1.5.5 - [contiv](https://github.com/contiv/install) v1.2.1 - [flanneld](https://github.com/coreos/flannel) v0.11.0 - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.5 @@ -134,7 +134,7 @@ Note: The list of validated [docker versions](https://github.com/kubernetes/kube Requirements ------------ -- **Minimum required version of Kubernetes is v1.13** +- **Minimum required version of Kubernetes is v1.14** - **Ansible v2.7.8 (or newer, but [not 2.8.x](https://github.com/kubernetes-sigs/kubespray/issues/4778)) and python-netaddr is installed on the machine that will run Ansible commands** - **Jinja 2.9 (or newer) is required to run the Ansible Playbooks** diff --git a/cluster.yml b/cluster.yml index bc0580331ab..3e8b7811515 100644 --- a/cluster.yml +++ b/cluster.yml @@ -19,14 +19,14 @@ - { role: kubespray-defaults} - { role: bastion-ssh-config, tags: ["localhost", "bastion"]} -- hosts: k8s-cluster:etcd:calico-rr +- hosts: k8s-cluster:etcd any_errors_fatal: "{{ any_errors_fatal | default(true) }}" gather_facts: false roles: - { role: kubespray-defaults} - { role: bootstrap-os, tags: bootstrap-os} -- hosts: k8s-cluster:etcd:calico-rr +- hosts: k8s-cluster:etcd any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} @@ -46,7 +46,7 @@ etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}" when: not etcd_kubeadm_enabled| default(false) -- hosts: k8s-cluster:calico-rr +- hosts: k8s-cluster any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: - { role: kubespray-defaults} @@ -79,6 +79,12 @@ - { role: kubernetes/kubeadm, tags: kubeadm} - { role: network_plugin, tags: network } +- hosts: calico-rr + any_errors_fatal: "{{ any_errors_fatal | default(true) }}" + roles: + - { role: kubespray-defaults} + - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr']} + - hosts: kube-master[0] any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: @@ -95,12 +101,6 @@ - { role: kubernetes-apps/ingress_controller, tags: ingress-controller } - { role: kubernetes-apps/external_provisioner, tags: external-provisioner } -- hosts: calico-rr - any_errors_fatal: "{{ any_errors_fatal | default(true) }}" - roles: - - { role: kubespray-defaults} - - { role: network_plugin/calico/rr, tags: network } - - hosts: kube-master any_errors_fatal: "{{ any_errors_fatal | default(true) }}" roles: diff --git a/contrib/metallb/README.md b/contrib/metallb/README.md index e3b19ac8224..c8b6691eacb 100644 --- a/contrib/metallb/README.md +++ b/contrib/metallb/README.md @@ -2,7 +2,7 @@ ``` MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation. In short, it allows you to create Kubernetes services of type “LoadBalancer” in clusters that don’t run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers. ``` -This playbook aims to automate [this](https://metallb.universe.tf/tutorial/layer2/tutorial). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer. +This playbook aims to automate [this](https://metallb.universe.tf/concepts/layer2/). It deploys MetalLB into kubernetes and sets up a layer 2 loadbalancer. ## Install ``` diff --git a/docs/calico.md b/docs/calico.md index 5eedef8acac..52586b18f17 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -119,13 +119,13 @@ recommended here: You need to edit your inventory and add: -* `calico-rr` group with nodes in it. At the moment it's incompatible with - `kube-node` due to BGP port conflict with `calico-node` container. So you - should not have nodes in both `calico-rr` and `kube-node` groups. +* `calico-rr` group with nodes in it. `calico-rr` can be combined with + `kube-node` and/or `kube-master`. `calico-rr` group also must be a child + group of `k8s-cluster` group. * `cluster_id` by route reflector node/group (see details [here](https://hub.docker.com/r/calico/routereflector/)) -Here's an example of Kubespray inventory with route reflectors: +Here's an example of Kubespray inventory with standalone route reflectors: ``` [all] @@ -154,6 +154,7 @@ node5 [k8s-cluster:children] kube-node kube-master +calico-rr [calico-rr] rr0 diff --git a/docs/getting-started.md b/docs/getting-started.md index 710b700021d..93c09a65ff9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -51,20 +51,27 @@ You may want to add worker, master or etcd nodes to your existing cluster. This Remove nodes ------------ -You may want to remove **worker** nodes to your existing cluster. This can be done by re-running the `remove-node.yml` playbook. First, all nodes will be drained, then stop some kubernetes services and delete some certificates, and finally execute the kubectl command to delete these nodes. This can be combined with the add node function, This is generally helpful when doing something like autoscaling your clusters. Of course if a node is not working, you can remove the node and install it again. - -Add worker nodes to the list under kube-node if you want to delete them (or utilize a [dynamic inventory](https://docs.ansible.com/ansible/intro_dynamic_inventory.html)). - - ansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \ - --private-key=~/.ssh/private_key - -Use `--extra-vars "node=,"` to select the node you want to delete. +You may want to remove **master**, **worker**, or **etcd** nodes from your +existing cluster. This can be done by re-running the `remove-node.yml` +playbook. First, all specified nodes will be drained, then stop some +kubernetes services and delete some certificates, +and finally execute the kubectl command to delete these nodes. +This can be combined with the add node function. This is generally helpful +when doing something like autoscaling your clusters. Of course, if a node +is not working, you can remove the node and install it again. + +Use `--extra-vars "node=,"` to select the node(s) you want to delete. ``` ansible-playbook -i inventory/mycluster/hosts.yml remove-node.yml -b -v \ --private-key=~/.ssh/private_key \ --extra-vars "node=nodename,nodename2" ``` +If a node is completely unreachable by ssh, add `--extra-vars reset_nodes=no` +to skip the node reset step. If one node is unavailable, but others you wish +to remove are able to connect via SSH, you could set reset_nodes=no as a host +var in inventory. + Connecting to Kubernetes ------------------------ diff --git a/inventory/local/hosts.ini b/inventory/local/hosts.ini index 425ad23ef38..7834d27c0ea 100644 --- a/inventory/local/hosts.ini +++ b/inventory/local/hosts.ini @@ -12,3 +12,4 @@ node1 [k8s-cluster:children] kube-node kube-master +calico-rr diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml index 663a9dc9ceb..ef9eff9e754 100644 --- a/inventory/sample/group_vars/k8s-cluster/addons.yml +++ b/inventory/sample/group_vars/k8s-cluster/addons.yml @@ -80,6 +80,7 @@ rbd_provisioner_enabled: false # Nginx ingress controller deployment ingress_nginx_enabled: false # ingress_nginx_host_network: false +ingress_publish_status_address: "" # ingress_nginx_nodeselector: # beta.kubernetes.io/os: "linux" # ingress_nginx_tolerations: diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index df8fcdba00e..3e19bf458ea 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -20,7 +20,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.15.1 +kube_version: v1.15.2 # kubernetes image repo define kube_image_repo: "gcr.io/google-containers" diff --git a/inventory/sample/inventory.ini b/inventory/sample/inventory.ini index 8e32a3a75fe..e5a53b05a11 100644 --- a/inventory/sample/inventory.ini +++ b/inventory/sample/inventory.ini @@ -28,6 +28,9 @@ # node5 # node6 +[calico-rr] + [k8s-cluster:children] kube-master kube-node +calico-rr diff --git a/remove-node.yml b/remove-node.yml index b25b770fd92..1f4a6616988 100644 --- a/remove-node.yml +++ b/remove-node.yml @@ -1,6 +1,7 @@ --- - hosts: localhost become: no + gather_facts: no tasks: - name: "Check ansible version >=2.7.8" assert: @@ -12,12 +13,8 @@ vars: ansible_connection: local -- hosts: all - vars: - ansible_ssh_pipelining: true - gather_facts: true - - hosts: "{{ node | default('etcd:k8s-cluster:calico-rr') }}" + gather_facts: no vars_prompt: name: "delete_nodes_confirmation" prompt: "Are you sure you want to delete nodes state? Type 'yes' to delete nodes." @@ -31,16 +28,20 @@ when: delete_nodes_confirmation != "yes" - hosts: kube-master + gather_facts: no roles: - { role: kubespray-defaults } - { role: remove-node/pre-remove, tags: pre-remove } - hosts: "{{ node | default('kube-node') }}" + gather_facts: no roles: - { role: kubespray-defaults } - - { role: reset, tags: reset } + - { role: reset, tags: reset, when: reset_nodes|default(True) } -- hosts: kube-master +# Currently cannot remove first master or etcd +- hosts: "{{ node | default('kube-master[1:]:etcd[:1]') }}" + gather_facts: no roles: - { role: kubespray-defaults } - { role: remove-node/post-remove, tags: post-remove } diff --git a/requirements.txt b/requirements.txt index 7f71b2e2057..26a69f4c1b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ansible==2.7.8 +ansible==2.7.12 jinja2==2.10.1 netaddr==0.7.19 pbr==5.2.0 diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 9f476357ee0..21401464ded 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -49,7 +49,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{ groups['kub image_arch: "{{host_architecture | default('amd64')}}" # Versions -kube_version: v1.15.1 +kube_version: v1.15.2 kubeadm_version: "{{ kube_version }}" etcd_version: v3.3.10 @@ -73,10 +73,10 @@ cni_version: "v0.8.1" weave_version: 2.5.2 pod_infra_version: 3.1 contiv_version: 1.2.1 -cilium_version: "v1.3.0" +cilium_version: "v1.5.5" kube_ovn_version: "v0.6.0" kube_router_version: "v0.2.5" -multus_version: "v3.1.autoconf" +multus_version: "v3.2.1" crictl_version: "v1.15.0" @@ -105,24 +105,30 @@ crictl_checksums: # Checksums hyperkube_checksums: arm: + v1.15.2: eeaa8e071541c7bcaa186ff1d2919d076b27ef70c9e9df70f910756eba55dc99 v1.15.1: fc5af96fd9341776d84c38675be7b8045dee20af327af9331972c422a4109918 v1.15.0: d923c781031bfd97d0fbe50311e4d7c3616aa5b6d466b99049931f09d73d07b9 + v1.14.5: 860b84dd32611a6008fe20fb998a2fc0a25ff44067eae556224827d05429c91e v1.14.4: 429a10369b2ef35a9c2d662347277339d53fa66ef55ffeabcc7d9b850e31056d v1.14.3: 3fac785261bcf79f7a80b12c4a1dda893ce8c0879caf57b36d4701730671b574 v1.14.2: 6929a59850c8702c04d62cd343d1143b17456da040f32317e09f8c25a08d2346 v1.14.1: 839a4abfeafbd5f5ab057ad0e8a0b0b488b3cde14a646eba040a7f579875f565 v1.14.0: d090b1da23564a7e9bb8f1f4264f2116536c52611ae203fe2ca13eaad0a8003e arm64: + v1.15.2: c4cf69f52c7013faee9d54e0f376e0732a4a7b0f7ffc7241e9b7e28bad0ac77f v1.15.1: 80ed372c5f6c5178df88616175310057c06bdc9d0905953814a1927eb3aaa657 v1.15.0: 824af7d925b87a5ade63575b98b59ee81005fc76eac1dc399602308d7a60bc3c + v1.14.5: 90c77847d64eb857c8e686e8593fe7a9e505bcbf960b0407217255827a9da59a v1.14.4: 9e0b4fde88a07c705e0937cd3161392684e3ca08535d14a99ae3b86bbf4c56b3 v1.14.3: f29211d668cbcf1aa415dfa64aad95ffc53b5410482a23cddb680caec4e907a3 v1.14.2: 959fb7d9c17fc8f7cb1a69920aaf08aefd62c0fbf6b5bdc46250f147ea6a5cd4 v1.14.1: d5236efc2547fd07c7cc2ed9345dfbcd1204385847ca686cf1c62d15056de399 v1.14.0: 708e00a41f6516d525dee00c91ebe3c3bf2feaf9b7f0af7689487e3e17e356c2 amd64: + v1.15.2: ab885606438748eb89a7738e219f5353d94c40c63a4935a539ce89760280f065 v1.15.1: 22b7b1e7f5f2a452d62e0ca4c2cba67119c51e04219aaeaf8452825f9177069e v1.15.0: 3cc72cc58517b97c608c7a59a20255675bc70f07217c9e11e58cac7746139283 + v1.14.5: 2c3410518980b8705ba9b7b708076a206f2bde37cb8bf5ba8f15c32c697f4d97 v1.14.4: 5f31434f3a884257a7b0e3178fc869720a7526c8637af5713d23433ddf2592dd v1.14.3: 6c6cb5c118b2129ba4e56697f42567be3587eb636a477cd342b69f87b3b049d1 v1.14.2: 05546057f2053e085fa8387ab82581c95fe4195cd783408ccbb4fc3487c50176 @@ -130,24 +136,30 @@ hyperkube_checksums: v1.14.0: af8b04504365dbe4ce6a1772f42eb390d4221a21149b522fc8a0c4b1cd3d97aa kubeadm_checksums: arm: + v1.15.2: 4b35ad0031c08a83de7c8d9f9bbed6a30d93a5c74e16ea9e6211ad2e0e12bdd1 v1.15.1: 855abd520291dcef0577a1a2ef87a70f522fd2b22603a12abcd86c2f7ec9c022 v1.15.0: 9464030a1d4e101de5f47348f3514d5a9eb95cbce2e5e31f53ada1ca485cf75e + v1.14.5: 0bb551f7468de2fa6f98ce60653495327be052364ac9f9e8917a4d1ad864412b v1.14.4: 36835488d7187406690ee6aa4b3c9c54855cb5c55d786d0574a508b955fe3a46 v1.14.3: 270b8c346aeaa309d11d65695c4a90f6bff5b1ea14bdec3c417ca2dfb3de0db3 v1.14.2: d2a59269aa68a4bace2a80b247b6f9a82f0542ec3004185fb0ba86e181fdfb29 v1.14.1: 4bd111411208f1270ed3af8780b87d24a3c17c9fdbe4b0f8c7a9a21cd765543e v1.14.0: 11f2cfa8bf7ee177dbac8073ab0f039dc265536baaa8dc0c4dea699f981f6fd1 arm64: + v1.15.2: d3b6ee2048b366726ca366d2db4c46b2cacc38e8ec09cc35781d16593753d930 v1.15.1: 44fbfad0f1026d249fc4f365f1e9562cd52d75360d4d1032731122ba5a4d57dc v1.15.0: fe3c79070814fe847a23209b1027672fe5c5e7e5c9611e329225058926836f96 + v1.14.5: 7dd1195d16980c4c888d13e49d97c3513f668e192bf2778bc0f0516e0f7fe2ac v1.14.4: 60745b3ac761d3aa55ab9a24677ecf4e7f48b5abed34c725047a174456e5a79b v1.14.3: 8edcc07c65f81eea3fc47cd237dd6560c6907c5e0ca52d71eab53ca1164e7d01 v1.14.2: bff0712b87796509129aa802ad3ac25b8cc83af01762b22b4dcca8dbdb26b520 v1.14.1: 5cf05464168e45ee4719264a267c65f9319fae1ceb9923fedab97a9d6a629e0b v1.14.0: 7ed9d706e50cd6d3fc618a7af3d19b691b8a5343ddedaeccb4ea09af3ecfae2c amd64: + v1.15.2: fe2a13a1dea73249560ea44ab54c0359a9722e9c66832f6bcad86798438cba2f v1.15.1: 3d42441ae177826f1181e559cd2a729464ca8efadef196cfa0e8053a615333b5 v1.15.0: fc4aa44b96dc143d7c3062124e25fed671cab884ebb8b2446edd10abb45e88c2 + v1.14.5: b3e840f7816f64e071d25f8a90b984eecd6251b68e568b420d85ef0a4dd514bb v1.14.4: 291790a1cef82c4de28cc3338a199ca8356838ca26f775f2c2acba165b633d9f v1.14.3: 026700dfff3c78be1295417e96d882136e5e1f095eb843e6575e57ef9930b5d3 v1.14.2: 77510f61352bb6e537e70730b670627963f2c314fbd36a644b0c435b97e9705a @@ -237,8 +249,10 @@ contiv_ovs_image_repo: "docker.io/contiv/ovs" contiv_ovs_image_tag: "latest" cilium_image_repo: "docker.io/cilium/cilium" cilium_image_tag: "{{ cilium_version }}" -cilium_init_image_repo: "docker.io/library/busybox" -cilium_init_image_tag: "1.28.4" +cilium_init_image_repo: "docker.io/cilium/cilium-init" +cilium_init_image_tag: "2019-04-05" +cilium_operator_image_repo: "docker.io/cilium/operator" +cilium_operator_image_tag: "{{ cilium_version }}" kube_ovn_db_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-db" kube_ovn_node_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-node" kube_ovn_cni_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-cni" @@ -261,7 +275,7 @@ coredns_version: "1.6.0" coredns_image_repo: "docker.io/coredns/coredns" coredns_image_tag: "{{ coredns_version }}" -nodelocaldns_version: "1.15.1" +nodelocaldns_version: "1.15.4" nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache" nodelocaldns_image_tag: "{{ nodelocaldns_version }}" @@ -415,6 +429,15 @@ downloads: groups: - k8s-cluster + cilium_operator: + enabled: "{{ kube_network_plugin == 'cilium' }}" + container: true + repo: "{{ cilium_operator_image_repo }}" + tag: "{{ cilium_operator_image_tag }}" + sha256: "{{ cilium_operator_digest_checksum|default(None) }}" + groups: + - k8s-cluster + multus: enabled: "{{ kube_network_plugin_multus }}" container: true diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 2470217bc86..e3f9c31dd6c 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -64,7 +64,7 @@ when: is_etcd_master and etcd_events_cluster_setup - name: Configure | Check if etcd cluster is healthy - shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_client_url }} cluster-health | grep -q 'cluster is healthy'" register: etcd_cluster_is_healthy until: etcd_cluster_is_healthy.rc == 0 retries: 4 @@ -72,7 +72,11 @@ ignore_errors: false changed_when: false check_mode: no - when: is_etcd_master and etcd_cluster_setup + delegate_to: "{{ groups['etcd'][0] }}" + run_once: yes + when: + - is_etcd_master + - etcd_cluster_setup tags: - facts environment: @@ -81,7 +85,7 @@ ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Check if etcd-events cluster is healthy - shell: "{{ bin_dir }}/etcdctl --endpoints={{ etcd_events_access_addresses }} cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_events_client_url }} cluster-health | grep -q 'cluster is healthy'" register: etcd_events_cluster_is_healthy until: etcd_events_cluster_is_healthy.rc == 0 retries: 4 @@ -89,7 +93,12 @@ ignore_errors: false changed_when: false check_mode: no - when: is_etcd_master and etcd_events_cluster_setup + delegate_to: "{{ groups['etcd'][0] }}" + run_once: yes + when: + - is_etcd_master + - etcd_events_cluster_setup + - etcd_cluster_setup tags: - facts environment: diff --git a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml index afd1e258bea..62908eb05bc 100644 --- a/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml +++ b/roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml @@ -36,3 +36,4 @@ - inventory_hostname == groups['kube-master'] | first tags: - nodelocaldns + - coredns diff --git a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml index 2dc10409355..11bec1f7a8a 100644 --- a/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml +++ b/roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml @@ -8,7 +8,7 @@ rbd_provisioner_secret_name: ceph-secret-admin rbd_provisioner_secret_token: ceph-key-admin rbd_provisioner_user_id: kube rbd_provisioner_user_secret_name: ceph-secret-user -rbd_provisioner_user_secret_token: ceph-key-user +rbd_provisioner_user_secret: ceph-key-user rbd_provisioner_user_secret_namespace: rbd-provisioner rbd_provisioner_fs_type: ext4 rbd_provisioner_image_format: "2" diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml index 05b72ea9068..57bd4830c99 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -1,6 +1,7 @@ --- ingress_nginx_namespace: "ingress-nginx" ingress_nginx_host_network: false +ingress_publish_status_address: "" ingress_nginx_nodeselector: beta.kubernetes.io/os: "linux" ingress_nginx_tolerations: [] diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 index 8c48089d55a..6f94680a6a5 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -47,6 +47,9 @@ spec: - --annotations-prefix=nginx.ingress.kubernetes.io {% if ingress_nginx_host_network %} - --report-node-internal-ip-address +{% endif %} +{% if ingress_publish_status_address != "" %} + - --publish-status-address={{ ingress_publish_status_address }} {% endif %} securityContext: capabilities: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 index 81efb98fc61..0110c058c91 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -17,3 +17,8 @@ discoveryTokenUnsafeSkipCAVerification: true nodeRegistration: name: {{ kube_override_hostname }} criSocket: {{ cri_socket }} +{% if 'calico-rr' in group_names and 'kube-node' not in group_names %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/calico-rr +{% endif %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 index 66583412b64..75f27fad3eb 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -21,3 +21,8 @@ caCertPath: {{ kube_cert_dir }}/ca.crt nodeRegistration: name: {{ kube_override_hostname }} criSocket: {{ cri_socket }} +{% if 'calico-rr' in group_names and 'kube-node' not in group_names %} + taints: + - effect: NoSchedule + key: node-role.kubernetes.io/calico-rr +{% endif %} diff --git a/roles/kubernetes/master/defaults/main/main.yml b/roles/kubernetes/master/defaults/main/main.yml index 28b4a0980dc..65e06c01e5c 100644 --- a/roles/kubernetes/master/defaults/main/main.yml +++ b/roles/kubernetes/master/defaults/main/main.yml @@ -41,6 +41,9 @@ kube_scheduler_bind_address: 0.0.0.0 # discovery_timeout modifies the discovery timeout discovery_timeout: 5m0s +# Instruct first master to refresh kubeadm token +kubeadm_refresh_token: true + # audit support kubernetes_audit: false # path to audit log file diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index d107c179ff4..2f63714f23b 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -146,6 +146,7 @@ when: - inventory_hostname == groups['kube-master']|first - kubeadm_token is defined + - kubeadm_refresh_token tags: - kubeadm_token diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 48938d06b4e..c52d0fa21bc 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -166,6 +166,26 @@ - inventory_hostname == groups['kube-master'][0] run_once: yes +- name: "Check that cluster_id is set if calico_rr enabled" + assert: + that: + - cluster_id is defined + msg: "A unique cluster_id is required if using calico_rr" + when: + - kube_network_plugin == 'calico' + - peer_with_calico_rr + - inventory_hostname == groups['kube-master'][0] + run_once: yes + +- name: "Check that calico_rr nodes are in k8s-cluster group" + assert: + that: + - '"k8s-cluster" in group_names' + msg: "calico-rr must be a child group of k8s-cluster group" + when: + - kube_network_plugin == 'calico' + - '"calico-rr" in group_names' + - name: "Check that kube_service_addresses is a network range" assert: that: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index fe22a68f9af..69fdad5455f 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -12,7 +12,7 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.15.1 +kube_version: v1.15.2 ## The minimum version working kube_version_min_required: v1.14.0 diff --git a/roles/network_plugin/calico/rr/defaults/main.yml b/roles/network_plugin/calico/rr/defaults/main.yml index 4871f34a803..dedda197cbc 100644 --- a/roles/network_plugin/calico/rr/defaults/main.yml +++ b/roles/network_plugin/calico/rr/defaults/main.yml @@ -2,15 +2,4 @@ # Global as_num (/calico/bgp/v1/global/as_num) # should be the same as in calico role global_as_num: "64512" - -calico_cert_dir: /etc/calico/certs - -# Limits for apps -calico_rr_memory_limit: 1000M -calico_rr_cpu_limit: 300m -calico_rr_memory_requests: 128M -calico_rr_cpu_requests: 150m - -kube_etcd_cacert_file: ca.pem -kube_etcd_cert_file: node-{{ inventory_hostname }}.pem -kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem +calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}" diff --git a/roles/network_plugin/calico/rr/handlers/main.yml b/roles/network_plugin/calico/rr/handlers/main.yml deleted file mode 100644 index 1436649e357..00000000000 --- a/roles/network_plugin/calico/rr/handlers/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -- name: restart calico-rr - command: /bin/true - notify: - - Calico-rr | reload systemd - - Calico-rr | reload calico-rr - -- name: Calico-rr | reload systemd - systemd: - daemon_reload: true - -- name: Calico-rr | reload calico-rr - service: - name: calico-rr - state: restarted diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 2b023a2aa8e..0c93a55060b 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -1,82 +1,29 @@ --- -# Required from inventory: -# calico_rr_ip - which specific IP to use for RR, defaults to -# "ip" from inventory or "ansible_default_ipv4.address" +- name: Calico-rr | Pre-upgrade tasks + include_tasks: pre.yml -- name: Calico-rr | Set IP fact - set_fact: - rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}" +- name: Calico-rr | Fetch current node object + command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -oyaml" + register: calico_rr_node -- name: Calico-rr | Create calico certs directory - file: - dest: "{{ calico_cert_dir }}" - state: directory - mode: 0750 - owner: root - group: root - -- name: Calico-rr | Link etcd certificates for calico-node - file: - src: "{{ etcd_cert_dir }}/{{ item.s }}" - dest: "{{ calico_cert_dir }}/{{ item.d }}" - state: hard - force: yes - with_items: - - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"} - - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} - -- name: Calico-rr | Create dir for logs - file: - path: /var/log/calico-rr - state: directory - mode: 0755 - owner: root - group: root - -- name: Calico-rr | Write calico-rr.env for systemd init file - template: - src: calico-rr.env.j2 - dest: /etc/calico/calico-rr.env - notify: restart calico-rr - -- name: Calico-rr | Write calico-rr systemd init file - template: - src: calico-rr-docker.service.j2 - dest: /etc/systemd/system/calico-rr.service - notify: restart calico-rr - when: - - container_manager in ['crio', 'docker', 'rkt'] - -- name: Calico-rr | Write calico-rr systemd init file - template: - src: calico-rr-containerd.service.j2 - dest: /etc/systemd/system/calico-rr.service - notify: restart calico-rr - when: - - container_manager == 'containerd' +# FIXME(mattymo): Use jsonpatch when ansible/ansible#52931 is merged +- name: Calico-rr | Set route reflector cluster ID + shell: >- + echo -e '{{ calico_rr_node.stdout }}' | + sed '/bgp:/a \ \ \ \ routeReflectorClusterID: {{ cluster_id }}' + register: calico_rr_node + when: '("routeReflectorClusterID: " + cluster_id|string) not in calico_rr_node.stdout_lines' - name: Calico-rr | Configure route reflector - command: |- - {{ bin_dir }}/etcdctl \ - --endpoints={{ etcd_access_addresses }} \ - put /calico/bgp/v1/rr_v4/{{ rr_ip }} \ - '{ - "ip": "{{ rr_ip }}", - "cluster_id": "{{ cluster_id }}" - }' - environment: - ETCDCTL_API: 3 - ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem" - ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem" + shell: |- + echo -e '{{ calico_rr_node.stdout }}' | + {{ bin_dir }}/calicoctl.sh replace -f- retries: 4 delay: "{{ retry_stagger | random + 3 }}" - delegate_to: "{{ groups['etcd'][0] }}" - -- meta: flush_handlers -- name: Calico-rr | Enable calico-rr - service: - name: calico-rr - state: started - enabled: yes +- name: Calico-rr | Set label for route reflector + command: >- + {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }} + 'i-am-a-route-reflector=true' --overwrite + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" diff --git a/roles/network_plugin/calico/rr/tasks/pre.yml b/roles/network_plugin/calico/rr/tasks/pre.yml new file mode 100644 index 00000000000..d8dbd807280 --- /dev/null +++ b/roles/network_plugin/calico/rr/tasks/pre.yml @@ -0,0 +1,15 @@ +--- +- name: Calico-rr | Disable calico-rr service if it exists + service: + name: calico-rr + state: stopped + enabled: no + failed_when: false + +- name: Calico-rr | Delete obsolete files + file: + path: "{{ item }}" + state: absent + with_items: + - /etc/calico/calico-rr.env + - /etc/systemd/system/calico-rr.service diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 deleted file mode 100644 index db719afd379..00000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 +++ /dev/null @@ -1,27 +0,0 @@ -[Unit] -Description=calico-rr -After=containerd.service -Requires=containerd.service - -[Service] -EnvironmentFile=/etc/calico/calico-rr.env -ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr -ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \ - --env IP=${IP} \ - --env IP6=${IP6} \ - --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ - --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \ - --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \ - --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \ - --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \ - --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \ - {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \ - calico-rr - -Restart=always -RestartSec=10s - -ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 deleted file mode 100644 index f6da04a4d95..00000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 +++ /dev/null @@ -1,28 +0,0 @@ -[Unit] -Description=calico-rr -After=docker.service -Requires=docker.service - -[Service] -EnvironmentFile=/etc/calico/calico-rr.env -ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr -ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ - --name=calico-rr \ - -e IP=${IP} \ - -e IP6=${IP6} \ - -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \ - -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \ - -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \ - -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ - -v /var/log/calico-rr:/var/log/calico \ - -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ - --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ - {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} - -Restart=always -RestartSec=10s - -ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr - -[Install] -WantedBy=multi-user.target diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 deleted file mode 100644 index 1cdb2659c4b..00000000000 --- a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 +++ /dev/null @@ -1,6 +0,0 @@ -ETCD_ENDPOINTS="{{ etcd_access_addresses }}" -ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt" -ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt" -ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem" -IP="{{ rr_ip }}" -IP6="" diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 7ee560095fb..c4b03822b79 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -163,16 +163,16 @@ - name: Calico | Configure peering with router(s) at global scope shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "global-{{ item.router_id }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "global-{{ item.router_id }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "peerIP": "{{ item.router_id }}" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -181,6 +181,46 @@ - inventory_hostname == groups['kube-master'][0] - peer_with_router|default(false) +- name: Calico | Configure peering with route reflectors at global scope + shell: | + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "peer-to-rrs" + }, + "spec": { + "nodeSelector": "!has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico-rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube-master'][0] + - peer_with_calico_rr|default(false) + +- name: Calico | Configure route reflectors to peer with each other + shell: > + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "rr-mesh" + }, + "spec": { + "nodeSelector": "has(i-am-a-route-reflector)", + "peerSelector": "has(i-am-a-route-reflector)" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: + - "{{ groups['calico-rr'] | default([]) }}" + when: + - inventory_hostname == groups['kube-master'][0] + - peer_with_calico_rr|default(false) + - name: Calico | Create calico manifests template: src: "{{ item.file }}.j2" @@ -234,18 +274,18 @@ - name: Calico | Configure node asNumber for per node peering shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "Node", - "metadata": { - "name": "{{ inventory_hostname }}" - }, - "spec": { - "bgp": { - "asNumber": "{{ local_as }}" - }, - "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] - }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }} + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "Node", + "metadata": { + "name": "{{ inventory_hostname }}" + }, + "spec": { + "bgp": { + "asNumber": "{{ local_as }}" + }, + "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}] + }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }} retries: 4 delay: "{{ retry_stagger | random + 3 }}" when: @@ -256,17 +296,17 @@ - name: Calico | Configure peering with router(s) at node scope shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "{{ inventory_hostname }}-{{ item.router_id }}" - }, - "spec": { - "asNumber": "{{ item.as }}", - "node": "{{ inventory_hostname }}", - "peerIP": "{{ item.router_id }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - + echo '{ + "apiVersion": "projectcalico.org/v3", + "kind": "BGPPeer", + "metadata": { + "name": "{{ inventory_hostname }}-{{ item.router_id }}" + }, + "spec": { + "asNumber": "{{ item.as }}", + "node": "{{ inventory_hostname }}", + "peerIP": "{{ item.router_id }}" + }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - retries: 4 delay: "{{ retry_stagger | random + 3 }}" with_items: @@ -274,25 +314,3 @@ when: - peer_with_router|default(false) - inventory_hostname in groups['k8s-cluster'] - -- name: Calico | Configure peering with route reflectors - shell: > - echo '{ - "apiVersion": "projectcalico.org/v3", - "kind": "BGPPeer", - "metadata": { - "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}" - }, - "spec": { - "asNumber": "{{ local_as | default(global_as_num) }}", - "node": "{{ inventory_hostname }}", - "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}" - }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f - - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - with_items: - - "{{ groups['calico-rr'] | default([]) }}" - when: - - peer_with_calico_rr|default(false) - - inventory_hostname in groups['k8s-cluster'] - - hostvars[item]['cluster_id'] == cluster_id diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml index 95cfdfcf2cb..9ed4b2bd484 100755 --- a/roles/network_plugin/cilium/defaults/main.yml +++ b/roles/network_plugin/cilium/defaults/main.yml @@ -1,7 +1,9 @@ --- # Log-level cilium_debug: false -cilium_disable_ipv4: false + +cilium_enable_ipv4: true +cilium_enable_ipv6: false # Etcd SSL dirs cilium_cert_dir: /etc/cilium/certs @@ -9,9 +11,6 @@ kube_etcd_cacert_file: ca.pem kube_etcd_cert_file: node-{{ inventory_hostname }}.pem kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem -# Cilium Network Policy directory -cilium_policy_dir: /etc/kubernetes/policy - # Limits for apps cilium_memory_limit: 500M cilium_cpu_limit: 500m @@ -20,3 +19,12 @@ cilium_cpu_requests: 100m # Optional features cilium_enable_prometheus: false +# Enable if you want to make use of hostPort mappings +cilium_enable_portmap: false + +# If upgrading from Cilium < 1.5, you may want to override some of these options +# to prevent service disruptions. See also: +# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action +cilium_preallocate_bpf_maps: false +cilium_tofqdns_enable_poller: false +cilium_enable_legacy_services: false diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml index e830818e9ef..8038bdf4276 100755 --- a/roles/network_plugin/cilium/tasks/main.yml +++ b/roles/network_plugin/cilium/tasks/main.yml @@ -34,6 +34,7 @@ - {name: cilium, file: cilium-crb.yml, type: clusterrolebinding} - {name: cilium, file: cilium-cr.yml, type: clusterrole} - {name: cilium, file: cilium-ds.yml, type: ds} + - {name: cilium, file: cilium-deploy.yml, type: deploy} - {name: cilium, file: cilium-sa.yml, type: sa} register: cilium_node_manifests when: @@ -48,7 +49,16 @@ mode: 0755 register: cni_bin_dir -- name: Cilium | Create network policy directory - file: - path: "{{ cilium_policy_dir }}" - state: directory +- name: Cilium | Copy CNI plugins + unarchive: + src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" + dest: "/opt/cni/bin" + mode: 0755 + remote_src: yes + when: cilium_enable_portmap + +- name: Cilium | Enable portmap addon + template: + src: 000-cilium-portmap.conflist.j2 + dest: /etc/cni/net.d/000-cilium-portmap.conflist + when: cilium_enable_portmap diff --git a/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 b/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 new file mode 100644 index 00000000000..982a7c94fe4 --- /dev/null +++ b/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 @@ -0,0 +1,13 @@ +{ + "cniVersion": "0.3.1", + "name": "cilium-portmap", + "plugins": [ + { + "type": "cilium-cni" + }, + { + "type": "portmap", + "capabilities": { "portMappings": true } + } + ] +} diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 old mode 100755 new mode 100644 index cf5758465dd..21a803cd475 --- a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2 @@ -25,18 +25,76 @@ data: key-file: "{{ cilium_cert_dir }}/key.pem" cert-file: "{{ cilium_cert_dir }}/cert.crt" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. +{% if cilium_enable_prometheus %} + prometheus-serve-addr: ":9090" +{% endif %} + # If you want to run cilium in debug mode change this value to true debug: "{{ cilium_debug }}" - disable-ipv4: "{{ cilium_disable_ipv4 }}" - # If you want to clean cilium state; change this value to true + enable-ipv4: "{{ cilium_enable_ipv4 }}" + enable-ipv6: "{{ cilium_enable_ipv6 }}" + # If a serious issue occurs during Cilium startup, this + # invasive option may be set to true to remove all persistent + # state. Endpoints will not be restored using knowledge from a + # prior Cilium run, so they may receive new IP addresses upon + # restart. This also triggers clean-cilium-bpf-state. clean-cilium-state: "false" - legacy-host-allows-world: "false" + # If you want to clean cilium BPF state, set this to true; + # Removes all BPF maps from the filesystem. Upon restart, + # endpoints are restored with the same IP addresses, however + # any ongoing connections may be disrupted briefly. + # Loadbalancing decisions will be reset, so any ongoing + # connections via a service may be loadbalanced to a different + # backend after restart. + clean-cilium-bpf-state: "false" + + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" # If you want cilium monitor to aggregate tracing for packets, set this level # to "low", "medium", or "maximum". The higher the level, the less packets # that will be seen in monitor output. monitor-aggregation-level: "none" + # ct-global-max-entries-* specifies the maximum number of connections + # supported across all endpoints, split by protocol: tcp or other. One pair + # of maps uses these values for IPv4 connections, and another pair of maps + # use these values for IPv6 connections. + # + # If these values are modified, then during the next Cilium startup the + # tracking of ongoing connections may be disrupted. This may lead to brief + # policy drops or a change in loadbalancing decisions for a connection. + # + # For users upgrading from Cilium 1.2 or earlier, to minimize disruption + # during the upgrade process, comment out these options. + bpf-ct-global-tcp-max: "524288" + bpf-ct-global-any-max: "262144" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # This may lead to policy drops or a change in loadbalancing decisions for a + # connection for some time. Endpoints may need to be recreated to restore + # connectivity. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "{{cilium_preallocate_bpf_maps}}" + # Regular expression matching compatible Istio sidecar istio-proxy # container image names sidecar-istio-proxy-image: "cilium/istio_proxy" @@ -47,3 +105,38 @@ data: # - vxlan (default) # - geneve tunnel: "vxlan" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + #cluster-id: 1 + + # DNS Polling periodically issues a DNS lookup for each `matchName` from + # cilium-agent. The result is used to regenerate endpoint policy. + # DNS lookups are repeated with an interval of 5 seconds, and are made for + # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP + # data is used instead. An IP change will trigger a regeneration of the Cilium + # policy for each endpoint and increment the per cilium-agent policy + # repository revision. + # + # This option is disabled by default starting from version 1.4.x in favor + # of a more powerful DNS proxy-based implementation, see [0] for details. + # Enable this option if you want to use FQDN policies but do not want to use + # the DNS proxy. + # + # To ease upgrade, users may opt to set this option to "true". + # Otherwise please refer to the Upgrade Guide [1] which explains how to + # prepare policy rules for upgrade. + # + # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based + # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action + tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}" + + # wait-bpf-mount makes init container wait until bpf filesystem is mounted + wait-bpf-mount: "false" + + # Enable legacy services (prior v1.5) to prevent from terminating existing + # connections with services when upgrading Cilium from < v1.5 to v1.5. + enable-legacy-services: "{{cilium_enable_legacy_services}}" diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 old mode 100755 new mode 100644 index 60cdb5b6e4c..9bdec5aed56 --- a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 @@ -1,11 +1,58 @@ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + # to get k8s version and status + - componentstatuses + verbs: + - get +- apiGroups: + - "" + resources: + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - "" + resources: + # to automatically read from k8s and import the node's pod CIDR to cilium's + # etcd so all nodes know how to reach another pod running in in a different + # node. + - nodes + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumendpoints + - ciliumendpoints/status + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: cilium rules: - apiGroups: - - "networking.k8s.io" + - networking.k8s.io resources: - networkpolicies verbs: @@ -34,11 +81,16 @@ rules: - list - watch - update + - apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - patch - apiGroups: - extensions resources: - - networkpolicies # FIXME remove this when we drop support for k8s NP-beta GH-1202 - - thirdpartyresources - ingresses verbs: - create @@ -46,7 +98,7 @@ rules: - list - watch - apiGroups: - - "apiextensions.k8s.io" + - apiextensions.k8s.io resources: - customresourcedefinitions verbs: @@ -63,12 +115,4 @@ rules: - ciliumendpoints - ciliumendpoints/status verbs: - - "*" - - apiGroups: - - policy - resourceNames: - - privileged - resources: - - podsecuritypolicies - verbs: - - use + - '*' diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 index 35994bc684d..8857d31a67a 100755 --- a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 @@ -1,6 +1,19 @@ --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: cilium-operator + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding metadata: name: cilium roleRef: @@ -8,8 +21,9 @@ roleRef: kind: ClusterRole name: cilium subjects: - - kind: ServiceAccount - name: cilium - namespace: kube-system - - kind: Group - name: system:nodes +- kind: ServiceAccount + name: cilium + namespace: kube-system +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes diff --git a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 new file mode 100644 index 00000000000..6d0e2352f64 --- /dev/null +++ b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 @@ -0,0 +1,122 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + io.cilium/app: operator + name: cilium-operator + name: cilium-operator + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + io.cilium/app: operator + name: cilium-operator + spec: + containers: + - args: + - --debug=$(CILIUM_DEBUG) + - --kvstore=etcd + - --kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config + command: + - cilium-operator + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: CILIUM_CLUSTER_NAME + valueFrom: + configMapKeyRef: + key: cluster-name + name: cilium-config + optional: true + - name: CILIUM_CLUSTER_ID + valueFrom: + configMapKeyRef: + key: cluster-id + name: cilium-config + optional: true + - name: CILIUM_DISABLE_ENDPOINT_CRD + valueFrom: + configMapKeyRef: + key: disable-endpoint-crd + name: cilium-config + optional: true + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: cilium-aws + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: cilium-aws + optional: true + - name: AWS_DEFAULT_REGION + valueFrom: + secretKeyRef: + key: AWS_DEFAULT_REGION + name: cilium-aws + optional: true + image: docker.io/cilium/operator:v1.5.5 + imagePullPolicy: IfNotPresent + name: cilium-operator + livenessProbe: + httpGet: + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /var/lib/etcd-config + name: etcd-config-path + readOnly: true + - mountPath: "{{cilium_cert_dir}}" + name: etcd-secrets + readOnly: true + dnsPolicy: ClusterFirst + priorityClassName: system-node-critical + restartPolicy: Always + serviceAccount: cilium-operator + serviceAccountName: cilium-operator + volumes: + # To read the etcd config stored in config maps + - configMap: + defaultMode: 420 + items: + - key: etcd-config + path: etcd.config + name: cilium-config + name: etcd-config-path + # To read the k8s etcd secrets in case the user might want to use TLS + - name: etcd-secrets + hostPath: + path: "{{cilium_cert_dir}}" diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index 6f62f95369c..87ef2150e78 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -1,219 +1,251 @@ ---- apiVersion: apps/v1 kind: DaemonSet metadata: + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" name: cilium namespace: kube-system spec: - updateStrategy: - type: "RollingUpdate" - rollingUpdate: - # Specifies the maximum number of Pods that can be unavailable during the update process. - # The current default value is 1 or 100% for daemonsets; Adding an explicit value here - # to avoid confusion, as the default value is specific to the type (daemonset/deployment). - maxUnavailable: "100%" selector: matchLabels: k8s-app: cilium kubernetes.io/cluster-service: "true" template: metadata: - labels: - k8s-app: cilium - kubernetes.io/cluster-service: "true" + annotations: {% if cilium_enable_prometheus %} - prometheus.io/scrape: "true" prometheus.io/port: "9090" + prometheus.io/scrape: "true" {% endif %} + # This annotation plus the CriticalAddonsOnly toleration makes + # cilium to be a critical pod in the cluster, which ensures cilium + # gets priority scheduling. + # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + scheduler.alpha.kubernetes.io/critical-pod: "" + scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]' + labels: + k8s-app: cilium + kubernetes.io/cluster-service: "true" spec: - priorityClassName: system-node-critical - serviceAccountName: cilium - initContainers: - - name: clean-cilium-state - image: {{ cilium_init_image_repo }}:{{ cilium_init_image_tag }} - imagePullPolicy: IfNotPresent - command: ['sh', '-c', 'if [ "${CLEAN_CILIUM_STATE}" = "true" ]; then rm -rf /var/run/cilium/state; rm -rf /sys/fs/bpf/tc/globals/cilium_*; fi'] - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - - name: cilium-run - mountPath: /var/run/cilium - env: - - name: "CLEAN_CILIUM_STATE" - valueFrom: - configMapKeyRef: - name: cilium-config - optional: true - key: clean-cilium-state containers: - - image: {{ cilium_image_repo }}:{{ cilium_image_tag }} - imagePullPolicy: Always - name: cilium-agent - command: ["cilium-agent"] - args: - - "--debug=$(CILIUM_DEBUG)" - - "--kvstore=etcd" - - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config" - - "--disable-ipv4=$(DISABLE_IPV4)" -{% if cilium_enable_prometheus %} - ports: - - name: prometheus - containerPort: 9090 -{% endif %} - lifecycle: - postStart: - exec: - command: - - "/cni-install.sh" - preStop: - exec: - command: - - "/cni-uninstall.sh" - env: - - name: "K8S_NODE_NAME" - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: "CILIUM_DEBUG" - valueFrom: - configMapKeyRef: - name: cilium-config - key: debug - - name: "DISABLE_IPV4" - valueFrom: - configMapKeyRef: - name: cilium-config - key: disable-ipv4 -{% if cilium_enable_prometheus %} - # Note: this variable is a no-op if not defined, and is used in the - # prometheus examples. - - name: "CILIUM_PROMETHEUS_SERVE_ADDR" - valueFrom: - configMapKeyRef: - name: cilium-metrics-config - optional: true - key: prometheus-serve-addr -{% endif %} - - name: "CILIUM_LEGACY_HOST_ALLOWS_WORLD" - valueFrom: - configMapKeyRef: - name: cilium-config - optional: true - key: legacy-host-allows-world - - name: "CILIUM_SIDECAR_ISTIO_PROXY_IMAGE" - valueFrom: - configMapKeyRef: - name: cilium-config - key: sidecar-istio-proxy-image - optional: true - - name: "CILIUM_TUNNEL" - valueFrom: - configMapKeyRef: - key: tunnel - name: cilium-config - optional: true - - name: "CILIUM_MONITOR_AGGREGATION_LEVEL" - valueFrom: - configMapKeyRef: - key: monitor-aggregation-level - name: cilium-config - optional: true - resources: - limits: - cpu: {{ cilium_cpu_limit }} - memory: {{ cilium_memory_limit }} - requests: - cpu: {{ cilium_cpu_requests }} - memory: {{ cilium_memory_requests }} - livenessProbe: + - args: + - --kvstore=etcd + - --kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config + - --config-dir=/tmp/cilium/config-map + command: + - cilium-agent + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + image: "{{cilium_image_repo}}:{{cilium_image_tag}}" + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: {{ cilium_cpu_limit }} + memory: {{ cilium_memory_limit }} + requests: + cpu: {{ cilium_cpu_requests }} + memory: {{ cilium_memory_requests }} + lifecycle: + postStart: exec: command: - - cilium - - status - initialDelaySeconds: 15 - failureThreshold: 10 - periodSeconds: 10 - readinessProbe: + - /cni-install.sh + preStop: exec: command: - - cilium - - status - initialDelaySeconds: 5 - periodSeconds: 5 - volumeMounts: - - name: bpf-maps - mountPath: /sys/fs/bpf - - name: cilium-run - mountPath: /var/run/cilium - - name: cni-path - mountPath: /host/opt/cni/bin - - name: etc-cni-netd - mountPath: /host/etc/cni/net.d + - /cni-uninstall.sh + livenessProbe: + exec: + command: + - cilium + - status + - --brief + failureThreshold: 10 + # The initial delay for the liveness probe is intentionally large to + # avoid an endless kill & restart cycle if in the event that the initial + # bootstrapping takes longer than expected. + initialDelaySeconds: 120 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + name: cilium-agent +{% if cilium_enable_prometheus %} + ports: + - containerPort: 9090 + hostPort: 9090 + name: prometheus + protocol: TCP +{% endif %} + readinessProbe: + exec: + command: + - cilium + - status + - --brief + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_MODULE + privileged: true + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + - mountPath: /host/opt/cni/bin + name: cni-path + - mountPath: /host/etc/cni/net.d + name: etc-cni-netd {% if container_manager == 'docker' %} - - name: docker-socket - mountPath: /var/run/docker.sock - readOnly: true + - mountPath: /var/run/docker.sock + name: docker-socket + readOnly: true {% else %} - name: "{{ container_manager }}-socket" mountPath: {{ cri_socket }} readOnly: true {% endif %} - - name: etcd-config-path - mountPath: /var/lib/etcd-config - readOnly: true - - name: cilium-certs - mountPath: {{ cilium_cert_dir }} - readOnly: true - securityContext: - capabilities: - add: - - "NET_ADMIN" - privileged: true - hostNetwork: true + - mountPath: /var/lib/etcd-config + name: etcd-config-path + readOnly: true + - mountPath: "{{cilium_cert_dir}}" + name: etcd-secrets + readOnly: true + - mountPath: /var/lib/cilium/clustermesh + name: clustermesh-secrets + readOnly: true + - mountPath: /tmp/cilium/config-map + name: cilium-config-path + readOnly: true + # Needed to be able to load kernel modules + - mountPath: /lib/modules + name: lib-modules + readOnly: true dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: false + initContainers: + - command: + - /init-container.sh + env: + - name: CLEAN_CILIUM_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-state + name: cilium-config + optional: true + - name: CLEAN_CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + key: clean-cilium-bpf-state + name: cilium-config + optional: true + - name: CILIUM_WAIT_BPF_MOUNT + valueFrom: + configMapKeyRef: + key: wait-bpf-mount + name: cilium-config + optional: true + image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}" + imagePullPolicy: IfNotPresent + name: clean-cilium-state + securityContext: + capabilities: + add: + - NET_ADMIN + privileged: true + volumeMounts: + - mountPath: /sys/fs/bpf + name: bpf-maps + - mountPath: /var/run/cilium + name: cilium-run + priorityClassName: system-node-critical + restartPolicy: Always + serviceAccount: cilium + serviceAccountName: cilium + terminationGracePeriodSeconds: 1 + tolerations: + - operator: Exists volumes: # To keep state between restarts / upgrades - - name: cilium-run - hostPath: - path: /var/run/cilium - # To keep state between restarts / upgrades - - name: bpf-maps - hostPath: - path: /sys/fs/bpf + - hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + name: cilium-run + # To keep state between restarts / upgrades for bpf maps + - hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + name: bpf-maps {% if container_manager == 'docker' %} # To read docker events from the node - - name: docker-socket - hostPath: - path: /var/run/docker.sock + - hostPath: + path: /var/run/docker.sock + type: Socket + name: docker-socket {% else %} # To read crio events from the node - - name: {{ container_manager }}-socket - hostPath: - path: {{ cri_socket }} + - hostPath: + path: {{ cri_socket }} + type: Socket + name: {{ container_manager }}-socket {% endif %} # To install cilium cni plugin in the host - - name: cni-path - hostPath: - path: /opt/cni/bin + - hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + name: cni-path # To install cilium cni configuration in the host - - name: etc-cni-netd - hostPath: - path: /etc/cni/net.d + - hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + name: etc-cni-netd + # To be able to load kernel modules + - hostPath: + path: /lib/modules + name: lib-modules # To read the etcd config stored in config maps - - name: etcd-config-path - configMap: - name: cilium-config - items: - - key: etcd-config - path: etcd.config + - configMap: + defaultMode: 420 + items: + - key: etcd-config + path: etcd.config + name: cilium-config + name: etcd-config-path # To read the k8s etcd secrets in case the user might want to use TLS - - name: cilium-certs - hostPath: - path: {{ cilium_cert_dir }} - - restartPolicy: Always - tolerations: - - operator: Exists - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - - key: CriticalAddonsOnly - operator: "Exists" + - name: etcd-secrets + hostPath: + path: "{{cilium_cert_dir}}" + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + defaultMode: 420 + optional: true + secretName: cilium-clustermesh + # To read the configuration from the config map + - configMap: + name: cilium-config + name: cilium-config-path + updateStrategy: + rollingUpdate: + # Specifies the maximum number of Pods that can be unavailable during the update process. + maxUnavailable: 2 + type: RollingUpdate diff --git a/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 index c03ac59b49b..2c5c0cb197a 100755 --- a/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 @@ -1,6 +1,12 @@ --- apiVersion: v1 kind: ServiceAccount +metadata: + name: cilium-operator + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount metadata: name: cilium namespace: kube-system diff --git a/roles/remove-node/post-remove/tasks/main.yml b/roles/remove-node/post-remove/tasks/main.yml index 530cb29dfd1..e36c2174219 100644 --- a/roles/remove-node/post-remove/tasks/main.yml +++ b/roles/remove-node/post-remove/tasks/main.yml @@ -1,9 +1,54 @@ --- +- name: Lookup node IP in kubernetes + shell: >- + {{ bin_dir }}/kubectl get nodes {{ node }} + -o jsonpath='{range.status.addresses[?(@.type=="InternalIP")]}{.address}{"\n"}{end}' + register: remove_node_ip + when: + - inventory_hostname in groups['etcd'] + - ip is not defined + - access_ip is not defined + delegate_to: "{{ groups['etcd']|first }}" + failed_when: false + +- name: Set node IP + set_fact: + node_ip: "{{ ip | default(access_ip | default(remove_node_ip.stdout)) | trim }}" - name: Delete node - command: "{{ bin_dir }}/kubectl delete node {{ item }}" - with_items: - - "{{ node.split(',') | default(groups['kube-node']) }}" + command: "{{ bin_dir }}/kubectl delete node {{ inventory_hostname }}" delegate_to: "{{ groups['kube-master']|first }}" run_once: true ignore_errors: yes + +- name: Lookup etcd member id + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member list | grep {{ node_ip }} | cut -d: -f1" + register: etcd_member_id + ignore_errors: true + changed_when: false + check_mode: no + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem" + ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" + delegate_to: "{{ groups['etcd']|first }}" + when: inventory_hostname in groups['etcd'] + +- name: Remove etcd member from cluster + shell: "{{ bin_dir }}/etcdctl --no-sync --endpoints={{ etcd_access_addresses }} member remove {{ etcd_member_id.stdout }}" + register: etcd_member_in_cluster + ignore_errors: true + changed_when: false + check_mode: no + tags: + - facts + environment: + ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem" + ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem" + ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" + delegate_to: "{{ groups['etcd']|first }}" + when: + - inventory_hostname in groups['etcd'] + - etcd_member_id.stdout | length > 0 diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 index a2f301def61..82293e0cdec 100644 --- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 +++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 @@ -50,5 +50,8 @@ instance-1 [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] [fake_hosts] diff --git a/tests/templates/inventory-aws.j2 b/tests/templates/inventory-aws.j2 index ee89bb5a472..92f107f653d 100644 --- a/tests/templates/inventory-aws.j2 +++ b/tests/templates/inventory-aws.j2 @@ -22,3 +22,6 @@ node2 [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] diff --git a/tests/templates/inventory-do.j2 b/tests/templates/inventory-do.j2 index 95b6f3027b8..83a749afcfe 100644 --- a/tests/templates/inventory-do.j2 +++ b/tests/templates/inventory-do.j2 @@ -43,6 +43,9 @@ {{droplets.results[0].droplet.name}} {% endif %} +[calico-rr] + [k8s-cluster:children] kube-node kube-master +calico-rr diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2 index 896cc157d44..503bb40914f 100644 --- a/tests/templates/inventory-gce.j2 +++ b/tests/templates/inventory-gce.j2 @@ -66,6 +66,9 @@ [k8s-cluster:children] kube-node kube-master +calico-rr + +[calico-rr] {% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %} [fake_hosts]