diff --git a/.envrc b/.envrc index 6c54d515581..c84c3f45b99 100644 --- a/.envrc +++ b/.envrc @@ -8,3 +8,5 @@ export ANSIBLE_COLLECTIONS_PATH=$(expand_path ./.venv/galaxy) export ANSIBLE_ROLES_PATH=$(expand_path ./.venv/galaxy/ansible_roles) export ANSIBLE_VARS_ENABLED="host_group_vars,community.sops.sops" export K8S_AUTH_KUBECONFIG="$(expand_path ./kubeconfig)" +# k0s +export DISABLE_TELEMETRY="true" diff --git a/.github/renovate.json5 b/.github/renovate.json5 index d2897040857..4e3ae0249fa 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -78,10 +78,10 @@ }, // custom versioning { - "description": "Use custom versioning for k3s", + "description": "Use custom versioning for k0s/k3s", "matchDatasources": ["github-releases"], - "versioning": "regex:^v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?\\+k3s)(?\\d+)$", - "matchPackagePatterns": ["k3s"] + "versioning": "regex:^v(?\\d+)\\.(?\\d+)\\.(?\\d+)(?\\+k.s)\\.?(?\\d+)$", + "matchPackagePatterns": ["k0s", "k3s"] }, // commit message topics { diff --git a/.gitignore b/.gitignore index 4ad3e0699bb..d7ae8c521ce 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ kubeconfig .venv* # Taskfile .tasks +# intellij +.idea + diff --git a/.taskfiles/BrewTasks.yaml b/.taskfiles/BrewTasks.yaml index 03b8fb4bb9e..bdb440d66c6 100644 --- a/.taskfiles/BrewTasks.yaml +++ b/.taskfiles/BrewTasks.yaml @@ -20,6 +20,7 @@ tasks: fluxcd/tap/flux helm jq + k0sproject/tap/k0sctl k9s kubernetes-cli kustomize diff --git a/.taskfiles/FluxTasks.yaml b/.taskfiles/FluxTasks.yaml index f01e722f249..c03b5965972 100644 --- a/.taskfiles/FluxTasks.yaml +++ b/.taskfiles/FluxTasks.yaml @@ -26,9 +26,15 @@ tasks: - kubectl apply --server-side --filename {{.KUBERNETES_DIR}}/flux/vars/cluster-settings-user.yaml - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/flux/config preconditions: - - { msg: "Flux already appears installed", sh: "exit $(( ! $(kubectl get namespace flux-system) ))" } + - { msg: "Flux already appears installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 1 || exit 0" } - { msg: "Age private key not found", sh: "test -f {{.ROOT_DIR}}/age.key" } + github-deploy-key: + cmds: + - sops --decrypt {{.KUBERNETES_DIR}}/bootstrap/github-deploy-key.sops.yaml | kubectl apply -f - + preconditions: + - { msg: "Flux is not installed", sh: "kubectl get namespace flux-system &>/dev/null && exit 0 || exit 1" } + apply: desc: Apply a Flux Kustomization resource for a cluster summary: | diff --git a/.taskfiles/K0sTasks.yaml b/.taskfiles/K0sTasks.yaml new file mode 100644 index 00000000000..6b13081f821 --- /dev/null +++ b/.taskfiles/K0sTasks.yaml @@ -0,0 +1,28 @@ +--- +version: "3" + +env: + DISABLE_TELEMETRY: "true" + +tasks: + + kubeconfig: + desc: Gets k0s cluster kubeconfig + cmds: + - k0sctl kubeconfig --config k0s-config.yaml > kubeconfig + preconditions: + - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" } + + apply: + desc: Apply k0s cluster k0s-config.yaml + cmds: + - k0sctl apply --config k0s-config.yaml + - task: kubeconfig + preconditions: + - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" } + + reset: + desc: Resets the k0s cluster + cmd: k0sctl reset --config k0s-config.yaml + preconditions: + - { msg: "k0s-config.yaml not found", sh: "test -f {{.ROOT_DIR}}/k0s-config.yaml" } diff --git a/README.md b/README.md index 7789ea0b309..555c8cbce46 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Deploy a Kubernetes cluster backed by Flux -Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state. +Welcome to my highly opinionated template for deploying a single Kubernetes ([k3s](https://k3s.io) or [k0s](https://github.com/k0sproject/k0s)) cluster with [Ansible](https://www.ansible.com) and using [Flux](https://toolkit.fluxcd.io) to manage its state. ## 👋 Introduction @@ -258,7 +258,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a > └─📁 apps # Apps deployed into the cluster grouped by namespace > ``` -### ⚡ Stage 4: Prepare your nodes for k3s +### ⚡ Stage 4: Prepare your nodes for Kubernetes 📍 _Here we will be running an Ansible playbook to prepare your nodes for running a Kubernetes cluster._ @@ -282,7 +282,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a task ansible:run playbook=cluster-prepare ``` -### ⛵ Stage 5: Use Ansible to install k3s +### ⛵ Stage 5: Install Kubernetes 📍 _Here we will be running a Ansible Playbook to install [k3s](https://k3s.io/) with [this](https://galaxy.ansible.com/xanmanning/k3s) Ansible galaxy role. If you run into problems, you can run `task ansible:run playbook=cluster-nuke` to destroy the k3s cluster and start over from this point._ @@ -298,13 +298,16 @@ Once you have installed Debian on your nodes, there are six stages to getting a task ansible:ping ``` -3. Install k3s with Ansible +3. Install Kubernetes ```sh + # k3s task ansible:run playbook=cluster-installation + # k0s + task k0s:apply ``` -4. Verify the nodes are online +5. Verify the nodes are online 📍 _If this command **fails** you likely haven't configured `direnv` as mentioned previously in the guide._ @@ -315,7 +318,7 @@ Once you have installed Debian on your nodes, there are six stages to getting a # k8s-1 Ready worker 1h v1.27.3+k3s1 ``` -5. The `kubeconfig` for interacting with your cluster should have been created in the root of your repository. +6. The `kubeconfig` for interacting with your cluster should have been created in the root of your repository. ### 🔹 Stage 6: Install Flux in your cluster diff --git a/Taskfile.yaml b/Taskfile.yaml index 8a31c904cf0..f207e3692ff 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -17,6 +17,7 @@ includes: aliases: ["k8s"] taskfile: .taskfiles/KubernetesTasks.yaml flux: .taskfiles/FluxTasks.yaml + k0s: .taskfiles/K0sTasks.yaml tasks: @@ -45,7 +46,6 @@ tasks: configure: desc: Configure repository from Ansible vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? - dir: "{{.BOOTSTRAP_DIR}}" - cmd: ansible-playbook configure.yaml + cmd: ./.venv/bin/ansible-playbook {{.BOOTSTRAP_DIR}}/configure.yaml env: ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false" diff --git a/bootstrap/configure.yaml b/bootstrap/configure.yaml index 9b3d9cb812b..e3679f6d5c3 100644 --- a/bootstrap/configure.yaml +++ b/bootstrap/configure.yaml @@ -17,11 +17,11 @@ ansible.builtin.set_fact: repository_path: "{{ repository.stdout }}" - - name: Override kube-vip address when there is a single master node and no address is defined - when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_vip_addr + - name: Override kubeapi address when there is a single master node and no address is defined + when: bootstrap_nodes.master | length == 1 and not bootstrap_kubeapi_addr ansible.builtin.set_fact: bootstrap_kube_vip_enabled: false - bootstrap_kube_vip_addr: "{{ bootstrap_nodes.master[0].address }}" + bootstrap_kubeapi_addr: "{{ bootstrap_nodes.master[0].address }}" - name: Verify configuration ansible.builtin.include_tasks: tasks/validation/main.yaml @@ -29,6 +29,10 @@ - name: Template Sops configuration ansible.builtin.include_tasks: tasks/sops/main.yaml + - name: Template k0s configuration + when: bootstrap_distribution == "k0s" + ansible.builtin.include_tasks: tasks/k0s/main.yaml + - name: Template Ansible configuration ansible.builtin.include_tasks: tasks/ansible/main.yaml diff --git a/bootstrap/tasks/addons/coredns.yaml b/bootstrap/tasks/addons/coredns.yaml new file mode 100644 index 00000000000..4599105980e --- /dev/null +++ b/bootstrap/tasks/addons/coredns.yaml @@ -0,0 +1,34 @@ +--- +- name: Set addon facts + ansible.builtin.set_fact: + addon_name: coredns + addon_namespace: kube-system + +- name: Ensure directories exist for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'directory' + ansible.builtin.file: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path }}" + state: directory + mode: "0755" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template unencrypted files for {{ addon_namespace }}/{{ addon_name }} + when: item.state == 'file' and 'sops' not in item.path + ansible.builtin.template: + src: "{{ item.src }}" + dest: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + mode: "0644" + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] + +- name: Template encrypted files for {{ addon_namespace }}/{{ addon_name }} + block: + - name: Template encrypted files + when: item.state == 'file' and 'sops' in item.path + community.sops.sops_encrypt: + path: "{{ repository_path }}/kubernetes/apps/{{ addon_namespace }}/{{ addon_name }}/{{ item.path | replace('.j2', '') }}" + encrypted_regex: ^(data|stringData)$ + age: ["{{ bootstrap_age_public_key }}"] + content_yaml: "{{ lookup('ansible.builtin.template', item.src) | from_yaml }}" + mode: "0644" + force: true + with_community.general.filetree: ["../templates/addons/{{ addon_name }}/"] diff --git a/bootstrap/tasks/addons/main.yaml b/bootstrap/tasks/addons/main.yaml index 23e4092bca0..52fac11e8c2 100644 --- a/bootstrap/tasks/addons/main.yaml +++ b/bootstrap/tasks/addons/main.yaml @@ -1,5 +1,9 @@ --- +- name: Process optional coredns + when: bootstrap_distribution == "k3s" + ansible.builtin.include_tasks: coredns.yaml + - name: Process addon csi-driver-nfs when: csi_driver_nfs.enabled | default(false) ansible.builtin.include_tasks: csi_driver_nfs.yaml @@ -21,7 +25,9 @@ ansible.builtin.include_tasks: kube_prometheus_stack.yaml - name: Process addon system-upgrade-controller - when: system_upgrade_controller.enabled | default(false) + when: + - bootstrap_distribution == "k3s" + - system_upgrade_controller.enabled | default(false) ansible.builtin.include_tasks: system_upgrade_controller.yaml - name: Process addon weave-gitops diff --git a/bootstrap/tasks/ansible/main.yaml b/bootstrap/tasks/ansible/main.yaml index 5c5a286fdcd..2bee0d440ac 100644 --- a/bootstrap/tasks/ansible/main.yaml +++ b/bootstrap/tasks/ansible/main.yaml @@ -27,13 +27,4 @@ mode: "0644" force: true with_community.general.filetree: ["../templates/ansible/"] - - name: Template encrypted node secrets - community.sops.sops_encrypt: - path: "{{ repository_path }}/ansible/inventory/host_vars/{{ item.name }}.sops.yaml" - age: ["{{ bootstrap_age_public_key }}"] - content_yaml: "{{ lookup('ansible.builtin.template', 'templates/node.sops.yaml.j2', template_vars=dict(password=item.password)) | from_yaml }}" - mode: "0644" - force: true - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" - loop_control: - label: "{{ item.address }}" + diff --git a/bootstrap/tasks/k0s/main.yaml b/bootstrap/tasks/k0s/main.yaml new file mode 100644 index 00000000000..5c2b5a105ac --- /dev/null +++ b/bootstrap/tasks/k0s/main.yaml @@ -0,0 +1,6 @@ +--- +- name: Template k0s configuration file + ansible.builtin.template: + src: "templates/k0s-config.yaml.j2" + dest: "{{ repository_path }}/k0s-config.yaml" + mode: "0644" diff --git a/bootstrap/tasks/validation/github.yaml b/bootstrap/tasks/validation/github.yaml index 7ffa295283c..d02d58f7979 100644 --- a/bootstrap/tasks/validation/github.yaml +++ b/bootstrap/tasks/validation/github.yaml @@ -14,6 +14,7 @@ fail_msg: Github user {{ bootstrap_github_username }} does not exist - name: Query Github repo + when: not bootstrap_private_github_repo | default(false) ansible.builtin.uri: url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} timeout: 5 @@ -22,12 +23,14 @@ register: result - name: Check if repo exists + when: not bootstrap_private_github_repo | default(false) ansible.builtin.assert: that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist - name: Query Github repo branch + when: not bootstrap_private_github_repo | default(false) ansible.builtin.uri: url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch | default('main', true) }} timeout: 5 @@ -36,6 +39,7 @@ register: result - name: Check if repo branch exists + when: not bootstrap_private_github_repo | default(false) ansible.builtin.assert: that: result.json.name == bootstrap_github_repository_branch | default('main', true) success_msg: Github repo branch {{ bootstrap_github_repository_branch | default('main', true) }} exists diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml index 7128b737bbf..c17ca08abcf 100644 --- a/bootstrap/tasks/validation/net.yaml +++ b/bootstrap/tasks/validation/net.yaml @@ -98,17 +98,17 @@ success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}. fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}. -- name: Verify kube-vip +- name: Verify kubeapi address ansible.builtin.assert: - that: bootstrap_kube_vip_addr is ansible.utils.ipv4 - success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is valid. - fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is invalid. + that: bootstrap_kubeapi_addr is ansible.utils.ipv4 + success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is valid. + fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is invalid. -- name: Verify kube-vip in node CIDR +- name: Verify kubeapi address in node CIDR ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_vip_addr) - success_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: kube-vip address {{ bootstrap_kube_vip_addr }} is not within {{ bootstrap_node_cidr }}. + that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kubeapi_addr) + success_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is within {{ bootstrap_node_cidr }}. + fail_msg: kubeapi address {{ bootstrap_kubeapi_addr }} is not within {{ bootstrap_node_cidr }}. - name: Verify all IP addresses are unique ansible.builtin.assert: @@ -117,7 +117,7 @@ bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, - bootstrap_kube_vip_addr + bootstrap_kubeapi_addr ] | unique | length == 4 success_msg: All IP addresses are unique. fail_msg: All IP addresses are not unique. @@ -133,12 +133,12 @@ loop_control: label: "{{ item.address }}" -- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kube-vip - when: bootstrap_kube_vip_enabled | default(true) +- name: Verify nodes are not the same IPs as k8s_gateway, ingress external/internal or kubeapi address + when: (bootstrap_distribution == "k3s") and (bootstrap_kube_vip_enabled | default(true)) ansible.builtin.assert: - that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_vip_addr) - success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kube-vip. - fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kube-vip. + that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kubeapi_addr) + success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or kubeapi. + fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or kubeapi. quiet: true loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" loop_control: diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml index 72730fbbae0..a8936bbd342 100644 --- a/bootstrap/tasks/validation/vars.yaml +++ b/bootstrap/tasks/validation/vars.yaml @@ -16,23 +16,24 @@ - bootstrap_cloudflare_tunnel_id - bootstrap_cloudflare_tunnel_secret - bootstrap_cluster_cidr + - bootstrap_distribution + - bootstrap_external_ingress_addr - bootstrap_flux_github_webhook_token - - bootstrap_github_repository_name - bootstrap_github_repository_branch + - bootstrap_github_repository_name - bootstrap_github_username - - bootstrap_external_ingress_addr - bootstrap_internal_ingress_addr - bootstrap_ipv6_enabled - bootstrap_k8s_gateway_addr - - bootstrap_kube_vip_addr - - bootstrap_local_path_provisioner_path + - bootstrap_kubeapi_addr + - bootstrap_local_storage_path - bootstrap_node_cidr - bootstrap_service_cidr - bootstrap_timezone - name: Verify bootstrap node names are valid ansible.builtin.assert: - that: item.name is match('^[a-z0-9-]+$') + that: item.name is match('^[a-z0-9-\.]+$') success_msg: Node name {{ item.name }} is valid fail_msg: Node name {{ item.name }} is not valid loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker | default([]) }}" diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/coredns/app/helmrelease.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/kube-system/coredns/app/helmrelease.yaml.j2 rename to bootstrap/templates/addons/coredns/app/helmrelease.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 b/bootstrap/templates/addons/coredns/app/kustomization.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/kube-system/coredns/app/kustomization.yaml.j2 rename to bootstrap/templates/addons/coredns/app/kustomization.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 b/bootstrap/templates/addons/coredns/ks.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/kube-system/coredns/ks.yaml.j2 rename to bootstrap/templates/addons/coredns/ks.yaml.j2 diff --git a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 index 639b4aead57..fc17ff89eb8 100644 --- a/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/grafana/app/helmrelease.yaml.j2 @@ -24,7 +24,7 @@ spec: uninstall: keepHistory: false dependsOn: - - name: local-path-provisioner + - name: openebs namespace: storage values: deploymentStrategy: @@ -168,6 +168,6 @@ spec: - *host persistence: enabled: true - storageClassName: local-hostpath + storageClassName: openebs-hostpath testFramework: enabled: false diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 index 515015acefc..e303ac94ed2 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmrelease.yaml.j2 @@ -27,7 +27,7 @@ spec: uninstall: keepHistory: false dependsOn: - - name: local-path-provisioner + - name: openebs namespace: storage valuesFrom: - name: kube-prometheus-stack-values diff --git a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 index 073cc199a08..f4d77532836 100644 --- a/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 +++ b/bootstrap/templates/addons/kube-prometheus-stack/app/helmvalues.yaml.j2 @@ -114,7 +114,7 @@ data: storageSpec: volumeClaimTemplate: spec: - storageClassName: local-hostpath + storageClassName: openebs-hostpath resources: requests: storage: 10Gi diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 index 3701a00475f..9618b3835d4 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/main.yaml.j2 @@ -14,7 +14,7 @@ k3s_etcd_datastore: true {% else %} k3s_etcd_datastore: false {% endif %} -k3s_registration_address: "{% raw %}{{ kube_vip_addr }}{% endraw %}" +k3s_registration_address: "{% raw %}{{ kubeapi_addr }}{% endraw %}" # /var/lib/rancher/k3s/server/manifests k3s_server_manifests_templates: - custom-cilium-helmchart.yaml.j2 diff --git a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 index 816dc3f58f0..260dee17132 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/kubernetes/supplemental.yaml.j2 @@ -3,7 +3,7 @@ timezone: "{{ bootstrap_timezone }}" github_username: "{{ bootstrap_github_username }}" coredns_addr: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" -kube_vip_addr: "{{ bootstrap_kube_vip_addr }}" +kubeapi_addr: "{{ bootstrap_kubeapi_addr }}" cluster_cidr: "{{ bootstrap_cluster_cidr.split(',')[0] }}" service_cidr: "{{ bootstrap_service_cidr.split(',')[0] }}" node_cidr: "{{ bootstrap_node_cidr }}" diff --git a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 index 777bfdba6e1..7ecd43ce839 100644 --- a/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 +++ b/bootstrap/templates/ansible/inventory/group_vars/master/main.yaml.j2 @@ -11,7 +11,7 @@ k3s_server: node-ip: "{% raw %}{{ ansible_host }}{% endraw %}" {% endif %} tls-san: - - "{% raw %}{{ kube_vip_addr }}{% endraw %}" + - "{% raw %}{{ kubeapi_addr }}{% endraw %}" docker: false flannel-backend: "none" # This needs to be in quotes disable: diff --git a/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 index 912dc066e0d..bd7d928e24a 100644 --- a/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 +++ b/bootstrap/templates/ansible/playbooks/cluster-nuke.yaml.j2 @@ -1,3 +1,4 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True --- - name: Cluster Nuke hosts: kubernetes @@ -21,6 +22,7 @@ ansible.builtin.pause: seconds: 5 tasks: + {% if bootstrap_distribution == "k3s" %} - name: Stop Kubernetes # noqa: ignore-errors ignore_errors: true block: @@ -30,6 +32,7 @@ public: true vars: k3s_state: stopped + {% endif %} # https://github.com/k3s-io/docs/blob/main/docs/installation/network-options.md - name: Networking @@ -55,6 +58,7 @@ path: /etc/cni/net.d state: absent + {% if bootstrap_distribution == "k3s" %} - name: Check to see if k3s-killall.sh exits ansible.builtin.stat: path: /usr/local/bin/k3s-killall.sh @@ -89,6 +93,12 @@ path: "{% raw %}{{ k3s_install_dir }}/{{ item }}{% endraw %}" state: absent loop: ["kubectl", "crictl", "ctr"] + {% endif %} + + - name: Remove local storage path + ansible.builtin.file: + path: "{% raw %}{{ bootstrap_local_storage_path }}{% endraw %}" + state: absent - name: Reboot ansible.builtin.reboot: diff --git a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 index b79b26d5c16..cec121c7864 100644 --- a/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/custom-cilium-helmchart.yaml.j2.j2 @@ -37,7 +37,7 @@ spec: ipv6: enabled: true {% endif %} - k8sServiceHost: "{% raw %}{{ kube_vip_addr }}{% endraw %}" + k8sServiceHost: "{% raw %}{{ kubeapi_addr }}{% endraw %}" k8sServicePort: 6443 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 @@ -59,6 +59,6 @@ spec: replicas: 1 rollOutPods: true rollOutCiliumPods: true + routingMode: native securityContext: privileged: true - tunnel: disabled diff --git a/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 b/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 index caf3a6baafb..b7a5841e1c0 100644 --- a/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 +++ b/bootstrap/templates/ansible/playbooks/templates/kube-vip-static-pod.yaml.j2.j2 @@ -15,7 +15,7 @@ spec: args: ["manager"] env: - name: address - value: "{% raw %}{{ kube_vip_addr }}{% endraw %}" + value: "{% raw %}{{ kubeapi_addr }}{% endraw %}" - name: vip_arp value: "true" - name: lb_enable diff --git a/bootstrap/templates/k0s-config.yaml.j2 b/bootstrap/templates/k0s-config.yaml.j2 new file mode 100644 index 00000000000..b86cecbdc94 --- /dev/null +++ b/bootstrap/templates/k0s-config.yaml.j2 @@ -0,0 +1,126 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True +--- +apiVersion: k0sctl.k0sproject.io/v1beta1 +kind: Cluster +metadata: + name: k0s-cluster +spec: + hosts: + {% for item in bootstrap_nodes.master %} + - role: {{ item.role | default('controller+worker') }} + ssh: + address: {{ item.address }} + user: {{ item.username }} + installFlags: + - --disable-components=metrics-server + {% if item.role | default('') == 'controller+worker' %} + - --no-taints + {% endif %} + {% endfor %} + {% if bootstrap_nodes.worker | default([]) | length > 0 %} + {% for item in bootstrap_nodes.worker %} + - role: worker + ssh: + address: {{ item.address }} + user: {{ item.username }} + {% endfor %} + {% endif %} + k0s: + # renovate: datasource=github-releases depName=k0sproject/k0s + version: "v1.28.4+k0s.0" + dynamicConfig: false + config: + spec: + telemetry: + enabled: false + controllerManager: + extraArgs: + # Required to monitor kube-controller-manager with kube-prometheus-stack + bind-address: "0.0.0.0" + scheduler: + extraArgs: + # Required to monitor kube-scheduler with kube-prometheus-stack + bind-address: "0.0.0.0" + api: + sans: + - {{ bootstrap_kubeapi_addr }} + {% if bootstrap_kubeapi_hostname is defined %} + - {{ bootstrap_kubeapi_hostname }} + {% endif %} + {% for item in bootstrap_nodes.master %} + {% if item.address != bootstrap_kubeapi_addr %} + - {{ item.address }} + {% endif %} + {% if (bootstrap_kubeapi_hostname is not defined) or (item.name != bootstrap_kubeapi_hostname) %} + - {{ item.name }} + {% endif %} + {% endfor %} + extensions: + helm: + repositories: + - name: cilium + url: https://helm.cilium.io + charts: + - name: cilium + chartname: cilium/cilium + # renovate: datasource=github-releases depName=cilium/cilium + version: "1.14.5" + namespace: kube-system + values: |2 + autoDirectNodeRoutes: true + bpf: + masquerade: true + bgp: + enabled: false + cluster: + name: home-cluster + id: 1 + containerRuntime: + integration: containerd + socketPath: /var/run/k0s/containerd.sock + endpointRoutes: + enabled: true + hubble: + enabled: false + ipam: + mode: kubernetes + ipv4NativeRoutingCIDR: "{{ bootstrap_cluster_cidr }}" + {% if bootstrap_ipv6_enabled | default(false) %} + ipv6NativeRoutingCIDR: "{{ bootstrap_cluster_cidr_v6 }}" + ipv6: + enabled: true + {% endif %} + k8sServiceHost: "{{ bootstrap_kubeapi_addr }}" + k8sServicePort: 6443 + kubeProxyReplacement: true + kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 + l2announcements: + {% if bootstrap_ipv6_enabled | default(false) %} + enabled: false + {% else %} + enabled: true + # https://github.com/cilium/cilium/issues/26586 + leaseDuration: 120s + leaseRenewDeadline: 60s + leaseRetryPeriod: 1s + {% endif %} + loadBalancer: + algorithm: maglev + mode: {{ bootstrap_cilium_loadbalancer_mode | default('dsr', true) }} + localRedirectPolicy: true + operator: + replicas: 1 + rollOutPods: true + rollOutCiliumPods: true + routingMode: native + securityContext: + privileged: true + network: + kubeProxy: + disabled: true + {% if bootstrap_nodes.master | length > 1 %} + nodeLocalLoadBalancing: + enabled: true + type: EnvoyProxy + {% endif %} + provider: custom diff --git a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 index 32c70548fe6..c9f09aebfa4 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/cilium/app/helmvalues.yaml.j2 @@ -16,7 +16,11 @@ data: id: 1 containerRuntime: integration: containerd + {% if bootstrap_distribution == "k3s" %} socketPath: /var/run/k3s/containerd/containerd.sock + {% else %} + socketPath: /var/run/k0s/containerd.sock + {% endif %} endpointRoutes: enabled: true hubble: @@ -63,7 +67,7 @@ data: ipv6: enabled: true {% endif %} - k8sServiceHost: "${KUBE_VIP_ADDR}" + k8sServiceHost: "${KUBEAPI_ADDR}" k8sServicePort: 6443 kubeProxyReplacement: true kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256 @@ -102,6 +106,6 @@ data: annotations: grafana_folder: Cilium rollOutCiliumPods: true + routingMode: native securityContext: privileged: true - tunnel: disabled diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index b338bd1781c..5cb2806e831 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -5,5 +5,7 @@ kind: Kustomization resources: - ./namespace.yaml - ./cilium/ks.yaml + {% if bootstrap_distribution == "k3s" %} - ./coredns/ks.yaml + {% endif %} - ./metrics-server/ks.yaml diff --git a/bootstrap/templates/kubernetes/apps/storage/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/kustomization.yaml.j2 index 2543a7a7eaa..b896982efa2 100644 --- a/bootstrap/templates/kubernetes/apps/storage/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/kustomization.yaml.j2 @@ -4,7 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./local-path-provisioner/ks.yaml + - ./openebs/ks.yaml - ./snapshot-controller/ks.yaml - ./volsync/ks.yaml {% if csi_driver_nfs.enabled | default(false) %} diff --git a/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/helmrelease.yaml.j2 deleted file mode 100644 index d6415c71f62..00000000000 --- a/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/helmrelease.yaml.j2 +++ /dev/null @@ -1,76 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2beta2 -kind: HelmRelease -metadata: - name: local-path-provisioner -spec: - interval: 30m - chart: - spec: - chart: democratic-csi - version: 0.14.3 - sourceRef: - name: democratic-csi - kind: HelmRepository - namespace: flux-system - install: - remediation: - retries: 3 - upgrade: - cleanupOnFail: true - remediation: - retries: 3 - uninstall: - keepHistory: false - values: - fullnameOverride: local-path-provisioner - controller: - strategy: node - externalProvisioner: - image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3 - extraArgs: - - --leader-election=false - - --node-deployment=true - - --node-deployment-immediate-binding=false - - --feature-gates=Topology=true - - --strict-topology=true - - --enable-capacity=true - - --capacity-ownerref-level=1 - externalResizer: - enabled: false - externalAttacher: - enabled: false - externalSnapshotter: - enabled: false - csiDriver: - name: local-hostpath.cluster.local - storageCapacity: true - attachRequired: false - fsGroupPolicy: File - storageClasses: - - name: local-hostpath - defaultClass: false - reclaimPolicy: Delete - volumeBindingMode: WaitForFirstConsumer - allowVolumeExpansion: true - driver: - config: - driver: local-hostpath - local-hostpath: - shareBasePath: &storagePath "{{ bootstrap_local_path_provisioner_path | default('/var/lib/rancher/k3s/local-hostpath', true) }}" - controllerBasePath: *storagePath - dirPermissionsMode: "0770" - dirPermissionsUser: 0 - dirPermissionsGroup: 0 - node: - driver: - image: ghcr.io/democratic-csi/democratic-csi:v1.8.4 - extraVolumeMounts: - - name: local-hostpath - mountPath: *storagePath - mountPropagation: Bidirectional - extraVolumes: - - name: local-hostpath - hostPath: - path: *storagePath - type: DirectoryOrCreate diff --git a/bootstrap/templates/kubernetes/apps/storage/openebs/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/openebs/app/helmrelease.yaml.j2 new file mode 100644 index 00000000000..2c420c2598e --- /dev/null +++ b/bootstrap/templates/kubernetes/apps/storage/openebs/app/helmrelease.yaml.j2 @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: openebs +spec: + interval: 30m + chart: + spec: + chart: openebs + version: 3.3.0 + sourceRef: + kind: HelmRepository + name: openebs + namespace: flux-system + maxHistory: 2 + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + localprovisioner: + hostpathClass: + enabled: true + name: openebs-hostpath + isDefaultClass: false + basePath: "{{ bootstrap_local_storage_path }}" + diff --git a/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/openebs/app/kustomization.yaml.j2 similarity index 100% rename from bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/app/kustomization.yaml.j2 rename to bootstrap/templates/kubernetes/apps/storage/openebs/app/kustomization.yaml.j2 diff --git a/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/storage/openebs/ks.yaml.j2 similarity index 77% rename from bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/ks.yaml.j2 rename to bootstrap/templates/kubernetes/apps/storage/openebs/ks.yaml.j2 index c518fb7b6c7..0b898957bd3 100644 --- a/bootstrap/templates/kubernetes/apps/storage/local-path-provisioner/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/storage/openebs/ks.yaml.j2 @@ -2,14 +2,14 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: &app local-path-provisioner + name: &app openebs namespace: flux-system spec: targetNamespace: storage commonMetadata: labels: app.kubernetes.io/name: *app - path: ./kubernetes/apps/storage/local-path-provisioner/app + path: ./kubernetes/apps/storage/openebs/app prune: true sourceRef: kind: GitRepository diff --git a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 index ecd2060cf7d..f4baa798935 100644 --- a/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/config/cluster.yaml.j2 @@ -1,3 +1,4 @@ +#jinja2: trim_blocks: True, lstrip_blocks: True --- apiVersion: source.toolkit.fluxcd.io/v1 kind: GitRepository @@ -8,7 +9,13 @@ spec: interval: 30m ref: branch: {{ bootstrap_github_repository_branch | default('main', true) }} + {% if bootstrap_private_github_repo | default(false) %} + secretRef: + name: github-deploy-key + url: "ssh://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}" + {% else %} url: "https://github.com/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}.git" + {% endif %} ignore: | # exclude all /* diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 index 8e30d2e3339..36110075368 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/kustomization.yaml.j2 @@ -8,7 +8,6 @@ resources: - ./cilium.yaml - ./coredns.yaml - ./csi-driver-nfs.yaml - - ./democratic-csi.yaml - ./descheduler.yaml - ./external-dns.yaml - ./grafana.yaml @@ -18,6 +17,7 @@ resources: - ./k8s-gateway.yaml - ./kubernetes-dashboard.yaml - ./metrics-server.yaml + - ./openebs.yaml - ./piraeus.yaml - ./prometheus-community.yaml - ./stakater.yaml diff --git a/bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 b/bootstrap/templates/kubernetes/flux/repositories/helm/openebs.yaml.j2 similarity index 63% rename from bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 rename to bootstrap/templates/kubernetes/flux/repositories/helm/openebs.yaml.j2 index a75d1fb31ea..d0f105e642a 100644 --- a/bootstrap/templates/kubernetes/flux/repositories/helm/democratic-csi.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/repositories/helm/openebs.yaml.j2 @@ -2,8 +2,8 @@ apiVersion: source.toolkit.fluxcd.io/v1beta2 kind: HelmRepository metadata: - name: democratic-csi + name: openebs namespace: flux-system spec: interval: 1h - url: https://democratic-csi.github.io/charts/ + url: https://openebs.github.io/charts diff --git a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 index 511c2bd27e4..759c9842942 100644 --- a/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 +++ b/bootstrap/templates/kubernetes/flux/vars/cluster-settings.yaml.j2 @@ -8,7 +8,7 @@ metadata: data: TIMEZONE: "{{ bootstrap_timezone }}" COREDNS_ADDR: "{{ bootstrap_service_cidr.split(',')[0] | ansible.utils.nthhost(10) }}" - KUBE_VIP_ADDR: "{{ bootstrap_kube_vip_addr }}" + KUBEAPI_ADDR: "{{ bootstrap_kubeapi_addr }}" CLUSTER_CIDR: "{{ bootstrap_cluster_cidr.split(',')[0] }}" SERVICE_CIDR: "{{ bootstrap_service_cidr.split(',')[0] }}" NODE_CIDR: "{{ bootstrap_node_cidr }}" diff --git a/bootstrap/templates/node.sops.yaml.j2 b/bootstrap/templates/node.sops.yaml.j2 deleted file mode 100644 index e538fd2e10e..00000000000 --- a/bootstrap/templates/node.sops.yaml.j2 +++ /dev/null @@ -1,2 +0,0 @@ ---- -ansible_become_pass: "{{ password }}" diff --git a/bootstrap/vars/addons.sample.yaml b/bootstrap/vars/addons.sample.yaml index 83ab4826e2f..7a0a5f23547 100644 --- a/bootstrap/vars/addons.sample.yaml +++ b/bootstrap/vars/addons.sample.yaml @@ -41,6 +41,7 @@ system_upgrade_controller: # WARNING: Only enable this if you also track the version of k3s in the # ansible configuration files. Running ansible against an already provisioned # cluster with this enabled might cause your cluster to be downgraded. + # Note that if bootstrap_distribution is set to k0s enable: true will be ignored. enabled: false # https://github.com/morphy2k/rss-forwarder diff --git a/bootstrap/vars/config.sample.yaml b/bootstrap/vars/config.sample.yaml index 597f9a57a4e..5718765158a 100644 --- a/bootstrap/vars/config.sample.yaml +++ b/bootstrap/vars/config.sample.yaml @@ -3,6 +3,10 @@ # Bootstrap configuration - config.yaml is gitignored # +# Distribution can either be 'k3s' or 'k0s' +# Note that changing this to k0s will ignore `system_upgrade_controller` in the addons. +bootstrap_distribution: k3s + # Github username (e.g. onedr0p) bootstrap_github_username: # Github repository (e.g. flux-cluster-template) @@ -38,9 +42,13 @@ bootstrap_cloudflare_tunnel_id: # CIDR your nodes are on (e.g. 192.168.1.0/24) bootstrap_node_cidr: -# The IP address to use with kube-vip, choose an available IP in your nodes network that is not being used -# (Optional) Leave this blank if you are deploying a single master node, this will disable kube-vip and use the master node IP instead -bootstrap_kube_vip_addr: +# The IP address of the kubeapi, choose an available IP in your nodes network that is not being used +# (Optional) Leave this blank if you are deploying a single master node, this will disable kube-vip in k3s +# or keepalived in k0s and use the master node IP instead +bootstrap_kubeapi_addr: +# The hostname of the kubeapi, set this if you intend to call the kubeapi by hostname rather than IP +# (Optional) This is currently only utilized in k0s and is added to the kubeapi cert sans +#bootstrap_kubeapi_hostname: # The Load balancer IP for k8s_gateway, choose an available IP in your nodes network that is not being used bootstrap_k8s_gateway_addr: # The Load balancer IP for external ingress, choose an available IP in your nodes network that is not being used @@ -63,23 +71,23 @@ bootstrap_cluster_cidr: 10.42.0.0/16 # (Advanced) For ipv6 use format 10.43.0.0/16,fd78:c889:47fb:e0::/112 bootstrap_service_cidr: 10.43.0.0/16 -# (Advanced) Path for democratic-csi local-hostpath to use for storage on ALL nodes -bootstrap_local_path_provisioner_path: /var/lib/rancher/k3s/local-hostpath +# (Advanced) Path for openebs openebs-hostpath to use for storage on ALL nodes +bootstrap_local_storage_path: /var/openebs/local # Node information bootstrap_nodes: # Use only 1, 3 or more odd master nodes, recommended is 3 master: - # - name: # name of the master node (must match [a-z0-9-]+) + # - name: # name or hostname of the master node (must match [a-z0-9-\.]+) # address: # ip address of the master node # username: # ssh username of the master node - # password: # password of ssh username for the master node - # # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field + # role: # (Optional) k0s only: can be controller+worker (default) or controller + # # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field # ... worker: # set to [] or omit if no workers are needed - # - name: # name of the worker node (must match [a-z0-9-]+) + # - name: # name or hostname of the worker node (must match [a-z0-9-\.]+) # address: # ip address of the worker node # username: # ssh username of the worker node - # password: # password of ssh username for the worker node - # # external_address: # Only use when SSH is not reachable on the IP specified in 'address' field + # # external_address: # (Optional) k3s only: Only use when SSH is not reachable on the IP specified in 'address' field # ... +