From 69797b74681b3a5d35480b0e14075f7d3fc72f11 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 20 Jan 2024 20:15:58 -0500 Subject: [PATCH 1/6] feat: migrate to python for data validation Signed-off-by: Devin Buhl --- .editorconfig | 6 +- .github/tests/config-k0s.yaml | 2 +- .github/tests/config-k3s-ipv4.yaml | 2 +- .github/tests/config-k3s-ipv6.yaml | 2 +- .github/tests/config-k3s-no-kube-vip.yaml | 2 +- .github/tests/config-talos.yaml | 2 +- .github/workflows/e2e.yaml | 28 +-- .taskfiles/Ansible/Taskfile.yaml | 41 ++-- Taskfile.yaml | 28 ++- bootstrap/scripts/loader.py | 26 ++- bootstrap/scripts/py_version_check.py | 9 - bootstrap/scripts/validation.py | 179 +++++++++++++++ bootstrap/tasks/validation/age.yaml | 21 -- bootstrap/tasks/validation/cli.yaml | 26 --- bootstrap/tasks/validation/cloudflare.yaml | 38 ---- bootstrap/tasks/validation/github.yaml | 48 ---- bootstrap/tasks/validation/main.yaml | 6 - bootstrap/tasks/validation/net.yaml | 214 ------------------ bootstrap/tasks/validation/vars.yaml | 48 ---- .../templates/ansible/requirements.txt.j2 | 7 + .../templates/ansible/requirements.yaml.j2 | 4 + bootstrap/validate.yaml | 28 --- requirements.txt | 7 +- 23 files changed, 267 insertions(+), 507 deletions(-) delete mode 100644 bootstrap/scripts/py_version_check.py create mode 100644 bootstrap/scripts/validation.py delete mode 100644 bootstrap/tasks/validation/age.yaml delete mode 100644 bootstrap/tasks/validation/cli.yaml delete mode 100644 bootstrap/tasks/validation/cloudflare.yaml delete mode 100644 bootstrap/tasks/validation/github.yaml delete mode 100644 bootstrap/tasks/validation/main.yaml delete mode 100644 bootstrap/tasks/validation/net.yaml delete mode 100644 bootstrap/tasks/validation/vars.yaml create mode 100644 bootstrap/templates/ansible/requirements.txt.j2 rename requirements.yaml => bootstrap/templates/ansible/requirements.yaml.j2 (72%) delete mode 100644 bootstrap/validate.yaml diff --git a/.editorconfig b/.editorconfig index 547304ee3c3..1de5cf77224 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,10 +9,6 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true -[Makefile] -indent_style = space -indent_size = 4 - -[*.{bash,sh}] +[*.{bash,py,sh}] indent_style = space indent_size = 4 diff --git a/.github/tests/config-k0s.yaml b/.github/tests/config-k0s.yaml index e9f39db2725..95510d197ee 100644 --- a/.github/tests/config-k0s.yaml +++ b/.github/tests/config-k0s.yaml @@ -1,5 +1,5 @@ --- -ci_test: true +skip_tests: true bootstrap_distribution: k0s bootstrap_github_username: onedr0p diff --git a/.github/tests/config-k3s-ipv4.yaml b/.github/tests/config-k3s-ipv4.yaml index 2c07a6e4dc6..ce75ee68b99 100644 --- a/.github/tests/config-k3s-ipv4.yaml +++ b/.github/tests/config-k3s-ipv4.yaml @@ -1,5 +1,5 @@ --- -ci_test: true +skip_tests: true bootstrap_distribution: k3s bootstrap_github_username: onedr0p diff --git a/.github/tests/config-k3s-ipv6.yaml b/.github/tests/config-k3s-ipv6.yaml index a63fd7610bc..0a509d750b0 100644 --- a/.github/tests/config-k3s-ipv6.yaml +++ b/.github/tests/config-k3s-ipv6.yaml @@ -1,5 +1,5 @@ --- -ci_test: true +skip_tests: true bootstrap_distribution: k3s bootstrap_github_username: onedr0p diff --git a/.github/tests/config-k3s-no-kube-vip.yaml b/.github/tests/config-k3s-no-kube-vip.yaml index 83a89d0ccf2..35e19fd0b69 100644 --- a/.github/tests/config-k3s-no-kube-vip.yaml +++ b/.github/tests/config-k3s-no-kube-vip.yaml @@ -1,5 +1,5 @@ --- -ci_test: true +skip_tests: true bootstrap_distribution: k3s bootstrap_github_username: onedr0p diff --git a/.github/tests/config-talos.yaml b/.github/tests/config-talos.yaml index f4eec082cb6..1ee3b64be9d 100644 --- a/.github/tests/config-talos.yaml +++ b/.github/tests/config-talos.yaml @@ -1,5 +1,5 @@ --- -ci_test: true +skip_tests: true bootstrap_distribution: talos bootstrap_github_username: onedr0p diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 5daef153af6..c8706e17dfb 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -77,7 +77,7 @@ jobs: shell: bash run: brew install go-task - - name: Install Brew dependencies + - name: Run Workstation tasks if: ${{ github.event_name == 'pull_request' && steps.cache-homebrew-packages.outputs.cache-hit != 'true' }} shell: bash run: task workstation:brew @@ -86,15 +86,11 @@ jobs: shell: bash run: direnv allow . - - name: Initialize Sops Age key + - name: Run Sops Age key task shell: bash run: task sops:age-keygen - - name: Install Ansible dependencies - shell: bash - run: task ansible:deps force=false - - - name: Generate bootstrap config file + - name: Run init tasks shell: bash run: | task init @@ -109,28 +105,26 @@ jobs: run: | echo "distribution=$(yq '.bootstrap_distribution' ./bootstrap/vars/config.yaml)" >> $GITHUB_OUTPUT - - name: Run configure + - name: Run configure task shell: bash run: task configure --yes - - name: Run Talos configure + - name: Run Talos tasks if: ${{ steps.config-env.outputs.distribution == 'talos' }} shell: bash run: | task talos:gensecret --yes task talos:genconfig - - name: Run Ansible lint - if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }} - shell: bash - run: task ansible:lint - - - name: List Hosts with Ansible + - name: Run Ansible tasks if: ${{ steps.config-env.outputs.distribution == 'k3s' || steps.config-env.outputs.distribution == 'k0s' }} shell: bash - run: task ansible:list + run: | + task ansible:deps force=false + task ansible:lint + task ansible:list - - name: Run repo clean and reset + - name: Run repo clean and reset tasks shell: bash run: | task repository:clean diff --git a/.taskfiles/Ansible/Taskfile.yaml b/.taskfiles/Ansible/Taskfile.yaml index 399181a435d..34184091e9e 100644 --- a/.taskfiles/Ansible/Taskfile.yaml +++ b/.taskfiles/Ansible/Taskfile.yaml @@ -14,17 +14,28 @@ env: vars: ANSIBLE_LINT_FILE: "{{.ANSIBLE_DIR}}/.ansible-lint" ANSIBLE_INVENTORY_FILE: "{{.ANSIBLE_DIR}}/inventory/hosts.yaml" - ANSIBLE_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.yaml" - PIP_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt" + ANSIBLE_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.yaml" + ANSIBLE_PIP_REQUIREMENTS_FILE: "{{.ANSIBLE_DIR}}/requirements.txt" tasks: deps: - desc: Set up Ansible dependencies for the environment + desc: Set up Ansible dependencies cmds: - - task: .setup-virtual-env - vars: - force: '{{.force | default "true"}}' + - task: :setup-virtual-env + - .venv/bin/python3 -m pip install --upgrade --requirement "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" + - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}} + preconditions: + - { msg: "Missing Ansible requirements file", sh: "test -f {{.ANSIBLE_REQUIREMENTS_FILE}}" } + - { msg: "Missing Pip requirements file", sh: "test -f {{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" } + sources: + - "{{.ANSIBLE_REQUIREMENTS_FILE}}" + - "{{.ANSIBLE_PIP_REQUIREMENTS_FILE}}" + generates: + - "{{.VIRTUAL_ENV}}/bin/ansible" + - "{{.VIRTUAL_ENV}}/bin/ansible-galaxy" + vars: + force: '{{.force | default "true"}}' run: desc: Run an Ansible playbook for configuring a cluster @@ -74,24 +85,6 @@ tasks: preconditions: - { msg: "Missing Ansible lint file", sh: "test -f {{.ANSIBLE_LINT_FILE}}" } - .setup-virtual-env: - internal: true - cmds: - - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" - - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel - - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}" - - .venv/bin/ansible-galaxy install --role-file "{{.ANSIBLE_REQUIREMENTS_FILE}}" {{if eq .force "true"}}--force{{end}} - sources: - - "{{.PIP_REQUIREMENTS_FILE}}" - - "{{.ANSIBLE_REQUIREMENTS_FILE}}" - generates: - - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" - preconditions: - - { msg: "Missing Ansible requirements file", sh: "test -f {{.ANSIBLE_REQUIREMENTS_FILE}}" } - - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } - vars: - force: '{{.force | default "true"}}' - .reset: internal: true cmd: rm -rf {{.ANSIBLE_DIR}} diff --git a/Taskfile.yaml b/Taskfile.yaml index ca722a33351..be3ae5fb52d 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -13,7 +13,7 @@ vars: BOOTSTRAP_CONFIG_FILE: "{{.BOOTSTRAP_DIR}}/vars/config.yaml" KUBECONFIG_FILE: "{{.ROOT_DIR}}/kubeconfig" MAKEJINJA_CONFIG_FILE: "{{.ROOT_DIR}}/makejinja.toml" - PYTHON_VERSION_CHECK_FILE: "{{.BOOTSTRAP_DIR}}/scripts/py_version_check.py" + PIP_REQUIREMENTS_FILE: "{{.ROOT_DIR}}/requirements.txt" SOPS_AGE_FILE: "{{.ROOT_DIR}}/age.key" # Binaries PYTHON_BIN: python3 @@ -37,8 +37,9 @@ tasks: default: task -l init: - desc: Initialize files and directories + desc: Initialize virtual env and configuration files cmds: + - task: setup-virtual-env - mkdir -p {{.PRIVATE_DIR}} - cp -n {{.BOOTSTRAP_ADDONS_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_ADDONS_FILE}} - cp -n {{.BOOTSTRAP_CONFIG_FILE | replace ".yaml" ".sample.yaml"}} {{.BOOTSTRAP_CONFIG_FILE}} @@ -50,20 +51,14 @@ tasks: silent: true - cmd: echo {{.BOOTSTRAP_ADDONS_FILE}} silent: true - status: - - test -f {{.BOOTSTRAP_ADDONS_FILE}} - - test -f {{.BOOTSTRAP_CONFIG_FILE}} configure: desc: Configure repository from Ansible vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? cmds: - - task: .pre-validate + # - task: .pre-validate - task: .template - - task: .post-validate - preconditions: - - { msg: "Missing Python version check script", sh: "test -f {{.PYTHON_VERSION_CHECK_FILE}}" } - - { msg: "Python version must be 3.11 or greater", sh: "{{.PYTHON_BIN}} {{.PYTHON_VERSION_CHECK_FILE}}" } + # - task: .post-validate .pre-validate: internal: true @@ -88,3 +83,16 @@ tasks: internal: true cmds: - task: kubernetes:kubeconform + + setup-virtual-env: + desc: Set up virtual environment + cmds: + - "{{.PYTHON_BIN}} -m venv {{.ROOT_DIR}}/.venv" + - .venv/bin/python3 -m pip install --upgrade pip setuptools wheel + - .venv/bin/python3 -m pip install --upgrade --requirement "{{.PIP_REQUIREMENTS_FILE}}" + sources: + - "{{.PIP_REQUIREMENTS_FILE}}" + generates: + - "{{.ROOT_DIR}}/.venv/pyvenv.cfg" + preconditions: + - { msg: "Missing Pip requirements file", sh: "test -f {{.PIP_REQUIREMENTS_FILE}}" } diff --git a/bootstrap/scripts/loader.py b/bootstrap/scripts/loader.py index 7509bbe3431..ad4283629d5 100644 --- a/bootstrap/scripts/loader.py +++ b/bootstrap/scripts/loader.py @@ -1,21 +1,39 @@ -import netaddr, bcrypt +import bcrypt +import netaddr + +import validation def nthhost(value: str, query: int) -> str: value = netaddr.IPNetwork(value) - try: nth = int(query) if value.size > nth: return str(value[nth]) - except ValueError: return False - return value def encrypt(value: str) -> str: return bcrypt.hashpw(value.encode(), bcrypt.gensalt(rounds=10)).decode("ascii") class Loader: + def __init__(self, data): + validation.validate_python_version() + if data.get("skip_tests", False): + return + validation.validate_cli_tools(data) + validation.validate_distribution(data) + validation.validate_github(data) + validation.validate_age(data) + validation.validate_timezone(data) + validation.validate_acme_email(data) + validation.validate_flux_github_webhook_token(data) + validation.validate_cloudflare(data) + validation.validate_host_network(data) + validation.validate_bootstrap_dns_server(data) + validation.validate_cilium_loadbalancer_mode(data) + validation.validate_local_storage_path(data) + validation.validate_cluster_cidrs(data) + def filters(self): return [nthhost, encrypt] diff --git a/bootstrap/scripts/py_version_check.py b/bootstrap/scripts/py_version_check.py deleted file mode 100644 index 3abb2ba5557..00000000000 --- a/bootstrap/scripts/py_version_check.py +++ /dev/null @@ -1,9 +0,0 @@ -import sys - -required_version = (3, 11, 0) - -if sys.version_info >= required_version: - print(f"Python version is greater than or equal to 3.11") -else: - print("Python version is below 3.11. Please upgrade.") - exit(1) diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py new file mode 100644 index 00000000000..6d47ed79149 --- /dev/null +++ b/bootstrap/scripts/validation.py @@ -0,0 +1,179 @@ +from email_validator import validate_email, EmailNotValidError +from functools import wraps +from pathvalidate import is_valid_filepath, ValidationError +from shutil import which +from typing import Callable, Optional +from zoneinfo import available_timezones +import CloudFlare +import netaddr +import re +import requests +import sys + +DISTRIBUTIONS = ["k0s", "k3s", "talos"] +CILIUM_LOADBALANCER_MODES = ["dsr", "snat"] +GLOBAL_CLI_TOOLS = ["age", "cloudflared", "flux", "sops", "jq", "kubeconform", "kustomize"] +TALOS_CLI_TOOLS = ["talosctl", "talhelper"] +K0S_CLI_TOOLS = ["k0sctl"] + +def required(*keys: str): + def wrapper_outter(func: Callable): + @wraps(func) + def wrapper(data: dict, *args, **kwargs) -> None: + for key in keys: + if data.get(key) is None: + raise ValueError(f"Missing required key {key}") + return func(*[data[key] for key in keys], **kwargs) + return wrapper + return wrapper_outter + +def _validate_ip(ip: str) -> None: + try: + netaddr.IPAddress(ip) + except netaddr.core.AddrFormatError as e: + raise ValueError(f"Invalid IP address {ip}") from e + +def _validate_cidr(cidr: str, family: int) -> None: + try: + network = netaddr.IPNetwork(cidr) + if network.version != family: + raise ValueError(f"Invalid CIDR family {network.version}") + except netaddr.core.AddrFormatError as e: + raise ValueError(f"Invalid CIDR {cidr}") from e + +def _validate_distribution(distribution: str) -> None: + if distribution not in DISTRIBUTIONS: + raise ValueError(f"Invalid distribution {distribution}") + return distribution + +def validate_python_version() -> None: + required_version = (3, 11, 0) + if sys.version_info < required_version: + raise ValueError(f"Python version is below 3.11. Please upgrade.") + +@required("bootstrap_distribution") +def validate_cli_tools(distribution: str) -> None: + distro = _validate_distribution(distribution) + for tool in GLOBAL_CLI_TOOLS: + if which(tool) is None: + raise ValueError(f"Missing required CLI tool {tool}") + for tool in TALOS_CLI_TOOLS if distro == "talos" else []: + if which(tool) is None: + raise ValueError(f"Missing required CLI tool {tool}") + for tool in K0S_CLI_TOOLS if distro == "k0s" else []: + if which(tool) is None: + raise ValueError(f"Missing required CLI tool {tool}") + +@required("bootstrap_distribution") +def validate_distribution(distribution: str) -> None: + _validate_distribution(distribution) + +@required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_github_repository_branch") +def validate_github(username: str, repository: str, branch: str, **_) -> None: + try: + request = requests.get(f"https://api.github.com/repos/{username}/{repository}/branches/{branch}") + if request.status_code != 200: + raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") + except requests.exceptions.RequestException as e: + raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") from e + +@required("bootstrap_age_public_key") +def validate_age(key: str) -> None: + if re.match(r"^age1[a-z0-9]{0,58}$", key) is None: + raise ValueError(f"Invalid Age public key {key}") + +@required("bootstrap_timezone") +def validate_timezone(timezone: str) -> None: + if not timezone in available_timezones(): + raise ValueError(f"Invalid timezone {timezone}") + +@required("bootstrap_ipv6_enabled", "bootstrap_cluster_cidr", "bootstrap_service_cidr") +def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: str) -> None: + if not isinstance(ipv6_enabled, bool): + raise ValueError(f"Invalid IPv6 enabled {ipv6_enabled}") + + if cluster_cidr == service_cidr: + raise ValueError(f"Cluster CIDR {cluster_cidr} is the same as service CIDR {service_cidr}") + + if ipv6_enabled: + if len(cluster_cidr.split(",")) != 2: + raise ValueError(f"Invalid cluster CIDR {cluster_cidr}") + if len(service_cidr.split(",")) != 2: + raise ValueError(f"Invalid service CIDR {service_cidr}") + cluster_ipv4,cluster_ipv6 = cluster_cidr.split(",") + _validate_cidr(cluster_ipv4, 4) + _validate_cidr(cluster_ipv6, 6) + service_ipv4,service_ipv6 = service_cidr.split(",") + _validate_cidr(service_ipv4, 4) + _validate_cidr(service_ipv6, 6) + return + + if len(cluster_cidr.split(",")) != 1: + raise ValueError(f"Invalid cluster CIDR {cluster_cidr}") + if len(service_cidr.split(",")) != 1: + raise ValueError(f"Invalid service CIDR {service_cidr}") + + _validate_cidr(cluster_cidr, 4) + _validate_cidr(service_cidr, 4) + +@required("bootstrap_acme_email", "bootstrap_acme_production_enabled") +def validate_acme_email(email: str, acme_production: bool) -> None: + try: + validate_email(email) + except EmailNotValidError as e: + raise ValueError(f"Invalid ACME email {email}") from e + +@required("bootstrap_flux_github_webhook_token") +def validate_flux_github_webhook_token(token: str) -> None: + if re.match(r"^[a-zA-Z0-9]+$", token) is None: + raise ValueError(f"Invalid Flux GitHub webhook token {token}") + +@required("bootstrap_cloudflare_domain", "bootstrap_cloudflare_token", "bootstrap_cloudflare_account_tag", "bootstrap_cloudflare_tunnel_secret", "bootstrap_cloudflare_tunnel_id") +def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret: str, tunnel_id: str) -> None: + try: + cf = CloudFlare.CloudFlare(token=token) + zones = cf.zones.get(params={"name": domain}) + if len(zones) == 0: + raise ValueError(f"Cloudflare domain {domain} not found or token does not have access to it") + except CloudFlare.exceptions.CloudFlareAPIError as e: + raise ValueError(f"Cloudflare domain {domain} not found or token does not have access to it") from e + +@required("bootstrap_cilium_loadbalancer_mode") +def validate_cilium_loadbalancer_mode(mode: str) -> None: + if mode not in CILIUM_LOADBALANCER_MODES: + raise ValueError(f"Invalid Cilium load balancer mode {mode}") + +@required("bootstrap_local_storage_path") +def validate_local_storage_path(path: str) -> None: + try: + if not is_valid_filepath(path, platform="linux"): + raise ValueError(f"Invalid local storage path {path}") + except ValidationError as e: + raise ValueError(f"Invalid local storage path {path}") from e + +@required("bootstrap_dns_server") +def validate_bootstrap_dns_server(dns_server: str) -> None: + _validate_ip(dns_server) + +@required("bootstrap_node_cidr", "bootstrap_kube_api_addr", "bootstrap_k8s_gateway_addr", "bootstrap_external_ingress_addr", "bootstrap_internal_ingress_addr") +def validate_host_network(node_cidr: str, api_addr: str, gateway_addr: str, external_ingress_addr: str, internal_ingress_addr: str) -> None: + _validate_cidr(node_cidr, 4) + _validate_ip(api_addr) + _validate_ip(gateway_addr) + _validate_ip(external_ingress_addr) + _validate_ip(internal_ingress_addr) + + addrs = [api_addr, gateway_addr, external_ingress_addr, internal_ingress_addr] + unique = set(addrs) + if len(addrs) != len(unique): + raise ValueError(f"{api_addr} and {gateway_addr} and {external_ingress_addr} and {internal_ingress_addr} are not unique") + + node_cidr = netaddr.IPNetwork(node_cidr) + if netaddr.IPAddress(api_addr) not in node_cidr: + raise ValueError(f"Kubernetes API address {api_addr} is not in the node CIDR {node_cidr}") + if netaddr.IPAddress(gateway_addr) not in node_cidr: + raise ValueError(f"Kubernetes gateway address {gateway_addr} is not in the node CIDR {node_cidr}") + if netaddr.IPAddress(external_ingress_addr) not in node_cidr: + raise ValueError(f"Kubernetes external ingress address {external_ingress_addr} is not in the node CIDR {node_cidr}") + if netaddr.IPAddress(internal_ingress_addr) not in node_cidr: + raise ValueError(f"Kubernetes internal ingress address {internal_ingress_addr} is not in the node CIDR {node_cidr}") diff --git a/bootstrap/tasks/validation/age.yaml b/bootstrap/tasks/validation/age.yaml deleted file mode 100644 index 81d6ae8e76a..00000000000 --- a/bootstrap/tasks/validation/age.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -- name: Query age key file - ansible.builtin.stat: - path: "{{ repository_path }}/age.key" - register: result - -- name: Check if age key file exists - ansible.builtin.assert: - that: result.stat.exists - success_msg: Age file {{ repository_path }}/age.key exists - fail_msg: Age file {{ repository_path }}/age.key does not exist - -- name: Query age key file contents - ansible.builtin.set_fact: - age_contents: "{{ lookup('file', repository_path + '/age.key') }}" - -- name: Check if age public keys match - ansible.builtin.assert: - that: bootstrap_age_public_key in age_contents - success_msg: Age public key {{ bootstrap_age_public_key }} exists - fail_msg: Age public key {{ bootstrap_age_public_key }} does not exist diff --git a/bootstrap/tasks/validation/cli.yaml b/bootstrap/tasks/validation/cli.yaml deleted file mode 100644 index 3147300baac..00000000000 --- a/bootstrap/tasks/validation/cli.yaml +++ /dev/null @@ -1,26 +0,0 @@ ---- -- name: Check if required CLI tools are present - ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 - loop: ["age", "cloudflared", "flux", "sops", "jq", "kubeconform", "kustomize"] - changed_when: false - check_mode: false - register: result - failed_when: result.rc != 0 and result.rc != 127 - -- name: Check if required CLI tools are present for k0s - when: bootstrap_distribution == "k0s" - ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 - loop: ["k0sctl"] - changed_when: false - check_mode: false - register: result - failed_when: result.rc != 0 and result.rc != 127 - -- name: Check if required CLI tools are present for talos - when: bootstrap_distribution == "talos" - ansible.builtin.shell: command -v {{ item }} >/dev/null 2>&1 - loop: ["talhelper", "talosctl"] - changed_when: false - check_mode: false - register: result - failed_when: result.rc != 0 and result.rc != 127 diff --git a/bootstrap/tasks/validation/cloudflare.yaml b/bootstrap/tasks/validation/cloudflare.yaml deleted file mode 100644 index 19ced4be374..00000000000 --- a/bootstrap/tasks/validation/cloudflare.yaml +++ /dev/null @@ -1,38 +0,0 @@ ---- -- name: Query Cloudflare zone - when: not ci_test|default(false) - ansible.builtin.uri: - url: https://api.cloudflare.com/client/v4/zones?name={{ bootstrap_cloudflare_domain }}&status=active - headers: - Authorization: Bearer {{ bootstrap_cloudflare_token }} - Content-Type: application/json - timeout: 5 - return_content: true - body_format: json - register: result - -- name: Check if Cloudflare zone exists - when: not ci_test|default(false) - ansible.builtin.assert: - that: result.json.success is true - success_msg: Cloudflare zone {{ bootstrap_cloudflare_domain }} exists - fail_msg: Cloudflare zone {{ bootstrap_cloudflare_domain }} does not exist - -- name: Query Cloudflared tunnel - when: not ci_test|default(false) - ansible.builtin.uri: - url: https://api.cloudflare.com/client/v4/accounts/{{ bootstrap_cloudflare_account_tag }}/cfd_tunnel/{{ bootstrap_cloudflare_tunnel_id }} - headers: - Authorization: Bearer {{ bootstrap_cloudflare_token }} - Content-Type: application/json - timeout: 5 - return_content: true - body_format: json - register: result - -- name: Check if Cloudflared tunnel exists - when: not ci_test|default(false) - ansible.builtin.assert: - that: result.json.success is true - success_msg: Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} exists - fail_msg: Cloudflared tunnel {{ bootstrap_cloudflare_tunnel_id }} does not exist diff --git a/bootstrap/tasks/validation/github.yaml b/bootstrap/tasks/validation/github.yaml deleted file mode 100644 index 1942ac022ec..00000000000 --- a/bootstrap/tasks/validation/github.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Query Github username - when: not ci_test|default(false) - ansible.builtin.uri: - url: https://api.github.com/users/{{ bootstrap_github_username }} - timeout: 5 - return_content: true - body_format: json - register: result - -- name: Check if username exists - when: not ci_test|default(false) - ansible.builtin.assert: - that: result.json.login == bootstrap_github_username - success_msg: Github user {{ bootstrap_github_username }} exists - fail_msg: Github user {{ bootstrap_github_username }} does not exist - -- name: Query Github repo - when: (not ci_test|default(false)) and (not bootstrap_private_github_repo|default(false)) - ansible.builtin.uri: - url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} - timeout: 5 - return_content: true - body_format: json - register: result - -- name: Check if repo exists - when: (not ci_test|default(false)) and (not bootstrap_private_github_repo|default(false)) - ansible.builtin.assert: - that: result.json.full_name == bootstrap_github_username + '/' + bootstrap_github_repository_name - success_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} exists - fail_msg: Github repo {{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }} does not exist - -- name: Query Github repo branch - when: (not ci_test|default(false)) and (not bootstrap_private_github_repo|default(false)) - ansible.builtin.uri: - url: https://api.github.com/repos/{{ bootstrap_github_username }}/{{ bootstrap_github_repository_name }}/branches/{{ bootstrap_github_repository_branch|default('main', true) }} - timeout: 5 - return_content: true - body_format: json - register: result - -- name: Check if repo branch exists - when: (not ci_test|default(false)) and (not bootstrap_private_github_repo|default(false)) - ansible.builtin.assert: - that: result.json.name == bootstrap_github_repository_branch|default('main', true) - success_msg: Github repo branch {{ bootstrap_github_repository_branch|default('main', true) }} exists - fail_msg: Github repo branch {{ bootstrap_github_repository_branch|default('main', true) }} does not exist diff --git a/bootstrap/tasks/validation/main.yaml b/bootstrap/tasks/validation/main.yaml deleted file mode 100644 index 38b0db1975e..00000000000 --- a/bootstrap/tasks/validation/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Verify configuration - ansible.builtin.include_tasks: "{{ task }}.yaml" - loop: [vars, age, cli, net, cloudflare, github] - loop_control: - loop_var: task diff --git a/bootstrap/tasks/validation/net.yaml b/bootstrap/tasks/validation/net.yaml deleted file mode 100644 index c845e77d470..00000000000 --- a/bootstrap/tasks/validation/net.yaml +++ /dev/null @@ -1,214 +0,0 @@ ---- -- name: Check if master node count is odd - ansible.builtin.assert: - that: - - bootstrap_nodes.master | length > 0 - - bootstrap_nodes.master | length is odd - success_msg: Master node count {{ bootstrap_nodes.master | length }} is correct. - fail_msg: Master node count {{ bootstrap_nodes.master | length }} is not greater than 0 or is not odd. - -- name: Check if node CIDR is ipv4 - ansible.builtin.assert: - that: bootstrap_node_cidr is ansible.utils.ipv4 - success_msg: Node CIDR {{ bootstrap_node_cidr }} is valid. - fail_msg: Node CIDR {{ bootstrap_node_cidr }} is invalid. - -- name: Check if cluster CIDR is ipv4 OR ipv6 - when: not bootstrap_ipv6_enabled|default(false) - ansible.builtin.assert: - that: bootstrap_cluster_cidr is ansible.utils.ipv4 or bootstrap_cluster_cidr is ansible.utils.ipv6 - success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid. - fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid. - -- name: Check if service CIDR is ipv4 OR ipv6 - when: not bootstrap_ipv6_enabled|default(false) - ansible.builtin.assert: - that: bootstrap_service_cidr is ansible.utils.ipv4 or bootstrap_service_cidr is ansible.utils.ipv6 - success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid. - fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid. - -- name: Check if cluster CIDR is ipv4 AND ipv6 - when: bootstrap_ipv6_enabled|default(false) - ansible.builtin.assert: - that: > - ( - bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv4 or - bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv4 - ) and ( - bootstrap_cluster_cidr.split(',')[1] is ansible.utils.ipv6 or - bootstrap_cluster_cidr.split(',')[0] is ansible.utils.ipv6 - ) - success_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is valid. - fail_msg: Cluster CIDR {{ bootstrap_cluster_cidr }} is invalid. - -- name: Check if service CIDR is ipv4 AND ipv6 - when: bootstrap_ipv6_enabled|default(false) - ansible.builtin.assert: - that: > - ( - bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv4 or - bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv4 - ) and ( - bootstrap_service_cidr.split(',')[1] is ansible.utils.ipv6 or - bootstrap_service_cidr.split(',')[0] is ansible.utils.ipv6 - ) - success_msg: Service CIDR {{ bootstrap_service_cidr }} is valid. - fail_msg: Service CIDR {{ bootstrap_service_cidr }} is invalid. - -- name: Check if k8s_gateway is ipv4 - ansible.builtin.assert: - that: bootstrap_k8s_gateway_addr is ansible.utils.ipv4 - success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is valid. - fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is invalid. - -- name: Check if k8s_gateway is in node CIDR - ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_k8s_gateway_addr) - success_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: k8s_gateway address {{ bootstrap_k8s_gateway_addr }} is not within {{ bootstrap_node_cidr }}. - -- name: Check if internal ingress is ipv4 - ansible.builtin.assert: - that: bootstrap_internal_ingress_addr is ansible.utils.ipv4 - success_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is valid. - fail_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is invalid. - -- name: Check if internal ingress is in node CIDR - ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_internal_ingress_addr) - success_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: internal ingress address {{ bootstrap_internal_ingress_addr }} is not within {{ bootstrap_node_cidr }}. - -- name: Check if external ingress is ipv4 - ansible.builtin.assert: - that: bootstrap_external_ingress_addr is ansible.utils.ipv4 - success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is valid. - fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is invalid. - -- name: Check external ingress is in node CIDR - ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_external_ingress_addr) - success_msg: external ingress address {{ bootstrap_external_ingress_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: external ingress address {{ bootstrap_external_ingress_addr }} is not within {{ bootstrap_node_cidr }}. - -- name: Check if Kube API address is ipv4 - ansible.builtin.assert: - that: bootstrap_kube_api_addr is ansible.utils.ipv4 - success_msg: Kube API address {{ bootstrap_kube_api_addr }} is valid. - fail_msg: Kube API address {{ bootstrap_kube_api_addr }} is invalid. - -- name: Check if Kube API address is in node CIDR - ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(bootstrap_kube_api_addr) - success_msg: Kube API address {{ bootstrap_kube_api_addr }} is within {{ bootstrap_node_cidr }}. - fail_msg: Kube API address {{ bootstrap_kube_api_addr }} is not within {{ bootstrap_node_cidr }}. - -- name: Check if DNS server is ipv4 - ansible.builtin.assert: - that: bootstrap_dns_server is ansible.utils.ipv4 - success_msg: DNS server {{ bootstrap_dns_server }} is valid. - fail_msg: DNS server {{ bootstrap_dns_server }} is invalid. - -- name: Check if all IP addresses are unique - ansible.builtin.assert: - that: > - [ - bootstrap_k8s_gateway_addr, - bootstrap_external_ingress_addr, - bootstrap_internal_ingress_addr, - bootstrap_kube_api_addr - ] | unique | length == 4 - success_msg: All IP addresses are unique. - fail_msg: All IP addresses are not unique. - -- name: Check if nodes are not the same IPs as k8s_gateway or ingress external/internal - when: not bootstrap_kube_vip_enabled|default(true) - ansible.builtin.assert: - that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr) - success_msg: Node address {{ item.address }} is different than k8s_gateway or ingress-nginx. - fail_msg: Node address {{ item.address }} is not different than k8s_gateway or ingress-nginx. - quiet: true - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" - -- name: Check if nodes are not the same IPs as k8s_gateway, ingress external/internal or Kube API address - when: (bootstrap_distribution == "k3s") and (bootstrap_kube_vip_enabled|default(true)) - ansible.builtin.assert: - that: item.address not in (bootstrap_k8s_gateway_addr, bootstrap_external_ingress_addr, bootstrap_internal_ingress_addr, bootstrap_kube_api_addr) - success_msg: Node address {{ item.address }} is different than k8s_gateway, ingress-nginx or Kube API. - fail_msg: Node address {{ item.address }} is not different than k8s_gateway, ingress-nginx or Kube API. - quiet: true - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" - -- name: Check if node addresses are ipv4 - ansible.builtin.assert: - that: item.address is ansible.utils.ipv4 - success_msg: Node address {{ item.address }} is valid. - fail_msg: Node address {{ item.address }} is invalid. - quiet: true - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" - -- name: Check if node addresses are in node CIDR - ansible.builtin.assert: - that: bootstrap_node_cidr | ansible.utils.network_in_usable(item.address) - success_msg: Node address {{ item.address }} is within {{ bootstrap_node_cidr }}. - fail_msg: Node address {{ item.address }} is not within {{ bootstrap_node_cidr }}. - quiet: true - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" - -- name: Check if node IP addresses are unique - ansible.builtin.assert: - that: > - ( - (bootstrap_nodes.master + bootstrap_nodes.worker|default([])) | map(attribute='address') | list - ) | unique | length - == - ( - (bootstrap_nodes.master + bootstrap_nodes.worker|default([])) | map(attribute='address') | list - ) | length - success_msg: All node IP addresses are unique. - fail_msg: All node IP addresses are not unique. - quiet: true - -- name: Check if node names are unique - ansible.builtin.assert: - that: > - ( - (bootstrap_nodes.master + bootstrap_nodes.worker|default([])) | map(attribute='name') | list - ) | unique | length - == - ( - (bootstrap_nodes.master + bootstrap_nodes.worker|default([])) | map(attribute='name') | list - ) | length - success_msg: All node names are unique. - fail_msg: All node names are not unique. - quiet: true - -- name: Check if SSH ports are reachable - when: (not ci_test|default(false)) and (not bootstrap_distribution == "talos") - ansible.builtin.wait_for: - host: "{{ item.address }}" - port: 22 - timeout: 10 - connection: local - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" - -- name: Check if Talos ports are reachable - when: (not ci_test|default(false)) and (bootstrap_distribution == "talos") - ansible.builtin.wait_for: - host: "{{ item.address }}" - port: 50000 - timeout: 10 - connection: local - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.address }}" diff --git a/bootstrap/tasks/validation/vars.yaml b/bootstrap/tasks/validation/vars.yaml deleted file mode 100644 index 32834154271..00000000000 --- a/bootstrap/tasks/validation/vars.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -- name: Check if required bootstrap vars are set - ansible.builtin.assert: - that: - - item in vars - - vars[item] != None - success_msg: Required bootstrap var {{ item }} exists and is defined - fail_msg: Required bootstrap var {{ item }} does not exists or is not defined - loop: - - bootstrap_acme_email - - bootstrap_age_public_key - - bootstrap_cilium_loadbalancer_mode - - bootstrap_cloudflare_account_tag - - bootstrap_cloudflare_domain - - bootstrap_cloudflare_token - - bootstrap_cloudflare_tunnel_id - - bootstrap_cloudflare_tunnel_secret - - bootstrap_cluster_cidr - - bootstrap_distribution - - bootstrap_dns_server - - bootstrap_external_ingress_addr - - bootstrap_flux_github_webhook_token - - bootstrap_github_repository_branch - - bootstrap_github_repository_name - - bootstrap_github_username - - bootstrap_internal_ingress_addr - - bootstrap_ipv6_enabled - - bootstrap_k8s_gateway_addr - - bootstrap_kube_api_addr - - bootstrap_local_storage_path - - bootstrap_node_cidr - - bootstrap_service_cidr - - bootstrap_timezone - -- name: Check if bootstrap distribution is valid - ansible.builtin.assert: - that: bootstrap_distribution in ['k0s', 'k3s', 'talos'] - success_msg: Distribution {{ bootstrap_distribution }} is valid - fail_msg: Distribution {{ bootstrap_distribution }} is not valid - -- name: Check if bootstrap node names are valid - ansible.builtin.assert: - that: item.name is match('^[a-z0-9-\.]+$') - success_msg: Node name {{ item.name }} is valid - fail_msg: Node name {{ item.name }} is not valid - loop: "{{ bootstrap_nodes.master + bootstrap_nodes.worker|default([]) }}" - loop_control: - label: "{{ item.name }}" diff --git a/bootstrap/templates/ansible/requirements.txt.j2 b/bootstrap/templates/ansible/requirements.txt.j2 new file mode 100644 index 00000000000..3fcc527463c --- /dev/null +++ b/bootstrap/templates/ansible/requirements.txt.j2 @@ -0,0 +1,7 @@ +#% if bootstrap_distribution in ['k3s', 'k0s'] %# +# Ansible +ansible-lint==6.22.2 +ansible==9.1.0 +jmespath==1.0.1 +openshift==0.13.2 +#% endif %# diff --git a/requirements.yaml b/bootstrap/templates/ansible/requirements.yaml.j2 similarity index 72% rename from requirements.yaml rename to bootstrap/templates/ansible/requirements.yaml.j2 index 50ef6d72e64..4ca4646fd40 100644 --- a/requirements.yaml +++ b/bootstrap/templates/ansible/requirements.yaml.j2 @@ -1,3 +1,4 @@ +#% if bootstrap_distribution in ['k3s', 'k0s'] %# --- collections: - name: ansible.posix @@ -8,7 +9,10 @@ collections: version: 8.2.0 - name: kubernetes.core version: 3.0.0 +#% if bootstrap_distribution == 'k3s' %# roles: - name: xanmanning.k3s src: https://github.com/PyratLabs/ansible-role-k3s version: v3.4.3 +#% endif %# +#% endif %# diff --git a/bootstrap/validate.yaml b/bootstrap/validate.yaml deleted file mode 100644 index c5929524c0f..00000000000 --- a/bootstrap/validate.yaml +++ /dev/null @@ -1,28 +0,0 @@ ---- -- name: Cluster Installation - hosts: localhost - connection: local - gather_facts: false - vars_files: - - vars/config.yaml - - vars/addons.yaml - tasks: - - name: Get absolute path to this Git repository # noqa: command-instead-of-module - ansible.builtin.command: git rev-parse --show-toplevel - changed_when: false - check_mode: false - register: repository - failed_when: repository.rc != 0 - - - name: Set facts - ansible.builtin.set_fact: - repository_path: "{{ repository.stdout }}" - - - name: Override Kube API address when there is a single master node and no address is defined - when: bootstrap_nodes.master | length == 1 and not bootstrap_kube_api_addr - ansible.builtin.set_fact: - bootstrap_kube_vip_enabled: false - bootstrap_kube_api_addr: "{{ bootstrap_nodes.master[0].address }}" - - - name: Verify configuration - ansible.builtin.include_tasks: tasks/validation/main.yaml diff --git a/requirements.txt b/requirements.txt index f7b575c3201..d6a8c68db18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ -ansible-lint==6.22.2 -ansible==9.1.0 bcrypt==4.1.2 # https://github.com/pyca/bcrypt/issues/684 -jmespath==1.0.1 +cloudflare==2.16.0 +email-validator==2.1.0 makejinja==2.3.5 netaddr==0.10.1 -openshift==0.13.2 passlib==1.7.4 +pathvalidate==3.2.0 From 31a6d8f4a1474e01a171a53f634416de88162c01 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 20 Jan 2024 20:56:40 -0500 Subject: [PATCH 2/6] fix: address PR comments Signed-off-by: Devin Buhl --- Taskfile.yaml | 2 +- bootstrap/scripts/validation.py | 16 ++++++++-------- .../apps/kube-system/kustomization.yaml.j2 | 2 +- .../kube-system/spegel/app/helmrelease.yaml.j2 | 2 +- .../kube-system/spegel/app/kustomization.yaml.j2 | 2 +- .../apps/kube-system/spegel/ks.yaml.j2 | 2 +- .../templates/kubernetes/k0s/k0sctl.yaml.j2 | 2 +- .../k0s/resources/containerd/spegel.toml.j2 | 2 +- .../templates/kubernetes/talos/talconfig.yaml.j2 | 2 +- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index be3ae5fb52d..2f46cb30108 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -73,7 +73,7 @@ tasks: .template: internal: true cmds: - - ./.venv/bin/makejinja + - ./.venv/bin/makejinja --undefined chainable - task: sops:encrypt preconditions: - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index 6d47ed79149..cb858bbd3d0 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -55,13 +55,13 @@ def validate_python_version() -> None: def validate_cli_tools(distribution: str) -> None: distro = _validate_distribution(distribution) for tool in GLOBAL_CLI_TOOLS: - if which(tool) is None: + if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") for tool in TALOS_CLI_TOOLS if distro == "talos" else []: - if which(tool) is None: + if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") for tool in K0S_CLI_TOOLS if distro == "k0s" else []: - if which(tool) is None: + if not which(tool): raise ValueError(f"Missing required CLI tool {tool}") @required("bootstrap_distribution") @@ -79,7 +79,7 @@ def validate_github(username: str, repository: str, branch: str, **_) -> None: @required("bootstrap_age_public_key") def validate_age(key: str) -> None: - if re.match(r"^age1[a-z0-9]{0,58}$", key) is None: + if not re.match(r"^age1[a-z0-9]{0,58}$", key): raise ValueError(f"Invalid Age public key {key}") @required("bootstrap_timezone") @@ -100,10 +100,10 @@ def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: raise ValueError(f"Invalid cluster CIDR {cluster_cidr}") if len(service_cidr.split(",")) != 2: raise ValueError(f"Invalid service CIDR {service_cidr}") - cluster_ipv4,cluster_ipv6 = cluster_cidr.split(",") + cluster_ipv4, cluster_ipv6 = cluster_cidr.split(",") _validate_cidr(cluster_ipv4, 4) _validate_cidr(cluster_ipv6, 6) - service_ipv4,service_ipv6 = service_cidr.split(",") + service_ipv4, service_ipv6 = service_cidr.split(",") _validate_cidr(service_ipv4, 4) _validate_cidr(service_ipv6, 6) return @@ -125,7 +125,7 @@ def validate_acme_email(email: str, acme_production: bool) -> None: @required("bootstrap_flux_github_webhook_token") def validate_flux_github_webhook_token(token: str) -> None: - if re.match(r"^[a-zA-Z0-9]+$", token) is None: + if not re.match(r"^[a-zA-Z0-9]+$", token): raise ValueError(f"Invalid Flux GitHub webhook token {token}") @required("bootstrap_cloudflare_domain", "bootstrap_cloudflare_token", "bootstrap_cloudflare_account_tag", "bootstrap_cloudflare_tunnel_secret", "bootstrap_cloudflare_tunnel_id") @@ -133,7 +133,7 @@ def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret try: cf = CloudFlare.CloudFlare(token=token) zones = cf.zones.get(params={"name": domain}) - if len(zones) == 0: + if not zones: raise ValueError(f"Cloudflare domain {domain} not found or token does not have access to it") except CloudFlare.exceptions.CloudFlareAPIError as e: raise ValueError(f"Cloudflare domain {domain} not found or token does not have access to it") from e diff --git a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 index 35a7454e583..f57d4645ad3 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/kustomization.yaml.j2 @@ -8,6 +8,6 @@ resources: - ./kubelet-csr-approver/ks.yaml #% endif %# - ./metrics-server/ks.yaml - #% if bootstrap_distribution in ['k0s', 'talos'] and spegel|default({}) and spegel.enabled|default(false) %# + #% if bootstrap_distribution in ['k0s', 'talos'] and spegel.enabled %# - ./spegel/ks.yaml #% endif %# diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 index 5cc8f200398..15e1021f404 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/helmrelease.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k0s', 'talos'] and spegel|default({}) and spegel.enabled|default(false) %# +#% if bootstrap_distribution in ['k0s', 'talos'] and spegel.enabled %# --- apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 index c64ca1b1bb4..2f07a61f6cf 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/app/kustomization.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k0s', 'talos'] and spegel|default({}) and spegel.enabled|default(false) %# +#% if bootstrap_distribution in ['k0s', 'talos'] and spegel.enabled %# --- apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 index 8a956eacb4b..0f15d88d036 100644 --- a/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 +++ b/bootstrap/templates/kubernetes/apps/kube-system/spegel/ks.yaml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution in ['k0s', 'talos'] and spegel|default({}) and spegel.enabled|default(false) %# +#% if bootstrap_distribution in ['k0s', 'talos'] and spegel.enabled %# --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 index 1d4d656f6d5..f3bd2ae0cbe 100644 --- a/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 +++ b/bootstrap/templates/kubernetes/k0s/k0sctl.yaml.j2 @@ -37,7 +37,7 @@ spec: before: - sudo bash /home/#{ item.username }#/k0s/hooks/apply-system.sh - sudo mv /home/#{ item.username }#/k0s/containerd/unprivileged-ports.toml /etc/k0s/containerd.d/unprivileged-ports.toml - #% if spegel|default({}) and spegel.enabled|default(false) %# + #% if spegel.enabled %# - sudo mv /home/#{ item.username }#/k0s/containerd/spegel.toml /etc/k0s/containerd.d/spegel.toml #% endif %# reset: diff --git a/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 index ab87567571e..73d0bdf0f56 100644 --- a/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 +++ b/bootstrap/templates/kubernetes/k0s/resources/containerd/spegel.toml.j2 @@ -1,4 +1,4 @@ -#% if bootstrap_distribution == 'k0s' and spegel|default({}) and spegel.enabled|default(false) %# +#% if bootstrap_distribution == 'k0s' and spegel.enabled %# [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/var/lib/k0s/containerd/certs.d" [plugins."io.containerd.grpc.v1.cri".containerd] diff --git a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 index 28d6435b39f..5586ae22055 100644 --- a/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 +++ b/bootstrap/templates/kubernetes/talos/talconfig.yaml.j2 @@ -77,7 +77,7 @@ controlPlane: [plugins."io.containerd.grpc.v1.cri"] enable_unprivileged_ports = true enable_unprivileged_icmp = true - #% if spegel|default({}) and spegel.enabled|default(false) %# + #% if spegel.enabled %# [plugins."io.containerd.grpc.v1.cri".containerd] discard_unpacked_layers = false [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] From f54cfb077c29fe15faa8fc0ad20159bbb04d3df9 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 20 Jan 2024 21:01:21 -0500 Subject: [PATCH 3/6] fix: add unused kwargs to validate functions Signed-off-by: Devin Buhl --- bootstrap/scripts/loader.py | 2 +- bootstrap/scripts/validation.py | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bootstrap/scripts/loader.py b/bootstrap/scripts/loader.py index ad4283629d5..a47f29dd42d 100644 --- a/bootstrap/scripts/loader.py +++ b/bootstrap/scripts/loader.py @@ -18,9 +18,9 @@ def encrypt(value: str) -> str: class Loader: def __init__(self, data): - validation.validate_python_version() if data.get("skip_tests", False): return + validation.validate_python_version() validation.validate_cli_tools(data) validation.validate_distribution(data) validation.validate_github(data) diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index cb858bbd3d0..0f5969b03ed 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -52,7 +52,7 @@ def validate_python_version() -> None: raise ValueError(f"Python version is below 3.11. Please upgrade.") @required("bootstrap_distribution") -def validate_cli_tools(distribution: str) -> None: +def validate_cli_tools(distribution: str, **_) -> None: distro = _validate_distribution(distribution) for tool in GLOBAL_CLI_TOOLS: if not which(tool): @@ -65,7 +65,7 @@ def validate_cli_tools(distribution: str) -> None: raise ValueError(f"Missing required CLI tool {tool}") @required("bootstrap_distribution") -def validate_distribution(distribution: str) -> None: +def validate_distribution(distribution: str, **_) -> None: _validate_distribution(distribution) @required("bootstrap_github_username", "bootstrap_github_repository_name", "bootstrap_github_repository_branch") @@ -78,17 +78,17 @@ def validate_github(username: str, repository: str, branch: str, **_) -> None: raise ValueError(f"GitHub repository {username}/{repository} branch {branch} not found") from e @required("bootstrap_age_public_key") -def validate_age(key: str) -> None: +def validate_age(key: str, **_) -> None: if not re.match(r"^age1[a-z0-9]{0,58}$", key): raise ValueError(f"Invalid Age public key {key}") @required("bootstrap_timezone") -def validate_timezone(timezone: str) -> None: +def validate_timezone(timezone: str, **_) -> None: if not timezone in available_timezones(): raise ValueError(f"Invalid timezone {timezone}") @required("bootstrap_ipv6_enabled", "bootstrap_cluster_cidr", "bootstrap_service_cidr") -def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: str) -> None: +def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: str, **_) -> None: if not isinstance(ipv6_enabled, bool): raise ValueError(f"Invalid IPv6 enabled {ipv6_enabled}") @@ -117,19 +117,19 @@ def validate_cluster_cidrs(ipv6_enabled: bool, cluster_cidr: str, service_cidr: _validate_cidr(service_cidr, 4) @required("bootstrap_acme_email", "bootstrap_acme_production_enabled") -def validate_acme_email(email: str, acme_production: bool) -> None: +def validate_acme_email(email: str, acme_production: bool, **_) -> None: try: validate_email(email) except EmailNotValidError as e: raise ValueError(f"Invalid ACME email {email}") from e @required("bootstrap_flux_github_webhook_token") -def validate_flux_github_webhook_token(token: str) -> None: +def validate_flux_github_webhook_token(token: str, **_) -> None: if not re.match(r"^[a-zA-Z0-9]+$", token): raise ValueError(f"Invalid Flux GitHub webhook token {token}") @required("bootstrap_cloudflare_domain", "bootstrap_cloudflare_token", "bootstrap_cloudflare_account_tag", "bootstrap_cloudflare_tunnel_secret", "bootstrap_cloudflare_tunnel_id") -def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret: str, tunnel_id: str) -> None: +def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret: str, tunnel_id: str, **_) -> None: try: cf = CloudFlare.CloudFlare(token=token) zones = cf.zones.get(params={"name": domain}) @@ -139,12 +139,12 @@ def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret raise ValueError(f"Cloudflare domain {domain} not found or token does not have access to it") from e @required("bootstrap_cilium_loadbalancer_mode") -def validate_cilium_loadbalancer_mode(mode: str) -> None: +def validate_cilium_loadbalancer_mode(mode: str, **_) -> None: if mode not in CILIUM_LOADBALANCER_MODES: raise ValueError(f"Invalid Cilium load balancer mode {mode}") @required("bootstrap_local_storage_path") -def validate_local_storage_path(path: str) -> None: +def validate_local_storage_path(path: str, **_) -> None: try: if not is_valid_filepath(path, platform="linux"): raise ValueError(f"Invalid local storage path {path}") @@ -152,11 +152,11 @@ def validate_local_storage_path(path: str) -> None: raise ValueError(f"Invalid local storage path {path}") from e @required("bootstrap_dns_server") -def validate_bootstrap_dns_server(dns_server: str) -> None: +def validate_bootstrap_dns_server(dns_server: str, **_) -> None: _validate_ip(dns_server) @required("bootstrap_node_cidr", "bootstrap_kube_api_addr", "bootstrap_k8s_gateway_addr", "bootstrap_external_ingress_addr", "bootstrap_internal_ingress_addr") -def validate_host_network(node_cidr: str, api_addr: str, gateway_addr: str, external_ingress_addr: str, internal_ingress_addr: str) -> None: +def validate_host_network(node_cidr: str, api_addr: str, gateway_addr: str, external_ingress_addr: str, internal_ingress_addr: str, **_) -> None: _validate_cidr(node_cidr, 4) _validate_ip(api_addr) _validate_ip(gateway_addr) From 54cd32877dfe10e57588a2a28a7430cf092bf6bc Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sat, 20 Jan 2024 21:07:06 -0500 Subject: [PATCH 4/6] chore: update renovate pip and ansible regex Signed-off-by: Devin Buhl --- .github/renovate.json5 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 418f08a3889..54dea8ddbd4 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -36,6 +36,16 @@ "(^|/)kustomization\\.ya?ml(\\.j2)?(\\.j2)?$" ] }, + "pip_requirements": { + "fileMatch": [ + "(^|/)[\\w-]*requirements(-\\w+)?\\.(txt|pip)(\\.j2)?(\\.j2)?$" + ] + }, + "ansible-galaxy": { + "fileMatch": [ + "(^|/)(galaxy|requirements)(\\.ansible)?\\.ya?ml(\\.j2)?(\\.j2)?$" + ] + }, // commit message topics "commitMessageTopic": "{{depName}}", "commitMessageExtra": "to {{newVersion}}", From 24e660c69f3c9bbbb1407422dfa3cea53894fae0 Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sun, 21 Jan 2024 08:22:06 -0500 Subject: [PATCH 5/6] feat: add bootstrap_nodes test Signed-off-by: Devin Buhl --- .github/workflows/e2e.yaml | 4 ++++ README.md | 29 ++++++++++++------------ Taskfile.yaml | 4 ++-- bootstrap/scripts/loader.py | 1 + bootstrap/scripts/validation.py | 40 +++++++++++++++++++++++++++++++-- requirements.txt | 2 +- 6 files changed, 60 insertions(+), 20 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c8706e17dfb..157485cd6b6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -90,6 +90,10 @@ jobs: shell: bash run: task sops:age-keygen + - name: Run setup-virtual-env task + shell: bash + run: task setup-virtual-env + - name: Run init tasks shell: bash run: | diff --git a/README.md b/README.md index b767fd012f8..c8b651c395f 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ If you are marching forward, now is a good time to choose whether you will deplo ### System requirements > [!IMPORTANT] -> 1. The included behaviour of Talos, k0s or k3s is that all nodes are able to run workloads, **including** control nodes. Worker nodes are therefore optional. +> 1. The included behaviour of Talos, k3s or k0s is that all nodes are able to run workloads, **including** control nodes. Worker nodes are therefore optional. > 2. Do you have 3 or more nodes? It is strongly recommended to make 3 of them control nodes for a highly available control plane. > 3. Running the cluster on Proxmox VE? My thoughts and recommendations about that are documented [here](https://onedr0p.github.io/home-ops/notes/proxmox-considerations.html). @@ -193,12 +193,12 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g go-task workstation:yay ``` -4. Setup a Python virual env and install Ansible by running the following task command. +4. Setup a Python virual env by running the following task command. 📍 _This commands requires Python 3.11+ to be installed._ ```sh - task ansible:deps + task setup-virtual-env ``` 5. Continue on to 🔧 [**Stage 3**](#-stage-3-do-bootstrap-configuration) @@ -278,14 +278,7 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g 7. Once done run the following command which will verify and generate all the files needed to continue. - > [!NOTE] - > The following configure task will create a `./ansible` directory for k3s or k0s and the following directories under `./kubernetes`. - > ```sh - > 📁 kubernetes # Kubernetes cluster defined as code - > ├─📁 bootstrap # Flux installation (not tracked by Flux) - > ├─📁 flux # Main Flux configuration of repository - > └─📁 apps # Apps deployed into the cluster grouped by namespace - > ``` + 📍 _The following configure task will create a `./ansible` directory for k3s or k0s and the following directories under `./kubernetes` for all distributions_ ```sh task configure @@ -305,25 +298,31 @@ Once you have installed Talos or Debian on your nodes, there are six stages to g 1. Ensure you are able to SSH into your nodes from your workstation using a private SSH key **without a passphrase** (for example using a SSH agent). This lets Ansible interact with your nodes. -2. Verify Ansible can view your config +3. Install the Ansible dependencies + + ```sh + task ansible:deps + ``` + +4. Verify Ansible can view your config ```sh task ansible:list ``` -3. Verify Ansible can ping your nodes +5. Verify Ansible can ping your nodes ```sh task ansible:ping ``` -4. Run the Ansible prepare playbook (nodes wil reboot when done) +6. Run the Ansible prepare playbook (nodes wil reboot when done) ```sh task ansible:run playbook=cluster-prepare ``` -5. Continue on to ⛵ [**Stage 5**](#-stage-5-install-kubernetes) +7. Continue on to ⛵ [**Stage 5**](#-stage-5-install-kubernetes) ### ⛵ Stage 5: Install Kubernetes diff --git a/Taskfile.yaml b/Taskfile.yaml index 2f46cb30108..a2ae83d9d5a 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -56,9 +56,9 @@ tasks: desc: Configure repository from Ansible vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? cmds: - # - task: .pre-validate + - task: .pre-validate - task: .template - # - task: .post-validate + - task: .post-validate .pre-validate: internal: true diff --git a/bootstrap/scripts/loader.py b/bootstrap/scripts/loader.py index a47f29dd42d..e5ba7f7e0c2 100644 --- a/bootstrap/scripts/loader.py +++ b/bootstrap/scripts/loader.py @@ -34,6 +34,7 @@ def __init__(self, data): validation.validate_cilium_loadbalancer_mode(data) validation.validate_local_storage_path(data) validation.validate_cluster_cidrs(data) + validation.validate_nodes(data) def filters(self): return [nthhost, encrypt] diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index 0f5969b03ed..639129393f7 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -8,6 +8,7 @@ import netaddr import re import requests +import socket import sys DISTRIBUTIONS = ["k0s", "k3s", "talos"] @@ -27,25 +28,44 @@ def wrapper(data: dict, *args, **kwargs) -> None: return wrapper return wrapper_outter -def _validate_ip(ip: str) -> None: +def _validate_ip(ip: str) -> str: try: netaddr.IPAddress(ip) except netaddr.core.AddrFormatError as e: raise ValueError(f"Invalid IP address {ip}") from e + return ip -def _validate_cidr(cidr: str, family: int) -> None: +def _validate_cidr(cidr: str, family: int) -> str: try: network = netaddr.IPNetwork(cidr) if network.version != family: raise ValueError(f"Invalid CIDR family {network.version}") except netaddr.core.AddrFormatError as e: raise ValueError(f"Invalid CIDR {cidr}") from e + return cidr def _validate_distribution(distribution: str) -> None: if distribution not in DISTRIBUTIONS: raise ValueError(f"Invalid distribution {distribution}") return distribution +def _validate_node(node: dict, node_cidr: str, distribution: str) -> None: + if not node.get("name"): + raise ValueError(f"Node {node.get('name')} is missing a name") + if not node.get("username") and distribution not in ["k0s", "k3s"]: + raise ValueError(f"Node {node.get('username')} is missing a username") + if not node.get("diskSerial") and distribution in ["talos"]: + raise ValueError(f"Node {node.get('diskSerial')} is missing a disk serial") + ip = _validate_ip(node.get("address")) + if netaddr.IPAddress(ip, 4) not in netaddr.IPNetwork(node_cidr): + raise ValueError(f"Node {node.get('address')} is not in the node CIDR {node_cidr}") + port = 50000 if distribution == "talos" else 22 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(5) + result = sock.connect_ex((ip, port)) + if result != 0: + raise ValueError(f"Node {ip} port {port} is not open") + def validate_python_version() -> None: required_version = (3, 11, 0) if sys.version_info < required_version: @@ -177,3 +197,19 @@ def validate_host_network(node_cidr: str, api_addr: str, gateway_addr: str, exte raise ValueError(f"Kubernetes external ingress address {external_ingress_addr} is not in the node CIDR {node_cidr}") if netaddr.IPAddress(internal_ingress_addr) not in node_cidr: raise ValueError(f"Kubernetes internal ingress address {internal_ingress_addr} is not in the node CIDR {node_cidr}") + +@required("bootstrap_node_cidr", "bootstrap_nodes", "bootstrap_distribution") +def validate_nodes(node_cidr: str, nodes: dict[list], distribution: str, **_) -> None: + node_cidr = _validate_cidr(node_cidr, 4) + + masters = nodes.get("master", []) + if len(masters) < 1: + raise ValueError(f"Must have at least one master node") + if len(masters) % 2 == 0: + raise ValueError(f"Must have an odd number of master nodes") + for node in masters: + _validate_node(node, node_cidr, distribution) + + workers = nodes.get("worker", []) + for node in workers: + _validate_node(node, node_cidr, distribution) diff --git a/requirements.txt b/requirements.txt index d6a8c68db18..b1a882ee62e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -bcrypt==4.1.2 # https://github.com/pyca/bcrypt/issues/684 +bcrypt==4.1.2 cloudflare==2.16.0 email-validator==2.1.0 makejinja==2.3.5 From 89182bc3c04f63cb8f5d4f3d6cddae4581d49c5c Mon Sep 17 00:00:00 2001 From: Devin Buhl Date: Sun, 21 Jan 2024 08:31:07 -0500 Subject: [PATCH 6/6] fix: update taskfiles Signed-off-by: Devin Buhl --- Taskfile.yaml | 16 ++++------------ bootstrap/scripts/validation.py | 2 +- makejinja.toml | 1 + 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index a2ae83d9d5a..c817d272711 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -56,28 +56,20 @@ tasks: desc: Configure repository from Ansible vars prompt: Any conflicting config in the root kubernetes and ansible directories will be overwritten... continue? cmds: - - task: .pre-validate - task: .template - task: .post-validate - .pre-validate: - internal: true - cmd: ./.venv/bin/ansible-playbook {{.BOOTSTRAP_DIR}}/validate.yaml - env: - ANSIBLE_DISPLAY_SKIPPED_HOSTS: "false" - preconditions: - - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } - - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" } - - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" } - .template: internal: true cmds: - - ./.venv/bin/makejinja --undefined chainable + - ./.venv/bin/makejinja - task: sops:encrypt preconditions: + - { msg: "Missing virtual environment", sh: "test -d {{.ROOT_DIR}}/.venv" } - { msg: "Missing Makejinja config file", sh: "test -f {{.MAKEJINJA_CONFIG_FILE}}" } - { msg: "Missing Makejinja loader file", sh: "test -f {{.BOOTSTRAP_DIR}}/scripts/loader.py" } + - { msg: "Missing bootstrap addons file", sh: "test -f {{.BOOTSTRAP_ADDONS_FILE}}" } + - { msg: "Missing bootstrap config file", sh: "test -f {{.BOOTSTRAP_CONFIG_FILE}}" } .post-validate: internal: true diff --git a/bootstrap/scripts/validation.py b/bootstrap/scripts/validation.py index 639129393f7..6d1de1d800b 100644 --- a/bootstrap/scripts/validation.py +++ b/bootstrap/scripts/validation.py @@ -146,7 +146,7 @@ def validate_acme_email(email: str, acme_production: bool, **_) -> None: @required("bootstrap_flux_github_webhook_token") def validate_flux_github_webhook_token(token: str, **_) -> None: if not re.match(r"^[a-zA-Z0-9]+$", token): - raise ValueError(f"Invalid Flux GitHub webhook token {token}") + raise ValueError(f"Invalid Flux GitHub webhook token ***") @required("bootstrap_cloudflare_domain", "bootstrap_cloudflare_token", "bootstrap_cloudflare_account_tag", "bootstrap_cloudflare_tunnel_secret", "bootstrap_cloudflare_tunnel_id") def validate_cloudflare(domain: str, token: str, account_tag: str, tunnel_secret: str, tunnel_id: str, **_) -> None: diff --git a/makejinja.toml b/makejinja.toml index 9a3e473e18a..2ec26fc7f74 100644 --- a/makejinja.toml +++ b/makejinja.toml @@ -7,6 +7,7 @@ import_paths = ["./bootstrap/scripts"] loaders = ["loader:Loader"] jinja_suffix = ".j2" force = true +undefined = "chainable" # Block and comment delimiters are changed to avoid conflicts with Renovate # Variable delimiters are changed to avoid conflicts with Renovate and Go templates