From 788af6ab2355d85c03cee164ee774eac83177699 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 15 May 2023 11:25:45 +0200 Subject: [PATCH 01/15] Introduce `gardener` role (#143) --- control-plane/roles/gardener/README.md | 134 +++++++ .../roles/gardener/defaults/main/certs.yaml | 37 ++ .../gardener/defaults/main/cloud_profile.yaml | 30 ++ .../defaults/main/control-plane-defaults | 1 + .../gardener/defaults/main/extensions.yaml | 59 +++ .../gardener/defaults/main/gardener.yaml | 87 +++++ .../gardener/defaults/main/global-defaults | 1 + .../defaults/main/virtual_garden.yaml | 16 + .../gardener/files/kube-apiserver/Chart.yaml | 18 + .../kube-apiserver/templates/_helpers.tpl | 76 ++++ .../templates/deployment-kube-apiserver.yaml | 248 ++++++++++++ .../poddisruptionbudget-kube-apiserver.yaml | 25 ++ .../templates/secret-identity-ca.yaml | 24 ++ .../templates/secret-kube-aggregator-ca.yaml | 23 ++ .../secret-kube-aggregator-client-tls.yaml | 23 ++ .../templates/secret-kube-apiserver-ca.yaml | 23 ++ .../secret-kube-apiserver-server-tls.yaml | 24 ++ .../secret-kube-apiserver-static-token.yaml | 9 + .../secret-kube-controller-manager-tls.yaml | 25 ++ .../secret-kubeconfig-for-admin.yaml | 25 ++ .../templates/secret-service-account-key.yaml | 22 ++ .../service-kube-apiserver-ingress.yaml | 37 ++ .../templates/service-kube-apiserver.yaml | 32 ++ .../gardener/files/kube-apiserver/values.yaml | 42 +++ .../roles/gardener/filter_plugins/common.py | 248 ++++++++++++ .../gardener/tasks/admission_controllers.yaml | 26 ++ .../tasks/control-plane-networks/gcp.yaml | 41 ++ .../tasks/control-plane-networks/metal.yaml | 12 + .../roles/gardener/tasks/extensions.yaml | 108 ++++++ .../roles/gardener/tasks/gardener.yaml | 49 +++ control-plane/roles/gardener/tasks/main.yaml | 182 +++++++++ control-plane/roles/gardener/tasks/seed.yaml | 101 +++++ .../roles/gardener/tasks/shooted_seed.yaml | 73 ++++ .../roles/gardener/tasks/virtual_garden.yaml | 44 +++ .../templates/30-cloud-profile-metal.yaml | 32 ++ .../templates/dns/controller-deployment.yaml | 22 ++ .../dns/controller-registration.yaml | 13 + .../roles/gardener/templates/etcd-values.j2 | 28 ++ .../extension-admission-metal-values.j2 | 22 ++ .../controller-deployment.yaml | 107 ++++++ .../controller-registration.yaml | 17 + .../gardener-control-plane-values.j2 | 136 +++++++ .../gardener/templates/gardenlet-values.j2 | 82 ++++ .../templates/kube-apiserver-values.j2 | 57 +++ .../roles/gardener/templates/managed-seed.j2 | 53 +++ .../controller-deployment.yaml | 13 + .../controller-registration.yaml | 13 + .../controller-deployment.yaml | 31 ++ .../controller-registration.yaml | 13 + .../controller-deployment.yaml | 12 + .../controller-registration.yaml | 15 + .../provider-gcp/controller-deployment.yaml | 11 + .../provider-gcp/controller-registration.yaml | 23 ++ .../controller-deployment.yaml | 21 ++ .../controller-registration.yaml | 14 + .../controller-deployment.yaml | 14 + .../controller-registration.yaml | 14 + .../roles/gardener/templates/shooted-seed.j2 | 86 +++++ .../roles/gardener/templates/vpa.yaml | 124 ++++++ control-plane/roles/gardener/test/__init__.py | 8 + .../roles/gardener/test/common_test.py | 354 ++++++++++++++++++ .../test/mock/gke_container_subnets.json | 44 +++ .../roles/gardener/test/mock/kubeconfig.yaml | 28 ++ .../roles/gardener/test/mock/secret.json | 22 ++ defaults/main.yaml | 32 ++ 65 files changed, 3386 insertions(+) create mode 100644 control-plane/roles/gardener/README.md create mode 100644 control-plane/roles/gardener/defaults/main/certs.yaml create mode 100644 control-plane/roles/gardener/defaults/main/cloud_profile.yaml create mode 120000 control-plane/roles/gardener/defaults/main/control-plane-defaults create mode 100644 control-plane/roles/gardener/defaults/main/extensions.yaml create mode 100644 control-plane/roles/gardener/defaults/main/gardener.yaml create mode 120000 control-plane/roles/gardener/defaults/main/global-defaults create mode 100644 control-plane/roles/gardener/defaults/main/virtual_garden.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/Chart.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-identity-ca.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml create mode 100644 control-plane/roles/gardener/files/kube-apiserver/values.yaml create mode 100644 control-plane/roles/gardener/filter_plugins/common.py create mode 100644 control-plane/roles/gardener/tasks/admission_controllers.yaml create mode 100644 control-plane/roles/gardener/tasks/control-plane-networks/gcp.yaml create mode 100644 control-plane/roles/gardener/tasks/control-plane-networks/metal.yaml create mode 100644 control-plane/roles/gardener/tasks/extensions.yaml create mode 100644 control-plane/roles/gardener/tasks/gardener.yaml create mode 100644 control-plane/roles/gardener/tasks/main.yaml create mode 100644 control-plane/roles/gardener/tasks/seed.yaml create mode 100644 control-plane/roles/gardener/tasks/shooted_seed.yaml create mode 100644 control-plane/roles/gardener/tasks/virtual_garden.yaml create mode 100644 control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml create mode 100644 control-plane/roles/gardener/templates/dns/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/dns/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/etcd-values.j2 create mode 100644 control-plane/roles/gardener/templates/extension-admission-metal-values.j2 create mode 100644 control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/extension-provider-metal/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/gardener-control-plane-values.j2 create mode 100644 control-plane/roles/gardener/templates/gardenlet-values.j2 create mode 100644 control-plane/roles/gardener/templates/kube-apiserver-values.j2 create mode 100644 control-plane/roles/gardener/templates/managed-seed.j2 create mode 100644 control-plane/roles/gardener/templates/networking-calico/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/networking-calico/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/networking-cilium/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/os-metal-extension/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/provider-gcp/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/shoot-cert-service/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/shoot-cert-service/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml create mode 100644 control-plane/roles/gardener/templates/shoot-dns-service/controller-registration.yaml create mode 100644 control-plane/roles/gardener/templates/shooted-seed.j2 create mode 100644 control-plane/roles/gardener/templates/vpa.yaml create mode 100644 control-plane/roles/gardener/test/__init__.py create mode 100644 control-plane/roles/gardener/test/common_test.py create mode 100644 control-plane/roles/gardener/test/mock/gke_container_subnets.json create mode 100644 control-plane/roles/gardener/test/mock/kubeconfig.yaml create mode 100644 control-plane/roles/gardener/test/mock/secret.json diff --git a/control-plane/roles/gardener/README.md b/control-plane/roles/gardener/README.md new file mode 100644 index 000000000..d887772ac --- /dev/null +++ b/control-plane/roles/gardener/README.md @@ -0,0 +1,134 @@ +# gardener + +Deploys Gardener into a virtual garden along with a dedicated ETCD and a set of extension controllers. + +Please refer to the metal-stack gardener integration in our [documentation](https://docs.metal-stack.io/stable/overview/kubernetes/). + +Check out the Gardener project for further documentation on [gardener.cloud](https://gardener.cloud/). + +## Variables + +| Name | Mandatory | Description | +| ------------------------------------------------------ | --------- | ----------------------------------------------------------------------------------------------------------------------------- | +| gardener_image_vector_overwrite | | Allows overriding the image vector to set custom image versions for gardener | +| gardener_component_image_vector_overwrite | | Allows overriding the image vector to set custom image versions for gardenlet components | +| gardener_apiserver_replicas | | Specifies the amount of gardener-apiserver replicas | +| gardener_apiserver_vpa | | Enables the VPA for the gardener-apiserver | +| gardener_apiserver_resources | | Set custom resource definitions for the gardener-apiserver | +| gardener_controller_manager_resources | | Set custom resource definitions for the gardener-controller-manager | +| gardener_scheduler_resources | | Set custom resource definitions for the gardener-scheduler | +| gardener_use_dns_records | | Enables the feature gate for Gardener `DNSRecords` | +| gardener_dns_domain | | Specifies the DNS domain on which the Gardener will manage DNS entries | +| gardener_dns_provider | yes | Specifies the DNS provider | +| gardener_backup_infrastructure | | Specifies the Gardener backup infrastructure | +| gardener_backup_infrastructure_secret | | Specifies the secret for the backup infrastructure | +| gardener_soil_name | | The name of the initial `Seed` (used for spinning up shooted seeds) | +| gardener_soil_kubeconfig_file_path | | The kubeconfig path to the initial seed cluster | +| gardener_soil_vertical_pod_autoscaler_enabled | | Enables the VPA for the intial seed cluster | +| gardener_soil_project_owner_name | | Specifies the owner name for the project that the initial seed uses to set up shooted seeds | +| gardener_gardenlet_shoot_concurrent_syncs | | Specifies the amount of concurrent shoot syncs for the Gardenlet | +| gardener_gardenlet_shoot_reconcile_in_maintenance_only | | Specifies whether to reconcile shoots only in their maintenance time windows for the Gardenlet | +| gardener_gardenlet_shoot_respect_sync_period_overwrite | | Specifies whether to allow sync period overwrites for shoot resources | +| gardener_shooted_seeds | | A list of definitions for shooted seeds reconcile by the initial seed cluster, will be turned into `ManagedSeeds` | +| gardener_shooted_seed_max_pods | | The max pods amount for the shooted seeds | +| gardener_shooted_seed_node_cidr_mask_size | | The node CIDR mask size used for the kubelets of the shooted seeds | +| gardener_shooted_seed_rollout_delay_minutes | | An optional delay between shooted seed rollouts (can be used to calm down bigger environments during an update) | +| gardener_kube_api_server_kubeconfig | | The kubeconfig for the Gardener Kubernetes API (virtual garden apiserver) | +| gardener_kube_apiserver_kubeconfig_path | | The acts on multiple Kubernetes APIs, this is where it puts the kubeconfig of the Gardener Kubernetes API | +| gardener_local_tmp_dir | | The acts on multiple Kubernetes APIs, this is a local folder in the deployment container to store the kubeconfigs (ephemeral) | + +### Virtual Garden + +These variables are related to spinning up the virtual garden, a dedicated kube-apiserver, kube-controller-manager and ETCD to host all Gardener resources. This one will have no worker nodes and cannot schedule pods. + +The deployment chart is taken from [garden-setup](https://github.com/gardener/garden-setup) and follows the same deployment approach. + +| Name | Mandatory | Description | +| ---------------------------------------------------- | --------- | ---------------------------------------------------------------------------------------------------------------------- | +| gardener_virtual_api_server_svc_cluster_ip_add | | An integer to "guess" a free IP for the service that allows the soil to internally communicate with the virtual garden | +| gardener_virtual_api_server_public_dns | | The DNS domain to reach the virtual garden API server on | +| gardener_virtual_api_server_healthcheck_static_token | yes | A static token for healthchecking the virtual garden API server | +| gardener_etcd_backup_schedule | | The backup schedule for the virtual garden ETCD | +| gardener_etcd_snapshot_period | | The snapshot period for the virtual garden ETCD | +| gardener_etcd_garbage_collection_period | | The priod for garbage collection for the virtual garden ETCD | +| gardener_etcd_resources | | Set custom resource definitions for the virtual garden ETCD | + +### Cloud Profile + +Variables for the metal-stack cloud profile. + +| Name | Mandatory | Description | +| --------------------------------------------------- | --------- | --------------------------------------------------------------------- | +| gardener_cloud_profile_stage_name | | The name of the metal-stack environment in the cloud profile | +| gardener_cloud_profile_metal_api_url | | The URL used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_metal_api_hmac | yes | The admin HMAC used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_machine_images | | The machine images available for shoots in the metal-api | +| gardener_cloud_profile_firewall_controller_versions | | The available firewall controller versions for metal-stack shoots | +| gardener_cloud_profile_kubernetes | | The available Kubernetes versions for metal-stack shoots | +| gardener_cloud_profile_machine_types | | The machine types available for shoots in the metal-api | +| gardener_cloud_profile_regions | | The regions available for shoots | +| gardener_cloud_profile_partitions | | The partitions available for shoots | +| gardener_os_cri_mapping | | A mapping to add available CRIs to the machine images | + +### Extensions + +These variable parametrize the Gardener extension controllers. + +This includes the metal-stack extension provider called [gardener-extension-provider-metal](https://github.com/metal-stack/gardener-extension-provider-metal) (GEPM). + +| Name | Mandatory | Description | +| ------------------------------------------------------------ | --------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| gardener_os_controller_repo_ref | | A repo reference for deploying the [os-metal-extension](https://github.com/metal-stack/os-metal-extension/) | +| gardener_networking_cilium_repo_ref | | A repo reference for deploying the [gardener-extension-networking-cilium](https://github.com/gardener/gardener-extension-networking-cilium) | +| gardener_extension_provider_metal_repo_ref | | A repo reference for deploying the [gardener-extension-provider-metal](https://github.com/metal-stack/gardener-extension-provider-metal) | +| gardener_metal_admission_replicas | | Specifies the amount of metal-admission webhook replicas | +| gardener_metal_admission_vpa | | Enables the VPA for the metal-admission webhook | +| gardener_extension_provider_metal_cluster_audit_enabled | | Enables the audit functionality of the GEPM | +| gardener_extension_provider_metal_audit_to_splunk_enabled | | Enables the audit to splunk feature gate of the GEPM | +| gardener_extension_provider_metal_audit_to_splunk | | Configuration for the audit to splunk feature gate of the GEPM | +| gardener_extension_provider_metal_etcd_backup_schedule | | The ETCD backup schedule for metal-stack shoot ETCDs | +| gardener_extension_provider_metal_etcd_delta_snapshot_period | | The ETCD delta snapshot period for metal-stack shoot ETCDs | +| gardener_extension_provider_metal_egress_destinations | | Sets allowed egress destinations for the `RestrictEgress` control plane feature gate of the GEPM | +| gardener_extension_provider_metal_duros_storage_enabled | | Enables the duros storage integration feature gate of the GEPM (Lightbits storage) | +| gardener_extension_provider_metal_duros_storage_config | | Configuration for the duros storage integration | +| gardener_extension_provider_metal_image_pull_secret | | Provide image pull secrets for deployed containers | +| gardener_cert_management_issuer_private_key | | The Let's Encrypt private key used by the cert-management extension controller to setup signed certificates | + +### Certificates + +Gardener requires quite a lot of certificates, which should be self-signed and have to be generated before the deployment. + +We use a small shell script as in the [mini-lab](https://github.com/metal-stack/mini-lab/blob/master/files/certs/roll_certs.sh) to generate the certificates. + +| Name | Mandatory | Description | +| -------------------------------------------- | --------- | ----------- | +| gardener_kube_api_server_ca | yes | - | +| gardener_kube_api_server_ca_key | yes | - | +| gardener_kube_api_server_cert | yes | - | +| gardener_kube_api_server_key | yes | - | +| gardener_kube_api_server_client_cert | yes | - | +| gardener_kube_api_server_client_key | yes | - | +| gardener_kube_aggregator_client_cert | yes | - | +| gardener_kube_aggregator_client_key | yes | - | +| gardener_kube_controller_manager_client_cert | yes | - | +| gardener_kube_controller_manager_client_key | yes | - | +| gardener_admin_client_cert | yes | - | +| gardener_admin_client_key | yes | - | +| gardener_service_account_client_key | yes | - | +| gardener_api_server_ca | yes | - | +| gardener_api_server_cert | yes | - | +| gardener_api_server_key | yes | - | +| gardener_admission_controller_ca | yes | - | +| gardener_admission_controller_cert | yes | - | +| gardener_admission_controller_key | yes | - | +| gardener_controller_manager_ca | yes | - | +| gardener_controller_manager_cert | yes | - | +| gardener_controller_manager_key | yes | - | +| gardener_metal_admission_controller_ca | yes | - | +| gardener_metal_admission_controller_cert | yes | - | +| gardener_metal_admission_controller_key | yes | - | +| gardener_etcd_ca_cert | yes | - | +| gardener_etcd_cert | yes | - | +| gardener_etcd_cert_key | yes | - | +| gardener_etcd_client_cert | yes | - | +| gardener_etcd_client_key | yes | - | diff --git a/control-plane/roles/gardener/defaults/main/certs.yaml b/control-plane/roles/gardener/defaults/main/certs.yaml new file mode 100644 index 000000000..7ad91b558 --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/certs.yaml @@ -0,0 +1,37 @@ +--- +gardener_kube_api_server_ca: +gardener_kube_api_server_ca_key: +gardener_kube_api_server_cert: +gardener_kube_api_server_key: +gardener_kube_api_server_client_cert: +gardener_kube_api_server_client_key: + +gardener_kube_aggregator_client_cert: +gardener_kube_aggregator_client_key: +gardener_kube_controller_manager_client_cert: +gardener_kube_controller_manager_client_key: +gardener_admin_client_cert: +gardener_admin_client_key: +gardener_service_account_client_key: + +gardener_api_server_ca: +gardener_api_server_cert: +gardener_api_server_key: + +gardener_admission_controller_ca: +gardener_admission_controller_cert: +gardener_admission_controller_key: + +gardener_controller_manager_ca: +gardener_controller_manager_cert: +gardener_controller_manager_key: + +gardener_metal_admission_controller_ca: +gardener_metal_admission_controller_cert: +gardener_metal_admission_controller_key: + +gardener_etcd_ca_cert: +gardener_etcd_cert: +gardener_etcd_cert_key: +gardener_etcd_client_cert: +gardener_etcd_client_key: diff --git a/control-plane/roles/gardener/defaults/main/cloud_profile.yaml b/control-plane/roles/gardener/defaults/main/cloud_profile.yaml new file mode 100644 index 000000000..3e5a95dbc --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/cloud_profile.yaml @@ -0,0 +1,30 @@ +--- +gardener_cloud_profile_stage_name: "{{ metal_control_plane_stage_name }}" +gardener_cloud_profile_metal_api_url: https://api.{{ metal_control_plane_ingress_dns }} +gardener_cloud_profile_metal_api_hmac: +gardener_cloud_profile_machine_images: "{{ metal_api_images | default([]) }}" +gardener_cloud_profile_firewall_controller_versions: [] +gardener_cloud_profile_kubernetes: +gardener_cloud_profile_machine_types: [] +gardener_cloud_profile_regions: +gardener_cloud_profile_partitions: {} + # : + # default-machine-types: + # firewall: + # - c1-xlarge-x86 + +gardener_os_cri_mapping: + ubuntu: + when: + cris: + - name: containerd + containerRuntimes: [] + - name: docker + containerRuntimes: [] + debian: + when: + cris: + - name: containerd + containerRuntimes: [] + - name: docker + containerRuntimes: [] diff --git a/control-plane/roles/gardener/defaults/main/control-plane-defaults b/control-plane/roles/gardener/defaults/main/control-plane-defaults new file mode 120000 index 000000000..a77f2709b --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/control-plane-defaults @@ -0,0 +1 @@ +../../../../control-plane-defaults/ \ No newline at end of file diff --git a/control-plane/roles/gardener/defaults/main/extensions.yaml b/control-plane/roles/gardener/defaults/main/extensions.yaml new file mode 100644 index 000000000..8f70e6cd8 --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/extensions.yaml @@ -0,0 +1,59 @@ +--- +gardener_os_controller_repo_ref: "{{ gardener_os_controller_image_tag }}" +# TODO: the ref to the official controller can be used as soon as we are compatible with g/g 1.59 +gardener_networking_cilium_repo_ref: "metal-stack/gardener-extension-networking-cilium/{{ gardener_networking_cilium_image_tag }}" + +gardener_metal_admission_replicas: 1 +gardener_metal_admission_vpa: true + +gardener_extension_provider_metal_repo_ref: "{{ gardener_extension_provider_metal_image_tag }}" + +gardener_extension_provider_metal_cluster_audit_enabled: false +gardener_extension_provider_metal_audit_to_splunk_enabled: false +gardener_extension_provider_metal_audit_to_splunk: + # ... + # hecToken: "" + # index: "" + # hecHost: "" + # hecPort: 443 + # tlsEnabled: true + # hecCAFile: "" + +gardener_extension_provider_metal_etcd_backup_schedule: "0,5,10,15,20,25,30,35,40,45,50,55 * * * *" +gardener_extension_provider_metal_etcd_delta_snapshot_period: "0s" + +gardener_extension_provider_metal_internal_networks: + - 10.0.0.0/8 + - 100.64.0.0/10 + +gardener_extension_provider_metal_egress_destinations: [] + +gardener_extension_provider_metal_duros_storage_enabled: false +gardener_extension_provider_metal_duros_storage_config: + # : + # apiEndpoint: + # apiCA: + # apiKey: + # apiCert: + # endpoints: [] + # adminKey: + # adminToken: + # storageClasses: + # - replicaCount: 2 + # name: duros-silver + # compression: true + # - replicaCount: 3 + # name: duros-gold + # compression: false + # encryption: true + # ... + +gardener_extension_provider_metal_image_pull_secret: + # auths: + # : + # username: "" + # password: "" + # auth: "" + # ... + +gardener_cert_management_issuer_private_key: "" diff --git a/control-plane/roles/gardener/defaults/main/gardener.yaml b/control-plane/roles/gardener/defaults/main/gardener.yaml new file mode 100644 index 000000000..0cab7970b --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/gardener.yaml @@ -0,0 +1,87 @@ +--- +gardener_image_vector_overwrite: +gardener_component_image_vector_overwrite: + +gardener_apiserver_replicas: 1 +gardener_apiserver_vpa: true + +gardener_apiserver_resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 400m + memory: 1Gi + +gardener_controller_manager_resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 750m + memory: 512Mi + +gardener_scheduler_resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 300m + memory: 256Mi + +gardener_use_dns_records: true +gardener_dns_domain: +gardener_dns_provider: + +gardener_backup_infrastructure: +gardener_backup_infrastructure_secret: + +gardener_soil_name: "{{ metal_control_plane_stage_name }}" +gardener_soil_kubeconfig_file_path: "{{ lookup('env', 'KUBECONFIG') }}" +gardener_soil_vertical_pod_autoscaler_enabled: false +gardener_soil_project_owner_name: admin + +gardener_gardenlet_shoot_concurrent_syncs: 5 +gardener_gardenlet_shoot_reconcile_in_maintenance_only: false +gardener_gardenlet_shoot_respect_sync_period_overwrite: true + +gardener_shooted_seeds: [] +# - name: shoot-1 +# project_id: 00000000-0000-0000-0000-000000000001 +# feature_gates: +# clusterAudit: false +# auditToSplunk: false +# region: region +# partition: partition +# networks: +# - internet +# - e77400fe-e993-47b1-a0dd-64c9b7457b76 +# k8s_version: 1.23.4 +# worker_groups: +# - worker_count: 1 +# worker_size: size +# worker_cri: containerd +# worker_max_surge: 3 +# worker_max_unavailable: 0 +# worker_image: +# name: debian +# version: "10.0" +# firewall_size: size +# firewall_image: firewall-ubuntu +# firewall_controller_version: auto +# api_server: +# autoscaler: +# min_replicas: 1 +# max_replicas: 5 +# verticalPodAutoscaler: false +# pod_cidr: +# service_cidr: + +gardener_shooted_seed_max_pods: 250 +gardener_shooted_seed_node_cidr_mask_size: 23 +# can be useful for rollouts where the shooted seed or managed seed resource changes: +gardener_shooted_seed_rollout_delay_minutes: + +gardener_kube_api_server_kubeconfig: "{{ 'garden-kube-apiserver' | kubeconfig_from_cert(gardener_kube_api_server_ca, gardener_kube_api_server_client_cert, gardener_kube_api_server_client_key, prepend_https=true) }}" +gardener_kube_apiserver_kubeconfig_path: "{{ gardener_local_tmp_dir }}/garden-kube-apiserver-kubeconfig" +gardener_local_tmp_dir: "{{ playbook_dir }}/.ansible/tmp" diff --git a/control-plane/roles/gardener/defaults/main/global-defaults b/control-plane/roles/gardener/defaults/main/global-defaults new file mode 120000 index 000000000..ac0cbf4d7 --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/global-defaults @@ -0,0 +1 @@ +../../../../../defaults \ No newline at end of file diff --git a/control-plane/roles/gardener/defaults/main/virtual_garden.yaml b/control-plane/roles/gardener/defaults/main/virtual_garden.yaml new file mode 100644 index 000000000..54d77e33a --- /dev/null +++ b/control-plane/roles/gardener/defaults/main/virtual_garden.yaml @@ -0,0 +1,16 @@ +--- +gardener_virtual_api_server_svc_cluster_ip_add: 20 +gardener_virtual_api_server_public_dns: gardener-kube-apiserver.{{ metal_control_plane_ingress_dns }} +gardener_virtual_api_server_healthcheck_static_token: + +gardener_etcd_backup_schedule: "0,5,10,15,20,25,30,35,40,45,50,55 * * * *" +gardener_etcd_snapshot_period: "0" +gardener_etcd_garbage_collection_period: "12h" + +gardener_etcd_resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 800m + memory: 8Gi diff --git a/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml b/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml new file mode 100644 index 000000000..54d65cdea --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: Helm chart for garden +name: garden +version: 0.1.0 diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl b/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl new file mode 100644 index 000000000..4ade798d8 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/_helpers.tpl @@ -0,0 +1,76 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "garden.kubeconfig-controller-manager" -}} +apiVersion: v1 +kind: Config +current-context: garden +contexts: +- context: + cluster: garden + user: kube-controller-manager + name: garden +clusters: +- cluster: + certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + server: https://localhost:443 + name: garden +users: +- name: kube-controller-manager + user: + client-certificate-data: {{ .Values.tls.kubeControllerManager.crt | b64enc }} + client-key-data: {{ .Values.tls.kubeControllerManager.key | b64enc }} +{{- end -}} + +{{- define "garden.kubeconfig-gardener" -}} +apiVersion: v1 +kind: Config +current-context: garden +contexts: +- context: + cluster: garden + user: gardener + name: garden +clusters: +- cluster: + certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + server: https://{{ .Values.apiServer.serviceName }}:443 + name: garden +users: +- name: gardener + user: + client-certificate-data: {{ .Values.tls.gardener.crt | b64enc }} + client-key-data: {{ .Values.tls.gardener.key | b64enc }} +{{- end -}} + +{{- define "garden.kubeconfig-admin" -}} +apiVersion: v1 +kind: Config +current-context: garden +contexts: +- context: + cluster: garden + user: admin + name: garden +clusters: +- cluster: + certificate-authority-data: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + server: https://{{ .Values.apiServer.hostname }}:443 + name: garden +users: +- name: admin + user: + client-certificate-data: {{ .Values.tls.admin.crt | b64enc }} + client-key-data: {{ .Values.tls.admin.key | b64enc }} +{{- end -}} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml new file mode 100644 index 000000000..6246ebaf7 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml @@ -0,0 +1,248 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garden-kube-apiserver + namespace: {{ .Release.Namespace }} + labels: + app: garden + component: kube-apiserver +spec: + revisionHistoryLimit: 0 + replicas: {{ .Values.replicas }} + selector: + matchLabels: + app: garden + component: kube-apiserver + template: + metadata: + annotations: + checksum/secret-kube-aggregator-ca: {{ include (print $.Template.BasePath "/secret-kube-aggregator-ca.yaml") . | sha256sum }} + checksum/secret-kube-aggregator-client: {{ include (print $.Template.BasePath "/secret-kube-aggregator-client-tls.yaml") . | sha256sum }} + checksum/secret-kube-apiserver-ca: {{ include (print $.Template.BasePath "/secret-kube-apiserver-ca.yaml") . | sha256sum }} + checksum/secret-kube-apiserver-server: {{ include (print $.Template.BasePath "/secret-kube-apiserver-server-tls.yaml") . | sha256sum }} + checksum/secret-kube-apiserver-static-token: {{ include (print $.Template.BasePath "/secret-kube-apiserver-static-token.yaml") . | sha256sum }} + checksum/secret-kube-controller-manager-client: {{ include (print $.Template.BasePath "/secret-kube-controller-manager-tls.yaml") . | sha256sum }} + checksum/secret-service-account-key: {{ include (print $.Template.BasePath "/secret-service-account-key.yaml") . | sha256sum }} + {{- if .Values.tls.identity.ca }} + checksum/secret-identity-ca: {{ include (print $.Template.BasePath "/secret-identity-ca.yaml") . | sha256sum }} + {{- end }} + labels: + app: garden + component: kube-apiserver + {{- if .Values.networkPolicies }} + networking.gardener.cloud/to-dns: allowed + networking.gardener.cloud/to-etcd: allowed + networking.gardener.cloud/to-gardener-apiserver: allowed + networking.gardener.cloud/to-gardener-controller-manager: allowed # needed for webhooks + networking.gardener.cloud/to-identity: allowed + networking.gardener.cloud/to-ingress: allowed # needed for communication to identity + networking.gardener.cloud/to-terminal-controller-manager: allowed # needed for webhooks + networking.gardener.cloud/to-world: allowed # necessary, except for GCP because GCP puts IP table rules on the nodes that allow in-cluster routing + {{- end }} + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - garden + - key: component + operator: In + values: + - kube-apiserver + tolerations: + - effect: NoExecute + operator: Exists + containers: + - name: kube-apiserver + image: {{ index .Values.images "apiserver" }} + imagePullPolicy: IfNotPresent + command: + - kube-apiserver + - --enable-admission-plugins=Priority,NamespaceLifecycle,LimitRanger,PodSecurityPolicy,ServiceAccount,NodeRestriction,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,StorageObjectInUseProtection,MutatingAdmissionWebhook,ValidatingAdmissionWebhook + - --disable-admission-plugins=PersistentVolumeLabel + - --allow-privileged=true + - --anonymous-auth=false + - --authorization-mode=Node,RBAC + - --token-auth-file=/srv/kubernetes/token/static_tokens.csv + - --client-ca-file=/srv/kubernetes/ca/ca.crt + - --enable-aggregator-routing=true + - --enable-bootstrap-token-auth=true + - --etcd-cafile=/srv/kubernetes/etcd/ca/ca.crt + - --etcd-certfile=/srv/kubernetes/etcd/client/tls.crt + - --etcd-keyfile=/srv/kubernetes/etcd/client/tls.key + - --etcd-servers={{ .Values.etcd.main.endpoints }} + - --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP + - --insecure-port=0 + - --profiling=false + - --proxy-client-cert-file=/srv/kubernetes/aggregator/tls.crt + - --proxy-client-key-file=/srv/kubernetes/aggregator/tls.key + - --requestheader-client-ca-file=/srv/kubernetes/aggregator-ca/ca.crt + - --requestheader-extra-headers-prefix=X-Remote-Extra- + - --requestheader-group-headers=X-Remote-Group + - --requestheader-username-headers=X-Remote-User + - --secure-port=443 + - --service-cluster-ip-range=100.64.0.0/13 + - --service-account-key-file=/srv/kubernetes/service-account-key/service_account.key + - --tls-cert-file=/srv/kubernetes/apiserver/tls.crt + - --tls-private-key-file=/srv/kubernetes/apiserver/tls.key + # - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA + - --v=2 + livenessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: 443 + httpHeaders: + - name: Authorization + value: Bearer {{ .Values.tls.kubeAPIServer.staticTokens.healthCheck }} + successThreshold: 1 + failureThreshold: 3 + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 15 + readinessProbe: + httpGet: + scheme: HTTPS + path: /healthz + port: 443 + httpHeaders: + - name: Authorization + value: Bearer {{ .Values.tls.kubeAPIServer.staticTokens.healthCheck }} + successThreshold: 1 + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 15 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + ports: + - name: https + containerPort: 443 + protocol: TCP + resources: + limits: + cpu: 1200m + memory: 2000Mi + requests: + cpu: 200m + memory: 512Mi + volumeMounts: + - name: ca-kube-apiserver + mountPath: /srv/kubernetes/ca + - name: ca-etcd + mountPath: /srv/kubernetes/etcd/ca + - name: ca-front-proxy + mountPath: /srv/kubernetes/aggregator-ca + - name: etcd-client-tls + mountPath: /srv/kubernetes/etcd/client + - name: kube-apiserver + mountPath: /srv/kubernetes/apiserver + - name: service-account-key + mountPath: /srv/kubernetes/service-account-key + - name: kube-apiserver-static-token + mountPath: /srv/kubernetes/token + - name: kube-aggregator + mountPath: /srv/kubernetes/aggregator + {{- if .Values.tls.identity.ca }} + - name: ca-identity + mountPath: /srv/kubernetes/identity-ca + {{- end }} + - name: kube-controller-manager + image: {{ index .Values.images "controllermanager" }} + imagePullPolicy: IfNotPresent + command: + - kube-controller-manager + - --cluster-signing-cert-file=/srv/kubernetes/ca/ca.crt + - --cluster-signing-key-file=/srv/kubernetes/ca/ca.key + - --controllers=namespace,serviceaccount,serviceaccount-token,clusterrole-aggregation,garbagecollector,csrapproving,csrcleaner,csrsigning,bootstrapsigner,tokencleaner + - --authorization-kubeconfig=/srv/kubernetes/controller-manager/kubeconfig + - --authentication-kubeconfig=/srv/kubernetes/controller-manager/kubeconfig + - --kubeconfig=/srv/kubernetes/controller-manager/kubeconfig + - --root-ca-file=/srv/kubernetes/ca/ca.crt + - --service-account-private-key-file=/srv/kubernetes/service-account-key/service_account.key + - --use-service-account-credentials=true + - --v=2 + livenessProbe: + failureThreshold: 2 + httpGet: + path: /healthz + port: 10252 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 15 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 200m + memory: 128Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: ca-kube-apiserver + mountPath: /srv/kubernetes/ca + - name: kube-controller-manager + mountPath: /srv/kubernetes/controller-manager + - name: service-account-key + mountPath: /srv/kubernetes/service-account-key + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - name: ca-kube-apiserver + secret: + secretName: garden-kube-apiserver-ca + {{- if .Values.tls.identity.ca }} + - name: ca-identity + secret: + secretName: identity-ca + {{- end }} + - name: ca-etcd + secret: + secretName: {{ .Values.etcd.secretNames.ca | default "garden-etcd-ca" }} + - name: ca-front-proxy + secret: + secretName: garden-kube-aggregator-ca + - name: kube-apiserver + secret: + secretName: garden-kube-apiserver + - name: kube-controller-manager + secret: + secretName: garden-kube-controller-manager + - name: etcd-client-tls + secret: + secretName: {{ .Values.etcd.secretNames.client | default "garden-etcd-client" }} + - name: kube-apiserver-static-token + secret: + secretName: garden-kube-apiserver-static-token + - name: service-account-key + secret: + secretName: garden-service-account-key + - name: kube-aggregator + secret: + secretName: garden-kube-aggregator diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml new file mode 100644 index 000000000..91c396e8f --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/poddisruptionbudget-kube-apiserver.yaml @@ -0,0 +1,25 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: garden-kube-apiserver + namespace: {{ .Release.Namespace }} +spec: + minAvailable: 2 + selector: + matchLabels: + app: garden + component: kube-apiserver diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-identity-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-identity-ca.yaml new file mode 100644 index 000000000..23670bbdf --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-identity-ca.yaml @@ -0,0 +1,24 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.tls.identity.ca }} +apiVersion: v1 +kind: Secret +metadata: + name: identity-ca + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.tls.identity.ca | b64enc }} +{{- end }} \ No newline at end of file diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml new file mode 100644 index 000000000..347014455 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-ca.yaml @@ -0,0 +1,23 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-aggregator-ca + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.tls.kubeAggregator.ca.crt | b64enc }} + ca.key: {{ .Values.tls.kubeAggregator.ca.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml new file mode 100644 index 000000000..9966853fb --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-aggregator-client-tls.yaml @@ -0,0 +1,23 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-aggregator + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.tls.kubeAggregator.client.crt | b64enc }} + tls.key: {{ .Values.tls.kubeAggregator.client.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml new file mode 100644 index 000000000..3dae3fcb6 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-ca.yaml @@ -0,0 +1,23 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-apiserver-ca + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + ca.key: {{ .Values.tls.kubeAPIServer.ca.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml new file mode 100644 index 000000000..0d6a31af2 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-server-tls.yaml @@ -0,0 +1,24 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-apiserver + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + tls.crt: {{ .Values.tls.kubeAPIServer.server.crt | b64enc }} + tls.key: {{ .Values.tls.kubeAPIServer.server.key | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml new file mode 100644 index 000000000..274a90fcb --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-apiserver-static-token.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-apiserver-static-token + namespace: {{ .Release.Namespace }} +type: Opaque +data: + static_tokens.csv: {{ printf "%s,kube-apiserver-health-check,kube-apiserver-health-check," .Values.tls.kubeAPIServer.staticTokens.healthCheck | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml new file mode 100644 index 000000000..4abad4067 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kube-controller-manager-tls.yaml @@ -0,0 +1,25 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kube-controller-manager + namespace: {{ .Release.Namespace }} +type: kubernetes.io/tls +data: + ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + tls.crt: {{ .Values.tls.kubeControllerManager.crt | b64enc }} + tls.key: {{ .Values.tls.kubeControllerManager.key | b64enc }} + kubeconfig: {{ include "garden.kubeconfig-controller-manager" . | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml new file mode 100644 index 000000000..6a9d84d57 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-kubeconfig-for-admin.yaml @@ -0,0 +1,25 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-kubeconfig-for-admin + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.tls.kubeAPIServer.ca.crt | b64enc }} + tls.crt: {{ .Values.tls.admin.crt | b64enc }} + tls.key: {{ .Values.tls.admin.key | b64enc }} + kubeconfig: {{ include "garden.kubeconfig-admin" . | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml new file mode 100644 index 000000000..b089acaf5 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/secret-service-account-key.yaml @@ -0,0 +1,22 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Secret +metadata: + name: garden-service-account-key + namespace: {{ .Release.Namespace }} +type: Opaque +data: + service_account.key: {{ .Values.tls.serviceAccountKey | b64enc }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml new file mode 100644 index 000000000..ef92f4552 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver-ingress.yaml @@ -0,0 +1,37 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/ssl-passthrough: "true" + name: apiserver-ingress + namespace: {{ .Release.Namespace }} +spec: + ingressClassName: nginx + rules: + - host: {{ .Values.apiServer.hostname }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ .Values.serviceName }} + port: + number: 443 + tls: + - hosts: + - {{ .Values.apiServer.hostname }} diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml new file mode 100644 index 000000000..f6f15d36a --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/service-kube-apiserver.yaml @@ -0,0 +1,32 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: garden + component: kube-apiserver +spec: + type: ClusterIP + selector: + app: garden + component: kube-apiserver + ports: + - name: kube-apiserver + protocol: TCP + port: 443 + targetPort: 443 diff --git a/control-plane/roles/gardener/files/kube-apiserver/values.yaml b/control-plane/roles/gardener/files/kube-apiserver/values.yaml new file mode 100644 index 000000000..e778874c1 --- /dev/null +++ b/control-plane/roles/gardener/files/kube-apiserver/values.yaml @@ -0,0 +1,42 @@ +# Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +namespace: xxx +serviceName: garden-kube-apiserver +images: + apiserver: k8s.gcr.io/kube-apiserver:v1.19.16 + controllermanager: k8s.gcr.io/kube-controller-manager:v1.19.16 + +replicas: 3 +apiServer: + hostname: 127.0.0.1 + serviceName: garden-kube-apiserver + +tls: + kubeAPIServer: + ca: + crt: ca-certificate + key: ca-key + server: + crt: server-certificate + key: server-key + staticTokens: + healthCheck: token + kubeControllerManager: + crt: client-certificate + key: client-key + admin: + crt: client-certificate + key: client-key + serviceAccountKey: key diff --git a/control-plane/roles/gardener/filter_plugins/common.py b/control-plane/roles/gardener/filter_plugins/common.py new file mode 100644 index 000000000..39751fb6f --- /dev/null +++ b/control-plane/roles/gardener/filter_plugins/common.py @@ -0,0 +1,248 @@ +from __future__ import (absolute_import, division, print_function) + +__metaclass__ = type + +import base64 +import ipaddress +import yaml + +from ansible.module_utils.six import PY3 +from ansible.plugins.test.core import version_compare +from ansible.errors import AnsibleFilterError + + +def b64decode(source): + content = base64.b64decode(source) + if PY3: + content = content.decode('utf-8') + return content + + +def b64encode(source): + if PY3: + source = source.encode('utf-8') + content = base64.b64encode(source) + if PY3: + content = content.decode('utf-8') + return content + + +def _extract_cluster_from_kubeconfig(kubeconfig_path): + with open(kubeconfig_path, 'r') as f: + kubeconfig = yaml.safe_load(f) + + current_context_name = kubeconfig['current-context'] + contexts = kubeconfig.get('contexts', []) + current_contexts = [context for context in contexts if context["name"] == current_context_name] + + if not current_contexts: + raise AnsibleFilterError("current context not found in kubeconfig") + + current_context = current_contexts[0] + cluster_name = current_context.get("context", dict()).get("cluster") + + if not cluster_name: + raise AnsibleFilterError("cluster name not defined in current context") + + clusters = kubeconfig.get('clusters', []) + current_clusters = [cluster for cluster in clusters if cluster["name"] == cluster_name] + + if not current_clusters: + raise AnsibleFilterError("current cluster not found in kubeconfig") + + return current_clusters[0].get("cluster", dict()) + + +def kubeconfig_for_sa(kubeconfig_path, secret): + cluster = _extract_cluster_from_kubeconfig(kubeconfig_path) + + server = cluster.get("server") + + secret_data = secret.get("data", dict()) + ca = str(secret_data.get("ca.crt")) + token = b64decode(secret_data.get("token")) + namespace = b64decode(secret_data.get("namespace")) + + return yaml.safe_dump({ + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": ca, + "server": server, + } + } + ], + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "namespace": namespace, + "user": "default-user", + } + } + ], + "current-context": "default-context", + "users": [ + { + "name": "default-user", + "user": { + "token": token, + } + } + ], + }) + + +def extract_gcp_node_network(subnets, region): + for subnet in subnets: + subnetwork = subnet.get("subnetwork") + if not subnetwork: + continue + + if region in subnetwork: + return subnet.get("ipCidrRange") + + raise AnsibleFilterError("no node network found for region: %s" % region) + + +def managed_seed_annotation(managed_seed_api_server, api_server=None): + if not managed_seed_api_server: + return "" + + settings = [] + + if api_server: + defaultReplicas = api_server.get("replicas") + if defaultReplicas: + settings.append("apiServer.replicas=" + str(defaultReplicas)) + + autoscaler = api_server.get("autoscaler", dict()) + + min_replicas = autoscaler.get("min_replicas") + if min_replicas: + settings.append("apiServer.autoscaler.minReplicas=" + str(min_replicas)) + + max_replicas = autoscaler.get("max_replicas") + if max_replicas: + settings.append("apiServer.autoscaler.maxReplicas=" + str(max_replicas)) + + return ",".join(settings) + + +def network_cidr_add(cidr, add): + return str(ipaddress.ip_network(cidr).network_address + add) + + +def kubeconfig_from_cert(server, ca, cert, key, prepend_https=False): + if prepend_https and not server.startswith("https"): + server = "https://" + server + + return yaml.safe_dump({ + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": b64encode(ca), + "server": server, + } + } + ], + "current-context": "default-context", + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "user": "default-user", + } + } + ], + "users": [ + { + "name": "default-user", + "user": { + "client-certificate-data": b64encode(cert), + "client-key-data": b64encode(key), + } + } + ], + }) + + +def machine_images_for_cloud_profile(image_list, cris=None): + images = dict() + for image in image_list: + if 'machine' not in image.get("features", list()): + continue + + if image.get('omit_from_cloud_profile', False): + continue + + image_id = image.get("id") + if image_id is None: + continue + + parts = image_id.split("-") + name = "-".join(parts[:-1]) + + version = parts[-1] + + version_parts = version.split(".") + # ubuntu-19.10.20200331 + # major = version_parts[0] + minor = ".".join(version_parts[:2]) + + image_versions = images.get(name, set()) + # Do not add the major version to the vector + # metal-api cannot match latest version if only major is given + # image_versions.add(major) + image_versions.add(minor) + image_versions.add(version) + images[name] = image_versions + + result = list() + for name, value in images.items(): + versions = list() + for v in sorted(list(value)): + version = dict( + version=v + ) + + if cris is not None and name in cris: + cri = cris[name].copy() + cri_config = cri.pop("cris", []) + cri_condition = cri.pop("when", None) + + if cri_condition is None: + version["cri"] = cri_config + else: + if version_compare(v, cri_condition["version"], cri_condition["operator"]): + version["cri"] = cri_config + + versions.append(version) + + image = dict( + name=name, + versions=versions, + ) + result.append(image) + + return result + + +class FilterModule(object): + def filters(self): + return { + 'network_cidr_add': network_cidr_add, + 'kubeconfig_from_cert': kubeconfig_from_cert, + 'machine_images_for_cloud_profile': machine_images_for_cloud_profile, + 'kubeconfig_for_sa': kubeconfig_for_sa, + 'extract_gcp_node_network': extract_gcp_node_network, + 'managed_seed_annotation': managed_seed_annotation, + } diff --git a/control-plane/roles/gardener/tasks/admission_controllers.yaml b/control-plane/roles/gardener/tasks/admission_controllers.yaml new file mode 100644 index 000000000..966c97a3a --- /dev/null +++ b/control-plane/roles/gardener/tasks/admission_controllers.yaml @@ -0,0 +1,26 @@ +--- +- name: Clone extension-provider-metal + git: + repo: "{{ gardener_extension_provider_metal_repo_url }}" + dest: "{{ gardener_local_tmp_dir }}/extension-provider-metal" + depth: 1 + version: "{{ gardener_extension_provider_metal_repo_ref }}" + +- name: "Register admission webhook: os extension admission metal" + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_chart: "{{ gardener_local_tmp_dir }}/extension-provider-metal/charts/gardener-extension-admission-metal/charts/runtime" + helm_release_name: admission-metal + helm_target_namespace: garden + helm_value_file_template: extension-admission-metal-values.j2 + +- name: "Register admission webhook: os extension admission metal (in virtual apiserver)" + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_chart: "{{ gardener_local_tmp_dir }}/extension-provider-metal/charts/gardener-extension-admission-metal/charts/application" + helm_release_name: admission-metal + helm_target_namespace: garden + helm_value_file_template: extension-admission-metal-values.j2 + helm_kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" diff --git a/control-plane/roles/gardener/tasks/control-plane-networks/gcp.yaml b/control-plane/roles/gardener/tasks/control-plane-networks/gcp.yaml new file mode 100644 index 000000000..475533e44 --- /dev/null +++ b/control-plane/roles/gardener/tasks/control-plane-networks/gcp.yaml @@ -0,0 +1,41 @@ +--- +# GKE sometimes changes the cidrs, which is a validation error for Gardener. +# Therefore, do not touch the cidrs once deployed +- name: Describe GCP cluster + command: gcloud --format json container clusters describe {{ gcp_cluster_name }} --region {{ gcp_region }} + register: gcp_cluster + changed_when: false + +- name: Describe GCP container subnets + command: gcloud --format json container subnets list-usable + register: gcp_subnets + changed_when: false + +# this config map provides the external api-server ip of the gke cluster. +# this information is unfortunately not provided by gke itself in the cluster. +# the ip is required by the gardener-extension-provider-metal to create a kubeconfig +# for the firewall-controller, which can then connect to the kube-apiserver for +# reconciling the firewall v2 resource. +- name: Deploy seed-api-server config map + k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: seed-api-server + namespace: garden + data: + url: "https://{{ (gcp_cluster.stdout | from_json)['endpoint'] }}" + apply: yes + +- name: Set node cidr + set_fact: + _gardener_soil_node_cidr: "{{ (gcp_subnets.stdout | from_json) | extract_gcp_node_network(region=gcp_region) }}" + +- name: Set pod cidr + set_fact: + _gardener_soil_pod_cidr: "{{ lookup('k8s', kind='Node')[0].get('spec', {}).get('podCIDR') }}" + +- name: Set service cidr + set_fact: + _gardener_soil_service_cidr: "{{ (gcp_cluster.stdout | from_json)['servicesIpv4Cidr'] }}" diff --git a/control-plane/roles/gardener/tasks/control-plane-networks/metal.yaml b/control-plane/roles/gardener/tasks/control-plane-networks/metal.yaml new file mode 100644 index 000000000..1d4d85b5a --- /dev/null +++ b/control-plane/roles/gardener/tasks/control-plane-networks/metal.yaml @@ -0,0 +1,12 @@ +--- +- name: Set node cidr + set_fact: + _gardener_soil_node_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['nodeNetwork'] }}" + +- name: Set pod cidr + set_fact: + _gardener_soil_pod_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['podNetwork'] }}" + +- name: Set service cidr + set_fact: + _gardener_soil_service_cidr: "{{ (lookup('k8s', api_version='v1', kind='ConfigMap', namespace='kube-system', resource_name='shoot-info') | default({}, true)).get('data', {})['serviceNetwork'] }}" diff --git a/control-plane/roles/gardener/tasks/extensions.yaml b/control-plane/roles/gardener/tasks/extensions.yaml new file mode 100644 index 000000000..2f4df867a --- /dev/null +++ b/control-plane/roles/gardener/tasks/extensions.yaml @@ -0,0 +1,108 @@ +--- +- name: "Register controller: extension provider metal" + k8s: + definition: "{{ lookup('template', 'extension-provider-metal/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + tags: extension-provider-metal + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: provider gcp (backups only)" + k8s: + definition: "{{ lookup('template', 'provider-gcp/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + tags: provider-gcp + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: external-dns" + k8s: + definition: "{{ lookup('template', 'dns/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: os extension provider metal" + k8s: + definition: "{{ lookup('template', 'os-metal-extension/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: networking calico" + k8s: + definition: "{{ lookup('template', 'networking-calico/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: networking cilium" + k8s: + definition: "{{ lookup('template', 'networking-cilium/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: shoot-cert-service" + k8s: + definition: "{{ lookup('template', 'shoot-cert-service/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + tags: shoot-cert-service + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml + +- name: "Register controller: shoot-dns-service" + k8s: + definition: "{{ lookup('template', 'shoot-dns-service/{{ item }}', split_lines=False) }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + tags: shoot-dns-service + register: result + until: result is success + retries: 10 + delay: 6 + loop: + - controller-deployment.yaml + - controller-registration.yaml diff --git a/control-plane/roles/gardener/tasks/gardener.yaml b/control-plane/roles/gardener/tasks/gardener.yaml new file mode 100644 index 000000000..68963212d --- /dev/null +++ b/control-plane/roles/gardener/tasks/gardener.yaml @@ -0,0 +1,49 @@ +--- +- name: Clone Gardener + git: + repo: "{{ gardener_repo_url }}" + dest: "{{ gardener_local_tmp_dir }}/gardener" + depth: 1 + version: "{{ gardener_repo_ref }}" + +- name: Create garden namespace (in virtual apiserver) + k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: garden + labels: + app: gardener + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + +- name: Deploy Gardener Control Plane (in virtual apiserver) + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_timeout: "600s" + helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/controlplane/charts/application" + helm_release_name: controlplane + helm_target_namespace: garden + helm_value_file_template: gardener-control-plane-values.j2 + helm_kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + +- name: Deploy Gardener Control Plane + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_timeout: "600s" + helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/controlplane/charts/runtime" + helm_release_name: controlplane + helm_target_namespace: garden + helm_value_file_template: gardener-control-plane-values.j2 + +- name: Wait until gardener runtime components are ready + command: echo + changed_when: false + retries: 10 + delay: 6 + until: + - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='gardener-apiserver').get('status', {}).get('readyReplicas', 0) >= 1 + - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='garden-kube-apiserver').get('status', {}).get('readyReplicas', 0) >= 3 + - lookup('k8s', api_version='apps/v1', kind='Deployment', namespace='garden', resource_name='gardener-controller-manager').get('status', {}).get('readyReplicas', 0) >= 1 diff --git a/control-plane/roles/gardener/tasks/main.yaml b/control-plane/roles/gardener/tasks/main.yaml new file mode 100644 index 000000000..8e72ed9fa --- /dev/null +++ b/control-plane/roles/gardener/tasks/main.yaml @@ -0,0 +1,182 @@ +--- +- name: Gather release versions + setup_yaml: + +- name: Check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: yes + that: + - gardener_repo_url is defined + - gardener_repo_ref is defined + - gardener_gardenlet_image_tag is defined + - gardener_gardenlet_image_name is defined + - gardener_etcd_repo_ref is defined + - gardener_apiserver_image_tag is defined + - gardener_controller_manager_image_tag is defined + - gardener_scheduler_image_tag is defined + - gardener_extension_provider_metal_image_tag is defined + - gardener_machine_controller_manager_image_tag is defined + - gardener_os_controller_image_tag is defined + - metal_cloud_controller_manager_image_tag is defined + - gardener_networking_calico_image_tag is defined + - gardener_external_dns_image_tag is defined + - csi_lvm_controller_image_tag is defined + - csi_lvm_provisioner_image_tag is defined + - gardener_api_server_ca is not none + - gardener_api_server_ca_key is not none + - gardener_api_server_cert is not none + - gardener_api_server_key is not none + - gardener_api_server_client_cert is not none + - gardener_api_server_client_key is not none + - gardener_kube_aggregator_client_cert is not none + - gardener_kube_aggregator_client_key is not none + - gardener_kube_controller_manager_client_cert is not none + - gardener_kube_controller_manager_client_key is not none + - gardener_admin_client_cert is not none + - gardener_admin_client_key is not none + - gardener_service_account_client_key is not none + - gardener_admission_controller_ca is not none + - gardener_admission_controller_cert is not none + - gardener_admission_controller_key is not none + - gardener_controller_manager_ca is not none + - gardener_controller_manager_cert is not none + - gardener_controller_manager_key is not none + - gardener_admission_metal_ca is not none + - gardener_admission_metal_cert is not none + - gardener_admission_metal_key is not none + - gardener_etcd_ca_cert is not none + - gardener_etcd_cert is not none + - gardener_etcd_cert_key is not none + - gardener_etcd_client_cert is not none + - gardener_etcd_client_key is not none + - gardener_virtual_api_server_public_dns is not none + - gardener_virtual_api_server_healthcheck_static_token is not none + - gardener_dns_domain is not none + - gardener_dns_provider is not none + - gardener_cloud_profile_metal_api_url is not none + - gardener_cloud_profile_metal_api_hmac is not none +# Gardener expects a VPA to be present, otherwise VPA enabled and soil bootstrapping do not work +# maybe this can be removed in a future version of Gardener? +- name: Deploy VerticalPodAutoscaler CRD if required + k8s: + definition: "{{ lookup('template', 'vpa.yaml') }}" + apply: yes + when: not lookup('k8s', api_version='apiextensions.k8s.io/v1', kind='CustomResourceDefinition', resource_name='verticalpodautoscalers.autoscaling.k8s.io') + +- name: Create garden namespace + k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: garden + +- name: Fail when provider unsupported + fail: + msg: "provider is unsupported: {{ metal_control_plane_host_provider }}" + when: + - metal_control_plane_host_provider not in ["gcp", "metal"] + +- name: GCP control plane service cidr + import_tasks: control-plane-networks/gcp.yaml + when: + - metal_control_plane_host_provider == "gcp" + +- name: metal control plane service cidr + import_tasks: control-plane-networks/metal.yaml + when: + - metal_control_plane_host_provider == "metal" + +- name: Deploy virtual garden + import_tasks: virtual_garden.yaml + +- name: Deploy Gardener + import_tasks: gardener.yaml + +- name: Register admission controllers + import_tasks: admission_controllers.yaml + +- name: Add Metal cloud profile + k8s: + definition: "{{ lookup('template', '30-cloud-profile-metal.yaml') }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + register: result + until: result is success + retries: 10 + delay: 6 + +- name: Register extension controllers + import_tasks: extensions.yaml + +- name: Register control plane as seed + import_tasks: seed.yaml + +- name: Create Gardener project for shooted seeds + k8s: + definition: + apiVersion: core.gardener.cloud/v1alpha1 + kind: Project + metadata: + name: "{{ gardener_soil_name }}" + labels: + gardener.cloud/role: "project" + project.gardener.cloud/name: "{{ gardener_soil_name }}" + spec: + namespace: garden + tolerations: + defaults: + - key: seed.gardener.cloud/protected + - key: seed.gardener.cloud/invisible + - key: seed.gardener.cloud/disable-capacity-reservation + whitelist: + - key: seed.gardener.cloud/protected + - key: seed.gardener.cloud/invisible + - key: seed.gardener.cloud/disable-capacity-reservation + owner: + apiGroup: rbac.authorization.k8s.io + kind: User + name: "{{ gardener_soil_project_owner_name }}" + members: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: "{{ gardener_soil_project_owner_name }}" + role: admin + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + when: not lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Project', resource_name=gardener_soil_name) + +- name: Add project labels to garden namespace + k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + labels: + gardener.cloud/role: "project" + project.gardener.cloud/name: "{{ gardener_soil_name }}" + name: garden + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + +- name: Add provider secret for shoot clusters + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "metal-provider-secret" + namespace: garden + labels: + cloudprofile.garden.sapcloud.io/name: metal + type: Opaque + data: + metalAPIHMac: "{{ gardener_cloud_profile_metal_api_hmac | b64encode }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Loop over shooted seeds + include_tasks: shooted_seed.yaml + loop: "{{ gardener_shooted_seeds }}" + loop_control: + loop_var: gardener_shooted_seed + label: "{{ gardener_shooted_seed.name }}" diff --git a/control-plane/roles/gardener/tasks/seed.yaml b/control-plane/roles/gardener/tasks/seed.yaml new file mode 100644 index 000000000..94d26b6c9 --- /dev/null +++ b/control-plane/roles/gardener/tasks/seed.yaml @@ -0,0 +1,101 @@ +--- +- name: Create backup infrastructure secret + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: backup-secret + namespace: garden + type: Opaque + data: "{{ gardener_backup_infrastructure_secret }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + when: gardener_backup_infrastructure_secret + +- name: Add service account for gardener seeds + k8s: + definition: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: gardener-seeds + namespace: garden + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Make seed service account cluster-admin + k8s: + definition: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: gardener-seeds + roleRef: + apiGroup: "" + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: gardener-seeds + namespace: garden + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Fetch service account token name + set_fact: + gardenlet_sa_token_name: "{{ lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, kind='ServiceAccount', namespace='garden', resource_name='gardener-seeds').get('secrets')[0].get('name') }}" + +- name: Add seed secret + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: gardener-seed-kubeconfig + namespace: garden + type: Opaque + data: + kubeconfig: "{{ gardener_soil_kubeconfig_file_path | kubeconfig_for_sa(secret=lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, kind='Secret', namespace='garden', resource_name=gardenlet_sa_token_name)) | b64encode }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Clone Gardener + git: + repo: "{{ gardener_repo_url }}" + dest: "{{ gardener_local_tmp_dir }}/gardener" + depth: 1 + version: "{{ gardener_repo_ref }}" + +- name: Fetch current seed cidrs + set_fact: + _gardener_current_soil_node_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('nodes', none) }}" + _gardener_current_soil_pod_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('pods', none) }}" + _gardener_current_soil_service_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('services', none) }}" + +- name: Set cidrs for seed + set_fact: + _gardener_gardenlet_node_cidr: "{{ _gardener_current_soil_node_cidr if _gardener_current_soil_node_cidr else _gardener_soil_node_cidr }}" + _gardener_gardenlet_pod_cidr: "{{ _gardener_current_soil_pod_cidr if _gardener_current_soil_pod_cidr else _gardener_soil_pod_cidr }}" + _gardener_gardenlet_service_cidr: "{{ _gardener_current_soil_service_cidr if _gardener_current_soil_service_cidr else _gardener_soil_service_cidr }}" + +- name: Deploy initial Gardenlet + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_chart: "{{ gardener_local_tmp_dir }}/gardener/charts/gardener/gardenlet" + helm_release_name: gardenlet + helm_target_namespace: garden + helm_value_file_template: gardenlet-values.j2 + helm_wait: false + +- name: Wait until seed is ready + command: echo + changed_when: false + retries: 60 + delay: 3 + until: + - "'conditions' in lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name).get('status', {})" + - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions | length > 2 + - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions[0].message == "Gardenlet is posting ready status." + - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions[2].message == "Seed cluster has been bootstrapped successfully." diff --git a/control-plane/roles/gardener/tasks/shooted_seed.yaml b/control-plane/roles/gardener/tasks/shooted_seed.yaml new file mode 100644 index 000000000..0a9cb6038 --- /dev/null +++ b/control-plane/roles/gardener/tasks/shooted_seed.yaml @@ -0,0 +1,73 @@ +--- +- name: Create backup infrastructure secret for shooted seed + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ gardener_shooted_seed.name }}-backup-secret" + namespace: garden + type: Opaque + data: "{{ gardener_backup_infrastructure_secret }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + when: gardener_backup_infrastructure_secret + +- name: Create backup infrastructure config for shooted seed + set_fact: + gardener_shooted_seed_backup_infratructure: + provider: "{{ gardener_backup_infrastructure.provider }}" + region: "{{ gardener_backup_infrastructure.region }}" + secretRef: + name: "{{ gardener_shooted_seed.name }}-backup-secret" + namespace: garden + when: gardener_backup_infrastructure_secret + +- name: Add seed provider secret + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ gardener_shooted_seed.name }}-provider-secret" + namespace: garden + labels: + cloudprofile.garden.sapcloud.io/name: metal + type: Opaque + data: + metalAPIHMac: "{{ gardener_cloud_profile_metal_api_hmac | b64encode }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Add secret binding + k8s: + definition: + apiVersion: core.gardener.cloud/v1alpha1 + kind: SecretBinding + metadata: + labels: + cloudprofile.garden.sapcloud.io/name: metal + name: "{{ gardener_shooted_seed.name }}-provider-secret" + namespace: garden + secretRef: + name: "{{ gardener_shooted_seed.name }}-provider-secret" + namespace: garden + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Deploy shooted seed + k8s: + definition: "{{ lookup('template', 'shooted-seed.j2') }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Deploy managed seed resource + k8s: + definition: "{{ lookup('template', 'managed-seed.j2') }}" + kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" + apply: yes + +- name: Delay shoot rollout + ansible.builtin.pause: + minutes: "{{ gardener_shooted_seed_rollout_delay_minutes }}" + when: gardener_shooted_seed_rollout_delay_minutes diff --git a/control-plane/roles/gardener/tasks/virtual_garden.yaml b/control-plane/roles/gardener/tasks/virtual_garden.yaml new file mode 100644 index 000000000..d0c0c11ad --- /dev/null +++ b/control-plane/roles/gardener/tasks/virtual_garden.yaml @@ -0,0 +1,44 @@ +--- +- name: Clone Gardener etcd (with backup restore) + git: + repo: "{{ gardener_etcd_repo_url }}" + dest: "{{ gardener_local_tmp_dir }}/etcd-backup-restore" + depth: 1 + version: "{{ gardener_etcd_repo_ref }}" + +- name: Deploy Gardener etcd (with backup restore) + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_timeout: "600s" + helm_chart: "{{ gardener_local_tmp_dir }}/etcd-backup-restore/chart/etcd-backup-restore" + helm_release_name: etcd-{{ metal_control_plane_stage_name }} + helm_target_namespace: garden + helm_value_file_template: etcd-values.j2 + +# The virtual garden setup requires one service through which the soil and the garden-apiserver can communicate +# This needs to have the same, static ip address, which needs to be chosen randomly from the service network +- name: Determine current gardener-apiserver service cluster ip + set_fact: + _gardener_api_server_current_service_ip: "{{ (lookup('k8s', api_version='v1', kind='Service', namespace='garden', resource_name='gardener-apiserver') | default({}, true)).get('spec', {}).get('clusterIP', none) }}" + +- name: Calculate gardener-kube-apiserver service cluster IP + set_fact: + gardener_virtual_api_server_svc_cluster_ip: "{{ _gardener_api_server_current_service_ip if _gardener_api_server_current_service_ip else _gardener_soil_service_cidr | network_cidr_add(add=gardener_virtual_api_server_svc_cluster_ip_add) }}" + +- name: Deploy kube-apiserver for virtual garden + include_role: + name: ansible-common/roles/helm-chart + vars: + helm_timeout: "600s" + helm_chart_custom_folder: "{{ ansible_parent_role_paths[0] }}/files/kube-apiserver" + helm_chart: "./kube-apiserver" + helm_release_name: virtual-garden + helm_target_namespace: garden + helm_value_file_template: kube-apiserver-values.j2 + helm_chart_inject_config_hash: yes + +- name: Read admin kubeconfig for garden-kube-apiserver + copy: + dest: "{{ gardener_kube_apiserver_kubeconfig_path }}" + content: "{{ lookup('k8s', api_version='v1', kind='Secret', namespace='garden', resource_name='garden-kubeconfig-for-admin').get('data', {}).get('kubeconfig') | b64decode }}" diff --git a/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml b/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml new file mode 100644 index 000000000..aabe91926 --- /dev/null +++ b/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: CloudProfile +metadata: + name: metal +spec: + type: metal + providerConfig: + apiVersion: metal.provider.extensions.gardener.cloud/v1alpha1 + kind: CloudProfileConfig + metalControlPlanes: + {{ gardener_cloud_profile_stage_name }}: + endpoint: {{ gardener_cloud_profile_metal_api_url }} + +{% if gardener_cloud_profile_machine_images | length > 0 %} + firewallImages: +{% for image in gardener_cloud_profile_machine_images %} +{% if 'firewall' in image.features %} + - "{{ image.id }}" +{% endif %} +{% endfor %} +{% endif %} + firewallControllerVersions: {{ gardener_cloud_profile_firewall_controller_versions | to_json }} + partitions: +{% for id, partition_config in gardener_cloud_profile_partitions.items() %} + {{ id }}: + firewallTypes: {{ partition_config.get('default-machine-types', {}).get('firewall', []) | to_json }} +{% endfor %} + kubernetes: {{ gardener_cloud_profile_kubernetes | to_json }} + machineImages: {{ gardener_cloud_profile_machine_images | machine_images_for_cloud_profile(cris=gardener_os_cri_mapping) | to_json }} + machineTypes: {{ gardener_cloud_profile_machine_types | to_json }} + regions: {{ gardener_cloud_profile_regions | to_json }} diff --git a/control-plane/roles/gardener/templates/dns/controller-deployment.yaml b/control-plane/roles/gardener/templates/dns/controller-deployment.yaml new file mode 100644 index 000000000..a81a49d5c --- /dev/null +++ b/control-plane/roles/gardener/templates/dns/controller-deployment.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: dns-external +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/external-dns-management/' + gardener_external_dns_image_tag + '/examples/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + createCRDs: false + image: + repository: "{{ gardener_external_dns_image_name }}" + tag: "{{ gardener_external_dns_image_tag }}" + configuration: + serverPortHttp: 8080 + controllers: compound + providerTypes: "{{ gardener_dns_provider }}" + leaseDuration: 30s + vpa: + minAllowed: + cpu: 50m + memory: 50Mi diff --git a/control-plane/roles/gardener/templates/dns/controller-registration.yaml b/control-plane/roles/gardener/templates/dns/controller-registration.yaml new file mode 100644 index 000000000..84751f0ba --- /dev/null +++ b/control-plane/roles/gardener/templates/dns/controller-registration.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: dns-external +spec: + deployment: + policy: Always + deploymentRefs: + - name: dns-external + resources: + - kind: DNSProvider + type: "{{ gardener_dns_provider }}" diff --git a/control-plane/roles/gardener/templates/etcd-values.j2 b/control-plane/roles/gardener/templates/etcd-values.j2 new file mode 100644 index 000000000..e29e1fdd0 --- /dev/null +++ b/control-plane/roles/gardener/templates/etcd-values.j2 @@ -0,0 +1,28 @@ +images: + etcdBackupRestore: + tag: "{{ gardener_etcd_repo_ref }}" + +{% if gardener_backup_infrastructure_secret %} +backup: + storageContainer: {{ gardener_backup_infrastructure.bucket }} +{% if metal_control_plane_host_provider == "gcp" %} + storageProvider: "GCS" + gcs: + serviceAccountJson: {{ gardener_backup_infrastructure_service_account_json | to_json }} +{% endif %} +{% endif %} + + schedule: {{ gardener_etcd_backup_schedule }} + deltaSnapshotPeriod: {{ gardener_etcd_snapshot_period }} + garbageCollectionPeriod: {{ gardener_etcd_garbage_collection_period }} + +etcdTLS: + caBundle: | + {{ gardener_etcd_ca_cert | indent(width=4, first=false) }} + crt: | + {{ gardener_etcd_cert | indent(width=4, first=false) }} + key: | + {{ gardener_etcd_cert_key | indent(width=4, first=false) }} + +resources: + etcd: {{ gardener_etcd_resources | to_json }} diff --git a/control-plane/roles/gardener/templates/extension-admission-metal-values.j2 b/control-plane/roles/gardener/templates/extension-admission-metal-values.j2 new file mode 100644 index 000000000..71b40590a --- /dev/null +++ b/control-plane/roles/gardener/templates/extension-admission-metal-values.j2 @@ -0,0 +1,22 @@ +global: + image: + repository: "{{ gardener_extension_provider_metal_image_name }}" + tag: "{{ gardener_extension_provider_metal_image_tag }}" + pullPolicy: {{ metal_control_plane_image_pull_policy }} + webhookConfig: + caBundle: | + {{ gardener_metal_admission_controller_ca | indent(width=6, first=false) }} + serverPort: 443 + tls: + crt: | + {{ gardener_metal_admission_controller_cert | indent(width=8, first=false) }} + key: | + {{ gardener_metal_admission_controller_key | indent(width=8, first=false) }} + kubeconfig: | + {{ gardener_kube_api_server_kubeconfig | indent(width=4, first=false) }} + virtualGarden: + enabled: true + replicaCount: {{ gardener_metal_admission_replicas }} + resources: {} + vpa: + enabled: {{ gardener_metal_admission_vpa }} diff --git a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml new file mode 100644 index 000000000..c541f93c6 --- /dev/null +++ b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: provider-metal +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/gardener-extension-provider-metal/' + gardener_extension_provider_metal_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_extension_provider_metal_image_name }}" + tag: "{{ gardener_extension_provider_metal_image_tag }}" + config: + machineImages: + {% for image in (gardener_cloud_profile_machine_images | machine_images_for_cloud_profile) -%} + {%- for image_version in image.versions -%} + - name: "{{ image.name }}" + version: "{{ image_version.version }}" + image: "{{ image.name }}-{{ image_version.version }}" + {% endfor -%} + {%- endfor %} + + clusterAudit: + enabled: {{ gardener_extension_provider_metal_cluster_audit_enabled }} + + auditToSplunk: + enabled: {{ gardener_extension_provider_metal_audit_to_splunk_enabled }} +{% if gardener_extension_provider_metal_audit_to_splunk_enabled %} + hecToken: {{ gardener_extension_provider_metal_audit_to_splunk.hecToken }} + index: {{ gardener_extension_provider_metal_audit_to_splunk.index }} + hecHost: {{ gardener_extension_provider_metal_audit_to_splunk.hecHost }} + hecPort: {{ gardener_extension_provider_metal_audit_to_splunk.hecPort }} + tlsEnabled: {{ gardener_extension_provider_metal_audit_to_splunk.tlsEnabled }} + hecCAFile: | + {{ gardener_extension_provider_metal_audit_to_splunk.hecCAFile | indent(width=10, first=false) }} +{% endif %} + + etcd: + backup: + schedule: "{{ gardener_extension_provider_metal_etcd_backup_schedule }}" + deltaSnapshotPeriod: "{{ gardener_extension_provider_metal_etcd_delta_snapshot_period }}" + + storage: + duros: + enabled: {{ gardener_extension_provider_metal_duros_storage_enabled }} +{% if gardener_extension_provider_metal_duros_storage_enabled %} + partitionConfig: {{ gardener_extension_provider_metal_duros_storage_config | to_json }} +{% endif %} + +{% if gardener_extension_provider_metal_image_pull_secret %} + imagePullSecret: + encodedDockerConfigJSON: {{ gardener_extension_provider_metal_image_pull_secret | to_json | b64encode }} +{% endif %} + +{% if gardener_extension_provider_metal_egress_destinations %} + # for restrictEgress control plane feature gate + egressDestinations: {{ gardener_extension_provider_metal_egress_destinations | to_json }} +{% endif %} + + imageVectorOverwrite: | + images: + - name: metalccm + sourceRepository: https://github.com/metal-stack/metal-ccm + repository: {{ metal_cloud_controller_manager_image_name }} + tag: {{ metal_cloud_controller_manager_image_tag }} + - name: droptailer + sourceRepository: https://github.com/metal-stack/droptailer + repository: {{ droptailer_image_name }} + tag: {{ droptailer_image_tag }} + - name: machine-controller-manager + sourceRepository: github.com/gardener/machine-controller-manager + repository: {{ gardener_machine_controller_manager_image_name }} + tag: {{ gardener_machine_controller_manager_image_tag }} + - name: firewall-controller-manager + sourceRepository: github.com/metal-stack/firewall-controller-manager + repository: {{ firewall_controller_manager_image_name }} + tag: {{ firewall_controller_manager_image_tag }} + - name: machine-controller-manager-provider-metal + sourceRepository: https://github.com/metal-stack/machine-controller-manager-provider-metal + repository: {{ gardener_mcm_provider_metal_image_name | default("") }} + tag: {{ gardener_mcm_provider_metal_image_tag | default("") }} + - name: csi-lvm-controller + sourceRepository: github.com/metal-stack/csi-lvm-controller + repository: {{ csi_lvm_controller_image_name }} + tag: {{ csi_lvm_controller_image_tag }} + - name: csi-lvm-provisioner + sourceRepository: github.com/metal-stack/csi-lvm-provisioner + repository: {{ csi_lvm_provisioner_image_name }} + tag: {{ csi_lvm_provisioner_image_tag }} +{% if gardener_extension_provider_metal_cluster_audit_enabled and audit_forwarder_image_tag is defined %} + - name: auditforwarder + sourceRepository: https://github.com/metal-stack/audit-forwarder + repository: {{ audit_forwarder_image_name }} + tag: {{ audit_forwarder_image_tag }} +{% endif %} +{% if gardener_extension_provider_metal_audit_to_splunk_enabled and fluentd_splunk_audit_image_tag is defined%} + - name: fluentd-splunk-audit + sourceRepository: https://github.com/fluent/fluentd-kubernetes-daemonset + repository: {{ fluentd_splunk_audit_image_name }} + tag: {{ fluentd_splunk_audit_image_tag }} +{% endif %} +{% if gardener_extension_provider_metal_duros_storage_enabled and duros_controller_image_tag is defined %} + - name: duros-controller + sourceRepository: https://github.com/metal-stack/duros-controller + repository: {{ duros_controller_image_name }} + tag: {{ duros_controller_image_tag }} +{% endif %} diff --git a/control-plane/roles/gardener/templates/extension-provider-metal/controller-registration.yaml b/control-plane/roles/gardener/templates/extension-provider-metal/controller-registration.yaml new file mode 100644 index 000000000..632bf534a --- /dev/null +++ b/control-plane/roles/gardener/templates/extension-provider-metal/controller-registration.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: provider-metal +spec: + deployment: + policy: Always + deploymentRefs: + - name: provider-metal + resources: + - kind: ControlPlane + type: metal + - kind: Infrastructure + type: metal + - kind: Worker + type: metal diff --git a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 new file mode 100644 index 000000000..a9d2905d6 --- /dev/null +++ b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 @@ -0,0 +1,136 @@ +--- +global: + admission: + image: + repository: {{ gardener_admission_controller_image_name }} + tag: {{ gardener_admission_controller_image_tag }} + enabled: true + kubeconfig: | + {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} + config: + server: + https: + tls: + caBundle: | + {{ gardener_admission_controller_ca | indent(width=14, first=false) }} + crt: | + {{ gardener_admission_controller_cert | indent(width=14, first=false) }} + key: | + {{ gardener_admission_controller_key | indent(width=14, first=false) }} + + apiserver: + replicaCount: {{ gardener_apiserver_replicas }} + clusterIdentity: gardener-soil-{{ metal_control_plane_stage_name }} + resources: {{ gardener_apiserver_resources | to_json }} + featureGates: + SeedChange: true + UseDNSRecords: {{ gardener_use_dns_records }} + WorkerPoolKubernetesVersion: true + image: + repository: {{ gardener_apiserver_image_name }} + tag: {{ gardener_apiserver_image_tag }} + etcd: + useSidecar: false + servers: https://etcd-{{ metal_control_plane_stage_name }}-etcd-client:2379 + caBundle: | + {{ gardener_etcd_ca_cert | indent(width=8, first=false) }} + tls: + crt: | + {{ gardener_etcd_client_cert | indent(width=10, first=false) }} + key: | + {{ gardener_etcd_client_key | indent(width=10, first=false) }} + insecureSkipTLSVerify: false + caBundle: | + {{ gardener_api_server_ca | indent(width=6, first=false) }} + tls: + crt: | + {{ gardener_api_server_cert | indent(width=8, first=false) }} + key: | + {{ gardener_api_server_key | indent(width=8, first=false) }} + kubeconfig: | + {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} + + vpa: {{ gardener_apiserver_vpa }} + # this requires the Hvpa resource in autoscaling.k8s.io/v1alpha1, which is not in the GKE cluster... how to actually use this? + # hvpa: + # enabled: true + # maxReplicas: 4 + # minReplicas: 1 + # targetAverageUtilizationCpu: 400 + # targetAverageUtilizationMemory: 400 + # vpaScaleUpStabilization: + # stabilizationDuration: "3m" + # minChange: + # cpu: + # value: 300m + # percentage: 80 + # memory: + # value: 600M + # percentage: 80 + # vpaScaleDownStabilization: + # stabilizationDuration: "15m" + # minChange: + # cpu: + # value: 200m + # percentage: 80 + # memory: + # value: 300M + # percentage: 80 + # limitsRequestsGapScaleParams: + # cpu: + # value: "1" + # percentage: 40 + # memory: + # value: "2G" + # percentage: 40 + + scheduler: + resources: {{ gardener_scheduler_resources | to_json }} + image: + repository: {{ gardener_scheduler_image_name }} + tag: {{ gardener_scheduler_image_tag }} + kubeconfig: | + {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} + + controller: + resources: {{ gardener_controller_manager_resources | to_json }} + image: + repository: {{ gardener_controller_manager_image_name }} + tag: {{ gardener_controller_manager_image_tag }} + + config: + featureGates: + UseDNSRecords: {{ gardener_use_dns_records }} + # ssh key pair rotation is removed in 1.51, but was enabled by default in 1.45 + # TODO: comment in this line prior to deploying g/g 1.45 + # RotateSSHKeypairOnMaintenance: false + server: + https: + tls: + caBundle: | + {{ gardener_controller_manager_ca | indent(width=14, first=false) }} + crt: | + {{ gardener_controller_manager_cert | indent(width=14, first=false) }} + key: | + {{ gardener_controller_manager_key | indent(width=14, first=false) }} + controllers: + shootMaintenance: + enableShootCoreAddonRestarter: false + kubeconfig: | + {{ gardener_kube_api_server_kubeconfig | indent(width=6, first=false) }} + + internalDomain: + provider: "{{ gardener_dns_provider }}" + domain: "{{ gardener_dns_domain }}" + credentials: {{ gardener_dns_credentials | to_json }} + + defaultDomains: + - provider: "{{ gardener_dns_provider }}" + domain: "{{ gardener_dns_domain }}" + credentials: {{ gardener_dns_credentials | to_json }} + + deployment: + virtualGarden: + enabled: true + clusterIP: "{{ gardener_virtual_api_server_svc_cluster_ip }}" + createNamespace: false diff --git a/control-plane/roles/gardener/templates/gardenlet-values.j2 b/control-plane/roles/gardener/templates/gardenlet-values.j2 new file mode 100644 index 000000000..3d38f7738 --- /dev/null +++ b/control-plane/roles/gardener/templates/gardenlet-values.j2 @@ -0,0 +1,82 @@ +global: + gardenlet: + image: + repository: {{ gardener_gardenlet_image_name }} + tag: {{ gardener_gardenlet_image_tag }} + pullPolicy: {{ metal_control_plane_image_pull_policy }} + config: + gardenClientConnection: + gardenClusterAddress: {{ (lookup('file', gardener_kube_apiserver_kubeconfig_path) | from_yaml).get("clusters")[0]["cluster"]["server"] }} + gardenClusterCACert: {{ (lookup('file', gardener_kube_apiserver_kubeconfig_path) | from_yaml).get("clusters")[0]["cluster"]["certificate-authority-data"] }} + kubeconfig: | + {{ lookup('file', gardener_kube_apiserver_kubeconfig_path) | indent(width=10, first=false) }} + + controllers: + backupEntry: + deletionGracePeriodHours: 72 + deletionGracePeriodShootPurposes: + - evaluation + - infrastructure + - production + shoot: + concurrentSyncs: {{ gardener_gardenlet_shoot_concurrent_syncs }} + reconcileInMaintenanceOnly: {{ gardener_gardenlet_shoot_reconcile_in_maintenance_only }} + # allow setting shoot ignore annotation: + respectSyncPeriodOverwrite: {{ gardener_gardenlet_shoot_respect_sync_period_overwrite }} + + seedConfig: + apiVersion: core.gardener.cloud/v1beta1 + kind: Seed + metadata: + name: {{ gardener_soil_name }} + labels: + name: {{ gardener_soil_name }} + spec: + provider: + type: {{ metal_control_plane_host_provider }} + region: local + secretRef: + name: gardener-seed-kubeconfig + namespace: garden + dns: + ingressDomain: {{ gardener_dns_domain }} + networks: + nodes: "{{ _gardener_gardenlet_node_cidr }}" + pods: "{{ _gardener_gardenlet_pod_cidr }}" + services: "{{ _gardener_gardenlet_service_cidr }}" + backup: {{ gardener_backup_infrastructure | to_json }} + blockCIDRs: [] + settings: + excessCapacityReservation: + enabled: false + scheduling: + visible: false + shootDNS: + enabled: true + verticalPodAutoscaler: + enabled: {{ gardener_soil_vertical_pod_autoscaler_enabled }} + taints: + - key: seed.gardener.cloud/protected + - key: seed.gardener.cloud/invisible + - key: seed.gardener.cloud/disable-capacity-reservation + + featureGates: + ManagedIstio: true + HVPA: false + HVPAForShootedSeed: false + UseDNSRecords: {{ gardener_use_dns_records }} +{% if gardener_use_dns_records %} + DisableDNSProviderManagement: true +{% endif %} + + vpa: {{ gardener_soil_vertical_pod_autoscaler_enabled }} + +{% if gardener_image_vector_overwrite %} + imageVectorOverwrite: | + images: + {{ gardener_image_vector_overwrite | to_yaml | indent(width=6, first=false) }} +{% endif %} +{% if gardener_component_image_vector_overwrite %} + componentImageVectorOverwrites: | + {{ gardener_component_image_vector_overwrite | to_yaml | indent(width=6, first=false) }} +{% endif %} diff --git a/control-plane/roles/gardener/templates/kube-apiserver-values.j2 b/control-plane/roles/gardener/templates/kube-apiserver-values.j2 new file mode 100644 index 000000000..76afe969c --- /dev/null +++ b/control-plane/roles/gardener/templates/kube-apiserver-values.j2 @@ -0,0 +1,57 @@ +--- +images: + hyperkube: {{ gardener_virtual_api_server_image_name }}:{{ gardener_virtual_api_server_image_tag }} + controllermanager: {{ gardener_virtual_controller_manager_image_name }}:{{ gardener_virtual_controller_manager_image_tag }} + +apiServer: + hostname: {{ gardener_virtual_api_server_public_dns }} + serviceName: garden-kube-apiserver + +tls: + kubeAPIServer: + ca: + crt: | + {{ gardener_kube_api_server_ca | indent(width=8, first=false) }} + key: | + {{ gardener_kube_api_server_ca_key | indent(width=8, first=false) }} + server: + crt: | + {{ gardener_kube_api_server_cert | indent(width=8, first=false) }} + key: | + {{ gardener_kube_api_server_key | indent(width=8, first=false) }} + staticTokens: + healthCheck: {{ gardener_virtual_api_server_healthcheck_static_token }} + kubeAggregator: + ca: + crt: | + {{ gardener_kube_api_server_ca | indent(width=8, first=false) }} + key: | + {{ gardener_kube_api_server_ca_key | indent(width=8, first=false) }} + client: + crt: | + {{ gardener_kube_aggregator_client_cert | indent(width=8, first=false) }} + key: | + {{ gardener_kube_aggregator_client_key | indent(width=8, first=false) }} + kubeControllerManager: + crt: | + {{ gardener_kube_controller_manager_client_cert | indent(width=6, first=false) }} + key: | + {{ gardener_kube_controller_manager_client_key | indent(width=6, first=false) }} + admin: + crt: | + {{ gardener_admin_client_cert | indent(width=6, first=false) }} + key: | + {{ gardener_admin_client_key | indent(width=6, first=false) }} + serviceAccountKey: | + {{ gardener_service_account_client_key | indent(width=4, first=false) }} + identity: + ca: + +etcd: + main: + endpoints: https://etcd-{{ metal_control_plane_stage_name }}-etcd-client:2379 + secretNames: + ca: etcd-{{ metal_control_plane_stage_name }}-etcd-ca + client: etcd-{{ metal_control_plane_stage_name }}-etcd-tls + +networkPolicies: false diff --git a/control-plane/roles/gardener/templates/managed-seed.j2 b/control-plane/roles/gardener/templates/managed-seed.j2 new file mode 100644 index 000000000..5759aff7b --- /dev/null +++ b/control-plane/roles/gardener/templates/managed-seed.j2 @@ -0,0 +1,53 @@ +apiVersion: seedmanagement.gardener.cloud/v1alpha1 +kind: ManagedSeed +metadata: + name: "{{ gardener_shooted_seed.name }}" + namespace: garden +spec: + gardenlet: + bootstrap: BootstrapToken + config: + apiVersion: gardenlet.config.gardener.cloud/v1alpha1 + kind: GardenletConfiguration + seedConfig: + spec: + backup: + provider: "{{ gardener_backup_infrastructure.provider }}" + region: "{{ gardener_backup_infrastructure.region }}" + secretRef: + name: "{{ gardener_shooted_seed.name }}-backup-secret" + namespace: garden + dns: + provider: + secretRef: + name: {{ lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='v1', kind='Secret', namespace='garden', label_selector='gardener.cloud/role=internal-domain').get('metadata', {}).get('name') }} + namespace: garden + type: {{ gardener_dns_provider }} + ingress: + domain: ingress.{{ gardener_shooted_seed.name }}.{{ gardener_soil_name }}.{{ gardener_dns_domain }} + controller: + kind: nginx + networks: + pods: "{{ gardener_shooted_seed.pod_cidr }}" + services: "{{ gardener_shooted_seed.service_cidr }}" + provider: + region: "{{ gardener_shooted_seed.region }}" + type: metal + settings: + excessCapacityReservation: + enabled: true + scheduling: + visible: {{ gardener_shooted_seed.visible | default(true) }} + shootDNS: + enabled: true + verticalPodAutoscaler: + enabled: {{ gardener_shooted_seed.verticalPodAutoscaler | default(true) }} + deployment: + image: + pullPolicy: IfNotPresent + replicaCount: 1 + revisionHistoryLimit: 1 + vpa: true + mergeWithParent: true + shoot: + name: "{{ gardener_shooted_seed.name }}" diff --git a/control-plane/roles/gardener/templates/networking-calico/controller-deployment.yaml b/control-plane/roles/gardener/templates/networking-calico/controller-deployment.yaml new file mode 100644 index 000000000..a81759535 --- /dev/null +++ b/control-plane/roles/gardener/templates/networking-calico/controller-deployment.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: networking-calico +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-networking-calico/' + gardener_networking_calico_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_networking_calico_image_name }}" + tag: "{{ gardener_networking_calico_image_tag }}" + pullPolicy: Always diff --git a/control-plane/roles/gardener/templates/networking-calico/controller-registration.yaml b/control-plane/roles/gardener/templates/networking-calico/controller-registration.yaml new file mode 100644 index 000000000..c3f466c58 --- /dev/null +++ b/control-plane/roles/gardener/templates/networking-calico/controller-registration.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: networking-calico +spec: + deployment: + policy: Always + deploymentRefs: + - name: networking-calico + resources: + - kind: Network + type: calico diff --git a/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml b/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml new file mode 100644 index 000000000..fdf027223 --- /dev/null +++ b/control-plane/roles/gardener/templates/networking-cilium/controller-deployment.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: networking-cilium +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/' + gardener_networking_cilium_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_networking_cilium_image_name }}" + tag: "{{ gardener_networking_cilium_image_tag }}" + pullPolicy: Always + imageVectorOverwrite: | + images: + - name: cilium-agent + sourceRepository: github.com/cilium/cilium + repository: quay.io/cilium/cilium + tag: v1.12.1 + - name: cilium-preflight + sourceRepository: github.com/cilium/cilium + repository: quay.io/cilium/cilium + tag: v1.12.1 + - name: cilium-operator + sourceRepository: github.com/cilium/cilium + repository: quay.io/cilium/operator + tag: v1.12.1 + - name: hubble-relay + sourceRepository: github.com/cilium/hubble-ui + repository: quay.io/cilium/hubble-relay + tag: v1.12.1 diff --git a/control-plane/roles/gardener/templates/networking-cilium/controller-registration.yaml b/control-plane/roles/gardener/templates/networking-cilium/controller-registration.yaml new file mode 100644 index 000000000..e3a721406 --- /dev/null +++ b/control-plane/roles/gardener/templates/networking-cilium/controller-registration.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: networking-cilium +spec: + deployment: + policy: Always + deploymentRefs: + - name: networking-cilium + resources: + - kind: Network + type: cilium diff --git a/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml b/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml new file mode 100644 index 000000000..9b4c8cd04 --- /dev/null +++ b/control-plane/roles/gardener/templates/os-metal-extension/controller-deployment.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: os-metal +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/metal-stack/os-metal-extension/' + gardener_os_controller_repo_ref + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_os_controller_image_name }}" + tag: "{{ gardener_os_controller_image_tag }}" diff --git a/control-plane/roles/gardener/templates/os-metal-extension/controller-registration.yaml b/control-plane/roles/gardener/templates/os-metal-extension/controller-registration.yaml new file mode 100644 index 000000000..a4f8155cf --- /dev/null +++ b/control-plane/roles/gardener/templates/os-metal-extension/controller-registration.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: os-metal +spec: + deployment: + deploymentRefs: + - name: os-metal + policy: Always + resources: + - kind: OperatingSystemConfig + type: ubuntu + - kind: OperatingSystemConfig + type: debian diff --git a/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml b/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml new file mode 100644 index 000000000..fa2375087 --- /dev/null +++ b/control-plane/roles/gardener/templates/provider-gcp/controller-deployment.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: provider-gcp +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-provider-gcp/' + gardener_extension_provider_gcp_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + tag: {{ gardener_extension_provider_gcp_image_tag }} diff --git a/control-plane/roles/gardener/templates/provider-gcp/controller-registration.yaml b/control-plane/roles/gardener/templates/provider-gcp/controller-registration.yaml new file mode 100644 index 000000000..e44f09712 --- /dev/null +++ b/control-plane/roles/gardener/templates/provider-gcp/controller-registration.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: provider-gcp +spec: + deployment: + deploymentRefs: + - name: provider-gcp + policy: Always + resources: + - kind: BackupBucket + type: gcp + - kind: BackupEntry + type: gcp + - kind: ControlPlane + type: gcp + - kind: Infrastructure + type: gcp + - kind: Worker + type: gcp + - kind: DNSRecord + type: google-clouddns diff --git a/control-plane/roles/gardener/templates/shoot-cert-service/controller-deployment.yaml b/control-plane/roles/gardener/templates/shoot-cert-service/controller-deployment.yaml new file mode 100644 index 000000000..434bdd7ca --- /dev/null +++ b/control-plane/roles/gardener/templates/shoot-cert-service/controller-deployment.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: extension-shoot-cert-service +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-shoot-cert-service/' + gardener_shoot_cert_service_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_shoot_cert_service_image_name }}" + tag: "{{ gardener_shoot_cert_service_image_tag }}" + certificateConfig: + # defaultRequestsPerDayQuota: 100 + defaultIssuer: + restricted: true # restrict default issuer to any sub-domain of shoot.spec.dns.domain + acme: + email: cert-expiry@metal-pod.io + server: https://acme-v02.api.letsencrypt.org/directory + privateKey: | + {{ gardener_cert_management_issuer_private_key | indent(width=12, first=false) }} diff --git a/control-plane/roles/gardener/templates/shoot-cert-service/controller-registration.yaml b/control-plane/roles/gardener/templates/shoot-cert-service/controller-registration.yaml new file mode 100644 index 000000000..7a9e99396 --- /dev/null +++ b/control-plane/roles/gardener/templates/shoot-cert-service/controller-registration.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: extension-shoot-cert-service +spec: + deployment: + policy: Always + deploymentRefs: + - name: extension-shoot-cert-service + resources: + - kind: Extension + type: shoot-cert-service + globallyEnabled: true diff --git a/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml b/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml new file mode 100644 index 000000000..48e98727d --- /dev/null +++ b/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerDeployment +metadata: + name: extension-shoot-dns-service +type: helm +providerConfig: + chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener-extension-shoot-dns-service/' + gardener_shoot_dns_service_image_tag + '/example/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + values: + image: + repository: "{{ gardener_shoot_dns_service_image_name }}" + tag: "{{ gardener_shoot_dns_service_image_tag }}" + dnsProviderManagement: + enabled: {{ gardener_use_dns_records }} diff --git a/control-plane/roles/gardener/templates/shoot-dns-service/controller-registration.yaml b/control-plane/roles/gardener/templates/shoot-dns-service/controller-registration.yaml new file mode 100644 index 000000000..04c999d0b --- /dev/null +++ b/control-plane/roles/gardener/templates/shoot-dns-service/controller-registration.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: core.gardener.cloud/v1beta1 +kind: ControllerRegistration +metadata: + name: extension-shoot-dns-service +spec: + deployment: + policy: Always + deploymentRefs: + - name: extension-shoot-dns-service + resources: + - kind: Extension + type: shoot-dns-service + globallyEnabled: true diff --git a/control-plane/roles/gardener/templates/shooted-seed.j2 b/control-plane/roles/gardener/templates/shooted-seed.j2 new file mode 100644 index 000000000..ec691a56d --- /dev/null +++ b/control-plane/roles/gardener/templates/shooted-seed.j2 @@ -0,0 +1,86 @@ +apiVersion: core.gardener.cloud/v1beta1 +kind: Shoot +metadata: + name: "{{ gardener_shooted_seed.name }}" + namespace: garden + annotations: + shoot.gardener.cloud/managed-seed-api-server: "{{ true | managed_seed_annotation(api_server=gardener_shooted_seed.api_server) }}" + cluster.metal-stack.io/name: "{{ gardener_shooted_seed.name }}" + cluster.metal-stack.io/project: "{{ gardener_shooted_seed.project_id }}" + cluster.metal-stack.io/tenant: "{{ metal_control_plane_provider_tenant }}" + cluster.metal-stack.io/partition: "{{ gardener_shooted_seed.partition }}" +spec: + seedName: "{{ gardener_soil_name }}" + secretBindingName: "{{ gardener_shooted_seed.name }}-provider-secret" + cloudProfileName: metal + region: "{{ gardener_shooted_seed.region }}" + purpose: infrastructure + tolerations: + - key: seed.gardener.cloud/protected + - key: seed.gardener.cloud/invisible + - key: seed.gardener.cloud/disable-capacity-reservation + addons: + kubernetes-dashboard: + enabled: false + nginx-ingress: + enabled: false + kubernetes: + version: "{{ gardener_shooted_seed.k8s_version }}" + kubelet: + maxPods: {{ gardener_shooted_seed_max_pods }} + kubeControllerManager: + nodeCIDRMaskSize: {{ gardener_shooted_seed_node_cidr_mask_size }} + provider: + type: metal + workers: +{% for worker_group in gardener_shooted_seed.worker_groups %} + - name: "group-cri-{{ loop.index0 }}" + cri: + name: {{ worker_group.worker_cri | default("containerd") }} + maximum: {{ worker_group.worker_count }} + minimum: {{ worker_group.worker_count }} + maxSurge: {{ worker_group.worker_max_surge }} + maxUnavailable: {{ worker_group.worker_max_unavailable }} + machine: + type: "{{ worker_group.worker_size }}" + image: {{ worker_group.worker_image | to_json }} +{% endfor %} + infrastructureConfig: + apiVersion: metal.provider.extensions.gardener.cloud/v1alpha1 + kind: InfrastructureConfig + firewall: + image: "{{ gardener_shooted_seed.firewall_image }}" + size: "{{ gardener_shooted_seed.firewall_size }}" + networks: {{ gardener_shooted_seed.networks | to_json }} +{% if gardener_shooted_seed.firewall_controller_version is defined %} + controllerVersion: "{{ gardener_shooted_seed.firewall_controller_version }}" +{% endif %} + projectID: "{{ gardener_shooted_seed.project_id }}" + partitionID: "{{ gardener_shooted_seed.partition }}" + controlPlaneConfig: + apiVersion: metal.provider.extensions.gardener.cloud/v1alpha1 + kind: ControlPlaneConfig +{% if 'storage_class_name' in gardener_shooted_seed %} + customDefaultStorageClass: + className: {{ gardener_shooted_seed.storage_class_name }} +{% endif %} + featureGates: {{ gardener_shooted_seed.feature_gates | to_json }} + networking: + type: calico + pods: {{ gardener_shooted_seed.pod_cidr }} + services: {{ gardener_shooted_seed.service_cidr }} + providerConfig: + apiVersion: calico.networking.extensions.gardener.cloud/v1alpha1 + kind: NetworkConfig + backend: none + ipv4: + mode: Never + typha: + enabled: false + maintenance: + timeWindow: + begin: 220000+0100 + end: 230000+0100 + autoUpdate: + kubernetesVersion: false + machineImageVersion: false diff --git a/control-plane/roles/gardener/templates/vpa.yaml b/control-plane/roles/gardener/templates/vpa.yaml new file mode 100644 index 000000000..138e9dee7 --- /dev/null +++ b/control-plane/roles/gardener/templates/vpa.yaml @@ -0,0 +1,124 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 + resources.gardener.cloud/keep-object: "true" + labels: + gardener.cloud/role: vpa + name: verticalpodautoscalers.autoscaling.k8s.io +spec: + conversion: + strategy: None + group: autoscaling.k8s.io + names: + kind: VerticalPodAutoscaler + listKind: VerticalPodAutoscalerList + plural: verticalpodautoscalers + shortNames: + - vpa + singular: verticalpodautoscaler + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + resourcePolicy: + properties: + containerPolicies: + items: + properties: + containerName: + type: string + controlledResources: + items: + enum: + - cpu + - memory + type: string + type: array + maxAllowed: + type: object + x-kubernetes-preserve-unknown-fields: true + minAllowed: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + enum: + - Auto + - "Off" + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + targetRef: + type: object + x-kubernetes-preserve-unknown-fields: true + updatePolicy: + properties: + updateMode: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + x-kubernetes-preserve-unknown-fields: true + served: true + storage: true + - name: v1beta2 + schema: + openAPIV3Schema: + properties: + spec: + properties: + resourcePolicy: + properties: + containerPolicies: + items: + properties: + containerName: + type: string + controlledResources: + items: + enum: + - cpu + - memory + type: string + type: array + maxAllowed: + type: object + x-kubernetes-preserve-unknown-fields: true + minAllowed: + type: object + x-kubernetes-preserve-unknown-fields: true + mode: + enum: + - Auto + - "Off" + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + targetRef: + type: object + x-kubernetes-preserve-unknown-fields: true + updatePolicy: + properties: + updateMode: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + x-kubernetes-preserve-unknown-fields: true + served: true + storage: false diff --git a/control-plane/roles/gardener/test/__init__.py b/control-plane/roles/gardener/test/__init__.py new file mode 100644 index 000000000..8dd4e633c --- /dev/null +++ b/control-plane/roles/gardener/test/__init__.py @@ -0,0 +1,8 @@ +import os + +FILTER_PLUGINS_PATH = os.path.join(os.path.dirname(os.path.abspath(os.path.dirname(__file__))), 'filter_plugins') + + +def read_mock_file(name): + with open(os.path.join(os.path.dirname(__file__), "mock", name), 'r') as f: + return f.read() diff --git a/control-plane/roles/gardener/test/common_test.py b/control-plane/roles/gardener/test/common_test.py new file mode 100644 index 000000000..6f9c79a9b --- /dev/null +++ b/control-plane/roles/gardener/test/common_test.py @@ -0,0 +1,354 @@ +from mock import patch, mock_open +import json +import sys +import unittest +import yaml + +from test import FILTER_PLUGINS_PATH, read_mock_file + +sys.path.insert(0, FILTER_PLUGINS_PATH) +from common import kubeconfig_from_cert, network_cidr_add, machine_images_for_cloud_profile, kubeconfig_for_sa, extract_gcp_node_network, managed_seed_annotation + + +class KubeconfigTest(unittest.TestCase): + def test_kubeconfig_from_cert(self): + actual = kubeconfig_from_cert("1.2.3.4", "ca", "cert", "key") + + self.maxDiff = None + + expected = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": "Y2E=", + "server": "1.2.3.4", + } + } + ], + "current-context": "default-context", + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "user": "default-user", + } + } + ], + "users": [ + { + "name": "default-user", + "user": { + "client-certificate-data": "Y2VydA==", + "client-key-data": "a2V5", + } + } + ], + } + + self.assertDictEqual(expected, yaml.safe_load(actual)) + + def test_kubeconfig_from_cert_prepend_https(self): + actual = kubeconfig_from_cert("1.2.3.4", "ca", "cert", "key", prepend_https=True) + + self.maxDiff = None + + expected = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": "Y2E=", + "server": "https://1.2.3.4", + } + } + ], + "current-context": "default-context", + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "user": "default-user", + } + } + ], + "users": [ + { + "name": "default-user", + "user": { + "client-certificate-data": "Y2VydA==", + "client-key-data": "a2V5", + } + } + ], + } + + self.assertDictEqual(expected, yaml.safe_load(actual)) + + +class NetworkTest(unittest.TestCase): + def test_network_cidr_add(self): + actual = network_cidr_add("10.0.0.0/28", 20) + expected = "10.0.0.20" + + self.assertEqual(actual, expected) + + +class ImageTest(unittest.TestCase): + def test_image_list(self): + self.maxDiff = None + + sample_data = """ + - id: firewall-ubuntu-2.0.20200714 + name: Firewall 2 Ubuntu 20200714 + description: Firewall 2 Ubuntu 20200714 + url: http://images.metal-stack.io/metal-os/master/firewall/2.0-ubuntu/20200714/img.tar.lz4 + features: + - firewall + - machine + - id: firewall-2.0.20200714 + name: Firewall 2 Debian 20200714 + description: Firewall 2 Debian 20200714 + url: http://images.metal-stack.io/metal-os/master/firewall/2.0/20200714/img.tar.lz4 + features: + - firewall + - id: ubuntu-19.10.20200331 + name: Ubuntu 19.10 20200331 + description: Ubuntu 19.10 20200331 + url: http://images.metal-stack.io/metal-os/ubuntu/19.10/20200331/img.tar.lz4 + features: + - machine + - id: ubuntu-19.10.20200701 + name: Ubuntu 19.10 20200701 + description: Ubuntu 19.10 20200701 + url: http://images.metal-stack.io/metal-os/ubuntu/19.10/20200701/img.tar.lz4 + features: + - machine + - id: ubuntu-20.04.20200331 + name: Ubuntu 20.04 20200331 + description: Ubuntu 20.04 20200331 + url: http://images.metal-stack.io/metal-os/ubuntu/20.04/20200331/img.tar.lz4 + features: + - machine + - id: debian-10.0.20200331 + name: Debian 10.0.20200331 + description: Debian 10 20200331 + url: http://images.metal-stack.io/metal-os/debian/10/20200331/img.tar.lz4 + features: + - machine + - id: centos-7.0.20210620 + name: Centos 7 20210620 + description: Centos 7 20210620 + url: http://images.metal-stack.io/metal-os/master/centos/7/20210620/img.tar.lz4 + omit_from_cloud_profile: yes + features: + - machine + """ + actual = machine_images_for_cloud_profile(yaml.safe_load(sample_data)) + expected = [ + { + "name": "firewall-ubuntu", + "versions": [ + {"version": "2.0"}, + {"version": "2.0.20200714"}, + ] + }, + { + "name": "ubuntu", + "versions": [ + {"version": "19.10"}, + {"version": "19.10.20200331"}, + {"version": "19.10.20200701"}, + {"version": "20.04"}, + {"version": "20.04.20200331"}, + ], + }, + { + "name": "debian", + "versions": [ + {"version": "10.0"}, + {"version": "10.0.20200331"}, + ], + }, + ] + + self.assertListEqual(actual, expected) + + def test_image_list_with_cri_mapping(self): + self.maxDiff = None + + sample_data = """ + - id: firewall-ubuntu-2.0.20200714 + name: Firewall 2 Ubuntu 20200714 + description: Firewall 2 Ubuntu 20200714 + url: http://images.metal-stack.io/metal-os/master/firewall/2.0-ubuntu/20200714/img.tar.lz4 + features: + - firewall + - machine + - id: firewall-2.0.20200714 + name: Firewall 2 Debian 20200714 + description: Firewall 2 Debian 20200714 + url: http://images.metal-stack.io/metal-os/master/firewall/2.0/20200714/img.tar.lz4 + features: + - firewall + - id: ubuntu-19.10.20200331 + name: Ubuntu 19.10 20200331 + description: Ubuntu 19.10 20200331 + url: http://images.metal-stack.io/metal-os/ubuntu/19.10/20200331/img.tar.lz4 + features: + - machine + - id: ubuntu-19.10.20200701 + name: Ubuntu 19.10 20200701 + description: Ubuntu 19.10 20200701 + url: http://images.metal-stack.io/metal-os/ubuntu/19.10/20200701/img.tar.lz4 + features: + - machine + - id: ubuntu-20.04.20200331 + name: Ubuntu 20.04 20200331 + description: Ubuntu 20.04 20200331 + url: http://images.metal-stack.io/metal-os/ubuntu/20.04/20200331/img.tar.lz4 + features: + - machine + """ + + cris = [ + { + "name": "containerd", + "containerRuntimes": [ + {"type": "gvisor"}, + {"type": "kata-containers"}, + ], + } + ] + cri_mapping = { + "ubuntu": { + "when": { + "operator": ">=", + "version": "20.04", + }, + "cris": cris, + } + } + + actual = machine_images_for_cloud_profile(yaml.safe_load(sample_data), cris=cri_mapping) + expected = [ + { + "name": "firewall-ubuntu", + "versions": [ + {"version": "2.0"}, + {"version": "2.0.20200714"}, + ] + }, + { + "name": "ubuntu", + "versions": [ + {"version": "19.10"}, + {"version": "19.10.20200331"}, + {"version": "19.10.20200701"}, + { + "version": "20.04", + "cri": [ + { + "name": "containerd", + "containerRuntimes": [ + {"type": "gvisor"}, + {"type": "kata-containers"}, + ], + } + ], + }, + { + "version": "20.04.20200331", + "cri": [ + { + "name": "containerd", + "containerRuntimes": [ + {"type": "gvisor"}, + {"type": "kata-containers"}, + ], + } + ], + }, + ], + }, + ] + + self.assertListEqual(actual, expected) + + +class KubeconfigTest(unittest.TestCase): + def test_kubeconfig_for_sa(self): + kubeconfig = read_mock_file("kubeconfig.yaml") + secret = json.loads(read_mock_file("secret.json")) + + with patch("builtins.open", mock_open(read_data=kubeconfig)) as mock_file: + actual = kubeconfig_for_sa("~/.kube/config", secret) + + self.maxDiff = None + + expected = { + "apiVersion": "v1", + "kind": "Config", + "clusters": [ + { + "name": "default-cluster", + "cluster": { + "certificate-authority-data": "Y2VydA==", + "server": "https://api-server-test.de", + } + } + ], + "contexts": [ + { + "name": "default-context", + "context": { + "cluster": "default-cluster", + "namespace": "garden", + "user": "default-user", + } + } + ], + "current-context": "default-context", + "users": [ + { + "name": "default-user", + "user": { + "token": "token", + } + } + ], + } + + self.assertDictEqual(expected, yaml.safe_load(actual)) + + +class GCPTest(unittest.TestCase): + def test_extract_gcp_node_network(self): + subnets = json.loads(read_mock_file("gke_container_subnets.json")) + + actual = extract_gcp_node_network(subnets, "europe-west3") + + self.assertEqual("0.0.0.3/32", actual) + + +class ShootedSeedTest(unittest.TestCase): + def test_managed_seed_annotation_no_shooted_seed(self): + actual = managed_seed_annotation(False) + + self.assertEqual("", actual) + + def test_managed_seed_api_server_configs(self): + actual = managed_seed_annotation(True, api_server=dict(autoscaler=dict(min_replicas=2, max_replicas=2))) + + self.assertEqual("apiServer.autoscaler.minReplicas=2,apiServer.autoscaler.maxReplicas=2", actual) + + def test_managed_seed_api_server_configs_with_default_replicas(self): + actual = managed_seed_annotation(True, api_server=dict(replicas=2,autoscaler=dict(min_replicas=2, max_replicas=2))) + + self.assertEqual("apiServer.replicas=2,apiServer.autoscaler.minReplicas=2,apiServer.autoscaler.maxReplicas=2", actual) diff --git a/control-plane/roles/gardener/test/mock/gke_container_subnets.json b/control-plane/roles/gardener/test/mock/gke_container_subnets.json new file mode 100644 index 000000000..822c85b70 --- /dev/null +++ b/control-plane/roles/gardener/test/mock/gke_container_subnets.json @@ -0,0 +1,44 @@ +[ + { + "ipCidrRange": "0.0.0.1/32", + "network": "projects/test-project/global/networks/default", + "subnetwork": "projects/test-project/regions/us-west1/subnetworks/default" + }, + { + "ipCidrRange": "0.0.0.2/32", + "network": "projects/test-project/global/networks/default", + "subnetwork": "projects/test-project/regions/southamerica-east1/subnetworks/default" + }, + { + "ipCidrRange": "0.0.0.3/32", + "network": "projects/test-project/global/networks/default", + "secondaryIpRanges": [ + { + "ipCidrRange": "0.0.0.4/32", + "rangeName": "prod-cluster-pods-a", + "status": "IN_USE_MANAGED_POD" + }, + { + "ipCidrRange": "0.0.0.5/32", + "rangeName": "prod-cluster-services-a", + "status": "IN_USE_SERVICE" + }, + { + "ipCidrRange": "0.0.0.6/32", + "rangeName": "test-cluster-pods-b", + "status": "IN_USE_MANAGED_POD" + }, + { + "ipCidrRange": "0.0.0.7/32", + "rangeName": "test-cluster-services-b", + "status": "IN_USE_SERVICE" + } + ], + "subnetwork": "projects/test-project/regions/europe-west3/subnetworks/default" + }, + { + "ipCidrRange": "0.0.0.8/32", + "network": "projects/test-project/global/networks/default", + "subnetwork": "projects/test-project/regions/asia-northeast2/subnetworks/default" + } +] diff --git a/control-plane/roles/gardener/test/mock/kubeconfig.yaml b/control-plane/roles/gardener/test/mock/kubeconfig.yaml new file mode 100644 index 000000000..1d503b79d --- /dev/null +++ b/control-plane/roles/gardener/test/mock/kubeconfig.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: abc + server: https://api-server-prod.de + name: prod +- cluster: + certificate-authority-data: cdf + server: https://api-server-test.de + name: test +contexts: +- context: + cluster: prod + user: Paul + name: prod-context +- context: + cluster: test + user: Raoul + name: test-context +current-context: test-context +users: +- name: Paul + user: + token: 123 +- name: Raoul + user: + token: 456 diff --git a/control-plane/roles/gardener/test/mock/secret.json b/control-plane/roles/gardener/test/mock/secret.json new file mode 100644 index 000000000..0ccd76c18 --- /dev/null +++ b/control-plane/roles/gardener/test/mock/secret.json @@ -0,0 +1,22 @@ +{ + "apiVersion": "v1", + "data": { + "ca.crt": "Y2VydA==", + "namespace": "Z2FyZGVu", + "token": "dG9rZW4=" + }, + "kind": "Secret", + "metadata": { + "annotations": { + "kubernetes.io/service-account.name": "gardener-local-seed", + "kubernetes.io/service-account.uid": "56c18c7f-c333-43af-95af-3c21a35ed39a" + }, + "creationTimestamp": "2019-10-16T07:47:44Z", + "name": "gardener-local-seed-token-svztd", + "namespace": "garden", + "resourceVersion": "29773005", + "selfLink": "/api/v1/namespaces/garden/secrets/gardener-local-seed-token-svztd", + "uid": "b1571590-42dd-4441-b043-4d2e37b98e3a" + }, + "type": "kubernetes.io/service-account-token" +} diff --git a/defaults/main.yaml b/defaults/main.yaml index 3b94b95a3..4eb104a06 100644 --- a/defaults/main.yaml +++ b/defaults/main.yaml @@ -91,6 +91,33 @@ metal_stack_release: promtail_image_name: "docker-images.third-party.partition.promtail.name" ztp_nginx_image_tag: "docker-images.third-party.partition.ztp-nginx.tag" ztp_nginx_image_name: "docker-images.third-party.partition.ztp-nginx.name" + # third-party gardener + gardener_admission_controller_image_tag: "docker-images.third-party.gardener.admission-controller.tag" + gardener_admission_controller_image_name: "docker-images.third-party.gardener.admission-controller.name" + gardener_apiserver_image_tag: "docker-images.third-party.gardener.api-server.tag" + gardener_apiserver_image_name: "docker-images.third-party.gardener.api-server.name" + gardener_controller_manager_image_tag: "docker-images.third-party.gardener.controller-manager.tag" + gardener_controller_manager_image_name: "docker-images.third-party.gardener.controller-manager.name" + gardener_scheduler_image_tag: "docker-images.third-party.gardener.scheduler.tag" + gardener_scheduler_image_name: "docker-images.third-party.gardener.scheduler.name" + gardener_gardenlet_image_tag: "docker-images.third-party.gardener.gardenlet.tag" + gardener_gardenlet_image_name: "docker-images.third-party.gardener.gardenlet.name" + gardener_virtual_api_server_image_tag: "docker-images.third-party.gardener.virtual-apiserver.tag" + gardener_virtual_api_server_image_name: "docker-images.third-party.gardener.virtual-apiserver.name" + gardener_virtual_controller_manager_image_tag: "docker-images.third-party.gardener.virtual-controller-manager.tag" + gardener_virtual_controller_manager_image_name: "docker-images.third-party.gardener.virtual-controller-manager.name" + gardener_extension_provider_gcp_image_tag: "docker-images.third-party.gardener.extension-provider-gcp.tag" + gardener_extension_provider_gcp_image_name: "docker-images.third-party.gardener.extension-provider-gcp.name" + gardener_external_dns_image_tag: "docker-images.third-party.gardener.external-dns.tag" + gardener_external_dns_image_name: "docker-images.third-party.gardener.external-dns.name" + gardener_networking_calico_image_tag: "docker-images.third-party.gardener.networking-calico.tag" + gardener_networking_calico_image_name: "docker-images.third-party.gardener.networking-calico.name" + gardener_networking_cilium_image_tag: "docker-images.third-party.gardener.networking-cilium.tag" + gardener_networking_cilium_image_name: "docker-images.third-party.gardener.networking-cilium.name" + gardener_shoot_cert_service_image_tag: "docker-images.third-party.gardener.shoot-cert-service.tag" + gardener_shoot_cert_service_image_name: "docker-images.third-party.gardener.shoot-cert-service.name" + gardener_shoot_dns_service_image_tag: "docker-images.third-party.gardener.shoot-dns-service.tag" + gardener_shoot_dns_service_image_name: "docker-images.third-party.gardener.shoot-dns-service.name" # helm-charts metal_helm_chart_version: "helm-charts.metal-stack.metal-control-plane.version" metal_helm_chart_repo: "helm-charts.metal-stack.metal-control-plane.repository" @@ -105,6 +132,11 @@ metal_stack_release: webhook_logger_image_tag: "docker-images.third-party.control-plane.webhook-logger.tag" metal_auditing_meili_chart_repo: "helm-charts.third-party.meilisearch.repository" metal_auditing_meili_chart_version: "helm-charts.third-party.meilisearch.version" + # repositories + gardener_repo_url: "repositories.third-party.gardener.gardener.url" + gardener_repo_ref: "repositories.third-party.gardener.gardener.ref" + gardener_etcd_repo_url: "repositories.third-party.gardener.etcd-backup-restore.url" + gardener_etcd_repo_ref: "repositories.third-party.gardener.etcd-backup-restore.ref" metal_auditing_meili_image_name: "helm-charts.third-party.meilisearch.image_name" metal_auditing_meili_image_tag: "helm-charts.third-party.meilisearch.image_tag" thanos_image_name: "docker-images.third-party.control-plane.thanos.name" From 41b8e61c98c4d6c07f1de9e9799ce0bcc41209b6 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Wed, 17 May 2023 15:11:25 +0200 Subject: [PATCH 02/15] Fix headscale api key creation for used namespace. (#174) --- control-plane/roles/headscale/tasks/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane/roles/headscale/tasks/main.yaml b/control-plane/roles/headscale/tasks/main.yaml index 86a45258e..9f184f732 100644 --- a/control-plane/roles/headscale/tasks/main.yaml +++ b/control-plane/roles/headscale/tasks/main.yaml @@ -75,7 +75,7 @@ - name: Create headscale API key kubernetes.core.k8s_exec: namespace: "{{ headscale_namespace }}" - pod: "{{ lookup('k8s', api_version='v1', namespace=headscale_namespace, kind='Pod')[0].get('metadata', {}).get('name') }}" + pod: "{{ lookup('k8s', api_version='v1', namespace=headscale_namespace, kind='Pod', label_selector='app=headscale').get('metadata', {}).get('name') }}" command: headscale apikeys create -e {{ headscale_api_key_expiration }} register: headscale_api_key when: From c8682b70ae7b0769d2fb4a2f5298a6269efaf23d Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 1 Jun 2023 16:23:27 +0200 Subject: [PATCH 03/15] Gardener DNSRecords Feature Gate was removed for g/g v1.50. (#175) --- control-plane/roles/gardener/README.md | 1 - control-plane/roles/gardener/defaults/main/gardener.yaml | 1 - .../roles/gardener/templates/gardener-control-plane-values.j2 | 2 -- control-plane/roles/gardener/templates/gardenlet-values.j2 | 4 ---- .../templates/shoot-dns-service/controller-deployment.yaml | 2 +- 5 files changed, 1 insertion(+), 9 deletions(-) diff --git a/control-plane/roles/gardener/README.md b/control-plane/roles/gardener/README.md index d887772ac..868821562 100644 --- a/control-plane/roles/gardener/README.md +++ b/control-plane/roles/gardener/README.md @@ -17,7 +17,6 @@ Check out the Gardener project for further documentation on [gardener.cloud](htt | gardener_apiserver_resources | | Set custom resource definitions for the gardener-apiserver | | gardener_controller_manager_resources | | Set custom resource definitions for the gardener-controller-manager | | gardener_scheduler_resources | | Set custom resource definitions for the gardener-scheduler | -| gardener_use_dns_records | | Enables the feature gate for Gardener `DNSRecords` | | gardener_dns_domain | | Specifies the DNS domain on which the Gardener will manage DNS entries | | gardener_dns_provider | yes | Specifies the DNS provider | | gardener_backup_infrastructure | | Specifies the Gardener backup infrastructure | diff --git a/control-plane/roles/gardener/defaults/main/gardener.yaml b/control-plane/roles/gardener/defaults/main/gardener.yaml index 0cab7970b..39e98e9ef 100644 --- a/control-plane/roles/gardener/defaults/main/gardener.yaml +++ b/control-plane/roles/gardener/defaults/main/gardener.yaml @@ -29,7 +29,6 @@ gardener_scheduler_resources: cpu: 300m memory: 256Mi -gardener_use_dns_records: true gardener_dns_domain: gardener_dns_provider: diff --git a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 index a9d2905d6..a4810e17f 100644 --- a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 +++ b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 @@ -24,7 +24,6 @@ global: resources: {{ gardener_apiserver_resources | to_json }} featureGates: SeedChange: true - UseDNSRecords: {{ gardener_use_dns_records }} WorkerPoolKubernetesVersion: true image: repository: {{ gardener_apiserver_image_name }} @@ -100,7 +99,6 @@ global: config: featureGates: - UseDNSRecords: {{ gardener_use_dns_records }} # ssh key pair rotation is removed in 1.51, but was enabled by default in 1.45 # TODO: comment in this line prior to deploying g/g 1.45 # RotateSSHKeypairOnMaintenance: false diff --git a/control-plane/roles/gardener/templates/gardenlet-values.j2 b/control-plane/roles/gardener/templates/gardenlet-values.j2 index 3d38f7738..9b570fca4 100644 --- a/control-plane/roles/gardener/templates/gardenlet-values.j2 +++ b/control-plane/roles/gardener/templates/gardenlet-values.j2 @@ -64,10 +64,6 @@ global: ManagedIstio: true HVPA: false HVPAForShootedSeed: false - UseDNSRecords: {{ gardener_use_dns_records }} -{% if gardener_use_dns_records %} - DisableDNSProviderManagement: true -{% endif %} vpa: {{ gardener_soil_vertical_pod_autoscaler_enabled }} diff --git a/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml b/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml index 48e98727d..138fffc92 100644 --- a/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/shoot-dns-service/controller-deployment.yaml @@ -11,4 +11,4 @@ providerConfig: repository: "{{ gardener_shoot_dns_service_image_name }}" tag: "{{ gardener_shoot_dns_service_image_tag }}" dnsProviderManagement: - enabled: {{ gardener_use_dns_records }} + enabled: true From 7f2fcdee27ac692ca4c66606e6f664aac3122af2 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 1 Jun 2023 17:06:31 +0200 Subject: [PATCH 04/15] Add flags for garden-kube-apiserver that are required for 1.20. (#176) --- .../kube-apiserver/templates/deployment-kube-apiserver.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml index 6246ebaf7..580645274 100644 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml @@ -103,6 +103,8 @@ spec: - --requestheader-username-headers=X-Remote-User - --secure-port=443 - --service-cluster-ip-range=100.64.0.0/13 + - --service-account-issuer=https://kubernetes.default.svc + - --service-account-signing-key-file=/srv/kubernetes/service-account-key/service_account.key - --service-account-key-file=/srv/kubernetes/service-account-key/service_account.key - --tls-cert-file=/srv/kubernetes/apiserver/tls.crt - --tls-private-key-file=/srv/kubernetes/apiserver/tls.key From f46907b1a255de853992c3f2892bd59b060c6c08 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 1 Jun 2023 17:14:17 +0200 Subject: [PATCH 05/15] Fix garden-kube-apiserver image override in values file. (#177) --- control-plane/roles/gardener/templates/kube-apiserver-values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane/roles/gardener/templates/kube-apiserver-values.j2 b/control-plane/roles/gardener/templates/kube-apiserver-values.j2 index 76afe969c..a99f2efa3 100644 --- a/control-plane/roles/gardener/templates/kube-apiserver-values.j2 +++ b/control-plane/roles/gardener/templates/kube-apiserver-values.j2 @@ -1,6 +1,6 @@ --- images: - hyperkube: {{ gardener_virtual_api_server_image_name }}:{{ gardener_virtual_api_server_image_tag }} + apiserver: {{ gardener_virtual_api_server_image_name }}:{{ gardener_virtual_api_server_image_tag }} controllermanager: {{ gardener_virtual_controller_manager_image_name }}:{{ gardener_virtual_controller_manager_image_tag }} apiServer: From 0b77b926700beedfd1886c9aead3299572224b37 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Wed, 7 Jun 2023 11:08:20 +0200 Subject: [PATCH 06/15] Update secret binding with provider type. (#178) --- Makefile | 1 + control-plane/roles/gardener/tasks/shooted_seed.yaml | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 15908b75c..58dfb98a3 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: test test: + python -m pip install mock for file in $(shell find . -name test -type d); do python -m unittest discover -v -p '*_test.py' -s $$(dirname $$file); done .PHONY: test-local diff --git a/control-plane/roles/gardener/tasks/shooted_seed.yaml b/control-plane/roles/gardener/tasks/shooted_seed.yaml index 0a9cb6038..4073d0d6a 100644 --- a/control-plane/roles/gardener/tasks/shooted_seed.yaml +++ b/control-plane/roles/gardener/tasks/shooted_seed.yaml @@ -42,7 +42,7 @@ - name: Add secret binding k8s: definition: - apiVersion: core.gardener.cloud/v1alpha1 + apiVersion: core.gardener.cloud/v1beta1 kind: SecretBinding metadata: labels: @@ -52,6 +52,8 @@ secretRef: name: "{{ gardener_shooted_seed.name }}-provider-secret" namespace: garden + provider: + type: metal kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}" apply: yes From 4f2635d686784ac6c63e4b54885f7ae32615bcdb Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 19 Jun 2023 12:21:50 +0200 Subject: [PATCH 07/15] Set GEPM image pull policy in controller configuration. (#181) --- control-plane/roles/gardener/README.md | 1 + control-plane/roles/gardener/defaults/main/extensions.yaml | 1 + .../extension-provider-metal/controller-deployment.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/control-plane/roles/gardener/README.md b/control-plane/roles/gardener/README.md index 868821562..3f519adff 100644 --- a/control-plane/roles/gardener/README.md +++ b/control-plane/roles/gardener/README.md @@ -90,6 +90,7 @@ This includes the metal-stack extension provider called [gardener-extension-prov | gardener_extension_provider_metal_egress_destinations | | Sets allowed egress destinations for the `RestrictEgress` control plane feature gate of the GEPM | | gardener_extension_provider_metal_duros_storage_enabled | | Enables the duros storage integration feature gate of the GEPM (Lightbits storage) | | gardener_extension_provider_metal_duros_storage_config | | Configuration for the duros storage integration | +| gardener_extension_provider_metal_image_pull_policy | | Sets the image pull policy for components deployed through this extension controller. | | gardener_extension_provider_metal_image_pull_secret | | Provide image pull secrets for deployed containers | | gardener_cert_management_issuer_private_key | | The Let's Encrypt private key used by the cert-management extension controller to setup signed certificates | diff --git a/control-plane/roles/gardener/defaults/main/extensions.yaml b/control-plane/roles/gardener/defaults/main/extensions.yaml index 8f70e6cd8..58efa751e 100644 --- a/control-plane/roles/gardener/defaults/main/extensions.yaml +++ b/control-plane/roles/gardener/defaults/main/extensions.yaml @@ -48,6 +48,7 @@ gardener_extension_provider_metal_duros_storage_config: # encryption: true # ... +gardener_extension_provider_metal_image_pull_policy: "{{ metal_control_plane_image_pull_policy | default('IfNotPresent') }}" gardener_extension_provider_metal_image_pull_secret: # auths: # : diff --git a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml index c541f93c6..4061346e0 100644 --- a/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/extension-provider-metal/controller-deployment.yaml @@ -47,6 +47,7 @@ providerConfig: partitionConfig: {{ gardener_extension_provider_metal_duros_storage_config | to_json }} {% endif %} + imagePullPolicy: {{ gardener_extension_provider_metal_image_pull_policy }} {% if gardener_extension_provider_metal_image_pull_secret %} imagePullSecret: encodedDockerConfigJSON: {{ gardener_extension_provider_metal_image_pull_secret | to_json | b64encode }} From 0ce0feaf60bcc553b45680754bbbc3f35b8c06d7 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 20 Jun 2023 14:01:46 +0200 Subject: [PATCH 08/15] Fix regression for gardener dns-external extension. (#179) --- .../roles/gardener/templates/dns/controller-deployment.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/control-plane/roles/gardener/templates/dns/controller-deployment.yaml b/control-plane/roles/gardener/templates/dns/controller-deployment.yaml index a81a49d5c..b566177dd 100644 --- a/control-plane/roles/gardener/templates/dns/controller-deployment.yaml +++ b/control-plane/roles/gardener/templates/dns/controller-deployment.yaml @@ -5,7 +5,9 @@ metadata: name: dns-external type: helm providerConfig: - chart: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/external-dns-management/' + gardener_external_dns_image_tag + '/examples/controller-registration.yaml', split_lines=False) | from_yaml_all | list)[0].providerConfig.chart }}" + # this is generated by hand because a regression! + # see our fork github.com/metal-stack/external-dns-management@v0.12.6-crd-fix + chart: H4sIAAAAAAAAA+w9a3fbNpb5rF+BUWfOJN2IsuTY2fWe7lnVdlOdurbWctJtv3QgEpI4pkgOAdpR2u5v33sB8E2KlCw7SUMcH0sC8bi4F7i4L4DsvWCBS52e5fLeirp0wVbMFf1ne0wHkF4fHclPSPlP+X1w+GowPBoeH2P+4NXw9eAZOdonEFUp5IIGhDwLPE9sKlf3/DNNrIL+xpI5K3vhegF7cB919Aey5+h//OoQ6H+wh/HVpi+c/l+RCRU4BTgRHlEEJ/dL5pJZaDuW7S6IT81bmBbc6HxFbpY2Jzz0fS8Q8AUmiUMWjjcjKyrMJZR+SQLmUGHfMagnlql86lrQgMsW8NRzyXM/YHP7PbPIvQ3l/vLCIFeusyaeK2siSMRnAXFslxkd42z661QAbNDEqbdaQQPvTqfEsgPeMRa26Mv/CvyOMfsQ9OX/KGO56OO/6Ce/c/tJQzMYX+iTue0w3vna4Pc+/J/RW/gvVvD9/6DoOxrYXsjJ+OwcOvQD75/MFB3Dthjtq3KQ1THuuOlZrN/52FRtnqrW/+mSBsJY05Xz8D7q1v+wuP6Hw3b9P0mivv2OBRxW5Am5G3So7yc/D4zB0DjuWIybge0LmTki38PGQEycHmTuBUQsGamYRMT0Vj6sZ1d0XLpiJ1XlOnf5Lj82Vr6cVLX+Lc80Ft5++qhZ/8PDo1e59X90+PpVu/6fIvW/7pCvYSf017BZLgUZHgz+g0xHEzI9J7C6qSt/0DnsjjYVTC5p6q4NMoKdX1bhsONzFtwxy1DiAW6kBD4d22Quhw0+dC2mGMUIZAn4mHpzcU9B0LhQRV6SO4MMgT2YzBeEcuJ6Aup5UCW4tzm05srqF+PT80sADHsAsBHyi8pO4rY1QyND44A8xwJd/aj74j+xibUXgpiyxk5JCJ2JeBAaIOgdhw0IcE2mxBWRdGBgGz/rNryZoFCcQgUffs3TBQkVGuilEP5Jv39/f29QCazhBYu+xhfv62H2AGBd4a0Lsgki+l+hHcBgZ2sCnBoq0BmA6dB7SatFwOAZinEuuQ9AHEKxi2tcYzOWzUVgz0KRwVcEHow6XQAwBtTvjqZkPO2Sb0fT8fQlNvLT+Ob7q7c35KfR9fXo8mZ8PiVX1+T06vJsfDO+uoRf35HR5c/kh/Hl2UvCbCQiYBLEPRgBgGkjJmGyYFtTxjIgRBsK95lpz20ThuYuQuBJZOHBJuFKcZQFK5sjRbmUKSWG+p1Ov7/wThbMZQHOU8Pow98SRLt+lNczPVcEnuOwoBewBQ5ViqIGXxLkfhEr1HXlFsf7VQxSl3p3fj2FUetf7D2F0QEJq7pCgYpE6exyOgm8OxAigxN6z3uBB4g/OszmA5EdL7Sw++yDD2HAKnL9wL7DAeefLjxv4QAesMH8M88HCgjAVw/2exCUEYnp57LS3KElnbpMOPZ8Xci33bk3c7z3hQcBW8ECR5KB9iGVCy1QMBcnNCdpSmptQ84NnYnTAKeJ6QUByOEkwTbJYLvjp1tvpYqSVDW9BYOJDOjne7AEbW//OR4ev27tP0+R6un/Kyj5wHO5IfwddcEa+h8Oj4+y9B8egALYyn9PkX77rf81ubNXJ4QzIaUesfbZNytECwgmJ7C1/vFHB0t1zt+D5GdJxovqXCTdSN5qdHS5HrHY3HZByKqYWQbW7ZJeUpqGjiCGMjhcYsPGO+qEjMuSV7DtB7BpkN+JCELXJMeH8qu9moYglL4n3V7SGAPw8LuC9zRguH/QuI956Dhr8q8Q9tQ5SCAoQsmRGJ2fmGpdlhfYB46KkxkzKYqE3AOwfghnMCAGa0KNH9pwLJBBQI517JUtlOwlpcfnIKAhbmDTw7K4X6FI88LojOdoIGOUayTi1gUSI9cboMyzBQiZIF7PGMqjFgrEVAKvoW2MaqyTRbc9j7EbPYwxHJWpLNCIBA5PWvqrHM7JN82pnIIzxoxqxbhWaFOVY1gzuVsDCIKSK+ak+zfe+xvv5lpT/W4z76q+Z+ZjitC4nLTtA4ksia1njirl0BlztiC4rNWtHF4a++p7pBv9DrMS+D2oN91/65Lur91t11sCG3TqgfKxNh3KuR5dZgJytoLsU1QkYel0/+sbMjAGr3oHCCCoQzPQNIUNEwPXm4bPeGOLCFRsiQNrskIH1pVx++/csL3+3SChblUfgwf3MWOCDrLTqKwYdfxlVK58OiTosldAvAx+lpRPpG2cdPmSDo+OT7rkeaz6df9qCLpAZS3K6hKZ9UK3QdJFgaoet4UXrPM1Uk/++OO/c9XKeyDFJbRLZyfbdVaCwI+9af6JUr38ZzogCwB79hy2oz+gTv47GAxz8t/gaHDUyn9PkXq9XiftAwhm1DRoKJbAwz8oLTrhsLe2a52QUzUhrmFCdFbAEC0q6EmHEGXjh8VtuzBnrEZyiYHLnKh9jmMjhKDj2eBLZX9p1p7e9nRjBAU74zaW1hD45rBl4CprykYziWuq5rISQ2Ud1b7Vm62ztaYsuLNNWTEIHQYY6EFt+03ghb5ER490u/ARMO6Fgcl0HlfVeOZHH/YTkGw/ACV1vhkwgd9hJ5zpmgsm5Kdjc/Ul9IF8TH69R1dtEQBElCvNbQoeGNS9F9wmG14JgPAQzX3s4b0DdYwFDSw0/BjSBlXsDcr42rTECxl9oJYI43wgcWCz/M9cIeq6npCTvywrV9i7d1P9ql+5IksPFoz1wXNhG3RsM9V/8UmuquOZt9kf6QLKkEZNQAU3WSDQZopMe+PDpIGtSIPfTCnCKnBgCsPXRvOV3QGecx2mmvLLSQ8/k7mn51qxbROYkbeKMqVUZUeUazg+DUoBAEmSdbFL37NgcYUo5KboGRVCTqAL7sQLU0CDQtAMwaArze3FivoVgHT1Jt6DJeEKgLtbxM7H3ona9DHSVvLfDPZ/5PtbioE18t9gOMzZf4eHB69et/LfU6SHyX/fqgnRioH7EwMBq9dsjjiI+P4GokCpokz+ABLwcIaBbVIOVQ1r0Eam6YWueBh58Qf3aRnOZDZpVfsnTw34f2DxhwUCbuT/gwNQ9l/n/T+DwXHL/58i5ZwCShQ+vT7juBjTG0OZNJ7aE6QUfq0lz7NYCi/ZGRKtyyhV7tpNoIO+IkTAQrH/UjS5kXQfbQFnl9NzwOtaZqGa80Mm+0IpPoT4ThhQ5ySrDBMCqmggLpNGIwVZPYRNPnRoENfCXrjp+TDymH1bSqeQE0S1AbqLZcl5QJ0J+iLQJO+EKzfuQ2mSqfjS7/7n7BI9i+g5S4aj0j+5506oWJ4QAzGE0wc7T5VQpIW6qTz0Z54QjGsCQaWi18haIAtX9CiVZiMqeZMtqDq++XlyvmXPlGOwDbMSEJ7H26ScqS8agVMAZXJ99W58dn69JTiStiQ2EFR3jB9FBExvRjdvt0W+6lNyHvS9CBuGL+jKL+8/YihGVOGmpLyCZvSmSA0rDXYekPkchB88ugBbwkLZzqoxUCyjJ8Ho+s35zbZIkHYjYlvKCYh+ZLogen1iPF8cnIYLg69B2FttWBmytbFVgO3qp8vz6/HZlrAhRRAkB1CzGSXCKaLj5iKVFzkHT8gQ9v4CHMgiFqnZnAME7WSAo40wYJkCEL9cXZ43h2IzNjBKT8YhSutQPcfQ5SeyuCHK+Mbk6mJ8+vOvOfaxTygxuEPZfuZ2hlvUwQsVx2X1MoBPz28yk2qfkPs0gK6EMrE2hbq0Ugbkyeh69ON0XzADB+IYZ6gyZ1F8InAoADSJas3xzOIodDsFoH88n06z3GwbcFUbsVtaZaLfekWTfRjjP0eT8bvDae4B9gUPA/SXp3OlghqJh5l8kkXO37FZ7VhXXm8u0aFlBWB3ChYVUCQ9wRgsDJNO8vdc01CIusSTeqoh1VNoBqWX0LHQDgo/BbRgegvX/hC3zVVoDsZKgxgmcm1KyQSZ6x1KwfKomozmDhj2QkI31d6dEpTJj3hQDiNcT2Q4Nz/p9xe2iARj01utQhCB1yoSGEOqvYD3LXbHnD7s+D0amEtbQOthwPqAyJ4E3ZWWa2NlfRUbeP+eg7V0OmKSUuBGOqBAiJ52qqursSTojqbt9fn0JrYiS5LkaSCxn1TkCSEQbYAVGVqOYfSBt1JHhFzL9wDPKrjFsZmbJwIPZytbqDgAIBFSzCCn0u8iI6Gkvd4yyNiF3BVzTlFofmwyILZ5D1HbnBBpladYQeEu8ygS99Opas1hMnFBX3jebeiPceYCGYuFcsR3ZHE106G85Emnl6Mfz3EtUkEw4A/RjGR37pQIMp6UtEpQpo+de/kEza6okBv58auS5+XbfAyxEuhrByNj4ZI4Pstb4aELN6sLZLssoRMmLSjV9vgw8awRKAGDVcNQmawDJi6JgMwwlFCJ0BFstrtksDcQKvRhDrUOS5rdNMswuRXUKICUDgiNwbNi5bNs1jdESwSGMhw2hUXZEx8JoChSqRya3uaJWNFxRn6pn46+UqpzwtIOFE4kpSrcplX4zW0lg9xAy9xIJlo+jQ78lEpuxQ42kC8jsTaaMNN0DdwgcdIk0nIyjUwvsLD9XeexLLAFVjQoKU1Dnrvz7mM5REYflxgB8m2DcIfjeUmYsTDIP+4Zntxj1j8eZwHUELFHqrSKdJmcppQFrYL+WievXT+qnKYoJ8/lToiH00ZR3ov46JoA3o6Pivp+kqLdU09jZpWUAjFjVbFwatCtHtMgoMUVjtDVDxeHoAdWNa6POQZRL7+kbRBSeIlo13jzfYhoUj3he6RohUxaKxP2pKa3jbjnUC7eSukXTV31kl6meBJGj5wktq1FPA0LF819SYqjZPIpwiY+7mGz24o7WtWtHc5uqnUjELyZOrf8Rp8qLOqx6ZHuJtJGvLl2nAVD8LajSVuma3vLGLyV4Bhf5pCzujfqfDvxJTG0tvJLIbXyS0nbX4j8ItlY/fKJnTRbs7ymAlLBFRIdvW7AKD4RIaERU3sYc0dfQy1Q0mnxUA5bMWvKJ3JPsqPOxtpq5zvBQ17RJOLCC3BHzuSFs0yoaVRSyjDktz86+fixfYYJpGKu21CBmlABZflObPHyDgS2gIqXyiG2bSzBKEa+zM8EFCTPyqIKstHzlZEFVD1MRxbQdJ/7Ci/I+0aj+XzN5kYyc1OrRREdHsvou8IyKniCNnWA6Cxr+ods/g4t54xMccs5XWTHliXGq5rPP2zQh3JvUbmlFJod5bNVgzPPg+nulre4TRjAoth2SqtpvWOtd6z1jv15vGOp7adMNmuuDjYSVqvM+Qk/LWu8Tqut5h1Ryl7KOBnHt5xpZU9jAe/xepDrpWzdlMKg1k9F7x9xIjYeaWNf12XK17UvNG/n4QIIMg6uArrP1H0b8XUSlZp1wVemzYXI7KI5bDyOkl0qe6UL5GSnbLel6N5kps1LpdmnqfXamC9tbchV4k+twjh2LXVQU3nmE8Bxd4pslvIKQGXw9gKznMDJtWiVaMzLWFFqap2F7cZEBzTKHfoUYqhMqYiJmcNW0XWKDAQQbcMtjRwgKbNuqxvX6sbqnHGrFu85gv4K0VpUeGV2ma4bH/6uVHO9oprr6U60hqvPbu1Jva0M/S3kfwy1TbcYHQChK32qLNvyW+Q9vNByTXQwe+/bmvfknFabnFV5/IHUb1tvQbQohjK/w0f1CMwA9fRh7Z+3PrsfDbbVWet01j1pqa1emkl7kP6kWyeSxiUztxPPAcp7qctulTCIRI4CABXBkbxRwOJzfffeCUouL0p63ywIwnYl+XylezwD/FUUl0bjOpLMcgAR4NGkVCCmSqoLS5w1ubOVaHvzvzfaa7mDDi2XQiN1KoZarZ40sjPQSfdpFGSpn7A0um+WLL7qMHbDxpqV2ubjBoDPWyk8PEx/3BA5WxiuvpSSPQZRGgBbpxeWx9MkDVcg6hOK5E1EiOaBnMp1mUwlveEBC1V8Aw/p6m9zR12Yhyx2Th3+COE5mxTpooCZNPaUavKeGeXWnFHPmVo4z2NvPegPtqlB3NxzrX1Qys3NoqHD1UzFgESTvNngEwRUeb+jEnuNvNnUWe3I8H1RVUd5i11VctwNj79IW0N8l1prbnhqL/wkHRqVMUlET8qsEunb8CoNE35QtEykIrEey/1eegC22cH5Rz2F/gmcP9fTOzrQtPnAqy5jRJUKvY8vTy/enp2f/Xp29eNovPWVCDEwGL+0GRRZogEgeAq7PXL7GRhhck23QQVtUMEXbrzRWv3NzUWtvB8ZAKBsEvaJ+kisZM6l6hafs0HDNd4iKpz9R6jqXaIB0Fxex85BtjEjO2zI5evFdBvypRbR4KjjFDaGJOFijmrJV7d5nKni8YstlNxbZo+q033Ye7nPNFJ+dJC59KMmGrV6DSyaxjSMQCAJXRk4mDZENGNqcGZgU2Sz7MF96KBix/AnM6qNWlYkaZ5Kn3FzO0kiciaaXnz4IuuAVnjaFjBC3vcS+b/n65cb9kL31gV1uafe+pJRzpKk7tyuCPrJHveR1zKj1CnPEegT+emXqaYMUNrKUUqFhQ3Ms0rZ3dv5bqkhweYCnB22CvkORPmawyR+RFo0o3PptHJWKQw9NLZkh4AZDWZaGlI5eiz3S9tU73VUIFaOQJpOo8OSCh87jmLzUdYmx7jk6S1tAksuENNMPJaIuQ9bq42svNImom9cS0xm1Z6CiuFIVrOXfQaB1lc6xMXKmSZPNljN/CQUL4mHbxIv3a/aDajdgDY92t5cvYVoSEHopfgOMjlt04JiTkb8aHJgDKGmZYN12HC5NIsiPdeFozn2ycyleIU0G0dsPfnUxrFxz2lPmtdblNQKib1PpWNJb8fbQvsUh9KbHTPNONh2HU4zsSDPdqpUy5bXlIzjc+Q1X6Qfrfiyn9ah9tQOte8lDX6BypPkyomMYy1foszBVvpCp0pP2/KD8g1lXG25JlIXau8vHlh5OMzSGFscIDnNPaqKs20PXrY+ktZH0vpINgpgfrPLfxLOGrtAuLokI7lEZ9fbgLCZKYqtp/jy8gqVvBSiTK0UYDgvIgUdv+euF88mJTGb2MojWASVEu7VXyUlx6ML54YStZEJlFNbUSVCdxHElQia2Qo3gnyWlNcwcjzhJi83TV3vimx2A7QRxEZ0AVFk7AMZxLFVuGdkBRRLYAoClhlwl1Q3H03kT1/f1Qxnk3SNNNYyIXBV91A9/ogQ0+OzZmP5RZUtHUV8+5PiEpmb5p9yRDsePK1gZr14LTbmx1ubJs2qKNGscySOo1RrQ+6QfnyDV7J2trYzoM1nKqHewqp0UVKpgW2plKQKZZ+AbelTOCfb1BjzNOePN6zWus0lvb1Ur/eS/SVzTfXGnbwBs6i7b7EEiknm1sU9waHYbEMIxmf76XbzIQ6p98YkqixS8S6lbCE1vPKF0CSMu4ybf5HWJ/ny6NbgtOcD4xeA1aINCXPL7EbR27wrTUVO0VDkqB7at63lO27ftvZZv21NnT2S622/7z2T1/a1Lz+Lgdjny8/aqHuSSe3VB5+f1bi9+uDJrz6IX0RUpi/s7Ua+7FYf9Rm7xaOYTcCxDwsOQw6lGTB7qL3UW75R03j6t1ahSNj4qLvaYdVZfwckfS+Iuc0qdITtO6zJTWFVF4pH8kAtNMk1EmpPzUsRUXqwTebP+DoReaF8iegVP8sIH0n/+zmQ/yks37kXIsN3c/pG7XAxze2Ai++o7TBLKmf4KrzaKfIdViJzWUtOBcfLXgcdt/7QCftEYXePaAZ9RIbUjPDtS2a+gJfMPP7bMR62mdVN1aff2dp3YGyqvS+r7W+/9VBBQDPis88tVdhh+yD0+FKhBN3Dd7y1NM6u6crZoQ/Q2w9eHx3JT0jZz8HRwdHBq2eDw1eD4dHw+Bjzh/Dr8Bk52PtoSxIoBTQg5FngeWJTubrnn2kqehx8nngYzmLal/gUmpjzUefJmPSTc3sF+706jidLtQ6J2CERMDwcTrkq+E4ZbHSmDN9UvcSBsZ2E2V0zabVlspU7Gwn8vY0cbn1hr2zA5VEnH1UkQ2IuUtjfK/J2QV/EiDR8OVNF6WX/5pKZtzxc9X3uZ+B87qNbhPzVuNGNGt9CZ2iZJF0sLBlc9wUA/DvhSzo8OoZWEAzk8PY8Rn/GgW/IbSSYeIH4Xgg/GauUlgDgJQvlSGE/pEiWLu4t3YpCvoeTvIsdNuytm95/VJNOhoB7JuFuayBtMsIIrvHk9CRzex1mXjJx7wW3xQeT8Vk2kzMzRKvwKegJmTegzvkb5cE7Pjo6TKSZIHRHG55ceu418PfcqW756C0gPFtHK1ypVw72Ei9rL9FcNBYTkcleSdGim0a7zOvGtJY/JVEzlSbARvW7FdNsQD4z/PhhmkiVGCJ4GNa7nwT2Hai2C3bOTepQbaLKXSdoUp/ObMcu6vQW6Pp5iWt0kfN2qA6sYrt49+WV66wR6d9BIWVlyaGfBouM57LXQyx/s9O2o5JexiCClq+t1DsTn6MVNyoVe4I5Y5YuJdYv0k0jeEl1BDKSS0l3pG3TNk8y9d2bFZ3Dw4o+/0LGInf6/CVZON5MHp+CelQQXH1C9oDHnqO1yXE+yxc7SKunVGhxgVJThokZ3Xqk5MarrgbtqasMesoqDkj4JjVDmzQl2RfMjw3olDjI+uP7qV7iJ+qLoa9aHWukQcOlpTOY3WFw9Q0VOPP/t3etv5HjRn4/918h2AE2d0i33X7NooF88Npzu0bm4dizOVwOh4XcktuC1VKvpPaM55L/PXxJIik+ihTtzSYt7KMtFqt+xUcVSRXJrg32nhyLoVJmaYE3widmKNhD/FEYC3AccKJFEzTIQgwO8H/rg2WM/pktqwaEj+SZoQyPqa28YkQiCkEvfMuFyqWeb0aPZ+B9ixoCJZ9S8s50KEecB7riBAjWtd5RYBwN3JQHzWTZylp6tb+/j6e0qPX0F30KnVdXUWIPxxA2KO975CCZk5QLpiWZrnsavcmwMLQ3JIld8oRtYfI9HufeZl8H1dYSzO4wxbRGJAZwJm6+0N7HX25SGv+gw7aOv6AqJzR2cGp+zujQ+AKbXXz7o7UMGS1utDO3AgVKCYDeUMxK+LAyh8oZocAN6ps5nr+l1fdb/InCBL/qiWd3mBqE3SQiDPK3ak+nxc48oyt6hZgw+P+8MTcbHvsvG1iL0bN3xtzNitlFYNdlmSt7akc4Y6eRoBFEmc8svRTC3h8zAKwTyLDobtNmuzFAq3E6BNaAkTOmz/VNuW3S02O7Rf5cTytK62iQYTLGQzeYYxV2mDUGSvFHb7XFHHYXUwwSEAS2zhBrgNvtMFBIEPAqK6wBbjbCAObOgL9uqxQ0WsKEHkMlCP+xoA29coga1idBEnxxW/tjh9qlNwKYBwCs64lKyPZ+CBIQALaqDyohm3uglbEXVLzIiTheFjWwG25oBs/uCBQXSBVb51TqAu+kUIEjtYF1WV4X164LkxVODWNH1ikC69BQeeGU0XZvnSL2bg6R46rAXVYk50lSpXUtf4TDkHHyNKbpUxx2rYdo4OQKihwE8anJZTTk/bRpcj0KVVZn8XheeZ/HsGFIT+1u/OCSguigN3w6JUBWz0HWKDVsFk9SwsHcgcWEwq8xdQYNrHbOQVQoLRQ2zqCB0cABRTgjL9cbHFto78WM0LH/QtiPxWzotQPQsP4KEuANu12YA4wdewW8V75dxQZUC1Ix/kvizoIDaGa1sEq9XAytm8ywKumMrlUpu+11lRtWMZUdtiplNscu8ryVga/D9ur4Lsg6Cg2nE8RKeK/UuoodrxbcRPgt4bpJDKqP1T74ru26Sg2qldE4+Cz6ukjz1oQsazmNIPxWg50EhtIGYhI8l4ndRI5VCG4MfNaPXaQF1MRqBvwWlt0kBtTHaADcV5zhkkbp4LACLWnjvRTtIz+wjmDL4L9G7YUgkJqO9sJz8dpDeHj9YFbEe1XbC0B4Le22xWu521mwr2YXupXmVgnIkrOBlzeufjnMxQCOWot2Fh1UOYDlG7dI7S48iH5gkzdi9dpVbmjFbLZu1LK2u+zQ6pmMnPd6t5tMX5UsYYydGg5BjADW3mgzcqGhcKw83nejRU3pp3iX/JQcXEA+EaIsAPh2Wd5qFPVFHtf6RkO2u2EKAEwFrxG4LtM8HuxAEXAlmAKEa8BrBC7cim7S+rlYXqdVVupNCe5kpJFWhHq6IeQgvCYZI6Gb+1YLGdiv1Cy9EVbPN9tCD656nlbbAgBrwMcX0Q9lucpTYvgSl8HNiuSjXxi9RzcOwsPqBxjfaBV0GuC4iA+jIniIIyvoMcaBSw6um22UY9IOPMxxkR5cQ9NAx6QdaKQDleqr1ZV2u22ng7jP2IedN7jivrzLyy9Os7mMZfKfy0HFBlQLYOfUejkZObDgAJqBzZugl4dtA8oMq5LNqmmVAps0sNywipmMmVYpkCUDyfNVBp+AT29xucCnsViGw/j8wCk9UWtKTm8BD4itcnwV+JA2eXb/7GTpCprH39ABhYbTCWDmlEo5WTmo2PFqgW0cr5SHiYNJDKqPzcDpNALbN6jUoFqZrJtOI5Bxg0jz1eTjJsVnqiwfL1N8nCbiDzcRZZt3mrSZ/UyFI4jwugJMh1FZJxPiCiOcumCTolLWw7S4IXgRPW2mxqYp2OS4ongRbU0myKYpyBS5SPfWEJ+snCX2VbySEbos5ZmY+wIGr5T6rJK+xBKptWQdCjRoQQq3gGrBMaopuR8CgFDHdSxM+BJ5C9lvoRwkL4gy5kYhKgFsHib2vpjBfszDZ72Eh3LwR16+52U8DdCvOPuQ8B7jhhwZBh+40iPG/MaqAFlh1ACMSWU9nIahEGnjNIF3U6qHT2+1CgqmgrXnDpWAd2CAsGCKGDvzUAlYn7YI8QePLwZLtnlq/kxedXTQr+UGzr5gb8mJiHZHSk9OdHKjBtb+aBWHP3EYjUc/aZl4o8EHSdVNtrSXXkvpVH4m9r6YTUGKoPDEQJGJOGCG3HGOrLe1+L5yxC4laBPiDh7/NYSJ35rQyLk85LIzrRVWsEsyIlDld4ax2aLx9H2WD2uqSzGAUOZ2xZD04WKmWYwQiAacugBYj0CrauAO4XIGTs6gaMhad+YtUhYNopY4bQCQRcZ1t0agoVpHbAALk+AJ3BbX5xPOB2DtjFYTvgeI2lNl9RCvHH4AgvNUWT3Ef4qrVdpoy6AhyYCi0PDxQGS1Go5RgRaWngiVlgIW/Kfh4AGE3A91215sDzK5LNOUXtU08zLBroKDaaYpdL1G1oqAyRmpgN7GiMhBNsfCeDRUs0EYAAYbCJCUcejffiEHwl+SO2Ot6FNKPaVXzILB64WMA/+n4W0BMmJ8WwAQpcRtHDRXo+JvTF7OikDNh7PZCG8vWN/At1KVFblmSnWPgwy87Yg025TcbtTfoRBEbGi1SCYvvZ5wTn/FBoJDaIZL67pK77MvQJVw5Uw3JIebKmpJoXTgbpgDqUDo3RUYiAmBn3yEvbIaKIaefOY13sUDlBEM+kdy86YTfHpZp4cKQ1kh1LhJ43xt9b1Mg4oQu2EfCggB+zZtrlZFWaWkaKD4a/RvRrLRynBUxSA0hE6meYpCG+jcBSbHQ4H2M/ANvbcRv4bParpv91Wf23d24wckuMaaqrNraq1CN7mBFNPPgtQagWZDQEHBVDDPjrSKgGdJTlLDaOXbucZ3qpfvTa7dyLv7vFy/Ac8GlAr5zQq8YLyUurpZAlxf+2zBD0hIjY2zB5Oq0FmEo+TQuulmFTbV7LMLN7Eh9dKOdU1KAca8DgJDqmMaLxo0go4b3eS6KkYjYq7Jt0vF5yaaPN206XqkBkbOmIzrkeD1Rz0bV0D3JRq0X1TJT5skbgbVTFKR6UymW5KuR6Tn44po1W8Uhpy/P+pYAAdZYdTQB9qN2/3vIm2cJrZAuxGb/OGCgqmgCbQbtZffRVgwRRSBdt5b9qFCnMFXyPOqZxQkyTpn0DBwxdFu/4WYF/9N+FApAdDrrcqIvfZgOSMUsBkT3y31QBFhkGtsiP/OebCYMPgVpsNvgzyIvTvmFb5TCPEErvyxDAS6z2qfk8Ag2qjtjkYLm9EB8B8BWrNiJ6C1r9KZGY6CZ1iNk0HCVuAg3P0Rm+MTeMTguQGEuT9gRUwCj9IYj6Dl4g/HxRj4GYGX6f2Qbu/U3cP2c9gKIw/WY1XRRVxIVZSrh1ZdLCuGTgLHaqNfGVSoAVoNhEkIgVu56qeBbVnpA7Efi1kTP6BAbI8dAPAOAlcdM6CFbIsXAMoYC129dqpAbVsvtTMeC9USG6DA7BAX4CBsrB7a9d2hBo7+KOA67uNwyGEcZjyOG1w8bu9S+mYgtUsxCFfm9sfAzim+TDd5+XxRJYO21pN25yEnhBgv2xpaGVCEP/KhzeSgmqykjoU/lPfZir67MpbfuiVDRhxUcjq+rkjzNK7TS/aXDJAkThOWqoelZeKFRjXeo0jMgzplZi8EN2mRfr5M4yTPCg2UCpOg5k5pLKC0/DzR0UgpfI6eDhylmOJD+qzY1Nw8oTXVs3o61CLDkV62KZCJlzMuyKmG7ocZWtm641y9S5/SwVZb9B6VHUowYRlmdRWP5+kN+nd4hm2fokegzu2KodCN0QEDcmVebwCoM8T5+XKJBhQfi3wwBuj3NuaYcBoTymmJSAEIDcydAcNPvPQ+6BIoYzx0/acE//MsoVL80du+I3geWwkTEAS25iOC9+mUUCFBwCu+IHgdQglh7gq4REzeYVcmI8QJ1CHqASkzOyNwP/Zy9GmXjjLDqaS3H+MPtXSVOl4rm10ZeXalm8Cg6mjszegjKl2FBlVKYYdGnUTpIsxZEcvJky4HTpp4ueLa5NtVVvyX4rAQmjI1nxaizu4MwvJZx+VTjomXDy5VZQHqSJXVWbzpzEroUZVaJr5otEgAIMIAsC6S9cdJOq2RwQR4wx6ukPU4TQtkGgbeOEzLYz0i4OqYlas3TNu3Xr+DT0HsR2FWGgyX802N3FyhVZYhjcPwxcRqBCzN0MRpGGJmNwKcYogBHk7o2TgDgh1C6nP2KID1OLT6GYLXEaMQ5n6AbYN/95NE7XxHI9V1Ho8DQyG8R+NV9SfHc0FtPP0w0uXEZVrhW8CWiHltidjTZXMJ3/MQHUo7Z7VGqBNGD+NprfBDWg2MXDHV5kNZHc5iNXByB1U9pdV1WTU/Ns3gEFaaiiYwVTN9QOkmSDo+PoiyZQoP6WUZvEN6nQQG0UZd/RotbE0BwH8EaE1Ir4DWHtJrZjgKniGkVwYJC+mFcPdHbA7p5RGDQ3ohzP0BK0J6eZTGWBstF384LsbAzwi8TO+HdHun7h62n8NCenmwHiG9LuJCqkIyOOvyhHP5KTMQOFYbfUivQg1QSC9MQgjcynABDWxLBAGI/VjMmpBeBWJ7SC+AdxC46pBeLWRbSC9Qxljo6pBeBWpbSK+d8ViolpBeBWaHkF4HYWP1MMw+ZA0c/VHAkN5adR+E5RqIQR5nobZrH5xuezByc4VG60PGQ9/qIQxz+cm1DQo8BgJ2xuOhKp2+j6MHsPZDq3fqLo7cxMsfl9Jpwx21gZEfJo1TBjtiLZcRcNRO18nRGrn5QVM7VaAj1bHwg2Jxmu6O0srWD6f1AzElc/s6DGDth1bXC4wfhVVZ/cSbPgczIMBvwWZ+fui04xuHMY2BkzOo4b1MxuuYmnHXMD2h7pAN94iw13qxinyuor8Cbn5yvfDJxlOBcX9/n7xapcg0xE2aRAJLIWsVF6s0+l2WfPlD9Lv7PF5Fiz92SOIkyXCOOL/gGYgqIeQ044sswONV93rBiUMZaQQ/pl5EHmvxiGlVNuWyzBfRp4trHcK0eOJzoT8XAmlT/k+8zgXyv0VFViRp0UTfWUojz55Q5dT1dVXepQsOGv7A8EPa8K8Q3rh5WEQHD8glNQ9fxST/Uogi/FVnnS6iHz99uuYSsgLVepyTbz23eB9XUi+i40OOosnWabltusRTk67tLp+aV0pRgh0dV47zQ0UbKqte223dlOvZU5lv1+n7cls0Ncev/7TWfsrlufGZZGh8S1WIEMpRoYoqj04pZZkNUNiUwV1jjQVd07aCvyXWB8tYqG88aF1EJGkqJGn6q/DKUvjQcmfUC123syk61WtBvwiKXYe++0CyoF6SFeTjQLSXfmnSChk2cl/TOi7iFeoJRTO73+Y5FrAXzZBU4dMuLwxg4MTCkQhN7aUG25HBC7YScb5c4qbgqzVjtqmyssqaZ/LhZyQvjPRz1jx0Ghdlkt6mORrtl9xeLP6t2tqqy0YqiIG0+P4eGzXu80v7Bi7FJqMp85RaXa62uZdekr75DT+aBnLQpOtNjmMcDlBCOe0GyrNnVCKOMg7R8+b0lPwfPfL/D88Oz76ZH5/Mj06Pzs7w+6Oj+fzom+jwRTSWHtSp4yqKvqnKsjHR2dJ/ow8ziDFqyb8vykYzQslw28/us7T6j45iFVcJHrui4UuaMArUeQfJ9EdPgDpMvMn+Qgfyiwgv43a05IzHg6d5nG8e4vnkEXW6RXT54ZbMnyfrtImTuIlxJ6X+hTbLdEoZTFsZLJ2s6NBx101KdkjNhIWeepMuMa+SLrYsIqKJBIYbtRlUUpKJBUPsRrxs0KByETXVNp30JuTXq397/38oazQ7EWZEjjbA1v9PDo+k/n98eHay6/+v8XDTy7b90grHV5FeswrHbXc6nbr32x9FVs+KLow7TsHW1NsOWQujC94TtymcRz6jPYs0zudhDvpepv8X8d7jH3v/b0eYJBjJx/1b+/8xSpP8/+Gbw13/f42H79L8xEGodbbqxiYLtHdf8xMPdb92nYWQz1sLNPNun8kqL+/wGgeJF1xE93Fep5MkrZdVtmkI5r0WR0SQRvfINpAwv+7KcSawmu39m3d15QPo//XGq9f3j6X/o05/Ivb/+Zs3u/7/Og/f/6mrRG78DvXm1otfl8ltutziLmby4e4rDuSTNptwP6T5elY/HCwfYrZSauVHSLnli3izISdaVUWKo/qz8gCOTVoJGbLK8PbpQjWZ0Oeh/JPp3bOY65Yu/7SjEHG1rmZFPUP84mpddntVBgTox7Jcb4S9LHGBpnDtQoa01KXl3CtAXveEuBVIOsV5Xn5OE3qFCDkeq15E31bbAq81H7Bg8G/h/FgOjt8iUvDjRms6rVTFQRYW8VtOeTxABSmVlMvHtGox/EGpI4i9n47sZzsiZhgv4k18l+VZk2GM//t/KIUxOE9UiciLPyGhqBm2zpOxum4T3tbLOI+pL20pqvSXbValyWVVbgSmE7yye/7u3URYIZ6y9Vv0E08dPqTN57J67Nnhl1fXF+KL66tLTuC2OK9/qlM24K+2OSqj98gy3+AE+g7PUZi1mKJ5P4J7dnp6fMLW69bxl/5Fnb7Liu0XnhlhdF7glYF6u9nkpN/H+Q9Vud3UYaTe14RbGGZVGif4PKcb5HXwIQT1M5pIrdm6QUj7b/f/aBSVjhsAWPz//Oz4WBr/H86Pdut/r/LI0/rqLl7O4m3zgAbVX+ni3+N3xI49tSOCG9QgQg0CIMt0u6ECyogNCiqBKcqd9VZrGu3tTaSPt1MWSLCONzWXRv3ahAUDuH5i66dTmCeaD94xXiti96cRvVfLAx/HiwQ7Kngsy7JKsoJvjUOmpMTCMeQ5vlIRfsZ71LnCfJX+D7P/d6jjZ8XK0w3Y7P/R6aFk/+cnp7v536s8nvb/e9ogdm7gtd0AKvub9J5MCZhNM9QZouJc9ogaqrck5pc4IMrxVghiePna/7X7yb/qY7f/dHInBNk4+gGb/T8+kcf/x/P5m539f40HGNglu4nOGdzSqX8IP6CI44JYBnxM3CL6uIl/QVPjFsMyni2lmEtBNRKYNsNU6IdiXUlN3N3IQH8D2LMcwwUsPtZCkZWGhM76wLhfpxIojDEV0eQ1pCaYvm1t0HyAImb51MW8+8BrfyD2n+639HcA1vH/0SD+A/3Y2f/XeID2X2N2SMsQ7I6JH2tJM+4zheK7hRzAAeXUR3icUK5CLGgQe+gYV0YNos6AtTpgKgSereNHexc5apJpdXW91wa2dJsr2rBqlrUNAZZ3FwhicCofLixuqhAjbX7FGdE/Qyzcv+MDtv8xnem9QPzvfH4mr/8czd/Md/b/NR7DmFKY4O8Wen6V0ID+OzdfG+dGB4o4Sr5Tn/tvEe8zFdOUdBjW3O7B3CMOKt42JdnTJDaYT+Vj2n1S3g3G/2kfu/1/2sQvG/81nx+fDuK/Tnbx36/ySHYGV7Zm2L+HezqOFsEfgrpvAnvMXfwFH4uKEq/L5JzRKXdseIzAESbHgXf7we6aiwnntiTToPZ2qxe3W5kG53z7n99O2l1g66w4p6E3/I7F5WYrDLhxqfWUM5Qs7vpcowF59WzKQinY0gc72uK+HZT3dYBcSY0/xOD3LMienNeAi428HOGa6UdHvsjom/dlkg6g88Sznm638vJbe3T2/4nW9TjDzx6z/T86nQ/if89Oznbr/6/yYBPwEY3oqgz38r29SWsX+pfqcPrJpEKmB5n8CzzcW0TzySRbowRqfzdlnTXE5qXb2WpZYWfRDiDxpRH4a+KBhjG2gCs0EzmczY9mZ3gFBEFidim6uv9QNtfIwGOLN8HnPeBIx4kQu4HjF9OaLZoQWz0/PFxPOEN8dvI+w/MMfM46R3Ykkc2PvkN0E2TwMA3ziywIb+gbCItTyqFjcXpIJJmN6x52mXuTibCzOvr/v0/4nclYz25DNE6c0PCWi5vLuh1oT/a1Q/3FZJ/NEx7T5/mC/iS9fM6nHPEpR4jhflRu6PEi+XNEzxOKNlWapAgJGiSoNnntD3eQEent8hWNMydi+RWofeos6SnCLNZmv/XR9O2svJ+tn2f3WVU35NCV2bJca4hqcu4FpSqrVUu1L9wb1EnYn0bx53paldsmPT3u3uLcV5cC2df50fHJKX3RbjpjqZga3+OaXsTLh/TTp3eoST2geuL30uLa34/w4tymwfc8tHedojr86cP78w/nP7y9pCTy1RUL4XV/QwR7j5DgXXioQgaXXmgpDEzk2x1MBCzq2kTy500romuS0jHgcrLu/S0+P5G9/Fzf0ArTqTwgGGjcUWgUVqWL+qooenW/bqvUUCdS8gAeS9eAG6aK0IbpIjAcBo7SVNexGqmUMHsyA1ol0RC0kqzDjsPRzpMEGf0an5hDXy5Jn2tybLYP6RvcDO/z2FQBShpZPYFIrZyORFBNR9QphncTILud6MBKyQOcbXrbBbX1aqAEMNUUgZFQLAgj6bA4bP1cT6jVxtLvjXRqXYx2oKMiXdJeLxKZVg1GZ1FiSKVWYUinVsBqNyzURnVsdsROrFfObFda0ouhKWlT+h5s1V1Jq9NcIDbrrSNVaq0jHugse+PoSEymx0P+lR/hZMWq38PDkbKLM9C4lg750Th/T6Yg55bhgXktp8i3GuBTzZREFOZcTKqeb7aFCtUPZbnKU1IcibXq1MS6uhOpzZWnpVXWnpZ6UH1X3QktCxpzzyUV9+VdXn6xt1gFpU5njtSssJpQqa2adKDqu3L5iNvgtr54SJePrJUIJB/018zbCHUKf9DeoQ6gU6qrpBxo+9F+AzY0g061YQ6zikZ6parGHEOVpWuA5U4+sBCD1GGpXIuzPlUSwPJcyzdYymbSXHC2YjIVyo36PkMTka6+b5TX91lo1IAVV9bJFMKtZMSjimV6K14RJpcone4hRyG+lU+tF6UqffdfFYe0ttnwBJ3EjfabdBZ4pyxN3mw3dPduZ1sT9f1bCzmxl8GOZO6+V6BsqGqW4s7bjszkZxOtf00GfjURLobq4ZvcbDJ0r+gVci/V8y1ZZNNcPiZwUJPzFSwRmZQS6TQKcUTi7VKLaEXcKFkrGhL/CUeZKtloandAw1WykKa+O0gjTHM3j5G6P0vfSsY+nRmoPranoNlo6PnveoXpoexGPtLB6FLrHpAPjAMiapfxbugqtLSgpKw2cxahFlWkbQs10/Ct00DpAhKCTtPWnHLQBmfNwrU6EC1telZS1mysdJLtpqvy9HQFbCjpS7MBuC9R07qokp/IGjzf+lb9gFu/ZqQmkr2tSKX2tloawdtqqTpvu6pQEbdWmHq9rB1Ma9VQUMg6cCRqBdQEAno1SQcd1Rlex9Pfadm7FSUpbxY4Aq07EWhUrqQnsLiRnpB3If1bdRcX0/naaN9b3IaJUHQZMqXCXahIRFchU8huQpnOuwiZQHIPcrLBNcikA7fwyNUDjrhj47rhq8FlF7yYnoxTs3/JXwPBkkkoymX7gadrr+R1b4bJnzdpkX6+TOMEGSvxPf2AiSe3wmvkCvnmk+snv3m5epc+pWjki7scebXuPi9REi6Ihg1SxddIepyfk8BxfPoIXyqFbUo9JJCtSWGZQivTBVuipOhMSYn+fIdLjcddQufPekJZjyGlWh8jnaCXkbLXT54Xk7ebfLvKCnxIDPtbaz83UncRv4S2Lb19q3hj7DYdFddruneKTsOl6k3+ZjjhpjOSSlnilaZ8K0VpVqYJdWWcSA9uauchGCfOqjve+RTrlelmYplKN/eWruXG5OIlFIvou8PvDhmt5dLpvra0Vzy3LW54u/LQQetuNe6aje4W4YGDFi71VWRXO+jhxbXye4uDtt63qqVUOGjtRaJaCtlBG67IVCsmOWjLnY+8CdDesdjXP7+YM7h6kOKh8Y+dfPUVfIZksYzlW+cUCWJ5NspSHNx/xmOVSkx5+RdfTpobt4YkAwBKOypdA4WRNfwCGAvYp4hVVxUtJhPNLUI0uqqN3MIcnrp4YCycpuD3UbRkG9fYgecLlkhsSxsjlXWJJHvUGXjyYtIepIeJpXMK+8IRj/rrQp/o9RwkMmo/ei63aPDzHH2OiyZqSnrrShQXUa8nO8IORz+f//ftzz/dvv354ubt5dsPn67O393+fPHj+dWHKKtxm53sM+Y40Kc7+W6f35LX4P0ObRxQe8fJfhsAzAZbNNrrBFVLl8JffbJ+nnZsxHtv9KL462RowoR3DrggU7Gg9rsbSBjfWAoSI3FOyfB47yW1wqxW22itIR16VdBDFHGoVlnO7uKKj9Paxx5LzrTZVpsSj9zY3hgjMS00WhqD/ep4YyTJTLdfvitj5Nxz7Mcr/J7sG2d6LuNOYfwaEd8eNrPZDHWTi/NoW6cJOU94mWd4iwzJ2eEic42evkTDZTQ8TROUB7UonYPmFphx22JV00bCYQ/cVwHfNFRa4hnRfVteLHfECcMCsA46FSXIUk58QIFUF4LKrgzoLtU36NmFhe+e3bN7ds/u2T27Z/fsnt2ze3bP7tk9u2f37J7ds3t2z+7ZPbtn9+ye3fNCzz8AOIseSQAIAgA= values: createCRDs: false image: From 1c8313c54f0804f3eb109a3054adede3b8b904ef Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 23 Jun 2023 10:38:32 +0200 Subject: [PATCH 09/15] Allow specifying custom firewall images in cloud profile. (#180) --- control-plane/roles/gardener/README.md | 26 ++--- .../gardener/defaults/main/cloud_profile.yaml | 2 + .../templates/30-cloud-profile-metal.yaml | 7 +- control-plane/roles/gardener/test/__init__.py | 5 + .../test/cloud_profile_template_test.py | 101 ++++++++++++++++++ .../gardener/test/mock/cloudprofile.yaml | 48 +++++++++ 6 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 control-plane/roles/gardener/test/cloud_profile_template_test.py create mode 100644 control-plane/roles/gardener/test/mock/cloudprofile.yaml diff --git a/control-plane/roles/gardener/README.md b/control-plane/roles/gardener/README.md index 3f519adff..890bf8882 100644 --- a/control-plane/roles/gardener/README.md +++ b/control-plane/roles/gardener/README.md @@ -56,18 +56,20 @@ The deployment chart is taken from [garden-setup](https://github.com/gardener/ga Variables for the metal-stack cloud profile. -| Name | Mandatory | Description | -| --------------------------------------------------- | --------- | --------------------------------------------------------------------- | -| gardener_cloud_profile_stage_name | | The name of the metal-stack environment in the cloud profile | -| gardener_cloud_profile_metal_api_url | | The URL used by the Gardener to communicate with the metal-api | -| gardener_cloud_profile_metal_api_hmac | yes | The admin HMAC used by the Gardener to communicate with the metal-api | -| gardener_cloud_profile_machine_images | | The machine images available for shoots in the metal-api | -| gardener_cloud_profile_firewall_controller_versions | | The available firewall controller versions for metal-stack shoots | -| gardener_cloud_profile_kubernetes | | The available Kubernetes versions for metal-stack shoots | -| gardener_cloud_profile_machine_types | | The machine types available for shoots in the metal-api | -| gardener_cloud_profile_regions | | The regions available for shoots | -| gardener_cloud_profile_partitions | | The partitions available for shoots | -| gardener_os_cri_mapping | | A mapping to add available CRIs to the machine images | +| Name | Mandatory | Description | +| ---------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------- | +| gardener_cloud_profile_stage_name | | The name of the metal-stack environment in the cloud profile | +| gardener_cloud_profile_metal_api_url | | The URL used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_metal_api_hmac | yes | The admin HMAC used by the Gardener to communicate with the metal-api | +| gardener_cloud_profile_machine_images | | The machine images available for shoots in the metal-api | +| gardener_cloud_profile_firewall_images | | The firewall images available for shoots in the metal-api | +| gardener_cloud_profile_firewall_images_from_machine_images | | If set to true, uses the passed machine images and adds those with firewall feature | +| gardener_cloud_profile_firewall_controller_versions | | The available firewall controller versions for metal-stack shoots | +| gardener_cloud_profile_kubernetes | | The available Kubernetes versions for metal-stack shoots | +| gardener_cloud_profile_machine_types | | The machine types available for shoots in the metal-api | +| gardener_cloud_profile_regions | | The regions available for shoots | +| gardener_cloud_profile_partitions | | The partitions available for shoots | +| gardener_os_cri_mapping | | A mapping to add available CRIs to the machine images | ### Extensions diff --git a/control-plane/roles/gardener/defaults/main/cloud_profile.yaml b/control-plane/roles/gardener/defaults/main/cloud_profile.yaml index 3e5a95dbc..f1ce66e5f 100644 --- a/control-plane/roles/gardener/defaults/main/cloud_profile.yaml +++ b/control-plane/roles/gardener/defaults/main/cloud_profile.yaml @@ -3,6 +3,8 @@ gardener_cloud_profile_stage_name: "{{ metal_control_plane_stage_name }}" gardener_cloud_profile_metal_api_url: https://api.{{ metal_control_plane_ingress_dns }} gardener_cloud_profile_metal_api_hmac: gardener_cloud_profile_machine_images: "{{ metal_api_images | default([]) }}" +gardener_cloud_profile_firewall_images: [] +gardener_cloud_profile_firewall_images_from_machine_images: true gardener_cloud_profile_firewall_controller_versions: [] gardener_cloud_profile_kubernetes: gardener_cloud_profile_machine_types: [] diff --git a/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml b/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml index aabe91926..65b37a434 100644 --- a/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml +++ b/control-plane/roles/gardener/templates/30-cloud-profile-metal.yaml @@ -12,13 +12,18 @@ spec: {{ gardener_cloud_profile_stage_name }}: endpoint: {{ gardener_cloud_profile_metal_api_url }} -{% if gardener_cloud_profile_machine_images | length > 0 %} +{% if (gardener_cloud_profile_firewall_images_from_machine_images and gardener_cloud_profile_machine_images | length > 0) or gardener_cloud_profile_firewall_images | length > 0 %} firewallImages: +{% for image in gardener_cloud_profile_firewall_images %} + - "{{ image }}" +{% endfor %} +{% if gardener_cloud_profile_firewall_images_from_machine_images %} {% for image in gardener_cloud_profile_machine_images %} {% if 'firewall' in image.features %} - "{{ image.id }}" {% endif %} {% endfor %} +{% endif %} {% endif %} firewallControllerVersions: {{ gardener_cloud_profile_firewall_controller_versions | to_json }} partitions: diff --git a/control-plane/roles/gardener/test/__init__.py b/control-plane/roles/gardener/test/__init__.py index 8dd4e633c..7fc64b41b 100644 --- a/control-plane/roles/gardener/test/__init__.py +++ b/control-plane/roles/gardener/test/__init__.py @@ -6,3 +6,8 @@ def read_mock_file(name): with open(os.path.join(os.path.dirname(__file__), "mock", name), 'r') as f: return f.read() + + +def read_template_file(name): + with open(os.path.join(os.path.dirname(__file__), "..", "templates", name), 'r') as f: + return f.read() diff --git a/control-plane/roles/gardener/test/cloud_profile_template_test.py b/control-plane/roles/gardener/test/cloud_profile_template_test.py new file mode 100644 index 000000000..fca2f4e94 --- /dev/null +++ b/control-plane/roles/gardener/test/cloud_profile_template_test.py @@ -0,0 +1,101 @@ +import unittest +import sys +import yaml + +from test import FILTER_PLUGINS_PATH,read_template_file, read_mock_file + +from ansible.template import Templar + + +sys.path.insert(0, FILTER_PLUGINS_PATH) +from common import machine_images_for_cloud_profile + + +class CloudProfileTemplate(unittest.TestCase): + def test_cloud_profile_template_template(self): + t = read_template_file("30-cloud-profile-metal.yaml") + + templar = Templar(loader=None, variables={ + "gardener_cloud_profile_stage_name": "prod", + "gardener_cloud_profile_metal_api_url": "https://metal-api", + "gardener_cloud_profile_metal_api_hmac": "hmac", + "gardener_cloud_profile_machine_images": [ + { + "id": "ubuntu-20.04.20210131", + "name": "Ubuntu 20.04 20210131", + "description": "Ubuntu 20.04 20210131", + "url": "http://images.metal-stack.io/metal-os/ubuntu", + "features": ["machine"], + }, + { + "id": "firewall-ubuntu-3.0.20300101", + "name": "Firewall Ubuntu 3.0 20300101", + "description": "Firewall Ubuntu 3.0 20300101", + "url": "http://images.metal-stack.io/metal-os/firewall", + "features": ["firewall"], + }, + ], + "gardener_cloud_profile_firewall_images": [ + "firewall-ubuntu-3.0", + ], + "gardener_cloud_profile_firewall_images_from_machine_images": True, + "gardener_cloud_profile_firewall_controller_versions": [ + { + "classification": "supported", + "url": "https://images.metal-stack.io/firewall-controller/v2.0.4/firewall-controller", + "version": "v2.0.4", + }, + ], + "gardener_cloud_profile_kubernetes": { + "versions": [ + { + "version": "1.18.0", + "expirationDate": "2020-12-01T00:00:00Z", + }, + { + "version": "1.24.15", + }, + ], + }, + "gardener_cloud_profile_machine_types": [ + { + "architecture": "amd64", + "cpu": "8", + "gpu": "0", + "memory": "32Gi", + "name": "n1-medium-x86", + "usable": True, + }, + ], + "gardener_cloud_profile_regions": [ + { + "name": "a", + "zones": [ + {"name": "partition-a"} + ], + }, + ], + "gardener_cloud_profile_partitions": { + "partition-a": { + "default-machine-types": { + "firewall": ["n1-medium-x86"], + }, + }, + }, + "gardener_os_cri_mapping": { + "ubuntu": { + "when": None, + "cris": [{"name": "containerd"}], + "containerRuntimes": [], + }, + }, + }) + + templar.environment.filters["machine_images_for_cloud_profile"] = machine_images_for_cloud_profile + + res = templar.template(t) + + expected = read_mock_file("cloudprofile.yaml") + + self.maxDiff = None + self.assertDictEqual(yaml.safe_load(expected), yaml.safe_load(res)) diff --git a/control-plane/roles/gardener/test/mock/cloudprofile.yaml b/control-plane/roles/gardener/test/mock/cloudprofile.yaml new file mode 100644 index 000000000..273fbfa21 --- /dev/null +++ b/control-plane/roles/gardener/test/mock/cloudprofile.yaml @@ -0,0 +1,48 @@ +apiVersion: core.gardener.cloud/v1beta1 +kind: CloudProfile +metadata: + name: metal +spec: + kubernetes: + versions: + - expirationDate: "2020-12-01T00:00:00Z" + version: 1.18.0 + - version: 1.24.15 + machineImages: + - name: ubuntu + versions: + - cri: + - name: containerd + version: "20.04" + - cri: + - name: containerd + version: 20.04.20210131 + machineTypes: + - architecture: amd64 + cpu: "8" + gpu: "0" + memory: 32Gi + name: n1-medium-x86 + usable: true + providerConfig: + apiVersion: metal.provider.extensions.gardener.cloud/v1alpha1 + kind: CloudProfileConfig + metalControlPlanes: + prod: + endpoint: https://metal-api + firewallControllerVersions: + - classification: supported + url: "https://images.metal-stack.io/firewall-controller/v2.0.4/firewall-controller" + version: v2.0.4 + firewallImages: + - firewall-ubuntu-3.0 + - firewall-ubuntu-3.0.20300101 + partitions: + partition-a: + firewallTypes: + - n1-medium-x86 + regions: + - name: a + zones: + - name: partition-a + type: metal From 93f2e602f942ac30b440403c0f31d560f32072d0 Mon Sep 17 00:00:00 2001 From: Valentin Knabel Date: Fri, 23 Jun 2023 13:20:13 +0200 Subject: [PATCH 10/15] feat(auditing): Allow defining pull secret #182 (#185) --- .../roles/auditing-meili/defaults/main/main.yaml | 8 ++++++++ control-plane/roles/auditing-meili/tasks/main.yaml | 13 +++++++++++++ .../roles/auditing-meili/templates/values.yaml | 3 +++ 3 files changed, 24 insertions(+) diff --git a/control-plane/roles/auditing-meili/defaults/main/main.yaml b/control-plane/roles/auditing-meili/defaults/main/main.yaml index da4d64bd6..6ed1a92a5 100644 --- a/control-plane/roles/auditing-meili/defaults/main/main.yaml +++ b/control-plane/roles/auditing-meili/defaults/main/main.yaml @@ -25,3 +25,11 @@ auditing_meili_persistence: # volume: # name: data # mountPath: /meili_data + +auditing_meili_registry_enabled: "{{ metal_registry_auth_enabled }}" +auditing_meili_registry_auth: + auths: + https://index.docker.io/v1/: + username: "{{ metal_registry_auth_user }}" + password: "{{ metal_registry_auth_password }}" + auth: "{{ (metal_registry_auth_user + ':' + metal_registry_auth_password) | b64encode }}" diff --git a/control-plane/roles/auditing-meili/tasks/main.yaml b/control-plane/roles/auditing-meili/tasks/main.yaml index 59250721e..c2a18e745 100644 --- a/control-plane/roles/auditing-meili/tasks/main.yaml +++ b/control-plane/roles/auditing-meili/tasks/main.yaml @@ -24,6 +24,19 @@ MEILI_MASTER_KEY: "{{ auditing_meili_secret }}" when: auditing_meili_secret +- name: Create registry pull secret + k8s: + definition: + apiVersion: v1 + kind: Secret + metadata: + name: metal-auditing-registry-pull-secret + namespace: "{{ auditing_meili_namespace }}" + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: "{{ auditing_meili_registry_auth | to_json | b64encode }}" + when: auditing_meili_registry_enabled + - name: Deploy meilisearch include_role: name: ansible-common/roles/helm-chart diff --git a/control-plane/roles/auditing-meili/templates/values.yaml b/control-plane/roles/auditing-meili/templates/values.yaml index cf2afc4e6..3ef01dec5 100644 --- a/control-plane/roles/auditing-meili/templates/values.yaml +++ b/control-plane/roles/auditing-meili/templates/values.yaml @@ -2,6 +2,9 @@ image: repository: "{{ metal_auditing_meili_image_name }}" tag: "{{ metal_auditing_meili_image_tag }}" +{% if auditing_meili_registry_enabled %} + pullSecret: "metal-auditing-registry-pull-secret" +{% endif %} readinessProbe: periodSeconds: 5 From 037e117ab0241c5941c596f0d7c829c0f4ffe205 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 23 Jun 2023 13:50:15 +0200 Subject: [PATCH 11/15] Adaptions for g/g v1.52. (#184) --- .../templates/deployment-kube-apiserver.yaml | 4 ++-- control-plane/roles/gardener/tasks/seed.yaml | 8 ++++---- .../gardener/templates/gardener-control-plane-values.j2 | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml index 580645274..16bf75a16 100644 --- a/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml +++ b/control-plane/roles/gardener/files/kube-apiserver/templates/deployment-kube-apiserver.yaml @@ -189,8 +189,8 @@ spec: failureThreshold: 2 httpGet: path: /healthz - port: 10252 - scheme: HTTP + port: 10257 + scheme: HTTPS initialDelaySeconds: 15 periodSeconds: 10 successThreshold: 1 diff --git a/control-plane/roles/gardener/tasks/seed.yaml b/control-plane/roles/gardener/tasks/seed.yaml index 94d26b6c9..9ea3b579b 100644 --- a/control-plane/roles/gardener/tasks/seed.yaml +++ b/control-plane/roles/gardener/tasks/seed.yaml @@ -69,9 +69,9 @@ - name: Fetch current seed cidrs set_fact: - _gardener_current_soil_node_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('nodes', none) }}" - _gardener_current_soil_pod_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('pods', none) }}" - _gardener_current_soil_service_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('services', none) }}" + _gardener_current_soil_node_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('nodes', none) }}" + _gardener_current_soil_pod_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('pods', none) }}" + _gardener_current_soil_service_cidr: "{{ (lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name) | default({}, true)).get('spec', {}).get('networks', {}).get('services', none) }}" - name: Set cidrs for seed set_fact: @@ -95,7 +95,7 @@ retries: 60 delay: 3 until: - - "'conditions' in lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1alpha1', kind='Seed', resource_name=gardener_soil_name).get('status', {})" + - "'conditions' in lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {})" - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions | length > 2 - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions[0].message == "Gardenlet is posting ready status." - lookup('k8s', kubeconfig=gardener_kube_apiserver_kubeconfig_path, api_version='core.gardener.cloud/v1beta1', kind='Seed', resource_name=gardener_soil_name).get('status', {}).conditions[2].message == "Seed cluster has been bootstrapped successfully." diff --git a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 index a4810e17f..3747a2c6e 100644 --- a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 +++ b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 @@ -24,7 +24,6 @@ global: resources: {{ gardener_apiserver_resources | to_json }} featureGates: SeedChange: true - WorkerPoolKubernetesVersion: true image: repository: {{ gardener_apiserver_image_name }} tag: {{ gardener_apiserver_image_tag }} From 9772a28adddc1f8454c24c81a11f2eab65d65de0 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 27 Jun 2023 10:52:04 +0200 Subject: [PATCH 12/15] Preparation for g/g v1.53. (#186) --- control-plane/roles/gardener/tasks/main.yaml | 2 +- control-plane/roles/gardener/templates/managed-seed.j2 | 2 -- control-plane/roles/gardener/templates/shooted-seed.j2 | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/control-plane/roles/gardener/tasks/main.yaml b/control-plane/roles/gardener/tasks/main.yaml index 8e72ed9fa..27b40451e 100644 --- a/control-plane/roles/gardener/tasks/main.yaml +++ b/control-plane/roles/gardener/tasks/main.yaml @@ -116,7 +116,7 @@ - name: Create Gardener project for shooted seeds k8s: definition: - apiVersion: core.gardener.cloud/v1alpha1 + apiVersion: core.gardener.cloud/v1beta1 kind: Project metadata: name: "{{ gardener_soil_name }}" diff --git a/control-plane/roles/gardener/templates/managed-seed.j2 b/control-plane/roles/gardener/templates/managed-seed.j2 index 5759aff7b..336a5297c 100644 --- a/control-plane/roles/gardener/templates/managed-seed.j2 +++ b/control-plane/roles/gardener/templates/managed-seed.j2 @@ -40,8 +40,6 @@ spec: visible: {{ gardener_shooted_seed.visible | default(true) }} shootDNS: enabled: true - verticalPodAutoscaler: - enabled: {{ gardener_shooted_seed.verticalPodAutoscaler | default(true) }} deployment: image: pullPolicy: IfNotPresent diff --git a/control-plane/roles/gardener/templates/shooted-seed.j2 b/control-plane/roles/gardener/templates/shooted-seed.j2 index ec691a56d..908b7c335 100644 --- a/control-plane/roles/gardener/templates/shooted-seed.j2 +++ b/control-plane/roles/gardener/templates/shooted-seed.j2 @@ -30,6 +30,8 @@ spec: maxPods: {{ gardener_shooted_seed_max_pods }} kubeControllerManager: nodeCIDRMaskSize: {{ gardener_shooted_seed_node_cidr_mask_size }} + verticalPodAutoscaler: + enabled: {{ gardener_shooted_seed.verticalPodAutoscaler | default(true) }} provider: type: metal workers: From 50751745bdb1c1f772adbd0b7973fe6d232bdab1 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 27 Jun 2023 13:51:16 +0200 Subject: [PATCH 13/15] Add `sonic` role to support SONiC switches. (#121) --- partition/roles/frr/README.md | 12 ++ partition/roles/frr/defaults/main.yaml | 3 + partition/roles/frr/files/daemons | 3 + partition/roles/frr/files/sysctl.conf | 55 +++++ partition/roles/frr/files/vtysh.conf | 1 + partition/roles/frr/handlers/main.yaml | 13 ++ partition/roles/frr/tasks/install/debian.yaml | 19 ++ partition/roles/frr/tasks/install/redhat.yaml | 5 + partition/roles/frr/tasks/main.yaml | 53 +++++ partition/roles/metal-core/tasks/main.yaml | 1 + partition/roles/mgmt-server/README.md | 29 +-- .../roles/mgmt-server/defaults/main.yaml | 7 +- .../roles/mgmt-server/handlers/main.yaml | 6 + .../roles/mgmt-server/tasks/install_frr.yaml | 7 + partition/roles/mgmt-server/tasks/main.yaml | 63 +++++- .../roles/mgmt-server/templates/frr.conf.j2 | 3 + .../roles/mgmt-server/templates/ssh_config.j2 | 8 + partition/roles/sonic-upgrade/README.md | 16 ++ .../roles/sonic-upgrade/defaults/main.yaml | 4 + partition/roles/sonic-upgrade/tasks/main.yaml | 42 ++++ partition/roles/sonic/README.md | 69 +++++++ partition/roles/sonic/defaults/main.yaml | 47 +++++ .../roles/sonic/files/bgp-validation@.service | 10 + .../roles/sonic/files/frr-reload.service | 10 + partition/roles/sonic/handlers/main.yaml | 39 ++++ partition/roles/sonic/tasks/main.yaml | 85 ++++++++ partition/roles/sonic/templates/frr.conf.j2 | 192 ++++++++++++++++++ .../roles/sonic/templates/iptables.json.j2 | 13 ++ partition/roles/sonic/templates/metal.yaml.j2 | 179 ++++++++++++++++ .../roles/sonic/templates/resolv.conf.j2 | 3 + partition/roles/sonic/test/__init__.py | 6 + partition/roles/sonic/test/data/exit/frr.conf | 108 ++++++++++ .../roles/sonic/test/data/exit/input.yaml | 107 ++++++++++ .../roles/sonic/test/data/exit/metal.yaml | 139 +++++++++++++ .../roles/sonic/test/data/mgmtleaf/frr.conf | 34 ++++ .../roles/sonic/test/data/mgmtleaf/input.yaml | 59 ++++++ .../sonic/test/data/mgmtleaf/iptables.json | 14 ++ .../roles/sonic/test/data/mgmtleaf/metal.yaml | 111 ++++++++++ .../roles/sonic/test/data/spine/frr.conf | 38 ++++ .../roles/sonic/test/data/spine/input.yaml | 27 +++ .../roles/sonic/test/data/spine/iptables.json | 10 + .../roles/sonic/test/data/spine/metal.yaml | 48 +++++ partition/roles/sonic/test/template_test.py | 46 +++++ partition/roles/systemd-networkd/README.md | 52 ++--- .../systemd-networkd/files/networkd.conf | 8 + .../files/rename-interfaces.sh | 23 +++ .../roles/systemd-networkd/handlers/main.yaml | 28 +++ .../roles/systemd-networkd/tasks/main.yaml | 11 +- .../roles/systemd-networkd/tasks/redhat.yaml | 17 ++ .../systemd-networkd/templates/network.j2 | 10 + .../roles/ztp/defaults/main/global-defaults | 1 + .../roles/ztp/defaults/{ => main}/main.yaml | 0 .../ztp/defaults/main/partition-defaults | 1 + partition/roles/ztp/tasks/main.yaml | 3 + 54 files changed, 1857 insertions(+), 41 deletions(-) create mode 100644 partition/roles/frr/README.md create mode 100644 partition/roles/frr/defaults/main.yaml create mode 100644 partition/roles/frr/files/daemons create mode 100644 partition/roles/frr/files/sysctl.conf create mode 100644 partition/roles/frr/files/vtysh.conf create mode 100644 partition/roles/frr/handlers/main.yaml create mode 100644 partition/roles/frr/tasks/install/debian.yaml create mode 100644 partition/roles/frr/tasks/install/redhat.yaml create mode 100644 partition/roles/frr/tasks/main.yaml create mode 100644 partition/roles/mgmt-server/templates/ssh_config.j2 create mode 100644 partition/roles/sonic-upgrade/README.md create mode 100644 partition/roles/sonic-upgrade/defaults/main.yaml create mode 100644 partition/roles/sonic-upgrade/tasks/main.yaml create mode 100644 partition/roles/sonic/README.md create mode 100644 partition/roles/sonic/defaults/main.yaml create mode 100644 partition/roles/sonic/files/bgp-validation@.service create mode 100644 partition/roles/sonic/files/frr-reload.service create mode 100644 partition/roles/sonic/handlers/main.yaml create mode 100644 partition/roles/sonic/tasks/main.yaml create mode 100644 partition/roles/sonic/templates/frr.conf.j2 create mode 100644 partition/roles/sonic/templates/iptables.json.j2 create mode 100644 partition/roles/sonic/templates/metal.yaml.j2 create mode 100644 partition/roles/sonic/templates/resolv.conf.j2 create mode 100644 partition/roles/sonic/test/__init__.py create mode 100644 partition/roles/sonic/test/data/exit/frr.conf create mode 100644 partition/roles/sonic/test/data/exit/input.yaml create mode 100644 partition/roles/sonic/test/data/exit/metal.yaml create mode 100644 partition/roles/sonic/test/data/mgmtleaf/frr.conf create mode 100644 partition/roles/sonic/test/data/mgmtleaf/input.yaml create mode 100644 partition/roles/sonic/test/data/mgmtleaf/iptables.json create mode 100644 partition/roles/sonic/test/data/mgmtleaf/metal.yaml create mode 100644 partition/roles/sonic/test/data/spine/frr.conf create mode 100644 partition/roles/sonic/test/data/spine/input.yaml create mode 100644 partition/roles/sonic/test/data/spine/iptables.json create mode 100644 partition/roles/sonic/test/data/spine/metal.yaml create mode 100644 partition/roles/sonic/test/template_test.py create mode 100644 partition/roles/systemd-networkd/files/networkd.conf create mode 100644 partition/roles/systemd-networkd/files/rename-interfaces.sh create mode 100644 partition/roles/systemd-networkd/tasks/redhat.yaml create mode 120000 partition/roles/ztp/defaults/main/global-defaults rename partition/roles/ztp/defaults/{ => main}/main.yaml (100%) create mode 120000 partition/roles/ztp/defaults/main/partition-defaults diff --git a/partition/roles/frr/README.md b/partition/roles/frr/README.md new file mode 100644 index 000000000..7d6915026 --- /dev/null +++ b/partition/roles/frr/README.md @@ -0,0 +1,12 @@ +# frr + +Configures and starts frr. + +This role can deploy on bare metal machines with Debian or Almalinux. It depends on fact gathering. + +## Variables + +| Name | Mandatory | Description | +| ----------- | --------- | ----------------------------------- | +| frr_version | | The version of FRR to be installed. | +| frr_repo | | The repository that contains FRR. | diff --git a/partition/roles/frr/defaults/main.yaml b/partition/roles/frr/defaults/main.yaml new file mode 100644 index 000000000..f17d38613 --- /dev/null +++ b/partition/roles/frr/defaults/main.yaml @@ -0,0 +1,3 @@ +--- +frr_version: 8.4 +frr_repo: frr-8 diff --git a/partition/roles/frr/files/daemons b/partition/roles/frr/files/daemons new file mode 100644 index 000000000..1fcb6cf80 --- /dev/null +++ b/partition/roles/frr/files/daemons @@ -0,0 +1,3 @@ +bgpd=yes +zebra=yes +bfdd=yes \ No newline at end of file diff --git a/partition/roles/frr/files/sysctl.conf b/partition/roles/frr/files/sysctl.conf new file mode 100644 index 000000000..6a62b049f --- /dev/null +++ b/partition/roles/frr/files/sysctl.conf @@ -0,0 +1,55 @@ +# source: https://github.com/FRRouting/frr/blob/master/doc/user/Useful_Sysctl_Settings.md +# The below sysctl values provide a logical set of defaults which can be further optimized. +# +# /etc/sysctl.d/99frr_defaults.conf +# Place this file at the location above and reload the device. +# or run the sysctl -p /etc/sysctl.d/99frr_defaults.conf + +# Enables IPv4/IPv6 Routing +net.ipv4.ip_forward = 1 +net.ipv6.conf.all.forwarding=1 + +# Routing +net.ipv6.route.max_size=131072 +net.ipv4.conf.all.ignore_routes_with_linkdown=1 +net.ipv6.conf.all.ignore_routes_with_linkdown=1 + +# Best Settings for Peering w/ BGP Unnumbered +# and OSPF Neighbors +net.ipv4.conf.all.rp_filter = 0 +net.ipv4.conf.default.rp_filter = 0 +net.ipv4.conf.lo.rp_filter = 0 +net.ipv4.conf.all.forwarding = 1 +net.ipv4.conf.default.forwarding = 1 +net.ipv4.conf.default.arp_announce = 2 +net.ipv4.conf.default.arp_notify = 1 +net.ipv4.conf.default.arp_ignore=1 +net.ipv4.conf.all.arp_announce = 2 +net.ipv4.conf.all.arp_notify = 1 +net.ipv4.conf.all.arp_ignore=1 +net.ipv4.icmp_errors_use_inbound_ifaddr=1 + +# Miscellaneous Settings +# Keep ipv6 permanent addresses on an admin down +net.ipv6.conf.all.keep_addr_on_down=1 + +# igmp +net.ipv4.igmp_max_memberships=1000 +net.ipv4.neigh.default.mcast_solicit = 10 + +# MLD +net.ipv6.mld_max_msf=512 + +# Garbage Collection Settings for ARP and Neighbors +net.ipv4.neigh.default.gc_thresh2=7168 +net.ipv4.neigh.default.gc_thresh3=8192 +net.ipv4.neigh.default.base_reachable_time_ms=14400000 +net.ipv6.neigh.default.gc_thresh2=3584 +net.ipv6.neigh.default.gc_thresh3=4096 +net.ipv6.neigh.default.base_reachable_time_ms=14400000 + +# Use neigh information on selection of nexthop for multipath hops +net.ipv4.fib_multipath_use_neigh=1 + +# Allows Apps to Work with VRF +net.ipv4.tcp_l3mdev_accept=1 diff --git a/partition/roles/frr/files/vtysh.conf b/partition/roles/frr/files/vtysh.conf new file mode 100644 index 000000000..f863f560f --- /dev/null +++ b/partition/roles/frr/files/vtysh.conf @@ -0,0 +1 @@ +service integrated-vtysh-config \ No newline at end of file diff --git a/partition/roles/frr/handlers/main.yaml b/partition/roles/frr/handlers/main.yaml new file mode 100644 index 000000000..29e30105a --- /dev/null +++ b/partition/roles/frr/handlers/main.yaml @@ -0,0 +1,13 @@ +--- +- name: reload frr + service: + name: frr + state: reloaded + +- name: restart frr + service: + name: frr + state: restarted + +- name: reload sysctls + command: sysctl --system diff --git a/partition/roles/frr/tasks/install/debian.yaml b/partition/roles/frr/tasks/install/debian.yaml new file mode 100644 index 000000000..97c0edf40 --- /dev/null +++ b/partition/roles/frr/tasks/install/debian.yaml @@ -0,0 +1,19 @@ +--- +- name: add FRR apt-key to verify FRR package + apt_key: + url: https://deb-us.frrouting.org/frr/keys.asc + state: present + +- name: add mainline FRR repository to install FRR + apt_repository: + repo: deb https://deb-us.frrouting.org/frr {{ ansible_distribution_release }} {{ frr_repo }} + state: present + filename: frr.list + +- name: install required packages to have the network stack in place + apt: + name: + - frr={{ frr_version }}* + - frr-pythontools={{ frr_version }}* + update_cache : yes + force_apt_get: yes diff --git a/partition/roles/frr/tasks/install/redhat.yaml b/partition/roles/frr/tasks/install/redhat.yaml new file mode 100644 index 000000000..46af5f9a3 --- /dev/null +++ b/partition/roles/frr/tasks/install/redhat.yaml @@ -0,0 +1,5 @@ +--- +- name: install frr + dnf: + name: frr + state: present diff --git a/partition/roles/frr/tasks/main.yaml b/partition/roles/frr/tasks/main.yaml new file mode 100644 index 000000000..b82e33d75 --- /dev/null +++ b/partition/roles/frr/tasks/main.yaml @@ -0,0 +1,53 @@ +--- +- name: check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: yes + that: + - frr_version is not none + - frr_repo is not none + - ansible_os_family in ['Debian', 'RedHat'] + +- name: gather package facts + package_facts: + manager: auto + +- name: check if frr is installed + set_fact: + frr_installed: "{{ ('packages' in ansible_facts and ('frr' in ansible_facts.packages) and (frr_version|string in (ansible_facts.packages['frr'][0].version))) | bool }}" + +- import_tasks: install/debian.yaml + when: not frr_installed and ansible_os_family == 'Debian' + +- import_tasks: install/redhat.yaml + when: not frr_installed and ansible_os_family == 'RedHat' + +- name: copy sysctls for frr + copy: + src: sysctl.conf + dest: /etc/sysctl.d/98-frr.conf + notify: reload sysctls + +- name: enable vtysh + copy: + src: vtysh.conf + dest: /etc/frr/ + notify: restart frr + +- name: enable frr daemons + copy: + src: daemons + dest: /etc/frr/daemons + notify: restart frr + +- name: ensure frr is started + systemd: + name: frr + enabled: yes + state: started + +- name: copy frr conf + copy: + content: "{{ frr_conf }}" + dest: /etc/frr/frr.conf + notify: reload frr diff --git a/partition/roles/metal-core/tasks/main.yaml b/partition/roles/metal-core/tasks/main.yaml index 8e1defefa..34207a9cb 100644 --- a/partition/roles/metal-core/tasks/main.yaml +++ b/partition/roles/metal-core/tasks/main.yaml @@ -56,6 +56,7 @@ systemd_docker_volumes: "{{ lookup('template', 'metal-core-volumes.j2') | from_yaml }}" systemd_docker_cap_add: - sys_admin + - net_admin systemd_service_environment: "{{ lookup('template', 'metal-core-env.j2') | from_yaml }}" systemd_external_config_changed: "{{ grpc_certs is changed }}" systemd_service_after: "{{ metal_core_service_dependency }}" diff --git a/partition/roles/mgmt-server/README.md b/partition/roles/mgmt-server/README.md index 4089ce0b0..47fbda800 100644 --- a/partition/roles/mgmt-server/README.md +++ b/partition/roles/mgmt-server/README.md @@ -4,15 +4,20 @@ Configures a server to act as management server for a metal-stack partition. ## Variables -| Name | Mandatory | Description | -|---------------------------------------|-----------|---------------------------------------------------------------------------------| -| mgmt_server_asn | yes | the ASN to use for routing. | -| mgmt_server_dns_over_tls | | whether to use DNSoverTLS (default is true). | -| mgmt_server_firewall_facing_interface | yes | the interface where the firewall is connected at the management server. | -| mgmt_server_firewall_ip | | the remote ip of the firewall for setting up a numbered BGP session. | -| mgmt_server_frr_match_interfaces | | announce the networks attached to the given interfaces over BGP. | -| mgmt_server_frr_rep | | the FRR repo to use. | -| mgmt_server_frr_version | | the FRR version to use. | -| mgmt_server_nameservers | | the nameservers to use (default is dns0.eu). | -| mgmt_server_router_id | yes | the router-id to use for routing. | -| mgmt_server_spine_facing_interface | yes | the interface where the management spine is connected at the management server. | \ No newline at end of file +| Name | Mandatory | Description | +| ------------------------------------- | --------- | ------------------------------------------------------------------------------------ | +| mgmt_server_asn | yes | the ASN to use for routing. | +| mgmt_server_dns_over_tls | | whether to use DNSoverTLS (default is true). | +| mgmt_server_firewall_facing_interface | yes | the interface where the firewall is connected at the management server. | +| mgmt_server_firewall_ip | | the remote ip of the firewall for setting up a numbered BGP session. | +| mgmt_server_frr_match_interfaces | | announce the networks attached to the given interfaces over BGP. | +| mgmt_server_frr_repo | | the FRR repo to use. | +| mgmt_server_frr_version | | the FRR version to use. | +| mgmt_server_nameservers | | the nameservers to use (default is dns0.eu). | +| mgmt_server_router_id | yes | the router-id to use for routing. | +| mgmt_server_spine_facing_interface | yes | the interface where the management spine is connected at the management server. | +| mgmt_server_metal_ssh_groups | | the ansible group to include into the ssh config | +| mgmt_server_metal_ssh_privkey | yes | the private SSH key of the `metal` admin user for connecting to the other components | +| mgmt_server_metal_ssh_pubkey | yes | the public SSH key of the `metal` admin user for connecting to the other components | +| mgmt_server_preserve_dhcp_route | no | preserve the dhcp (default) route the mgmt server got from the mgmt firewall | +| mgmt_server_provide_default_route | no | provide the default route with bgp (`network 0.0.0.0/0`) | diff --git a/partition/roles/mgmt-server/defaults/main.yaml b/partition/roles/mgmt-server/defaults/main.yaml index 16b853aec..a67a28a65 100644 --- a/partition/roles/mgmt-server/defaults/main.yaml +++ b/partition/roles/mgmt-server/defaults/main.yaml @@ -3,7 +3,7 @@ # mgmt_server_router_id: # mgmt_server_spine_facing_interface: # mgmt_server_firewall_facing_interface: -# mgmt_server_firewall_ip: +# mgmt_server_firewall_ip: mgmt_server_registry_mirror: https://mirror.gcr.io @@ -17,5 +17,8 @@ mgmt_server_dns_over_tls: true mgmt_server_frr_match_interfaces: - lo -mgmt_server_frr_version: 8.4.2 +mgmt_server_frr_version: 8.5-0 mgmt_server_frr_repo: frr-8 +mgmt_server_provide_default_route: false + +mgmt_server_metal_ssh_groups: "{{ groups.all }}" diff --git a/partition/roles/mgmt-server/handlers/main.yaml b/partition/roles/mgmt-server/handlers/main.yaml index 76d364c72..f250b3a8c 100644 --- a/partition/roles/mgmt-server/handlers/main.yaml +++ b/partition/roles/mgmt-server/handlers/main.yaml @@ -33,3 +33,9 @@ - name: save iptables v6 rules shell: ip6tables-save > /etc/iptables/rules.v6 listen: persist iptables + +- name: restart sshd + service: + name: sshd + enabled: true + state: restarted diff --git a/partition/roles/mgmt-server/tasks/install_frr.yaml b/partition/roles/mgmt-server/tasks/install_frr.yaml index 4b05e6865..9ffc37147 100644 --- a/partition/roles/mgmt-server/tasks/install_frr.yaml +++ b/partition/roles/mgmt-server/tasks/install_frr.yaml @@ -1,4 +1,11 @@ --- +- name: ensure gpg is installed to handle FRR repo key + apt: + name: + - gpg + update_cache : yes + force_apt_get: yes + - name: add FRR apt-key to verify FRR package apt_key: url: https://deb-us.frrouting.org/frr/keys.asc diff --git a/partition/roles/mgmt-server/tasks/main.yaml b/partition/roles/mgmt-server/tasks/main.yaml index cfd03b959..0dca5988b 100644 --- a/partition/roles/mgmt-server/tasks/main.yaml +++ b/partition/roles/mgmt-server/tasks/main.yaml @@ -7,6 +7,8 @@ - mgmt_server_asn is not none - mgmt_server_router_id is not none - mgmt_server_spine_facing_interface is not none + - mgmt_server_metal_ssh_privkey is not none + - mgmt_server_metal_ssh_pubkey is not none - name: gather package facts package_facts: @@ -33,7 +35,7 @@ name: - net-tools - ipmitool - - docker-ce + - docker.io - iptables-persistent update_cache : yes force_apt_get: yes @@ -85,9 +87,66 @@ notify: - reload frr +- name: Copy ssh key pair for metal user + copy: + content: "{{ item.content }}" + dest: "{{ item.dest }}" + mode: "{{ item.mode}}" + owner: metal + group: metal + loop: + - content: "{{ mgmt_server_metal_ssh_privkey }}" + dest: /home/metal/.ssh/id_rsa + mode: "0600" + - content: "{{ mgmt_server_metal_ssh_pubkey }}" + dest: /home/metal/.ssh/id_rsa.pub + mode: "0644" + +# This is so that self connect and cross connect to the other mgmtserver is possible +- name: Add own ssh key to authorized_keys + lineinfile: + path: /home/metal/.ssh/authorized_keys + regexp: '^{{ mgmt_server_metal_ssh_pubkey }}$' + line: '{{ mgmt_server_metal_ssh_pubkey }}' + +- name: configure sshd to avoid root login and password authentication, hide OS information + lineinfile: + path: /etc/ssh/sshd_config + state: present + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + validate: /usr/sbin/sshd -t -f %s + notify: restart sshd + loop: + - { regexp: "^PasswordAuthentication .+", line: "PasswordAuthentication no" } + - { regexp: "^PermitRootLogin .+", line: "PermitRootLogin no" } + - { regexp: "^DebianBanner .+", line: "DebianBanner no" } + +- name: Add IP's of all hosts to /etc/hosts to use those hostnames for connections + lineinfile: + dest: /etc/hosts + regexp: '.*\s{{ item }}$' + line: "{{ hostvars[item].ansible_host }} {{ item }} {{ hostvars[item].host_alias|default('') }}" + owner: root + group: root + mode: 0644 + when: + - hostvars[item].ansible_host is defined + loop: "{{ mgmt_server_metal_ssh_groups }}" + +- name: Create ssh configuration for easier access to mgmt components + template: + src: ssh_config.j2 + dest: /home/metal/.ssh/config + mode: 0644 + owner: metal + group: metal + - name: flush handlers to complete the mgmt-server setup meta: flush_handlers - name: flush dhcp routes if we have a bgp session to the firewall command: ip route flush proto dhcp - when: mgmt_server_firewall_ip is defined + when: + - mgmt_server_firewall_ip is defined + - mgmt_server_preserve_dhcp_route is undefined or not mgmt_server_preserve_dhcp_route diff --git a/partition/roles/mgmt-server/templates/frr.conf.j2 b/partition/roles/mgmt-server/templates/frr.conf.j2 index 75da5ee8b..f920c523a 100644 --- a/partition/roles/mgmt-server/templates/frr.conf.j2 +++ b/partition/roles/mgmt-server/templates/frr.conf.j2 @@ -28,6 +28,9 @@ router bgp {{ mgmt_server_asn }} ! address-family ipv4 unicast redistribute connected route-map LOCAL_INTERFACES + {% if mgmt_server_provide_default_route %} + network 0.0.0.0/0 + {% endif %} exit-address-family ! route-map LOCAL_INTERFACES permit 10 diff --git a/partition/roles/mgmt-server/templates/ssh_config.j2 b/partition/roles/mgmt-server/templates/ssh_config.j2 new file mode 100644 index 000000000..eaea27ff6 --- /dev/null +++ b/partition/roles/mgmt-server/templates/ssh_config.j2 @@ -0,0 +1,8 @@ +{% for host in mgmt_server_metal_ssh_groups %} +{% if hostvars[host].ansible_host is defined %} +{% if hostvars[host].ansible_user is defined %} +Host {{ host }} {{ hostvars[host].host_alias|default('') }} + User {{ hostvars[host].ansible_user }} +{% endif %} +{% endif %} +{% endfor %} diff --git a/partition/roles/sonic-upgrade/README.md b/partition/roles/sonic-upgrade/README.md new file mode 100644 index 000000000..d8e340570 --- /dev/null +++ b/partition/roles/sonic-upgrade/README.md @@ -0,0 +1,16 @@ +# sonic-upgrade + +Performs an upgrade of the SONiC OS on a device and reboots it to complete the installation. + +It depends on the `switch_facts` module from `ansible-common`, so make sure modules from `ansible-common` are included before executing this role. + +## Variables + +| Name | Mandatory | Description | +| ------------------------ | --------- | ------------------------------------------------------------------------------------------------------------------- | +| sonic_upgrade_host | yes | The host from which to dowload the image. | +| sonic_upgrade_image_path | | The path to the image. If this is given and not `sonic_upgrade_host`, the image is pushed to the device by ansible. | +| sonic_upgrade_vrf | | The vrf used for pulling the upgrade image. | +| sonic_upgrade_protocol | | The protocol (http or https) to use when downloading the sonic image. | +| sonic_upgrade_port | | The port on which the image server listens. | +| sonic_upgrade_image | yes | The file name of the sonic image. | diff --git a/partition/roles/sonic-upgrade/defaults/main.yaml b/partition/roles/sonic-upgrade/defaults/main.yaml new file mode 100644 index 000000000..98fd02336 --- /dev/null +++ b/partition/roles/sonic-upgrade/defaults/main.yaml @@ -0,0 +1,4 @@ +--- +sonic_upgrade_protocol: "http" +sonic_upgrade_port: 8080 +sonic_upgrade_vrf: "mgmt" diff --git a/partition/roles/sonic-upgrade/tasks/main.yaml b/partition/roles/sonic-upgrade/tasks/main.yaml new file mode 100644 index 000000000..4d69d9de5 --- /dev/null +++ b/partition/roles/sonic-upgrade/tasks/main.yaml @@ -0,0 +1,42 @@ +--- +- name: Gather switch facts + switch_facts: + +- name: Check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: yes + that: + - (sonic_upgrade_host is defined and sonic_upgrade_image_path is not defined) or (sonic_upgrade_host is not defined and sonic_upgrade_image_path is defined) + - sonic_upgrade_image is defined + - metal_stack_switch_os_is_sonic + +- name: Download the new sonic image + command: ip vrf exec {{ sonic_upgrade_vrf }} wget {{ sonic_upgrade_protocol }}://{{ sonic_upgrade_host }}:{{ sonic_upgrade_port }}/{{ sonic_upgrade_image }} -O /{{ sonic_upgrade_image }} + when: sonic_upgrade_host is defined and sonic_upgrade_image_path is not defined + +- name: Push the new sonic image to the device + copy: + src: "{{ sonic_upgrade_image_path }}/{{ sonic_upgrade_image }}" + dest: "/{{ sonic_upgrade_image }}" + when: sonic_upgrade_host is not defined and sonic_upgrade_image_path is defined + +- name: Install new sonic image + command: "sonic-installer install -y --skip_migration --skip-package-migration /{{ sonic_upgrade_image }}" + +- name: Reboot the system to complete the upgrade + reboot: + +- name: Wait until sonic has become ready + shell: "show system status" + register: result + until: result is not failed + retries: 30 + delay: 20 + +- name: Extra delay to be extra safe + wait_for: + timeout: 60 + +- name: clean up images + command: sonic-installer cleanup -y diff --git a/partition/roles/sonic/README.md b/partition/roles/sonic/README.md new file mode 100644 index 000000000..766111df9 --- /dev/null +++ b/partition/roles/sonic/README.md @@ -0,0 +1,69 @@ +# sonic + +Deploys a SONiC switch. It can run on all switches inside the network topology, i.e. spines, exits, leaves. It is supposed to run on Edgecore SONiC switches. + +It depends on the `switch_facts` module from `ansible-common`, so make sure modules from `ansible-common` are included before executing this role. + +## Variables + +| Name | Mandatory | Description | +| ---------------------------------------------------------- | --------- | -------------------------------------------------------------------------------------------------------------------- | +| sonic_mgmt_vrf | yes | Boolean flag to enable / disable the management vrf on the switch. | +| sonic_ntpservers | yes | The time servers to use on the switch. | +| sonic_nameservers | yes | The name servers to use on the switch. | +| sonic_loopback_address | yes | The loopback address use for this router. Is used to identify routers with bgp unnumbered. | +| sonic_asn | yes | The autonomous system number of the router. | +| sonic_mgmtif_ip | | The fixed IP address of the management interface in `IP/netmask` format. If not given, defaults to DHCP. | +| sonic_mgmtif_gateway | | If using a fixed management IP, the default gateway for the management interface. | +| sonic_ip_masquerade | | Enable ip masquerading on eth0. | +| sonic_breakouts | | The breakout configuration for ports, e.g. `dict('Ethernet0'='4x25G')` | +| sonic_ports | | Special configuration for ports (mtu, fec, have highest precedence) | +| sonic_ports.name | | The port name. | +| sonic_ports.speed | | Speed of the port. | +| sonic_ports.mtu | | MTU of the port. | +| sonic_ports.fec | | FEC used for the port. | +| sonic_ports.vrf | | VRF where the port should bind to. | +| sonic_ports.ips | | IPs to assign to the interface directly. | +| sonic_ports_default_mtu | | MTU default value for ports | +| sonic_ports_default_fec | | FEC default value for ports | +| sonic_ports_default_speed | | Speed default value for ports | +| sonic_bgp_ports | | Ports for the underlay BGP sessions | +| sonic_frr_render | | Render the frr config | +| sonic_frr_debug_options | | Debugging options for FRR. | +| sonic_frr_syslog_level | | Log level of FRR | +| sonic_frr_l2vpn_evpn | | Enable l2vpn evpn as address family. | +| sonic_frr_route_map | | Configure a route map | +| sonic_frr_route_map.name | | Name of the route map | +| sonic_frr_route_map.match | | The matcher of the route map | +| sonic_frr_static_routes | | Static routes to be injected through FRR. | +| sonic_frr_static_routes_mgmt | | Static routes to be injected to the mgmt VRF. | +| sonic_vlans | | VLANs to configure. | +| sonic_vlans.id | | The VLAN ID. | +| sonic_vlans.ip | | The IP of the SVI of this VLAN. | +| sonic_vlans.dhcp_servers | | DHCP servers to relay to. | +| sonic_vlans.untagged_ports | | Array of untagged ports to bind to this VLAN. | +| sonic_vlans.vrf | | The VRF to bind the VLANs SVI to. | +| sonic_vteps | | VTEPs to configure. If defined FRR will automatically advertise all VNIs. | +| sonic_vteps.comment | | Description for the VTEP. | +| sonic_vteps.vlan | | The local VLAN interface. | +| sonic_vteps.vni | | The global VNI within the CLOS topology. | +| sonic_lldp_hello_timer | | Interval for the lldp daemon on the switch to send hello to neighbors | +| sonic_interconnects | | Configure connections to other BGP parties (e.g. Internet or MPLS routers) | +| sonic_interconnects.sonic_interconnects_default_peer_group | | The default peer-group name where to put connecting parties. | +| sonic_interconnects.sonic_interconnects_default_bgp_timers | | Default bgp timers for connecting parties. | +| sonic_interconnects.announcements | | BGP announcements to the connecting parties. (e.g. a static network announcement `network w.x.y.z/24`) | +| sonic_interconnects.bgp_md5_password | | Use a MD5 password for the BGP session with the remote party. | +| sonic_interconnects.bgp_timers | | Use specific BGP timer values for the BGP session with the remote party. | +| sonic_interconnects.neighbor_ip | | Connect to this BGP neighbors IP. | +| sonic_interconnects.neighbors | | Connect to this BGP neighbors - supports multiple neighbors and also BGP unnumbered by giving `Ethernet0 interface`. | +| sonic_interconnects.unnumbered_interfaces | | Connect with BGP unnumbered on these interfaces - also sets IPv6 options to make unnumbered work right. | +| sonic_interconnects.peer_group | | Put the neighbor in this peer group. | +| sonic_interconnects.prefixlists | | BGP prefix lists to configure. | +| sonic_interconnects.remote_as | | The AS of the BGP neighbor. | +| sonic_interconnects.routemap_in | | Apply an incoming routemap for this BGP session. | +| sonic_interconnects.routemap_out | | Apply an outgoing routemap for this BGP session. | +| sonic_interconnects.vni | | This BGP session will connect the specified VNI within the CLOS topology with the given peer. | +| sonic_interconnects.vrf | | Use a dedicated BGP session fenced with an VRF for this connection. Also it declares the virtual network as layer-3. | +| sonic_ssh_sourceranges | | The source ranges from which the switch should be reachable over SSH on its prod (non-management) addresses | +| sonic_extended_cacl.ipv4 | | Iptables ipv4 rules that should be added as extended Control Plane ACLs (Edgecore Sonic specific feature) | +| sonic_extended_cacl.ipv6 | | Iptables ipv6 rules that should be added as extended Control Plane ACLs (Edgecore Sonic specific feature) | diff --git a/partition/roles/sonic/defaults/main.yaml b/partition/roles/sonic/defaults/main.yaml new file mode 100644 index 000000000..3acbb8164 --- /dev/null +++ b/partition/roles/sonic/defaults/main.yaml @@ -0,0 +1,47 @@ +--- +sonic_mgmt_vrf: true +sonic_ntpservers: [] +sonic_nameservers: [] +sonic_ip_masquerade: false +sonic_timezone: Europe/Berlin + +## Physical settings +sonic_breakouts: {} +sonic_ports: [] +#- name: Ethernet50 +# speed: 10000 +sonic_ports_default_speed: 10000 +sonic_ports_default_mtu: 9216 +sonic_ports_default_fec: none + +## BGP related settings +sonic_loopback_address: +sonic_asn: +sonic_bgp_ports: [] +sonic_frr_render: true +sonic_frr_debug_options: [] +sonic_frr_syslog_level: informational +sonic_frr_l2vpn_evpn: false +#sonic_frr_route_map: +# name: LOOPBACKS +# match: interface name Loopback0 +sonic_frr_static_routes: [] + +# VLAN settings +sonic_vlan_members: true +sonic_vlans: [] +#- id: 1 +# ip: '' +# untagged_ports: [] +# dhcp_servers: [] + +sonic_vteps: [] + +# LLDP Hello timer is 30s by default on the switch, but metal-hammer expects 10s or so +sonic_lldp_hello_timer: 10 + +sonic_frr_static_routes_mgmt: [] +sonic_interconnects: {} + +sonic_interconnects_default_peer_group: EXTERNAL +sonic_interconnects_default_bgp_timers: '1 3' \ No newline at end of file diff --git a/partition/roles/sonic/files/bgp-validation@.service b/partition/roles/sonic/files/bgp-validation@.service new file mode 100644 index 000000000..d2e9e2764 --- /dev/null +++ b/partition/roles/sonic/files/bgp-validation@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger a validation run of a frr configuration file %I + +[Service] +Type=oneshot +ExecStart=/usr/bin/vtysh --dryrun --inputfile %I +StandardOutput=journal + +[Install] +WantedBy=multi-user.target diff --git a/partition/roles/sonic/files/frr-reload.service b/partition/roles/sonic/files/frr-reload.service new file mode 100644 index 000000000..a5f286fba --- /dev/null +++ b/partition/roles/sonic/files/frr-reload.service @@ -0,0 +1,10 @@ +[Unit] +Description=Reload FRR + +[Service] +Type=oneshot +ExecStart=/usr/bin/docker exec bgp /usr/lib/frr/frr-reload +StandardOutput=journal + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/partition/roles/sonic/handlers/main.yaml b/partition/roles/sonic/handlers/main.yaml new file mode 100644 index 000000000..a89d76201 --- /dev/null +++ b/partition/roles/sonic/handlers/main.yaml @@ -0,0 +1,39 @@ +--- +- name: reload systemd + systemd: + daemon_reload: yes + +- name: write metal conf to db + ansible.builtin.command: sonic-cfggen --yaml /etc/sonic/metal.yaml --write-to-db + notify: config save + +- name: config save + ansible.builtin.command: config save -y + +- name: config save and reload + systemd: + name: config-save-reload + state: started + +- name: restart bgp + systemd: + name: bgp + state: restarted + +- name: restart caclmgrd + systemd: + name: caclmgrd + state: restarted + +- name: config reload + ansible.builtin.command: config reload -y + async: 120 + poll: 0 + notify: wait for new connection + +- name: wait for new connection + wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 30 + timeout: 300 diff --git a/partition/roles/sonic/tasks/main.yaml b/partition/roles/sonic/tasks/main.yaml new file mode 100644 index 000000000..c9fd14c63 --- /dev/null +++ b/partition/roles/sonic/tasks/main.yaml @@ -0,0 +1,85 @@ +--- +- name: Gather switch facts + switch_facts: + +- name: Check mandatory variables for this role are set + assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: yes + that: + - sonic_mgmt_vrf is defined + - sonic_loopback_address is defined + - sonic_asn is defined + - sonic_ntpservers is defined + - sonic_nameservers is defined + - metal_stack_switch_os_is_sonic + +- name: render resolv.conf + template: + src: resolv.conf.j2 + dest: /etc/resolv.conf + +- name: Activate IP MASQUERADE on eth0 + ansible.builtin.iptables: + chain: POSTROUTING + jump: MASQUERADE + out_interface: eth0 + table: nat + when: sonic_ip_masquerade + +- name: Activate ipv4 forwarding on eth0 + ansible.posix.sysctl: + name: net.ipv4.conf.eth0.forwarding + reload: no + sysctl_set: yes + value: "1" + when: sonic_ip_masquerade + +- name: Install services + copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + notify: reload systemd + loop: + - bgp-validation@.service + - frr-reload.service + +# Dependencies are returned by config. +- name: Configure breakouts + command: "config interface breakout --yes {{ item.key }} '{{ item.value }}'" + register: breakout_result + changed_when: "'Breakout process got successfully completed.' in breakout_result.stdout" + failed_when: "breakout_result.rc != 0 or 'Dependecies Exist. No further action will be taken' in breakout_result.stdout" + with_dict: "{{ sonic_breakouts }}" + when: sonic_breakouts is defined + +- name: Render metal config for config_db.json + template: + src: metal.yaml.j2 + dest: /etc/sonic/metal.yaml + notify: + - write metal conf to db + - config reload + +- name: Set NTP timezone + timezone: + name: "{{ sonic_timezone }}" + +- name: Render frr config + template: + src: frr.conf.j2 + dest: /etc/sonic/frr/frr.conf + when: sonic_frr_render + notify: + - restart bgp + +- name: render iptables.json + template: + src: iptables.json.j2 + dest: /etc/sonic/iptables.json + when: sonic_extended_cacl is defined + notify: + - restart caclmgrd + +- name: Flush handlers + meta: flush_handlers diff --git a/partition/roles/sonic/templates/frr.conf.j2 b/partition/roles/sonic/templates/frr.conf.j2 new file mode 100644 index 000000000..7226b4ef6 --- /dev/null +++ b/partition/roles/sonic/templates/frr.conf.j2 @@ -0,0 +1,192 @@ +#jinja2: lstrip_blocks: "True", trim_blocks: "True" +frr defaults datacenter +hostname {{ inventory_hostname }} +! +service integrated-vtysh-config +! +log syslog {{ sonic_frr_syslog_level }} +{% if sonic_frr_debug_options is defined %} +{% for option in sonic_frr_debug_options %} +{{ option }} +{% endfor %} +{% endif %} +{% for k, i in sonic_interconnects.items() %} +{% if i.vrf is defined %} +! +vrf {{ i.vrf }} + vni {{ i.vni }} +exit-vrf +{% endif %} +{% endfor %} +{% for port in sonic_bgp_ports %} +! +interface {{ port }} + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +{% endfor %} +{% for k, i in sonic_interconnects.items() %} +{% for n in i.unnumbered_interfaces|default([]) %} +! +interface {{ n }} + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +{% endfor %} +{% endfor %} +! +router bgp {{ sonic_asn }} + bgp router-id {{ sonic_loopback_address }} + bgp bestpath as-path multipath-relax + neighbor FABRIC peer-group + neighbor FABRIC remote-as external + neighbor FABRIC timers 1 3 +{% for port in sonic_bgp_ports %} + neighbor {{ port }} interface peer-group FABRIC +{% endfor %} +{% for k, i in sonic_interconnects.items() %} +{% if i.vrf is not defined %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} peer-group + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} remote-as external + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} timers {{ i.bgp_timers | default(sonic_interconnects_default_bgp_timers) }} + {% if i.bfd_parameters is defined %} + neighbor {{ i.peer_group }} bfd {{ i.bfd_parameters }} + {% endif %} + {% if i.neighbor_ip is defined %} + neighbor {{ i.neighbor_ip }} peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% if i.bgp_md5_password is defined %} + neighbor {{ i.neighbor_ip }} password {{ i.bgp_md5_password }} + {% endif %} + {% endif %} + {% for n in i.neighbors|default([]) %} + neighbor {{ n }} peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% endfor %} + {% for n in i.unnumbered_interfaces|default([]) %} + neighbor {{ n }} interface peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% endfor %} +{% endif %} +{% endfor %} + ! + address-family ipv4 unicast + redistribute connected route-map {{ sonic_frr_route_map.name | default("DENY_MGMT") }} +{% if sonic_frr_static_routes|length > 0 %} + redistribute static +{% endif %} +{% for k, i in sonic_interconnects.items() %} +{% if i.vrf is not defined %} + {% if i.announcements is defined %} + {% for a in i.announcements %} + {{ a }} + {% endfor %} + {% endif %} + {% if i.routemap_in is defined %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} route-map {{ i.routemap_in.name }} in + {% endif %} + {% if i.routemap_out is defined %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} route-map {{ i.routemap_out.name }} out + {% endif %} +{% endif %} +{% endfor %} + exit-address-family +{% if sonic_frr_l2vpn_evpn %} + ! + address-family l2vpn evpn +{% if sonic_vteps is defined and sonic_vteps|length > 0 %} + advertise-all-vni +{% endif %} + neighbor FABRIC activate + neighbor FABRIC allowas-in 2 + exit-address-family +{% endif %} +{% for k, i in sonic_interconnects.items() %} +{% if i.vrf is defined %} +! +router bgp {{ sonic_asn }} vrf {{ i.vrf }} + bgp router-id {{ sonic_loopback_address }} + bgp bestpath as-path multipath-relax + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} peer-group + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} remote-as external + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} timers {{ i.bgp_timers | default(sonic_interconnects_default_bgp_timers) }} + {% if i.bfd_parameters is defined %} + neighbor {{ i.peer_group }} bfd {{ i.bfd_parameters }} + {% endif %} + {% if i.neighbor_ip is defined %} + neighbor {{ i.neighbor_ip }} peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% if i.bgp_md5_password is defined %} + neighbor {{ i.neighbor_ip }} password {{ i.bgp_md5_password }} + {% endif %} + {% endif %} + {% for n in i.neighbors|default([]) %} + neighbor {{ n }} peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% endfor %} + {% for n in i.unnumbered_interfaces|default([]) %} + neighbor {{ n }} interface peer-group {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} + {% endfor %} + ! + address-family ipv4 unicast + {% if i.announcements is defined %} + {% for a in i.announcements %} + {{ a }} + {% endfor %} + {% endif %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} remove-private-AS all + {% if i.routemap_in is defined %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} route-map {{ i.routemap_in.name }} in + {% endif %} + {% if i.routemap_out is defined %} + neighbor {{ i.peer_group | default(sonic_interconnects_default_peer_group) }} route-map {{ i.routemap_out.name }} out + {% endif %} + exit-address-family + ! + address-family l2vpn evpn + advertise ipv4 unicast + exit-address-family +{% endif %} +{% endfor %} +{% for k, i in sonic_interconnects.items() %} +{% if i.prefixlists is defined %} +! +{% for p in i.prefixlists %} +{{ p }} +{% endfor %} +{% endif %} +{% if i.routemap_in is defined %} +! +route-map {{ i.routemap_in.name }} permit 10 +{% for e in i.routemap_in.entries %} + {{ e }} +{% endfor %} +{% endif %} +{% if i.routemap_out is defined %} +! +route-map {{ i.routemap_out.name }} permit 20 +{% for e in i.routemap_out.entries %} + {{ e }} +{% endfor %} +{% endif %} +{% endfor %} +{% if sonic_frr_route_map is defined %} +! +route-map {{ sonic_frr_route_map.name }} permit 10 + match {{ sonic_frr_route_map.match }} +{% else %} +! +route-map DENY_MGMT deny 10 + match interface eth0 +route-map DENY_MGMT permit 20 +{% endif %} +{% if sonic_frr_static_routes_mgmt is defined and sonic_frr_static_routes_mgmt|length > 0 %} +! +vrf mgmt + {% for route in sonic_frr_static_routes_mgmt %} + ip route {{ route }} + {% endfor %} +exit-vrf +{% endif %} +{% if sonic_frr_static_routes|length > 0 %} +! +{% for route in sonic_frr_static_routes %} +ip route {{ route }} +{% endfor %} +{% endif %} +! +line vty +! \ No newline at end of file diff --git a/partition/roles/sonic/templates/iptables.json.j2 b/partition/roles/sonic/templates/iptables.json.j2 new file mode 100644 index 000000000..70fb8c091 --- /dev/null +++ b/partition/roles/sonic/templates/iptables.json.j2 @@ -0,0 +1,13 @@ +{# jinja2: lstrip_blocks: "False", trim_blocks: "False" #} +{ + "ipv4":[ +{% for rule in sonic_extended_cacl.ipv4|default([]) %} + "{{ rule }}"{% if not loop.last %},{% endif +%} +{% endfor %} + ], + "ipv6":[ +{% for rule in sonic_extended_cacl.ipv6|default([]) %} + "{{ rule }}"{% if not loop.last %},{% endif +%} +{% endfor %} + ] +} diff --git a/partition/roles/sonic/templates/metal.yaml.j2 b/partition/roles/sonic/templates/metal.yaml.j2 new file mode 100644 index 000000000..1b2a2fa07 --- /dev/null +++ b/partition/roles/sonic/templates/metal.yaml.j2 @@ -0,0 +1,179 @@ +#jinja2: lstrip_blocks: "False", trim_blocks: "False" +--- +DEVICE_METADATA: + localhost: + docker_routing_config_mode: split + hostname: "{{ inventory_hostname }}" + type: "LeafRouter" + frr_mgmt_framework_config: "true" + +FEATURE: + dhcp_relay: + auto_restart: enabled + state: enabled + +NTP: + global: + src_intf: "eth0" + +NTP_SERVER: + {% for ntpserver in sonic_ntpservers %} + {{ ntpserver }}: {} + {% endfor %} + +LOOPBACK_INTERFACE: + Loopback0: {} + Loopback0|{{ sonic_loopback_address }}/32: {} + +{% if sonic_mgmtif_ip is defined %} +MGMT_INTERFACE: + eth0|{{ sonic_mgmtif_ip }}: + gwaddr: "{{ sonic_mgmtif_gateway }}" + +{% endif %} +MGMT_VRF_CONFIG: + vrf_global: + mgmtVrfEnabled: "{{ sonic_mgmt_vrf | lower }}" +{% if sonic_ports|length > 0 %} + +INTERFACE: + {% for port in sonic_ports %} + {% if port.name in sonic_bgp_ports|default([]) or port.vrf is defined %} + {{ port.name }}: + {% if port.name in sonic_bgp_ports|default([]) %} + ipv6_use_link_local_only: enable + {% endif %} + {% if port.vrf is defined %} + vrf_name: "{{ port.vrf }}" + {% endif %} + {% elif port.ips is defined %} + {{ port.name }}: {} + {% endif %} + {% for ip in port.ips|default([]) %} + {{ port.name }}|{{ ip }}: {} + {% endfor %} + {% endfor %} +{% endif %} +{% if sonic_ports|length > 0 %} + +PORT: + {% for port in sonic_ports %} + {{ port.name }}: + admin_status: up + speed: "{{ port.speed|default(sonic_ports_default_speed) }}" + mtu: "{{ port.mtu|default(sonic_ports_default_mtu) }}" + fec: "{{ port.fec|default(sonic_ports_default_fec)|string|lower }}" + {% endfor %} +{% endif %} +{% if sonic_vlans is defined and sonic_vlans|length > 0 %} + +VLAN: + {% for vlan in sonic_vlans %} + Vlan{{ vlan.id }}: + {% if vlan.dhcp_servers is defined and vlan.dhcp_servers|length > 0 %} + dhcp_servers: {{ vlan.dhcp_servers }} + {% endif %} + vlanid: {{ vlan.id }} + {% endfor %} + +VLAN_INTERFACE: + {% for vlan in sonic_vlans %} + {% if vlan.vrf is defined %} + Vlan{{ vlan.id }}: + vrf_name: "{{ vlan.vrf }}" + {% else %} + Vlan{{ vlan.id }}: {} + {% endif %} + {% if vlan.ip is defined %} + Vlan{{ vlan.id }}|{{ vlan.ip }}: {} + {% endif %} + {% endfor %} + +{% if sonic_vlan_members %} +VLAN_MEMBER: + {% for vlan in sonic_vlans %} + {% for untagged_port in vlan.untagged_ports|default([]) %} + Vlan{{ vlan.id }}|{{ untagged_port }}: + tagging_mode: untagged + {% endfor %} + {% endfor %} +{% endif %} +{% endif %} +{% if sonic_vteps is defined and sonic_vteps|length > 0 %} + +VXLAN_EVPN_NVO: + nvo: + source_vtep: vtep + +VXLAN_TUNNEL: + vtep: + src_ip: "{{ sonic_loopback_address }}" + +VXLAN_TUNNEL_MAP: +{% for vtep in sonic_vteps %} + {% if vtep.comment is defined %} + # {{ vtep.comment }} + {% endif %} + "vtep|map_{{ vtep.vni }}_{{ vtep.vlan }}": + vlan: "{{ vtep.vlan }}" + vni: "{{ vtep.vni }}" +{% endfor %} +{% endif %} +{% if sonic_interconnects is defined and sonic_interconnects|length > 0 %} + +VRF: +{% for k, i in sonic_interconnects.items() %} + {% if i.vrf is defined %} + {% if i.vni is defined %} + {{ i.vrf }}: + vni: "{{ i.vni }}" + {% else %} + {{ i.vrf }}: {} + {% endif %} + {% endif %} +{% endfor %} +{% endif %} +{% if sonic_lldp_hello_timer is defined %} + +LLDP: + Global: + hello_timer: {{ sonic_lldp_hello_timer }} +{% endif %} + +{% if sonic_ssh_sourceranges is defined %} +ACL_RULE: + ALLOW_SSH|DEFAULT_RULE: + ETHER_TYPE: "2048" + PACKET_ACTION: "DROP" + PRIORITY: "1" +{% for sourcerange in sonic_ssh_sourceranges %} + ALLOW_SSH|RULE_{{ loop.index }}: + PACKET_ACTION: "ACCEPT" + PRIORITY: "9{{ loop.index }}" + SRC_IP: "{{ sourcerange }}" +{% endfor %} + ALLOW_NTP|DEFAULT_RULE: + ETHER_TYPE: "2048" + PACKET_ACTION: "DROP" + PRIORITY: "1" + ALLOW_NTP|RULE_1: + PACKET_ACTION: "ACCEPT" + PRIORITY: "99" + SRC_IP: "0.0.0.0/0" + +ACL_TABLE: + ALLOW_SSH: + policy_desc: "Allow SSH access" + ports: [] + services: + - "SSH" + stage: "ingress" + type: "CTRLPLANE" + ALLOW_NTP: + policy_desc: "Allow NTP" + ports: [] + services: + - "NTP" + stage: "ingress" + type: "CTRLPLANE" +{% endif %} diff --git a/partition/roles/sonic/templates/resolv.conf.j2 b/partition/roles/sonic/templates/resolv.conf.j2 new file mode 100644 index 000000000..84f00e712 --- /dev/null +++ b/partition/roles/sonic/templates/resolv.conf.j2 @@ -0,0 +1,3 @@ +{% for ns in sonic_nameservers %} +nameserver {{ ns }} +{% endfor %} diff --git a/partition/roles/sonic/test/__init__.py b/partition/roles/sonic/test/__init__.py new file mode 100644 index 000000000..4795a2386 --- /dev/null +++ b/partition/roles/sonic/test/__init__.py @@ -0,0 +1,6 @@ +import os + + +def read_template_file(name): + with open(os.path.join(os.path.dirname(__file__), "..", "templates", name), 'r') as f: + return f.read() diff --git a/partition/roles/sonic/test/data/exit/frr.conf b/partition/roles/sonic/test/data/exit/frr.conf new file mode 100644 index 000000000..bace1ba34 --- /dev/null +++ b/partition/roles/sonic/test/data/exit/frr.conf @@ -0,0 +1,108 @@ +frr defaults datacenter +hostname exit01 +! +service integrated-vtysh-config +! +log syslog informational +! +vrf VrfMpls + vni 104000 +exit-vrf +! +interface Ethernet112 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +interface Ethernet116 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +interface Ethernet2 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +router bgp 4200000000 + bgp router-id 10.0.0.1 + bgp bestpath as-path multipath-relax + neighbor FABRIC peer-group + neighbor FABRIC remote-as external + neighbor FABRIC timers 1 3 + neighbor Ethernet112 interface peer-group FABRIC + neighbor Ethernet116 interface peer-group FABRIC + neighbor PXE peer-group + neighbor PXE remote-as external + neighbor PXE timers 1 3 + neighbor Ethernet1 interface peer-group PXE + neighbor PXE2 peer-group + neighbor PXE2 remote-as external + neighbor PXE2 timers 1 3 + neighbor Ethernet2 interface peer-group PXE2 + ! + address-family ipv4 unicast + redistribute connected route-map LOOPBACKS + redistribute static + neighbor PXE route-map ALLOW-PXE-IN in + neighbor PXE route-map ALLOW-PXE-OUT out + neighbor PXE2 route-map ALLOW-PXE2-IN in + neighbor PXE2 route-map ALLOW-PXE2-OUT out + exit-address-family + ! + address-family l2vpn evpn + advertise-all-vni + neighbor FABRIC activate + neighbor FABRIC allowas-in 2 + exit-address-family +! +router bgp 4200000000 vrf VrfMpls + bgp router-id 10.0.0.1 + bgp bestpath as-path multipath-relax + neighbor MPLS peer-group + neighbor MPLS remote-as external + neighbor MPLS timers 1 3 + neighbor 10.0.0.3 peer-group MPLS + neighbor 10.0.0.3 password test + ! + address-family ipv4 unicast + neighbor MPLS remove-private-AS all + neighbor MPLS route-map ALLOW-MPLS-IN in + neighbor MPLS route-map ALLOW-MPLS-OUT out + exit-address-family + ! + address-family l2vpn evpn + advertise ipv4 unicast + exit-address-family +! +ip prefix-list MPLS_PREFIX_IN seq 10 permit 10.0.0.0/8 +ip prefix-list MPLS_PREFIX_IN seq 20 permit 100.100.0.0/16 +! +route-map ALLOW-MPLS-IN permit 10 + match ip address prefix-list MPLS_PREFIX_IN +! +route-map ALLOW-MPLS-OUT permit 20 + match ip address prefix-list MPLS_PREFIX_OUT +! +ip prefix-list PXE_PREFIX_IN seq 10 permit 0.0.0.0/0 +ip prefix-list PXE_PREFIX_OUT seq 10 permit 10.255.0.0/16 ge 16 +! +route-map ALLOW-PXE-IN permit 10 + match ip address prefix-list PXE_PREFIX_IN +! +route-map ALLOW-PXE-OUT permit 20 + match ip address prefix-list PXE_PREFIX_OUT +! +ip prefix-list PXE2_PREFIX_IN seq 10 permit 0.0.0.0/0 +ip prefix-list PXE2_PREFIX_OUT seq 10 permit 10.255.0.0/16 ge 16 +! +route-map ALLOW-PXE2-IN permit 10 + match ip address prefix-list PXE2_PREFIX_IN +! +route-map ALLOW-PXE2-OUT permit 20 + match ip address prefix-list PXE2_PREFIX_OUT +! +route-map LOOPBACKS permit 10 + match interface Loopback0 +! +ip route 0.0.0.0/0 10.1.2.1 +! +line vty +! \ No newline at end of file diff --git a/partition/roles/sonic/test/data/exit/input.yaml b/partition/roles/sonic/test/data/exit/input.yaml new file mode 100644 index 000000000..509a9b304 --- /dev/null +++ b/partition/roles/sonic/test/data/exit/input.yaml @@ -0,0 +1,107 @@ +--- +inventory_hostname: exit01 +sonic_asn: 4200000000 +sonic_loopback_address: 10.0.0.1 + +sonic_breakouts: + Ethernet0: "4x10G" + +sonic_ports: +- name: Ethernet0 + speed: 10000 + mtu: 1500 + vrf: VrfMpls + ips: + - 10.0.0.2/32 +# Uplink for pxe VLAN +- name: Ethernet1 + speed: 10000 +- name: Ethernet2 + speed: 10000 +- name: Ethernet3 + speed: 10000 + ips: + - 10.1.2.2/30 +# spine uplinks +- name: Ethernet112 +- name: Ethernet116 + +sonic_bgp_ports: +- Ethernet112 +- Ethernet116 + +sonic_vlans: +- id: 4000 + vrf: vrfMpls + +sonic_vteps: +- comment: MPLS + vlan: 4000 + vni: 104000 + +# Default route from PXE VLAN +sonic_frr_static_routes: +- "0.0.0.0/0 10.1.2.1" + +sonic_interconnects: + mpls: + neighbor_ip: 10.0.0.3 + bgp_md5_password: test + remote_as: 65000 + peer_group: MPLS + vrf: VrfMpls + vni: 104000 + announcements: [] + routemap_in: + name: ALLOW-MPLS-IN + entries: + - "match ip address prefix-list MPLS_PREFIX_IN" + routemap_out: + name: ALLOW-MPLS-OUT + entries: + - "match ip address prefix-list MPLS_PREFIX_OUT" + prefixlists: + - "ip prefix-list MPLS_PREFIX_IN seq 10 permit 10.0.0.0/8" + - "ip prefix-list MPLS_PREFIX_IN seq 20 permit 100.100.0.0/16" + pxe: + peer_group: PXE + neighbors: + - "Ethernet1 interface" + announcements: [] + routemap_in: + name: ALLOW-PXE-IN + entries: + - "match ip address prefix-list PXE_PREFIX_IN" + routemap_out: + name: ALLOW-PXE-OUT + entries: + - "match ip address prefix-list PXE_PREFIX_OUT" + prefixlists: + - "ip prefix-list PXE_PREFIX_IN seq 10 permit 0.0.0.0/0" + - "ip prefix-list PXE_PREFIX_OUT seq 10 permit 10.255.0.0/16 ge 16" + pxe2: + peer_group: PXE2 + unnumbered_interfaces: + - "Ethernet2" + announcements: [] + routemap_in: + name: ALLOW-PXE2-IN + entries: + - "match ip address prefix-list PXE2_PREFIX_IN" + routemap_out: + name: ALLOW-PXE2-OUT + entries: + - "match ip address prefix-list PXE2_PREFIX_OUT" + prefixlists: + - "ip prefix-list PXE2_PREFIX_IN seq 10 permit 0.0.0.0/0" + - "ip prefix-list PXE2_PREFIX_OUT seq 10 permit 10.255.0.0/16 ge 16" + + +sonic_frr_l2vpn_evpn: true +sonic_frr_route_map: + name: LOOPBACKS + match: interface Loopback0 + +sonic_ssh_sourceranges: +- "1.2.3.4/32" +- "10.11.0.0/16" \ No newline at end of file diff --git a/partition/roles/sonic/test/data/exit/metal.yaml b/partition/roles/sonic/test/data/exit/metal.yaml new file mode 100644 index 000000000..4b2d1dba4 --- /dev/null +++ b/partition/roles/sonic/test/data/exit/metal.yaml @@ -0,0 +1,139 @@ +--- +DEVICE_METADATA: + localhost: + docker_routing_config_mode: split + hostname: "exit01" + type: "LeafRouter" + frr_mgmt_framework_config: "true" + +FEATURE: + dhcp_relay: + auto_restart: enabled + state: enabled + +NTP: + global: + src_intf: "eth0" + +NTP_SERVER: + +LOOPBACK_INTERFACE: + Loopback0: {} + Loopback0|10.0.0.1/32: {} + +MGMT_VRF_CONFIG: + vrf_global: + mgmtVrfEnabled: "true" + +INTERFACE: + Ethernet0: + vrf_name: "VrfMpls" + Ethernet0|10.0.0.2/32: {} + Ethernet3: {} + Ethernet3|10.1.2.2/30: {} + Ethernet112: + ipv6_use_link_local_only: enable + Ethernet116: + ipv6_use_link_local_only: enable + +PORT: + Ethernet0: + admin_status: up + speed: "10000" + mtu: "1500" + fec: "none" + Ethernet1: + admin_status: up + speed: "10000" + mtu: "9216" + fec: "none" + Ethernet2: + admin_status: up + speed: "10000" + mtu: "9216" + fec: "none" + Ethernet3: + admin_status: up + speed: "10000" + mtu: "9216" + fec: "none" + Ethernet112: + admin_status: up + speed: "10000" + mtu: "9216" + fec: "none" + Ethernet116: + admin_status: up + speed: "10000" + mtu: "9216" + fec: "none" + +VLAN: + Vlan4000: + vlanid: 4000 + +VLAN_INTERFACE: + Vlan4000: + vrf_name: "vrfMpls" + +VLAN_MEMBER: + +VXLAN_EVPN_NVO: + nvo: + source_vtep: vtep + +VXLAN_TUNNEL: + vtep: + src_ip: "10.0.0.1" + +VXLAN_TUNNEL_MAP: + # MPLS + "vtep|map_104000_4000": + vlan: "4000" + vni: "104000" + +VRF: + VrfMpls: + vni: "104000" + +LLDP: + Global: + hello_timer: 10 + +ACL_RULE: + ALLOW_SSH|DEFAULT_RULE: + ETHER_TYPE: "2048" + PACKET_ACTION: "DROP" + PRIORITY: "1" + ALLOW_SSH|RULE_1: + PACKET_ACTION: "ACCEPT" + PRIORITY: "91" + SRC_IP: "1.2.3.4/32" + ALLOW_SSH|RULE_2: + PACKET_ACTION: "ACCEPT" + PRIORITY: "92" + SRC_IP: "10.11.0.0/16" + ALLOW_NTP|DEFAULT_RULE: + ETHER_TYPE: "2048" + PACKET_ACTION: "DROP" + PRIORITY: "1" + ALLOW_NTP|RULE_1: + PACKET_ACTION: "ACCEPT" + PRIORITY: "99" + SRC_IP: "0.0.0.0/0" + +ACL_TABLE: + ALLOW_SSH: + policy_desc: "Allow SSH access" + ports: [] + services: + - "SSH" + stage: "ingress" + type: "CTRLPLANE" + ALLOW_NTP: + policy_desc: "Allow NTP" + ports: [] + services: + - "NTP" + stage: "ingress" + type: "CTRLPLANE" diff --git a/partition/roles/sonic/test/data/mgmtleaf/frr.conf b/partition/roles/sonic/test/data/mgmtleaf/frr.conf new file mode 100644 index 000000000..436860eb4 --- /dev/null +++ b/partition/roles/sonic/test/data/mgmtleaf/frr.conf @@ -0,0 +1,34 @@ +frr defaults datacenter +hostname r01mgmtleaf +! +service integrated-vtysh-config +! +log syslog informational +! +interface Ethernet50 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +interface Ethernet51 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +router bgp 420000000 + bgp router-id 10.0.0.1 + bgp bestpath as-path multipath-relax + neighbor FABRIC peer-group + neighbor FABRIC remote-as external + neighbor FABRIC timers 1 3 + neighbor Ethernet50 interface peer-group FABRIC + neighbor Ethernet51 interface peer-group FABRIC + ! + address-family ipv4 unicast + redistribute connected route-map DENY_MGMT + exit-address-family +! +route-map DENY_MGMT deny 10 + match interface eth0 +route-map DENY_MGMT permit 20 +! +line vty +! \ No newline at end of file diff --git a/partition/roles/sonic/test/data/mgmtleaf/input.yaml b/partition/roles/sonic/test/data/mgmtleaf/input.yaml new file mode 100644 index 000000000..b3ef81cae --- /dev/null +++ b/partition/roles/sonic/test/data/mgmtleaf/input.yaml @@ -0,0 +1,59 @@ +--- +inventory_hostname: r01mgmtleaf + +sonic_mgmtif_ip: '10.255.255.254/30' +sonic_mgmtif_gateway: 10.255.255.253 + +sonic_loopback_address: 10.0.0.1 +sonic_asn: 420000000 +sonic_ntpservers: ['1.2.3.4'] +sonic_breakouts: + Ethernet0: "4x25G" + +sonic_ports_default_speed: 1000 +sonic_ports_default_mtu: 9000 +sonic_ports: +- name: Ethernet0 +- name: Ethernet1 +- name: Ethernet2 +- name: Ethernet3 +- name: Ethernet50 + speed: 100000 + mtu: 9216 + fec: rs +- name: Ethernet51 + speed: 100000 + +sonic_bgp_ports: +- Ethernet50 +- Ethernet51 + +sonic_vlans: +- id: 1 + dhcp_servers: ['10.255.255.254'] + ip: '10.255.255.1/24' + untagged_ports: + - Ethernet0 + - Ethernet1 + - Ethernet2 + - Ethernet3 +- id: 2 + untagged_ports: + - Ethernet4 + +sonic_vteps: +- comment: "test-vtep" + vlan: Vlan2 + vni: 10002 + +sonic_extended_cacl: + ipv4: + - "-A INPUT -s 1.2.3.9/32 -p tcp -m tcp --dport 2112 -j ACCEPT" + - "-A INPUT -s 1.2.3.13/32 -p tcp -m tcp --dport 9080 -j ACCEPT" + - "-A INPUT -s 1.2.3.25/32 -p tcp -m tcp --dport 9100 -j ACCEPT" + - "-A INPUT -s 1.2.3.29/32 -p tcp -m tcp --dport 9342 -j ACCEPT" + ipv6: + - "-A INPUT -s 2001:db8::1/128 -p tcp -m tcp --dport 2112 -j ACCEPT" + - "-A INPUT -s 2001:db8:1::1/128 -p tcp -m tcp --dport 9080 -j ACCEPT" + - "-A INPUT -s 2001:db8:2::1/128 -p tcp -m tcp --dport 9100 -j ACCEPT" + - "-A INPUT -s 2001:db8:3::1/128 -p tcp -m tcp --dport 9342 -j ACCEPT" diff --git a/partition/roles/sonic/test/data/mgmtleaf/iptables.json b/partition/roles/sonic/test/data/mgmtleaf/iptables.json new file mode 100644 index 000000000..b08e0c708 --- /dev/null +++ b/partition/roles/sonic/test/data/mgmtleaf/iptables.json @@ -0,0 +1,14 @@ +{ + "ipv4":[ + "-A INPUT -s 1.2.3.9/32 -p tcp -m tcp --dport 2112 -j ACCEPT", + "-A INPUT -s 1.2.3.13/32 -p tcp -m tcp --dport 9080 -j ACCEPT", + "-A INPUT -s 1.2.3.25/32 -p tcp -m tcp --dport 9100 -j ACCEPT", + "-A INPUT -s 1.2.3.29/32 -p tcp -m tcp --dport 9342 -j ACCEPT" + ], + "ipv6":[ + "-A INPUT -s 2001:db8::1/128 -p tcp -m tcp --dport 2112 -j ACCEPT", + "-A INPUT -s 2001:db8:1::1/128 -p tcp -m tcp --dport 9080 -j ACCEPT", + "-A INPUT -s 2001:db8:2::1/128 -p tcp -m tcp --dport 9100 -j ACCEPT", + "-A INPUT -s 2001:db8:3::1/128 -p tcp -m tcp --dport 9342 -j ACCEPT" + ] +} diff --git a/partition/roles/sonic/test/data/mgmtleaf/metal.yaml b/partition/roles/sonic/test/data/mgmtleaf/metal.yaml new file mode 100644 index 000000000..1ec4be26e --- /dev/null +++ b/partition/roles/sonic/test/data/mgmtleaf/metal.yaml @@ -0,0 +1,111 @@ +--- +DEVICE_METADATA: + localhost: + docker_routing_config_mode: split + hostname: "r01mgmtleaf" + type: "LeafRouter" + frr_mgmt_framework_config: "true" + +FEATURE: + dhcp_relay: + auto_restart: enabled + state: enabled + +NTP: + global: + src_intf: "eth0" + +NTP_SERVER: + 1.2.3.4: {} + +LOOPBACK_INTERFACE: + Loopback0: {} + Loopback0|10.0.0.1/32: {} + +MGMT_INTERFACE: + eth0|10.255.255.254/30: + gwaddr: "10.255.255.253" + +MGMT_VRF_CONFIG: + vrf_global: + mgmtVrfEnabled: "true" + +INTERFACE: + Ethernet50: + ipv6_use_link_local_only: enable + Ethernet51: + ipv6_use_link_local_only: enable + +PORT: + Ethernet0: + admin_status: up + speed: "1000" + mtu: "9000" + fec: "none" + Ethernet1: + admin_status: up + speed: "1000" + mtu: "9000" + fec: "none" + Ethernet2: + admin_status: up + speed: "1000" + mtu: "9000" + fec: "none" + Ethernet3: + admin_status: up + speed: "1000" + mtu: "9000" + fec: "none" + Ethernet50: + admin_status: up + speed: "100000" + mtu: "9216" + fec: "rs" + Ethernet51: + admin_status: up + speed: "100000" + mtu: "9000" + fec: "none" + +VLAN: + Vlan1: + dhcp_servers: ['10.255.255.254'] + vlanid: 1 + Vlan2: + vlanid: 2 + +VLAN_INTERFACE: + Vlan1: {} + Vlan1|10.255.255.1/24: {} + Vlan2: {} + +VLAN_MEMBER: + Vlan1|Ethernet0: + tagging_mode: untagged + Vlan1|Ethernet1: + tagging_mode: untagged + Vlan1|Ethernet2: + tagging_mode: untagged + Vlan1|Ethernet3: + tagging_mode: untagged + Vlan2|Ethernet4: + tagging_mode: untagged + +VXLAN_EVPN_NVO: + nvo: + source_vtep: vtep + +VXLAN_TUNNEL: + vtep: + src_ip: "10.0.0.1" + +VXLAN_TUNNEL_MAP: + # test-vtep + "vtep|map_10002_Vlan2": + vlan: "Vlan2" + vni: "10002" + +LLDP: + Global: + hello_timer: 10 diff --git a/partition/roles/sonic/test/data/spine/frr.conf b/partition/roles/sonic/test/data/spine/frr.conf new file mode 100644 index 000000000..d1e04a6a7 --- /dev/null +++ b/partition/roles/sonic/test/data/spine/frr.conf @@ -0,0 +1,38 @@ +frr defaults datacenter +hostname spine01 +! +service integrated-vtysh-config +! +log syslog informational +! +interface Ethernet50 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +interface Ethernet51 + ipv6 nd ra-interval 6 + no ipv6 nd suppress-ra +! +router bgp 420000000 + bgp router-id 10.0.0.1 + bgp bestpath as-path multipath-relax + neighbor FABRIC peer-group + neighbor FABRIC remote-as external + neighbor FABRIC timers 1 3 + neighbor Ethernet50 interface peer-group FABRIC + neighbor Ethernet51 interface peer-group FABRIC + ! + address-family ipv4 unicast + redistribute connected route-map LOOPBACKS + exit-address-family + ! + address-family l2vpn evpn + neighbor FABRIC activate + neighbor FABRIC allowas-in 2 + exit-address-family +! +route-map LOOPBACKS permit 10 + match interface Loopback0 +! +line vty +! \ No newline at end of file diff --git a/partition/roles/sonic/test/data/spine/input.yaml b/partition/roles/sonic/test/data/spine/input.yaml new file mode 100644 index 000000000..30842c381 --- /dev/null +++ b/partition/roles/sonic/test/data/spine/input.yaml @@ -0,0 +1,27 @@ +--- +inventory_hostname: spine01 + +sonic_loopback_address: 10.0.0.1 +sonic_asn: 420000000 + +sonic_bgp_ports: +- Ethernet50 +- Ethernet51 + +sonic_ports_default_speed: 100000 +sonic_ports_default_mtu: 9216 +sonic_ports: +- name: Ethernet50 +- name: Ethernet51 + +sonic_frr_l2vpn_evpn: true +sonic_frr_route_map: + name: LOOPBACKS + match: interface Loopback0 + +sonic_extended_cacl: + ipv4: + - "-A INPUT -s 1.2.3.9/32 -p tcp -m tcp --dport 2112 -j ACCEPT" + - "-A INPUT -s 1.2.3.13/32 -p tcp -m tcp --dport 9080 -j ACCEPT" + - "-A INPUT -s 1.2.3.25/32 -p tcp -m tcp --dport 9100 -j ACCEPT" + - "-A INPUT -s 1.2.3.29/32 -p tcp -m tcp --dport 9342 -j ACCEPT" diff --git a/partition/roles/sonic/test/data/spine/iptables.json b/partition/roles/sonic/test/data/spine/iptables.json new file mode 100644 index 000000000..2807e94d0 --- /dev/null +++ b/partition/roles/sonic/test/data/spine/iptables.json @@ -0,0 +1,10 @@ +{ + "ipv4":[ + "-A INPUT -s 1.2.3.9/32 -p tcp -m tcp --dport 2112 -j ACCEPT", + "-A INPUT -s 1.2.3.13/32 -p tcp -m tcp --dport 9080 -j ACCEPT", + "-A INPUT -s 1.2.3.25/32 -p tcp -m tcp --dport 9100 -j ACCEPT", + "-A INPUT -s 1.2.3.29/32 -p tcp -m tcp --dport 9342 -j ACCEPT" + ], + "ipv6":[ + ] +} diff --git a/partition/roles/sonic/test/data/spine/metal.yaml b/partition/roles/sonic/test/data/spine/metal.yaml new file mode 100644 index 000000000..7fc90636d --- /dev/null +++ b/partition/roles/sonic/test/data/spine/metal.yaml @@ -0,0 +1,48 @@ +--- +DEVICE_METADATA: + localhost: + docker_routing_config_mode: split + hostname: "spine01" + type: "LeafRouter" + frr_mgmt_framework_config: "true" + +FEATURE: + dhcp_relay: + auto_restart: enabled + state: enabled + +NTP: + global: + src_intf: "eth0" + +NTP_SERVER: + +LOOPBACK_INTERFACE: + Loopback0: {} + Loopback0|10.0.0.1/32: {} + +MGMT_VRF_CONFIG: + vrf_global: + mgmtVrfEnabled: "true" + +INTERFACE: + Ethernet50: + ipv6_use_link_local_only: enable + Ethernet51: + ipv6_use_link_local_only: enable + +PORT: + Ethernet50: + admin_status: up + speed: "100000" + mtu: "9216" + fec: "none" + Ethernet51: + admin_status: up + speed: "100000" + mtu: "9216" + fec: "none" + +LLDP: + Global: + hello_timer: 10 diff --git a/partition/roles/sonic/test/template_test.py b/partition/roles/sonic/test/template_test.py new file mode 100644 index 000000000..5952262bf --- /dev/null +++ b/partition/roles/sonic/test/template_test.py @@ -0,0 +1,46 @@ +import unittest + +import yaml +import os +from test import read_template_file + +from ansible.template import Templar + + +def readYaml(f): + with open(os.path.join(os.path.dirname(__file__), f), "r") as stream: + return yaml.safe_load(stream) + + +def readFile(f): + with open(os.path.join(os.path.dirname(__file__), f), "r") as stream: + return stream.read() + + +class SonicRoleTemplates(unittest.TestCase): + def test_sonic_role_templates(self): + self.maxDiff = None + defaults = readYaml('../defaults/main.yaml') + metal_t = read_template_file('metal.yaml.j2') + frr_t = read_template_file('frr.conf.j2') + iptables_t = read_template_file('iptables.json.j2') + + for tc in next(os.walk(os.path.join(os.path.dirname(__file__), 'data')))[1]: + if tc.startswith("."): + continue + + vars = defaults | readYaml(f'./data/{tc}/input.yaml') + templar = Templar(loader=None, variables=vars) + + metal_exp = readFile(f'./data/{tc}/metal.yaml') + metal_res = templar.template(metal_t) + self.assertEqual(metal_exp.strip(), metal_res.strip(), 'detected a diff for metal.yaml rendering - tc ' + tc) + + frr_exp = readFile(f'./data/{tc}/frr.conf') + frr_res = templar.template(frr_t) + self.assertEqual(frr_exp.strip(), frr_res.strip(), 'detected a diff for frr.conf rendering - tc ' + tc) + + if 'sonic_extended_cacl' in vars: + iptables_exp = readFile(f'./data/{tc}/iptables.json') + iptables_res = templar.template(iptables_t,convert_data=False) + self.assertEqual(iptables_exp.strip(), iptables_res.strip(), 'detected a diff for iptables.json rendering - tc ' + tc) diff --git a/partition/roles/systemd-networkd/README.md b/partition/roles/systemd-networkd/README.md index 6df15b38b..d3ef6f59f 100644 --- a/partition/roles/systemd-networkd/README.md +++ b/partition/roles/systemd-networkd/README.md @@ -2,32 +2,36 @@ Deploys network-configuration for systems using systemd-networkd. +This role can deploy on bare metal machines with Debian or Almalinux. It depends on fact gathering. + ## Variables -| Name | Mandatory | Description | -|-------------------------------------|-----------|-----------------------------------------------------------------------------------------------------| -| systemd_networkd_mtu | no | The MTU to use for interfaces. | -| systemd_networkd_vrfs | no | An array of VRFs to be configured. | -| systemd_networkd_vrfs.name | no | The name of the VRF. | -| systemd_networkd_vrfs.table | no | The routing table id of the VRF. | -| systemd_networkd_nics | no | An array of network interfaces to be configured. Mac or Name is mandatory. | -| systemd_networkd_nics.mac | no | The MAC of the network interface. | -| systemd_networkd_nics.mtu | no | The MTU to use for this interface. | -| systemd_networkd_nics.name | no | The name this interface will be renamed to. | -| systemd_networkd_nics.dhcp | no | Configure the interface addresses with DHCP. | -| systemd_networkd_nics.addresses | no | array of IP addresses for the interfaces in CIDR notation. | -| systemd_networkd_nics.vrf | no | The VRF to bind this interface to. | -| systemd_networkd_nics.vxlans | no | array of VXLANs to terminate on a physical interface. | -| systemd_networkd_vxlans | no | VXLANs to terminate on a server. | -| systemd_networkd_vxlans.vtep.iface | no | The VXLAN interface that should serve as VTEP. | -| systemd_networkd_vxlans.vtep.vni | no | The network identifier of a VXLAN - should be unique within a BGP/EVPN-CLOS topology. | -| systemd_networkd_vxlans.vtep.ip | no | The IP address of the tunnel endpoint (usually the loopback address when used with bgp unnumbered). | -| systemd_networkd_vxlans.vtep.mtu | no | The MTU for the VXLAN interface. | -| systemd_networkd_vxlans.svi.iface | no | The VLAN interface that should be attached to the VTEP. | -| systemd_networkd_vxlans.svi.vlanid | no | The local VLAN ID. | -| systemd_networkd_vxlans.svi.vrf | no | The VRF that should be used as master device for the VLAN interface. | -| systemd_networkd_vxlans.svi.address | no | The IP address that should be configured at the VLAN interface. | -| systemd_networkd_vxlans.svi.mtu | no | The MTU for the VLAN interface. | +| Name | Mandatory | Description | +| ---------------------------------------- | --------- | --------------------------------------------------------------------------------------------------- | +| systemd_networkd_mtu | | The MTU to use for interfaces. | +| systemd_networkd_vrfs | | An array of VRFs to be configured. | +| systemd_networkd_vrfs.name | | The name of the VRF. | +| systemd_networkd_vrfs.table | | The routing table id of the VRF. | +| systemd_networkd_nics | | An array of network interfaces to be configured. Mac or Name is mandatory. | +| systemd_networkd_nics.mac | | The MAC of the network interface. | +| systemd_networkd_nics.mtu | | The MTU to use for this interface. | +| systemd_networkd_nics.name | | The name this interface will be renamed to. | +| systemd_networkd_nics.dhcp | | Configure the interface addresses with DHCP. | +| systemd_networkd_nics.dhcpv4routemetrics | | The metric to apply to routes learned through DHCPv4. | +| systemd_networkd_nics.addresses | | array of IP addresses for the interfaces in CIDR notation. | +| systemd_networkd_nics.gateways | | array of Gateways IPs for the interfaces. | +| systemd_networkd_nics.vrf | | The VRF to bind this interface to. | +| systemd_networkd_nics.vxlans | | array of VXLANs to terminate on a physical interface. | +| systemd_networkd_vxlans | | VXLANs to terminate on a server. | +| systemd_networkd_vxlans.vtep.iface | | The VXLAN interface that should serve as VTEP. | +| systemd_networkd_vxlans.vtep.vni | | The network identifier of a VXLAN - should be unique within a BGP/EVPN-CLOS topology. | +| systemd_networkd_vxlans.vtep.ip | | The IP address of the tunnel endpoint (usually the loopback address when used with bgp unnumbered). | +| systemd_networkd_vxlans.vtep.mtu | | The MTU for the VXLAN interface. | +| systemd_networkd_vxlans.svi.iface | | The VLAN interface that should be attached to the VTEP. | +| systemd_networkd_vxlans.svi.vlanid | | The local VLAN ID. | +| systemd_networkd_vxlans.svi.vrf | | The VRF that should be used as master device for the VLAN interface. | +| systemd_networkd_vxlans.svi.address | | The IP address that should be configured at the VLAN interface. | +| systemd_networkd_vxlans.svi.mtu | | The MTU for the VLAN interface. | ## Examples diff --git a/partition/roles/systemd-networkd/files/networkd.conf b/partition/roles/systemd-networkd/files/networkd.conf new file mode 100644 index 000000000..fe49e25ee --- /dev/null +++ b/partition/roles/systemd-networkd/files/networkd.conf @@ -0,0 +1,8 @@ +[Network] +#SpeedMeter=no +#SpeedMeterIntervalSec=10sec +ManageForeignRoutes=no + +[DHCP] +#DUIDType=vendor +#DUIDRawData= diff --git a/partition/roles/systemd-networkd/files/rename-interfaces.sh b/partition/roles/systemd-networkd/files/rename-interfaces.sh new file mode 100644 index 000000000..c855856ab --- /dev/null +++ b/partition/roles/systemd-networkd/files/rename-interfaces.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# Look up the systemd-networkd interface indexes of all ethernet interfaces +ethernets=$(networkctl list |grep ether |awk '{ print $1 }') + +for interface in $ethernets; do + networkctl down $interface +done + +# Wait for the interfaces to actually be down +while (networkctl list |grep ether |grep -v off); do + sleep 10 +done +# extra sleep for good measure +sleep 10 + +systemctl restart systemd-udev-trigger.service + +sleep 10 + +for interface in $ethernets; do + networkctl up $interface +done diff --git a/partition/roles/systemd-networkd/handlers/main.yaml b/partition/roles/systemd-networkd/handlers/main.yaml index 8c18e1d43..281119e03 100644 --- a/partition/roles/systemd-networkd/handlers/main.yaml +++ b/partition/roles/systemd-networkd/handlers/main.yaml @@ -3,8 +3,36 @@ service: name: systemd-networkd state: restarted + async: 1 + poll: 0 + notify: wait for new connection - name: restart systemd-udev-trigger service: name: systemd-udev-trigger state: restarted + async: 1 + poll: 0 + notify: wait for new connection + +- name: wait for new connection + wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 5 + timeout: 300 + +- name: copy rename script to host + copy: + src: rename-interfaces.sh + dest: /root/rename-interfaces.sh + owner: root + group: root + mode: "0755" + listen: rename interfaces + +- name: execute interface rename script + command: /root/rename-interfaces.sh + async: 300 + poll: 10 + listen: rename interfaces diff --git a/partition/roles/systemd-networkd/tasks/main.yaml b/partition/roles/systemd-networkd/tasks/main.yaml index 5bff4167a..1169907c9 100644 --- a/partition/roles/systemd-networkd/tasks/main.yaml +++ b/partition/roles/systemd-networkd/tasks/main.yaml @@ -1,10 +1,19 @@ --- +- import_tasks: redhat.yaml + when: ansible_os_family == 'RedHat' + - name: Enable systemd-networkd systemd: name: systemd-networkd enabled: yes state: started +- name: Copy systemd-networkd conf + copy: + src: networkd.conf + dest: /etc/systemd/ + notify: restart systemd-networkd + - name: Copy default file for VRFs copy: src: 99-vrf.network @@ -29,7 +38,7 @@ when: item.name != 'lo' notify: - restart systemd-networkd - - restart systemd-udev-trigger + - rename interfaces loop: "{{ systemd_networkd_nics }}" loop_control: index_var: i diff --git a/partition/roles/systemd-networkd/tasks/redhat.yaml b/partition/roles/systemd-networkd/tasks/redhat.yaml new file mode 100644 index 000000000..9a9770d29 --- /dev/null +++ b/partition/roles/systemd-networkd/tasks/redhat.yaml @@ -0,0 +1,17 @@ +--- +- name: Ensure epel is activated on RedHat derivatives + dnf: + name: epel-release + state: present + +- name: Ensure NetworkManager is deactivated on RedHat derivatives + systemd: + name: NetworkManager + enabled: no + masked: true + state: stopped + +- name: Ensure systemd-networkd is installed on RedHat derivatives + dnf: + name: systemd-networkd + state: present diff --git a/partition/roles/systemd-networkd/templates/network.j2 b/partition/roles/systemd-networkd/templates/network.j2 index fc7b9ef5d..7e86426cb 100644 --- a/partition/roles/systemd-networkd/templates/network.j2 +++ b/partition/roles/systemd-networkd/templates/network.j2 @@ -16,8 +16,18 @@ VRF={{ item.vrf }} {% for vxlan in item.vxlans|default([]) %} VXLAN={{ vxlan }} {% endfor %} +{% if item.dhcp is defined and item.dhcpv4routemetrics is defined %} + +[DHCPv4] +RouteMetric={{ item.dhcpv4routemetrics }} +{% endif %} {% for address in item.addresses|default([]) %} [Address] Address={{ address }} {% endfor %} +{% for gateway in item.gateways|default([]) %} + +[Route] +Gateway={{ gateway}} +{% endfor %} diff --git a/partition/roles/ztp/defaults/main/global-defaults b/partition/roles/ztp/defaults/main/global-defaults new file mode 120000 index 000000000..ac0cbf4d7 --- /dev/null +++ b/partition/roles/ztp/defaults/main/global-defaults @@ -0,0 +1 @@ +../../../../../defaults \ No newline at end of file diff --git a/partition/roles/ztp/defaults/main.yaml b/partition/roles/ztp/defaults/main/main.yaml similarity index 100% rename from partition/roles/ztp/defaults/main.yaml rename to partition/roles/ztp/defaults/main/main.yaml diff --git a/partition/roles/ztp/defaults/main/partition-defaults b/partition/roles/ztp/defaults/main/partition-defaults new file mode 120000 index 000000000..3a655898e --- /dev/null +++ b/partition/roles/ztp/defaults/main/partition-defaults @@ -0,0 +1 @@ +../../../../partition-defaults \ No newline at end of file diff --git a/partition/roles/ztp/tasks/main.yaml b/partition/roles/ztp/tasks/main.yaml index a49490219..9b62e94f4 100644 --- a/partition/roles/ztp/tasks/main.yaml +++ b/partition/roles/ztp/tasks/main.yaml @@ -1,4 +1,7 @@ --- +- name: Gather release versions + setup_yaml: + - name: Check mandatory variables for this role are set assert: fail_msg: "not all mandatory variables given, check role documentation" From ccfbba57918e8eed3396d5d29d8051a46ecf8b57 Mon Sep 17 00:00:00 2001 From: Robert Volkmann <20912167+robertvolkmann@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:39:34 +0200 Subject: [PATCH 14/15] Add GitHub OAuth2 configuration settings for Grafana (#187) --- control-plane/roles/monitoring/README.md | 4 +++- .../templates/prometheus-stack-values.yaml | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/control-plane/roles/monitoring/README.md b/control-plane/roles/monitoring/README.md index 86ea42b1f..cbfc7c09e 100644 --- a/control-plane/roles/monitoring/README.md +++ b/control-plane/roles/monitoring/README.md @@ -36,8 +36,10 @@ The following variables can be set to configure the role: | monitoring_grafana_admin_password | | Sets the admin password for Grafana | | monitoring_grafana_dashboard_timezone | | Sets the default's dashboard timezone for Grafana | | monitoring_grafana_additional_datasources | | Configures additional datasources for Grafana | +| monitoring_grafana_github_oauth | | [Configure GitHub OAuth2 authentication](https://grafana.com/docs/grafana/latest/setup-grafana/configure-security/configure-authentication/github/) | +| monitoring_grafana_extra_secret_mounts | | Extra secret mounts | | monitoring_slack_channel_url | | Slack channel url to add on alertmanager | | monitoring_slack_channel | | Slack channel to add on alertmanager | | monitoring_metal_api_url | | The URL where to reach metal-api | | monitoring_metal_api_hmac | | The hmac to authenticate against metal-api | -| monitoring_thanos_object_store_config | | Object storage used by Thanos, see [Official Documentation](https://thanos.io/tip/thanos/storage.md/#supported-clients) | \ No newline at end of file +| monitoring_thanos_object_store_config | | Object storage used by Thanos, see [Official Documentation](https://thanos.io/tip/thanos/storage.md/#supported-clients) | diff --git a/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml b/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml index 238e748c7..cf22409a8 100644 --- a/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml +++ b/control-plane/roles/monitoring/templates/prometheus-stack-values.yaml @@ -37,6 +37,19 @@ grafana: defaultDashboardsTimezone: {{ monitoring_grafana_dashboard_timezone }} additionalDataSources: {{ monitoring_grafana_additional_datasources | to_json }} + grafana.ini: + server: +{% if monitoring_ingress_grafana_tls %} + root_url: https://{{ monitoring_grafana_ingress_dns }} +{% else %} + root_url: http://{{ monitoring_grafana_ingress_dns }} +{% endif %} +{% if monitoring_grafana_github_oauth %} + auth.github: {{ monitoring_grafana_github_oauth | to_json }} +{% endif %} +{% if monitoring_grafana_extra_secret_mounts %} + extraSecretMounts: {{ monitoring_grafana_extra_secret_mounts | to_json }} +{% endif %} sidecar: datasources: url: http://thanos-query-frontend:9090 From 95c1436da7f42c79af487a407bb1c169ef64c41a Mon Sep 17 00:00:00 2001 From: Valentin Knabel Date: Thu, 29 Jun 2023 14:17:16 +0200 Subject: [PATCH 15/15] fix: auditing-meili did not allow null values in persistence (#188) --- control-plane/roles/auditing-meili/templates/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/control-plane/roles/auditing-meili/templates/values.yaml b/control-plane/roles/auditing-meili/templates/values.yaml index 3ef01dec5..88bf301e2 100644 --- a/control-plane/roles/auditing-meili/templates/values.yaml +++ b/control-plane/roles/auditing-meili/templates/values.yaml @@ -20,5 +20,5 @@ environment: auth: existingMasterKeySecret: metal-auditing-master-key -persistence: {{ auditing_meili_persistence }} -ingress: {{ auditing_meili_ingress }} +persistence: {{ auditing_meili_persistence | to_json }} +ingress: {{ auditing_meili_ingress | to_json }}