diff --git a/multi-cluster-utilities/README.md b/multi-cluster-utilities/README.md new file mode 100644 index 00000000..34674cba --- /dev/null +++ b/multi-cluster-utilities/README.md @@ -0,0 +1,54 @@ +# Multicluster Configuration +The create-kubeconfig-secret.sh is copied from [multicluster-runtime project](https://github.com/kubernetes-sigs/multicluster-runtime/tree/main), especially, from the `Kubeconfig Provider Example`. + +This Readme cover only what is relevant for setting up a `Management cluster` which hosts the kubernetes operator, and `Resource Clusters` which host the Netbox Operator Resources. For more information over the scripts, and how a multicluster setup could be setup with Kubeconfig provider please read [this](https://github.com/kubernetes-sigs/multicluster-runtime/blob/main/examples/kubeconfig/README.md) + +## 1. Create Management Cluster + +Follow the guide in project's root folder. +If all the prerequisites are in place, then `make create-kind` creates the cluster which is going to be used for the all the controller's dependencies (netbox backend, databases etc). +This cluster is also configured with the netbox-operator CRDs but the CRs are hosted and reconciled only in Resource Clusters. + +## 2. Create Resource Clusters + +Create your Resource Clusters & Provision Netbox-Operator Custom Resource Definitions. For each resource cluster you want to create, execute: + +- `kind create cluster --name ` +- `make install` + +Sidenote: kind create command changes the default context in which the kubectl commands point to. When you are done from this step, the kubectl context will be set at the last cluster you created. + +## 3. Establish cross-cluster access + +Set kubectl context back to `management cluster` +`kubectl config use-context kind-kind` + +Execute RBAC scripts for kubeconfig provider, towards each cluster you created in the previous step. +- `./create-kubeconfig-secret.sh -c kind-` + +For each cluster that gets configured as a 'Resource' cluster, a secret is populated in the 'Management' cluster. +Make sure that the appropriate secrets are populated in the kind-kind cluster, with names `kind-`. + +## 4-1. Execute manager process locally +At this point, you should be able to successfully start netbox operator process locally after: +- Establishing a port forward from Management cluster to your host for the Netbox service: `kubectl port-forward deploy/netbox 8080:8080 -n default` +- Setting environment variable `export NETBOX_HOST=localhost:8080` + +## 4-2. Execute manager process on management cluster +Deploying the manager in the management cluster involves additional manual steps. + +From the project's parent directory, execute `make deploy-kind` + +### Patch cluster role of netbox operator manager +ClusterRole `manager-role` needs to allow reading, listing and watching secrets. + - Execute `patch-netbox-clusterrole.sh` + +### Update secret in controller cluster +The secrets generated from the `create-kubeconfig-secret` needs to refer to the correct ip:port for each resource cluster. Currently it's pointing a localhost ip, which is only reachable from the host machine. +- Execute script `create-kubeconfig-secret-cluster.sh -c kind- --skip-create-rbac` + - This script updates the secret on management cluster, to use the IP of control-plane node of the resource cluster, retrieved from ``docker inspect -control-plane | jq '.[0].NetworkSettings.Networks.kind.IPAddress'`` + - The port of the K8s API server is assumed to be `6443`. You can check it with `docker inspect -control-plane | jq '.[0].NetworkSettings.Ports'` + +## 5. Test Reconciliation +Apply an example CR in resource cluster and check if it's getting reconciled. +`kubectl --context kind- apply -f config/samples/netbox_v1_ipaddress.yaml` diff --git a/multi-cluster-utilities/create-kubeconfig-secret-cluster.sh b/multi-cluster-utilities/create-kubeconfig-secret-cluster.sh new file mode 100755 index 00000000..62bf5f49 --- /dev/null +++ b/multi-cluster-utilities/create-kubeconfig-secret-cluster.sh @@ -0,0 +1,324 @@ +# File copied from https://github.com/kubernetes-sigs/multicluster-runtime/blob/main/examples/kubeconfig/README.md +// Modified by Swisscom (Schweiz) AG. +#!/bin/bash + +# Script to create a kubeconfig secret for the pod lister controller + +set -e + +# Default values +NAMESPACE="default" +SERVICE_ACCOUNT="multicluster-kubeconfig-provider" +KUBECONFIG_CONTEXT="" +SECRET_NAME="" +ROLE_TYPE="clusterrole" +RULES_FILE="" +CREATE_RBAC="true" + +# Check for yq +if ! command -v yq &>/dev/null; then + echo "ERROR: 'yq' is required but not installed. Please install yq (https://mikefarah.gitbook.io/yq/) and try again." + exit 1 +fi + +# Function to display usage information +function show_help { + echo "Usage: $0 [options]" + echo " -c, --context CONTEXT Kubeconfig context to use (required)" + echo " --name NAME Name for the secret (defaults to context name)" + echo " -n, --namespace NS Namespace to create the secret in (default: ${NAMESPACE})" + echo " -a, --service-account SA Service account name to use (default: ${SERVICE_ACCOUNT})" + echo " -t, --role-type TYPE Create Role or ClusterRole (role|clusterrole) (default: clusterrole)" + echo " -r, --rules-file FILE Path to rules file (default: rules.yaml in script directory)" + echo " --skip-create-rbac Skip creating RBAC resources (Role/ClusterRole and bindings)" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 -c prod-cluster" + echo " $0 -c prod-cluster -t role -r ./custom-rules.yaml" + echo " $0 -c prod-cluster -t clusterrole" + echo " $0 -c prod-cluster --skip-create-rbac" +} + +# Function to create Role or ClusterRole +function create_rbac { + local role_type="$1" + local rules_file="$2" + local role_name="$3" + local namespace="$4" + + if [ ! -f "$rules_file" ]; then + echo "ERROR: Rules file not found: $rules_file" + exit 1 + fi + + echo "Creating ${role_type} '${role_name}'..." + + if [ "$role_type" = "role" ]; then + # Create Role + ROLE_YAML=$(cat </dev/null; then + echo "Service account '${service_account}' not found in namespace '${namespace}'. Creating..." + + # Create the service account + SERVICE_ACCOUNT_YAML=$(cat < "$TEMP_KUBECONFIG" + +# echo "$NEW_KUBECONFIG" +# # Verify the kubeconfig works +# echo "Verifying kubeconfig..." +# if kubectl --kubeconfig="$TEMP_KUBECONFIG" version &>/dev/null; then +# rm "$TEMP_KUBECONFIG" +# echo "ERROR: Failed to verify kubeconfig - unable to connect to cluster." +# echo "- Ensure that the service account '${NAMESPACE}/${SERVICE_ACCOUNT}' on cluster '${KUBECONFIG_CONTEXT}' exists and is properly configured." +# echo "- You may specify a namespace using the -n flag." +# echo "- You may specify a service account using the -a flag." +# exit 1 +# fi +# echo "Kubeconfig verified successfully!" + +# Encode the verified kubeconfig +KUBECONFIG_B64=$(cat "$TEMP_KUBECONFIG" | base64 -w0) +rm "$TEMP_KUBECONFIG" + +# Generate and apply the secret +SECRET_YAML=$(cat </dev/null; then + echo "ERROR: 'yq' is required but not installed. Please install yq (https://mikefarah.gitbook.io/yq/) and try again." + exit 1 +fi + +# Function to display usage information +function show_help { + echo "Usage: $0 [options]" + echo " -c, --context CONTEXT Kubeconfig context to use (required)" + echo " --name NAME Name for the secret (defaults to context name)" + echo " -n, --namespace NS Namespace to create the secret in (default: ${NAMESPACE})" + echo " -a, --service-account SA Service account name to use (default: ${SERVICE_ACCOUNT})" + echo " -t, --role-type TYPE Create Role or ClusterRole (role|clusterrole) (default: clusterrole)" + echo " -r, --rules-file FILE Path to rules file (default: rules.yaml in script directory)" + echo " --skip-create-rbac Skip creating RBAC resources (Role/ClusterRole and bindings)" + echo " -h, --help Show this help message" + echo "" + echo "Examples:" + echo " $0 -c prod-cluster" + echo " $0 -c prod-cluster -t role -r ./custom-rules.yaml" + echo " $0 -c prod-cluster -t clusterrole" + echo " $0 -c prod-cluster --skip-create-rbac" +} + +# Function to create Role or ClusterRole +function create_rbac { + local role_type="$1" + local rules_file="$2" + local role_name="$3" + local namespace="$4" + + if [ ! -f "$rules_file" ]; then + echo "ERROR: Rules file not found: $rules_file" + exit 1 + fi + + echo "Creating ${role_type} '${role_name}'..." + + if [ "$role_type" = "role" ]; then + # Create Role + ROLE_YAML=$(cat </dev/null; then + echo "Service account '${service_account}' not found in namespace '${namespace}'. Creating..." + + # Create the service account + SERVICE_ACCOUNT_YAML=$(cat < "$TEMP_KUBECONFIG" + +# Verify the kubeconfig works +echo "Verifying kubeconfig..." +if ! kubectl --kubeconfig="$TEMP_KUBECONFIG" version &>/dev/null; then + rm "$TEMP_KUBECONFIG" + echo "ERROR: Failed to verify kubeconfig - unable to connect to cluster." + echo "- Ensure that the service account '${NAMESPACE}/${SERVICE_ACCOUNT}' on cluster '${KUBECONFIG_CONTEXT}' exists and is properly configured." + echo "- You may specify a namespace using the -n flag." + echo "- You may specify a service account using the -a flag." + exit 1 +fi +echo "Kubeconfig verified successfully!" + +# Encode the verified kubeconfig +KUBECONFIG_B64=$(cat "$TEMP_KUBECONFIG" | base64 -w0) +rm "$TEMP_KUBECONFIG" + +# Generate and apply the secret +SECRET_YAML=$(cat <