replace kubespray with rke2

MakefileVar.mk

@@ -14,6 +14,12 @@ AETHERCHARTDIR					?= $(CHARTDIR)/aether-helm-charts
 SDRANCHARTDIR					?= $(CHARTDIR)/sdran-helm-charts
 RESOURCEDIR						?= $(RIABDIR)/resources
 
+# Proxy config
+PROXY_ENABLED					?= false
+HTTP_PROXY						?= ${http_proxy}
+HTTPS_PROXY						?= ${https_proxy}
+NO_PROXY						?= ${no_proxy}
+
 # Commit IDs
 AETHERCHARTCID-V1.0.0			?= 6b3a267e428402d6bb8531bd921c1d202bb338b2
 AETHERCHARTCID-V1.3.0			?= 9f79ab87a96ae9ee2bb9a1540f4cd1574261611b
@@ -75,7 +81,12 @@ HELM_OPENCORD_URL				?= https://charts.opencord.org
 HELM_SDRAN_URL					?= https://sdrancharts.onosproject.org
 
 # Infrastructure component version
-KUBESPRAY_VERSION				?= release-2.18
+RKE2_K8S_VERSION				?= v1.23.15+rke2r1
+LPP_VERSION						?= v0.0.24
+KUBECTL_VERSION					?= v1.23.15
+GET_HELM						= get_helm.sh
+HELM_VERSION					?= v3.10.3
+
 
 # OMEC parameters
 UE_IP_POOL						?= 172.250.0.0

mk/infra.mk

@@ -2,51 +2,55 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # PHONY definitions
-INFRA_PHONY					:= infra-kubespray infra-k8s infra-fabric infra-atomix infra-onos-op infra-fabric-cu-du infra-prom-op-servicemonitor
+INFRA_PHONY					:= infra-k8s infra-fabric infra-atomix infra-onos-op infra-fabric-cu-du infra-prom-op-servicemonitor
 
-infra-kubespray: $(BUILD)/kubespray $(M)/kubespray-requirements
-infra-k8s: infra-kubespray $(M)/k8s-ready $(M)/helm-ready
+infra-k8s: $(M)/k8s-ready $(M)/helm-ready
 infra-fabric: $(M)/fabric
 infra-fabric-cu-du: $(M)/fabric-cu-du
 infra-atomix: $(M)/atomix
 infra-onos-op: $(M)/onos-operator
 infra-prom-op-servicemonitor: $(M)/prom-op-servicemonitor
 
-$(BUILD)/kubespray: | $(M)/setup
-	mkdir -p $(BUILD)
-	cd $(BUILD); git clone https://github.com/kubernetes-incubator/kubespray.git -b $(KUBESPRAY_VERSION)
-
-$(VENV)/bin/activate: | $(M)/setup
-	sudo pip3 install virtualenv
-	virtualenv $(VENV)
-
-$(M)/kubespray-requirements: $(BUILD)/kubespray | $(VENV)/bin/activate
-	source "$(VENV)/bin/activate" && \
-	pip3 install -r $(BUILD)/kubespray/requirements.txt
-	touch $@
-
-$(M)/k8s-ready: | $(M)/setup $(BUILD)/kubespray $(VENV)/bin/activate $(M)/kubespray-requirements
-	source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
-	ansible-playbook -b -i inventory/local/hosts.ini \
-		-e "{'override_system_hostname' : False, 'disable_swap' : True}" \
-		-e "{'docker_iptables_enabled' : True}" \
-		-e "{'kube_network_plugin_multus' : True, 'multus_version' : stable, 'multus_cni_version' : 0.3.1}" \
-		-e "{'kube_proxy_metrics_bind_address' : '0.0.0.0:10249'}" \
-		-e "{'kube_pods_subnet' : 192.168.84.0/24, 'kube_service_addresses' : 192.168.85.0/24}" \
-		-e "{'kube_apiserver_node_port_range' : 2000-36767}" \
-		-e "{'kubeadm_enabled': True}" \
-		-e "{'kubelet_custom_flags' : [--allowed-unsafe-sysctls=net.*]}" \
-		-e "{'dns_min_replicas' : 1}" \
-		-e "{'helm_enabled' : True}" \
-		cluster.yml
+$(M)/k8s-ready: | $(M)/setup
+	sudo mkdir -p /etc/rancher/rke2/
+	[ -d /usr/local/etc/emulab ] && [ ! -e /var/lib/rancher ] && sudo ln -s /var/lib/rancher /mnt/extra/rancher || true  # that link gets deleted on cleanup
+	echo "cni: multus,calico" >> config.yaml
+	echo "cluster-cidr: 192.168.84.0/24" >> config.yaml
+	echo "service-cidr: 192.168.85.0/24" >> config.yaml
+	echo "kubelet-arg:" >> config.yaml
+	echo "- --allowed-unsafe-sysctls="net.*"" >> config.yaml
+	echo "- --node-ip="$(NODE_IP)"" >> config.yaml
+	echo "pause-image: k8s.gcr.io/pause:3.3" >> config.yaml
+	echo "kube-proxy-arg:" >> config.yaml
+	echo "- --metrics-bind-address="0.0.0.0:10249"" >> config.yaml
+	echo "- --proxy-mode="ipvs"" >> config.yaml
+	echo "kube-apiserver-arg:" >> config.yaml
+	echo "- --service-node-port-range="2000-36767"" >> config.yaml
+	sudo mv config.yaml /etc/rancher/rke2/
+	curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_VERSION=$(RKE2_K8S_VERSION) sh -
+	sudo systemctl enable rke2-server.service
+	sudo systemctl start rke2-server.service
+	sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml wait nodes --for=condition=Ready --all --timeout=300s
+	sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml wait deployment -n kube-system --for=condition=available --all --timeout=300s
+	@$(eval STORAGE_CLASS := $(shell /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml get storageclass -o name))
+	@echo "STORAGE_CLASS: ${STORAGE_CLASS}"
+	if [ "$(STORAGE_CLASS)" == "" ]; then \
+		sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/$(LPP_VERSION)/deploy/local-path-storage.yaml --wait=true; \
+		sudo /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'; \
+	fi
+	curl -LO "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/linux/amd64/kubectl"
+	sudo chmod +x kubectl
+	sudo mv kubectl /usr/local/bin/
+	kubectl version --client
 	mkdir -p $(HOME)/.kube
-	sudo cp -f /etc/kubernetes/admin.conf $(HOME)/.kube/config
-	sudo chown $(shell id -u):$(shell id -g) $(HOME)/.kube/config
-	kubectl wait pod -n kube-system --for=condition=Ready --all --timeout=600s
-	kubectl get namespace $(RIAB_NAMESPACE) 2> /dev/null || kubectl create namespace $(RIAB_NAMESPACE)
+	sudo cp /etc/rancher/rke2/rke2.yaml $(HOME)/.kube/config
+	sudo chown -R $(shell id -u):$(shell id -g) $(HOME)/.kube
 	touch $@
 
 $(M)/helm-ready: | $(M)/k8s-ready
+	curl -fsSL -o ${GET_HELM} https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+	chmod 700 ${GET_HELM}
+	sudo DESIRED_VERSION=$(HELM_VERSION) ./${GET_HELM}
 	helm repo add incubator $(HELM_INCUBATOR_URL)
 	helm repo add cord $(HELM_OPENCORD_URL)
 	helm repo add sdran $(HELM_SDRAN_URL)

mk/preliminaries.mk

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # PHONY definitions
-PRELIMINARIES_PHONY			:= preliminaries
+PRELIMINARIES_PHONY			:= preliminaries $(M)/proxy-setting
 
 preliminaries: $(M) $(M)/system-check $(M)/setup
 
@@ -40,7 +40,31 @@ $(M)/system-check: | $(M) $(M)/repos
 	fi
 	touch $@
 
-$(M)/setup: | $(M)/system-check
+$(M)/setup: | $(M)/system-check $(M)/proxy-setting
 	sudo $(SCRIPTDIR)/cloudlab-disksetup.sh
 	sudo apt update; sudo apt install -y software-properties-common python3-pip jq httpie ipvsadm ethtool net-tools
+	systemctl list-units --full -all | grep "docker.service" || sudo apt install -y docker.io
+	sudo adduser $(USER) docker || true
 	touch $@
+
+ifeq ($(PROXY_ENABLED),true)
+$(M)/proxy-setting: | $(M)
+	echo "Defaults env_keep += \"HTTP_PROXY HTTPS_PROXY NO_PROXY http_proxy https_proxy no_proxy\"" | sudo EDITOR='tee -a' visudo -f /etc/sudoers.d/proxy
+	echo "HTTP_PROXY=$(HTTP_PROXY)" >> rke2-server
+	echo "HTTPS_PROXY=$(HTTPS_PROXY)" >> rke2-server
+	echo "NO_PROXY=$(NO_PROXY),.cluster.local,.svc,$(NODE_IP),192.168.84.0/24,192.168.85.0/24,$(RAN_SUBNET)" >> rke2-server
+	sudo mv rke2-server /etc/default/
+	echo "[Service]" >> http-proxy.conf
+	echo "Environment='HTTP_PROXY=$(HTTP_PROXY)'" >> http-proxy.conf
+	echo "Environment='HTTPS_PROXY=$(HTTPS_PROXY)'" >> http-proxy.conf
+	echo "Environment='NO_PROXY=$(NO_PROXY)'" >> http-proxy.conf
+	sudo mkdir -p /etc/systemd/system/docker.service.d
+	sudo mv http-proxy.conf /etc/systemd/system/docker.service.d
+	sudo systemctl daemon-reload
+	sudo systemctl restart docker
+	touch $(M)/proxy-setting
+else
+$(M)/proxy-setting: | $(M)
+	@echo -n ""
+	touch $(M)/proxy-setting
+endif

mk/reset-clean.mk

@@ -77,15 +77,8 @@ reset-test: reset-oai-test reset-5gc reset-ransim-test reset-prom-op-servicemoni
 
 clean: reset-test
 	helm repo remove sdran || true
-	@if [[ $(OS_VENDOR) =~ (Debian) ]]; then \
-		cp $(RESOURCEDIR)/kubespray-reset-defaults.yml $(BUILD)/kubespray/roles/reset/defaults/main.yml; \
-	fi
-	source "$(VENV)/bin/activate" && cd $(BUILD)/kubespray; \
-	ansible-playbook --extra-vars "reset_confirmation=yes" -b -i inventory/local/hosts.ini reset.yml || true
-	@if [ -d /usr/local/etc/emulab ]; then \
-		mount | grep /mnt/extra/kubelet/pods | cut -d" " -f3 | sudo xargs umount; \
-		sudo rm -rf /mnt/extra/kubelet; \
-	fi
+	sudo /usr/local/bin/rke2-uninstall.sh || true
+	sudo rm -rf /usr/local/bin/kubectl
 	rm -rf $(M)
 
 clean-all: clean