From ddc233f946f0a0a1b7df82378fd531c0903310f2 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Mon, 28 Apr 2025 13:36:43 +0200 Subject: [PATCH 01/32] Adapting deploy-netbox.sh to a vCluster deployment --- kind/deploy-netbox.sh | 98 ++++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 33 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index e2ef7423..76a4da81 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -3,14 +3,24 @@ set -e -u -o pipefail NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # default value -if [[ $# -ne 3 ]]; then - echo "Usage: $0 " +if [[ $# -lt 3 || $# -gt 4 ]]; then + echo "Usage: $0 [--vcluster]" exit 1 fi CLUSTER=$1 VERSION=$2 NAMESPACE=$3 +VCLUSTER_MODE=${4:-} + +# Choose kubectl and helm commands depending if we run on vCluster +if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then + KUBECTL="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- kubectl" + HELM="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- helm" +else + KUBECTL="kubectl" + HELM="helm" +fi # load remote images if [[ "${VERSION}" == "3.7.8" ]] ;then @@ -66,35 +76,57 @@ else exit 1 fi -for img in "${Remote_Images[@]}"; do - docker pull "$img" - kind load docker-image "$img" --name "${CLUSTER}" -done +if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then + echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." +else + echo "[Running in Kind mode] pulling and loading remote images into kind cluster..." + for img in "${Remote_Images[@]}"; do + docker pull "$img" + kind load docker-image "$img" --name "${CLUSTER}" + done +fi # build image for loading local data via NetBox API -cd ./kind/load-data-job && docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . && cd - - -# load local images -declare -a Local_Images=( \ -"netbox-load-local-data:1.0" \ -) -for img in "${Local_Images[@]}"; do - kind load docker-image "$img" --name "${CLUSTER}" -done - -# install helm charts -helm upgrade --install --namespace="${NAMESPACE}" postgres-operator \ -https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz - -kubectl apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" -kubectl wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db - -kubectl create configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts --from-file="$(dirname "$0")/load-data-job" -o yaml --dry-run=client | kubectl apply -f - -kubectl apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/load-data-job.yaml" -kubectl wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-demo-data-load-job -kubectl delete configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts +cd ./kind/load-data-job +docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . +cd - + +# Load local images into Kind only if not vCluster +if [[ "$VCLUSTER_MODE" != "--vcluster" ]]; then + echo "Loading local images into kind cluster..." + declare -a Local_Images=( \ + "netbox-load-local-data:1.0" \ + ) + for img in "${Local_Images[@]}"; do + kind load docker-image "$img" --name "${CLUSTER}" + done +else + echo "Skipping local image loading into Kind (vCluster mode)." +fi -helm upgrade --install --namespace="${NAMESPACE}" netbox \ +# Install Postgres Operator +${HELM} upgrade --install postgres-operator \ + --namespace="${NAMESPACE}" \ + --create-namespace \ + --set podPriorityClassName.create=false \ + --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ + --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ + https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz + +# Deploy the database +${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" +${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db + +# Load demo data +${KUBECTL} create configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts --from-file="$(dirname "$0")/load-data-job" -o yaml --dry-run=client | ${KUBECTL} apply -f - +${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/load-data-job.yaml" +${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-demo-data-load-job +${KUBECTL} delete configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts + +# Install NetBox +${HELM} upgrade --install netbox \ + --namespace="${NAMESPACE}" \ + --create-namespace \ --set postgresql.enabled="false" \ --set externalDatabase.host="netbox-db.${NAMESPACE}.svc.cluster.local" \ --set externalDatabase.existingSecretName="netbox.netbox-db.credentials.postgresql.acid.zalan.do" \ @@ -106,12 +138,12 @@ helm upgrade --install --namespace="${NAMESPACE}" netbox \ --set resources.limits.memory="2Gi" \ ${NETBOX_HELM_CHART} -kubectl rollout status --namespace="${NAMESPACE}" deployment netbox +${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox -# load local data -kubectl create job netbox-load-local-data --image=netbox-load-local-data:1.0 -kubectl wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data -docker rmi netbox-load-local-data:1.0 +# Load local data +${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found +${KUBECTL} create job netbox-load-local-data --namespace="${NAMESPACE}" --image=netbox-load-local-data:1.0 +${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data # clean up rm $(dirname "$0")/load-data-job/load-data.sh From 6a998ec26baf9ad02d1afe3df63656bfe329e523 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Mon, 28 Apr 2025 17:32:42 +0200 Subject: [PATCH 02/32] Debugging demo data --- kind/deploy-netbox.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 76a4da81..a10f9b58 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -78,6 +78,7 @@ fi if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." + sleep 15 else echo "[Running in Kind mode] pulling and loading remote images into kind cluster..." for img in "${Remote_Images[@]}"; do @@ -87,7 +88,7 @@ else fi # build image for loading local data via NetBox API -cd ./kind/load-data-job +cd "$(dirname "$0")/load-data-job" docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . cd - @@ -118,10 +119,26 @@ ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db # Load demo data -${KUBECTL} create configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts --from-file="$(dirname "$0")/load-data-job" -o yaml --dry-run=client | ${KUBECTL} apply -f - -${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/load-data-job.yaml" -${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-demo-data-load-job -${KUBECTL} delete configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts +if [[ -d "$(dirname "$0")/load-data-job" ]]; then + echo "load-data-job directory found, creating ConfigMap." + + echo "Generating ConfigMap YAML:" + ${KUBECTL} create configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts --from-file="/tmp/netbox-operator/kind/load-data-job" -o yaml --dry-run=client | ${KUBECTL} apply -f - + + echo "Applying ConfigMap to Kubernetes:" + ${KUBECTL} apply -f /tmp/netbox-configmap.yaml + + echo "Deploying load-data-job.yaml:" + ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/load-data-job.yaml" + + echo "Waiting for load-data-job to complete:" + ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-demo-data-load-job + + echo "Cleaning up ConfigMap:" + ${KUBECTL} delete configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts +else + echo " Skipping load-data-job because directory does not exist." +fi # Install NetBox ${HELM} upgrade --install netbox \ From 619d9aad611eb12eb54fd38d3930351992c9e730 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 29 Apr 2025 16:18:19 +0200 Subject: [PATCH 03/32] Adding steps to load demo-data into NetBox inside the vcluster --- kind/deploy-netbox.sh | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index a10f9b58..9740ecc5 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -118,27 +118,28 @@ ${HELM} upgrade --install postgres-operator \ ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db -# Load demo data -if [[ -d "$(dirname "$0")/load-data-job" ]]; then - echo "load-data-job directory found, creating ConfigMap." - - echo "Generating ConfigMap YAML:" - ${KUBECTL} create configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts --from-file="/tmp/netbox-operator/kind/load-data-job" -o yaml --dry-run=client | ${KUBECTL} apply -f - - - echo "Applying ConfigMap to Kubernetes:" - ${KUBECTL} apply -f /tmp/netbox-configmap.yaml - - echo "Deploying load-data-job.yaml:" - ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/load-data-job.yaml" - - echo "Waiting for load-data-job to complete:" - ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-demo-data-load-job - - echo "Cleaning up ConfigMap:" - ${KUBECTL} delete configmap --namespace="${NAMESPACE}" netbox-demo-data-load-job-scripts -else - echo " Skipping load-data-job because directory does not exist." -fi +echo "loading demo-data into NetBox inside the vcluster" + +# 1) Generate the ConfigMap locally, pipe into vcluster’s kubectl, +# targetting the $NAMESPACE +kubectl create configmap netbox-demo-data-load-job-scripts \ + --from-file="$(dirname "$0")/load-data-job" \ + --dry-run=client -o yaml \ +| vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" -f - + +# 2) Apply the Job YAML into the same namespace +vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" \ + -f "$(dirname "$0")/load-data-job.yaml" + +# 3) Wait for it in THAT namespace +vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl wait \ + -n "${NAMESPACE}" \ + --for=condition=complete \ + --timeout=600s job/netbox-demo-data-load-job + +# 4) Clean up the helper ConfigMap +vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete configmap \ + netbox-demo-data-load-job-scripts -n "${NAMESPACE}" # Install NetBox ${HELM} upgrade --install netbox \ From dd0d2588cba45f3ad25afe1ee13699d32e5dec20 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 29 Apr 2025 16:21:06 +0200 Subject: [PATCH 04/32] Adding description to script --- kind/deploy-netbox.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 9740ecc5..02e77183 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -1,6 +1,10 @@ #!/bin/bash set -e -u -o pipefail +# Deploy NetBox (with its PostgreSQL operator and demo data) into either: +# • a local kind cluster (preloading images), or +# • a virtual cluster via vcluster (skipping image loads). + NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # default value if [[ $# -lt 3 || $# -gt 4 ]]; then From f59c19a03068645d2ec0b95f4c1e8f63e0cfe44d Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 29 Apr 2025 16:28:00 +0200 Subject: [PATCH 05/32] Adding kind path --- kind/deploy-netbox.sh | 53 +++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 02e77183..4bd3468c 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -122,28 +122,37 @@ ${HELM} upgrade --install postgres-operator \ ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db -echo "loading demo-data into NetBox inside the vcluster" - -# 1) Generate the ConfigMap locally, pipe into vcluster’s kubectl, -# targetting the $NAMESPACE -kubectl create configmap netbox-demo-data-load-job-scripts \ - --from-file="$(dirname "$0")/load-data-job" \ - --dry-run=client -o yaml \ -| vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" -f - - -# 2) Apply the Job YAML into the same namespace -vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" \ - -f "$(dirname "$0")/load-data-job.yaml" - -# 3) Wait for it in THAT namespace -vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl wait \ - -n "${NAMESPACE}" \ - --for=condition=complete \ - --timeout=600s job/netbox-demo-data-load-job - -# 4) Clean up the helper ConfigMap -vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete configmap \ - netbox-demo-data-load-job-scripts -n "${NAMESPACE}" +echo "loading demo-data into NetBox…" + +if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then + # — vCluster — + echo " → inside the vcluster" + kubectl create configmap netbox-demo-data-load-job-scripts \ + --from-file="$(dirname "$0")/load-data-job" \ + --dry-run=client -o yaml \ + | vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -f - + + vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -f "$(dirname "$0")/load-data-job.yaml" + + vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl wait \ + --for=condition=complete --timeout=600s job/netbox-demo-data-load-job + + vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete configmap netbox-demo-data-load-job-scripts + +else + # — Kind — + echo " → on the Kind cluster" + kubectl create configmap netbox-demo-data-load-job-scripts \ + --from-file="$(dirname "$0")/load-data-job" \ + --dry-run=client -o yaml \ + | kubectl apply -f - + + kubectl apply -f "$(dirname "$0")/load-data-job.yaml" + + kubectl wait --for=condition=complete --timeout=600s job/netbox-demo-data-load-job + + kubectl delete configmap netbox-demo-data-load-job-scripts +fi # Install NetBox ${HELM} upgrade --install netbox \ From 94af0750b30b294a1d4ddf8a92e8b4d0e32937f6 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 29 Apr 2025 16:33:22 +0200 Subject: [PATCH 06/32] Adding namespaces --- kind/deploy-netbox.sh | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 4bd3468c..42a75b43 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -121,7 +121,6 @@ ${HELM} upgrade --install postgres-operator \ # Deploy the database ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db - echo "loading demo-data into NetBox…" if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then @@ -130,28 +129,38 @@ if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then kubectl create configmap netbox-demo-data-load-job-scripts \ --from-file="$(dirname "$0")/load-data-job" \ --dry-run=client -o yaml \ - | vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -f - + | vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" -f - - vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -f "$(dirname "$0")/load-data-job.yaml" + vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" \ + -f "$(dirname "$0")/load-data-job.yaml" vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl wait \ - --for=condition=complete --timeout=600s job/netbox-demo-data-load-job + -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job - vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete configmap netbox-demo-data-load-job-scripts + vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete \ + -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts else - # — Kind — - echo " → on the Kind cluster" + # — Kind — + echo " → on the Kind cluster (${NAMESPACE})" kubectl create configmap netbox-demo-data-load-job-scripts \ --from-file="$(dirname "$0")/load-data-job" \ + --namespace="${NAMESPACE}" \ --dry-run=client -o yaml \ | kubectl apply -f - - kubectl apply -f "$(dirname "$0")/load-data-job.yaml" + kubectl apply \ + --namespace="${NAMESPACE}" \ + -f "$(dirname "$0")/load-data-job.yaml" - kubectl wait --for=condition=complete --timeout=600s job/netbox-demo-data-load-job + kubectl wait \ + --namespace="${NAMESPACE}" \ + --for=condition=complete \ + --timeout=600s job/netbox-demo-data-load-job - kubectl delete configmap netbox-demo-data-load-job-scripts + kubectl delete \ + --namespace="${NAMESPACE}" \ + configmap/netbox-demo-data-load-job-scripts fi # Install NetBox From b0d8f09019c5d05e4e822910c45db6be10d48088 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 29 Apr 2025 16:34:06 +0200 Subject: [PATCH 07/32] Removing sleep used for testing purposes --- kind/deploy-netbox.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 42a75b43..7b3415f3 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -82,7 +82,6 @@ fi if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." - sleep 15 else echo "[Running in Kind mode] pulling and loading remote images into kind cluster..." for img in "${Remote_Images[@]}"; do From 3ba83af37160074b487fb522b049ba8b885e7f43 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Wed, 30 Apr 2025 08:45:18 +0200 Subject: [PATCH 08/32] Fixing script comment --- kind/deploy-netbox.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 7b3415f3..9f2d61af 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -2,8 +2,8 @@ set -e -u -o pipefail # Deploy NetBox (with its PostgreSQL operator and demo data) into either: -# • a local kind cluster (preloading images), or -# • a virtual cluster via vcluster (skipping image loads). +# • a local kind cluster (preloading images) +# • a virtual cluster using vcluster: https://github.com/loft-sh/vcluster ( used for testing pipeline, loading of images not needed ) NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # default value @@ -82,6 +82,7 @@ fi if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." + sleep 15 else echo "[Running in Kind mode] pulling and loading remote images into kind cluster..." for img in "${Remote_Images[@]}"; do From b1233753bcb389296b345034a728c2db345494c5 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Wed, 30 Apr 2025 08:49:23 +0200 Subject: [PATCH 09/32] Adding variable IS_VCLUSTER to easily check in all the if statements --- kind/deploy-netbox.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 9f2d61af..351a3a1e 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -15,10 +15,15 @@ fi CLUSTER=$1 VERSION=$2 NAMESPACE=$3 -VCLUSTER_MODE=${4:-} + +# Treat the optional fourth argument "--vcluster" as a boolean flag +IS_VCLUSTER=false +if [[ "${4:-}" == "--vcluster" ]]; then + IS_VCLUSTER=true +fi # Choose kubectl and helm commands depending if we run on vCluster -if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then +if $IS_VCLUSTER; then KUBECTL="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- kubectl" HELM="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- helm" else @@ -80,7 +85,7 @@ else exit 1 fi -if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then +if $IS_VCLUSTER; then echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." sleep 15 else @@ -97,7 +102,7 @@ docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f cd - # Load local images into Kind only if not vCluster -if [[ "$VCLUSTER_MODE" != "--vcluster" ]]; then +if ! $IS_VCLUSTER; then echo "Loading local images into kind cluster..." declare -a Local_Images=( \ "netbox-load-local-data:1.0" \ @@ -121,9 +126,10 @@ ${HELM} upgrade --install postgres-operator \ # Deploy the database ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db + echo "loading demo-data into NetBox…" -if [[ "$VCLUSTER_MODE" == "--vcluster" ]]; then +if $IS_VCLUSTER; then # — vCluster — echo " → inside the vcluster" kubectl create configmap netbox-demo-data-load-job-scripts \ From c2fdea6995af5d2504c494d635db3734c4a16eb4 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Wed, 30 Apr 2025 14:45:08 +0200 Subject: [PATCH 10/32] Refactoring part of loading demo-data into NetBox --- kind/deploy-netbox.sh | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 351a3a1e..f4e3737b 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -128,46 +128,37 @@ ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db echo "loading demo-data into NetBox…" - +# We use plain `kubectl create … --dry-run=client -o yaml` here to generate +# the ConfigMap manifest locally (no cluster connection needed), then pipe +# that YAML into `${KUBECTL} apply` so it’s applied against the selected +# target (Kind or vCluster) via our `${KUBECTL}` wrapper. if $IS_VCLUSTER; then # — vCluster — echo " → inside the vcluster" kubectl create configmap netbox-demo-data-load-job-scripts \ --from-file="$(dirname "$0")/load-data-job" \ --dry-run=client -o yaml \ - | vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" -f - - - vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl apply -n "${NAMESPACE}" \ - -f "$(dirname "$0")/load-data-job.yaml" - - vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl wait \ - -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job - - vcluster connect "${CLUSTER}" -n "${NAMESPACE}" -- kubectl delete \ - -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts + | ${KUBECTL} apply -n "${NAMESPACE}" -f - else # — Kind — echo " → on the Kind cluster (${NAMESPACE})" - kubectl create configmap netbox-demo-data-load-job-scripts \ + ${KUBECTL} create configmap netbox-demo-data-load-job-scripts \ --from-file="$(dirname "$0")/load-data-job" \ --namespace="${NAMESPACE}" \ --dry-run=client -o yaml \ - | kubectl apply -f - + | ${KUBECTL} apply -f - +fi - kubectl apply \ - --namespace="${NAMESPACE}" \ +${KUBECTL} apply -n "${NAMESPACE}" \ -f "$(dirname "$0")/load-data-job.yaml" - kubectl wait \ - --namespace="${NAMESPACE}" \ - --for=condition=complete \ - --timeout=600s job/netbox-demo-data-load-job +${KUBECTL} wait \ + -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job + +${KUBECTL} delete \ + -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts - kubectl delete \ - --namespace="${NAMESPACE}" \ - configmap/netbox-demo-data-load-job-scripts -fi # Install NetBox ${HELM} upgrade --install netbox \ From 75a35c7df19f75cc26650cbcee30b10b81b79163 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Fri, 2 May 2025 14:16:00 +0200 Subject: [PATCH 11/32] Parametrizing deploy-netbox.sh to allower using a customize artifactory instead of public repositories for binaries and helm charts --- kind/deploy-netbox.sh | 27 ++++++++++++++++++++------- kind/job/kustomization.yaml | 9 +++++++++ kind/{ => job}/load-data-job.yaml | 0 kind/load-data-job/dockerfile.orig | 27 ++++++++++++++++++++++++--- kind/load-data-job/load-data.orig.sh | 11 ++++++++++- 5 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 kind/job/kustomization.yaml rename kind/{ => job}/load-data-job.yaml (100%) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index f4e3737b..c46485dc 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -5,7 +5,8 @@ set -e -u -o pipefail # • a local kind cluster (preloading images) # • a virtual cluster using vcluster: https://github.com/loft-sh/vcluster ( used for testing pipeline, loading of images not needed ) -NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # default value +# Allow override via environment variable, otherwise fallback to default +NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz}" if [[ $# -lt 3 || $# -gt 4 ]]; then echo "Usage: $0 [--vcluster]" @@ -42,7 +43,8 @@ if [[ "${VERSION}" == "3.7.8" ]] ;then "ghcr.io/zalando/postgres-operator:v1.12.2" \ "ghcr.io/zalando/spilo-16:3.2-p3" \ ) - NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz" + # Allow override via environment variable, otherwise fallback to default + NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz}" # patch load-data.sh sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh @@ -59,7 +61,8 @@ elif [[ "${VERSION}" == "4.0.11" ]] ;then "ghcr.io/zalando/postgres-operator:v1.12.2" \ "ghcr.io/zalando/spilo-16:3.2-p3" \ ) - NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz" + # Allow override via environment variable, otherwise fallback to default + NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz}" # patch load-data.sh sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh @@ -98,7 +101,12 @@ fi # build image for loading local data via NetBox API cd "$(dirname "$0")/load-data-job" -docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . +docker build -t netbox-load-local-data:1.0 \ + --load --no-cache --progress=plain \ + --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE:-python:3.12}" \ + --build-arg ARTIFACTORY_PYPI_URL="${ARTIFACTORY_PYPI_URL:-}" \ + --build-arg ARTIFACTORY_TRUSTED_HOST="${ARTIFACTORY_TRUSTED_HOST:-}" \ + -f ./dockerfile . cd - # Load local images into Kind only if not vCluster @@ -115,13 +123,15 @@ else fi # Install Postgres Operator +# Allow override via environment variable, otherwise fallback to default +POSTGRESS_OPERATOR_HELM_CHART="${POSTGRESS_OPERATOR_HELM_CHART:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz}" ${HELM} upgrade --install postgres-operator \ --namespace="${NAMESPACE}" \ --create-namespace \ --set podPriorityClassName.create=false \ --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ - https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz + "${POSTGRESS_OPERATOR_HELM_CHART}" # Deploy the database ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" @@ -150,8 +160,11 @@ else | ${KUBECTL} apply -f - fi -${KUBECTL} apply -n "${NAMESPACE}" \ - -f "$(dirname "$0")/load-data-job.yaml" +JOB_DIR="$(dirname "$0")/job" + +cd "$JOB_DIR" +kustomize edit set image ghcr.io/zalando/spilo-16="${SPILO_IMAGE:-ghcr.io/zalando/spilo-16:3.2-p3}" +kustomize build . | ${KUBECTL} apply -n "${NAMESPACE}" -f - ${KUBECTL} wait \ -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml new file mode 100644 index 00000000..33d8017c --- /dev/null +++ b/kind/job/kustomization.yaml @@ -0,0 +1,9 @@ +resources: +- load-data-job.yaml + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: ghcr.io/zalando/spilo-16 + newName: mobile-docker-virtual.artifactory.swisscom.com/zalando/spilo-16 + newTag: 3.2-p3 \ No newline at end of file diff --git a/kind/load-data-job.yaml b/kind/job/load-data-job.yaml similarity index 100% rename from kind/load-data-job.yaml rename to kind/job/load-data-job.yaml diff --git a/kind/load-data-job/dockerfile.orig b/kind/load-data-job/dockerfile.orig index 447d0eb9..c1853744 100644 --- a/kind/load-data-job/dockerfile.orig +++ b/kind/load-data-job/dockerfile.orig @@ -1,4 +1,25 @@ -FROM python:3.12 +ARG PYTHON_BASE_IMAGE=python:3.12 +FROM ${PYTHON_BASE_IMAGE} + +# Optional Artifactory pip index +ARG ARTIFACTORY_PYPI_URL="" +ARG ARTIFACTORY_TRUSTED_HOST="" + +# Set envs so they’re usable in RUN commands +ENV ARTIFACTORY_PYPI_URL=${ARTIFACTORY_PYPI_URL} +ENV ARTIFACTORY_TRUSTED_HOST=${ARTIFACTORY_TRUSTED_HOST} + ADD main.py . -RUN pip install -Iv pynetbox==7.4.1 -CMD ["python", "./main.py"] + +# Conditional pip install using Artifactory if set +RUN if [ -n "$ARTIFACTORY_PYPI_URL" ]; then \ + echo "Using Artifactory pip index: $ARTIFACTORY_PYPI_URL"; \ + pip install -Iv pynetbox==7.4.1 \ + --index-url="$ARTIFACTORY_PYPI_URL" \ + --trusted-host="$ARTIFACTORY_TRUSTED_HOST"; \ + else \ + echo "Using public PyPI"; \ + pip install -Iv pynetbox==7.4.1; \ + fi + +CMD ["python", "./main.py"] diff --git a/kind/load-data-job/load-data.orig.sh b/kind/load-data-job/load-data.orig.sh index a8cfe3bf..670c8d80 100755 --- a/kind/load-data-job/load-data.orig.sh +++ b/kind/load-data-job/load-data.orig.sh @@ -1,8 +1,17 @@ #!/bin/sh set -o errexit +# Allow override of demo SQL file URL +NETBOX_SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" + TMP_SQL_FILE=$(mktemp /tmp/netbox-data-dump.XXXXXXX.sql) || exit 1 -curl -k https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql > "${TMP_SQL_FILE}" + +# Download the SQL dump +curl -k "${NETBOX_SQL_DUMP_URL}" > "${TMP_SQL_FILE}" + +# Load it into the database psql "user=netbox host=netbox-db.${NAMESPACE}.svc.cluster.local" netbox -q -f "${TMP_SQL_FILE}" rm "${TMP_SQL_FILE}" + +# Load additional local data psql "user=netbox host=netbox-db.${NAMESPACE}.svc.cluster.local" netbox -q -f /load-data-job/local-data-setup.sql From c73320c65574ec901ead43f21bc5bde16d2078cc Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Fri, 2 May 2025 16:03:33 +0200 Subject: [PATCH 12/32] Create patch to inject the NETBOX_SQL_DUMP_URL inside the Docker container --- kind/deploy-netbox.sh | 24 ++++++++++++++++++++++++ kind/job/kustomization.yaml | 4 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index c46485dc..cfa951d1 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -164,6 +164,30 @@ JOB_DIR="$(dirname "$0")/job" cd "$JOB_DIR" kustomize edit set image ghcr.io/zalando/spilo-16="${SPILO_IMAGE:-ghcr.io/zalando/spilo-16:3.2-p3}" + +# Create a patch file to inject NETBOX_SQL_DUMP_URL (from env or default) +SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" + +# Create patch +cat > sql-env-patch.yaml < Date: Fri, 2 May 2025 16:08:05 +0200 Subject: [PATCH 13/32] Removing unnecessary code & comments --- kind/deploy-netbox.sh | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index f4e3737b..ee126e2a 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -87,7 +87,6 @@ fi if $IS_VCLUSTER; then echo "[Running in vCluster mode] skipping docker pull and kind load for remote images." - sleep 15 else echo "[Running in Kind mode] pulling and loading remote images into kind cluster..." for img in "${Remote_Images[@]}"; do @@ -101,7 +100,6 @@ cd "$(dirname "$0")/load-data-job" docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . cd - -# Load local images into Kind only if not vCluster if ! $IS_VCLUSTER; then echo "Loading local images into kind cluster..." declare -a Local_Images=( \ @@ -132,23 +130,10 @@ echo "loading demo-data into NetBox…" # the ConfigMap manifest locally (no cluster connection needed), then pipe # that YAML into `${KUBECTL} apply` so it’s applied against the selected # target (Kind or vCluster) via our `${KUBECTL}` wrapper. -if $IS_VCLUSTER; then - # — vCluster — - echo " → inside the vcluster" - kubectl create configmap netbox-demo-data-load-job-scripts \ - --from-file="$(dirname "$0")/load-data-job" \ - --dry-run=client -o yaml \ - | ${KUBECTL} apply -n "${NAMESPACE}" -f - - -else - # — Kind — - echo " → on the Kind cluster (${NAMESPACE})" - ${KUBECTL} create configmap netbox-demo-data-load-job-scripts \ - --from-file="$(dirname "$0")/load-data-job" \ - --namespace="${NAMESPACE}" \ - --dry-run=client -o yaml \ - | ${KUBECTL} apply -f - -fi +kubectl create configmap netbox-demo-data-load-job-scripts \ + --from-file="$(dirname "$0")/load-data-job" \ + --dry-run=client -o yaml \ +| ${KUBECTL} apply -n "${NAMESPACE}" -f - ${KUBECTL} apply -n "${NAMESPACE}" \ -f "$(dirname "$0")/load-data-job.yaml" From 3fd18b202173b5f35fbe3ed750df35859986e72f Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Mon, 5 May 2025 10:41:31 +0200 Subject: [PATCH 14/32] Removing from kustomization.yaml the new changes applied in the script --- kind/job/kustomization.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml index 2b1c70d7..34c4c561 100644 --- a/kind/job/kustomization.yaml +++ b/kind/job/kustomization.yaml @@ -5,7 +5,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: ghcr.io/zalando/spilo-16 - newName: mobile-docker-virtual.artifactory.swisscom.com/zalando/spilo-16 - newTag: 3.2-p3 patches: - path: sql-env-patch.yaml From a70a5de5f36aa7516a97d95926e42773d2c36392 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Mon, 5 May 2025 10:41:56 +0200 Subject: [PATCH 15/32] Adding comment to script and renaming SQL_DUMP_URL to NETBOX_SQL_DUMP_URL --- kind/deploy-netbox.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index cfa951d1..d2466bf2 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -160,13 +160,13 @@ else | ${KUBECTL} apply -f - fi +# Set the image of the kustomization.yaml to the one specified (from env or default) JOB_DIR="$(dirname "$0")/job" - cd "$JOB_DIR" kustomize edit set image ghcr.io/zalando/spilo-16="${SPILO_IMAGE:-ghcr.io/zalando/spilo-16:3.2-p3}" # Create a patch file to inject NETBOX_SQL_DUMP_URL (from env or default) -SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" +NETBOX_SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" # Create patch cat > sql-env-patch.yaml < Date: Mon, 5 May 2025 11:34:08 +0200 Subject: [PATCH 16/32] Adding comment to clarify that vCluster will be deployed to the NAMESPACE passed as an argument --- kind/deploy-netbox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index d2466bf2..88c899ba 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -15,6 +15,7 @@ fi CLUSTER=$1 VERSION=$2 +# The specified namespace will be used for both the NetBox deployment and the vCluster creation NAMESPACE=$3 # Treat the optional fourth argument "--vcluster" as a boolean flag From 67e654ea2e8e581af56b75d77c2e41fbba4f6d7c Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Mon, 5 May 2025 11:34:32 +0200 Subject: [PATCH 17/32] Adding comment to clarify that vCluster will be deployed to the NAMESPACE passed as an argument --- kind/deploy-netbox.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index ee126e2a..417718d5 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -14,6 +14,7 @@ fi CLUSTER=$1 VERSION=$2 +# The specified namespace will be used for both the NetBox deployment and the vCluster creation NAMESPACE=$3 # Treat the optional fourth argument "--vcluster" as a boolean flag From 48619c787f266aaac862ee9f0fb569e6ffb3b6d1 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 10:24:43 +0200 Subject: [PATCH 18/32] Adding image.registry to the values of the two helm charts to be installed --- kind/deploy-netbox.sh | 10 +++++----- kind/job/kustomization.yaml | 2 ++ 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 1c40ebed..69e0e855 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -123,14 +123,14 @@ fi # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default -POSTGRESS_OPERATOR_HELM_CHART="${POSTGRESS_OPERATOR_HELM_CHART:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz}" -${HELM} upgrade --install postgres-operator \ +POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_CHART:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz}" +${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --namespace="${NAMESPACE}" \ --create-namespace \ --set podPriorityClassName.create=false \ --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ - "${POSTGRESS_OPERATOR_HELM_CHART}" + --set image.registry="$IMAGE_REGISTRY" # Deploy the database ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" @@ -185,7 +185,7 @@ ${KUBECTL} delete \ # Install NetBox -${HELM} upgrade --install netbox \ +${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ --namespace="${NAMESPACE}" \ --create-namespace \ --set postgresql.enabled="false" \ @@ -197,7 +197,7 @@ ${HELM} upgrade --install netbox \ --set resources.requests.memory="512Mi" \ --set resources.limits.cpu="2000m" \ --set resources.limits.memory="2Gi" \ - ${NETBOX_HELM_CHART} + --set image.registry="$IMAGE_REGISTRY" ${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml index 34c4c561..2b1c70d7 100644 --- a/kind/job/kustomization.yaml +++ b/kind/job/kustomization.yaml @@ -5,5 +5,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: ghcr.io/zalando/spilo-16 + newName: mobile-docker-virtual.artifactory.swisscom.com/zalando/spilo-16 + newTag: 3.2-p3 patches: - path: sql-env-patch.yaml From 84ce6c2816964bf7890135372fa5f5a256399274 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 10:25:33 +0200 Subject: [PATCH 19/32] Removing newImage & newTag from kustomization.yaml --- kind/job/kustomization.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml index 2b1c70d7..34c4c561 100644 --- a/kind/job/kustomization.yaml +++ b/kind/job/kustomization.yaml @@ -5,7 +5,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: ghcr.io/zalando/spilo-16 - newName: mobile-docker-virtual.artifactory.swisscom.com/zalando/spilo-16 - newTag: 3.2-p3 patches: - path: sql-env-patch.yaml From 63be4c540649832e4d4ad9c7b58f93a46210b2d7 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 10:43:32 +0200 Subject: [PATCH 20/32] Simplifying use of env variables --- kind/deploy-netbox.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 69e0e855..274476eb 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -101,9 +101,17 @@ fi # build image for loading local data via NetBox API cd "$(dirname "$0")/load-data-job" +# Append image registry prefix only if defined +PYTHON_IMAGE_NAME="python:3.12" +if [ -n "$IMAGE_REGISTRY" ]; then + PYTHON_BASE_IMAGE="${IMAGE_REGISTRY}/${PYTHON_IMAGE_NAME}" +else + PYTHON_BASE_IMAGE="$PYTHON_IMAGE_NAME" +fi + docker build -t netbox-load-local-data:1.0 \ --load --no-cache --progress=plain \ - --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE:-python:3.12}" \ + --build-arg PYTHON_BASE_IMAGE="$PYTHON_BASE_IMAGE" \ --build-arg ARTIFACTORY_PYPI_URL="${ARTIFACTORY_PYPI_URL:-}" \ --build-arg ARTIFACTORY_TRUSTED_HOST="${ARTIFACTORY_TRUSTED_HOST:-}" \ -f ./dockerfile . @@ -147,9 +155,12 @@ kubectl create configmap netbox-demo-data-load-job-scripts \ | ${KUBECTL} apply -n "${NAMESPACE}" -f - # Set the image of the kustomization.yaml to the one specified (from env or default) +SPILO_IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io}" +SPILO_IMAGE="${SPILO_IMAGE_REGISTRY}/zalando/spilo-16:3.2-p3" + JOB_DIR="$(dirname "$0")/job" cd "$JOB_DIR" -kustomize edit set image ghcr.io/zalando/spilo-16="${SPILO_IMAGE:-ghcr.io/zalando/spilo-16:3.2-p3}" +kustomize edit set image ghcr.io/zalando/spilo-16="$SPILO_IMAGE" # Create a patch file to inject NETBOX_SQL_DUMP_URL (from env or default) NETBOX_SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" From 9e96f6bdc1973bc76a96fdfbef71db0c34c80498 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 14:14:37 +0200 Subject: [PATCH 21/32] Assign IMAGE_REGISTRY to the helm upgrade from the env variable if this is set, otherwise, don't pass the flag and use the default one from the chart --- kind/deploy-netbox.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 274476eb..6b0cd620 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e -u -o pipefail +set -e -o pipefail # Deploy NetBox (with its PostgreSQL operator and demo data) into either: # • a local kind cluster (preloading images) @@ -129,6 +129,15 @@ else echo "Skipping local image loading into Kind (vCluster mode)." fi +# Assign IMAGE_REGISTRY from env if set, else empty +POSTGRES_IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" + +# Build optional set flag if registry is not defined +REGISTRY_ARG="" +if [ -n "$POSTGRES_IMAGE_REGISTRY" ]; then + REGISTRY_ARG="--set image.registry=$POSTGRES_IMAGE_REGISTRY" +fi + # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_CHART:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz}" @@ -138,7 +147,7 @@ ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --set podPriorityClassName.create=false \ --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ - --set image.registry="$IMAGE_REGISTRY" + $REGISTRY_ARG # Deploy the database ${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" @@ -194,6 +203,14 @@ ${KUBECTL} wait \ ${KUBECTL} delete \ -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts +# Assign IMAGE_REGISTRY from env if set, else empty +NETBOX_IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" + +# Build optional set flag if registry is not defined +REGISTRY_ARG="" +if [ -n "$NETBOX_IMAGE_REGISTRY" ]; then + REGISTRY_ARG="--set image.registry=$NETBOX_IMAGE_REGISTRY" +fi # Install NetBox ${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ @@ -208,7 +225,8 @@ ${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ --set resources.requests.memory="512Mi" \ --set resources.limits.cpu="2000m" \ --set resources.limits.memory="2Gi" \ - --set image.registry="$IMAGE_REGISTRY" + --set image.registry="$IMAGE_REGISTRY" \ + $REGISTRY_ARG ${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox From 648cea83cb59d79855efab92275999d4d9397cbb Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 14:24:26 +0200 Subject: [PATCH 22/32] Fix: Assign IMAGE_REGISTRY to the helm upgrade from the env variable if this is set, otherwise, don't pass the flag and use the default one from the chart --- kind/deploy-netbox.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 6b0cd620..ae6c334c 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -225,7 +225,6 @@ ${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ --set resources.requests.memory="512Mi" \ --set resources.limits.cpu="2000m" \ --set resources.limits.memory="2Gi" \ - --set image.registry="$IMAGE_REGISTRY" \ $REGISTRY_ARG ${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox From 53a6ec49d461d4041bdb502aca3b3610c63d24d2 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 6 May 2025 15:11:05 +0200 Subject: [PATCH 23/32] Setting a SCRIPT_DIR that it's used all over the script --- kind/deploy-netbox.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index ae6c334c..0e3455a1 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -5,6 +5,8 @@ set -e -o pipefail # • a local kind cluster (preloading images) # • a virtual cluster using vcluster: https://github.com/loft-sh/vcluster ( used for testing pipeline, loading of images not needed ) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + # Allow override via environment variable, otherwise fallback to default NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz}" @@ -48,10 +50,10 @@ if [[ "${VERSION}" == "3.7.8" ]] ;then NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz}" # patch load-data.sh - sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh # patch dockerfile (See README at https://github.com/netbox-community/pynetbox for the supported version matrix) - sed 's/RUN pip install -Iv pynetbox==7.4.1/RUN pip install -Iv pynetbox==7.3.4/g' $(dirname "$0")/load-data-job/dockerfile.orig > $(dirname "$0")/load-data-job/dockerfile + sed 's/RUN pip install -Iv pynetbox==7.4.1/RUN pip install -Iv pynetbox==7.3.4/g' $SCRIPT_DIR/load-data-job/dockerfile.orig > $SCRIPT_DIR/load-data-job/dockerfile elif [[ "${VERSION}" == "4.0.11" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -66,9 +68,9 @@ elif [[ "${VERSION}" == "4.0.11" ]] ;then NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz}" # patch load-data.sh - sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh - cp $(dirname "$0")/load-data-job/dockerfile.orig $(dirname "$0")/load-data-job/dockerfile + cp $SCRIPT_DIR/load-data-job/dockerfile.orig $SCRIPT_DIR/load-data-job/dockerfile elif [[ "${VERSION}" == "4.1.8" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -81,9 +83,9 @@ elif [[ "${VERSION}" == "4.1.8" ]] ;then ) # create load-data.sh - cp $(dirname "$0")/load-data-job/load-data.orig.sh $(dirname "$0")/load-data-job/load-data.sh + cp $SCRIPT_DIR/load-data-job/load-data.orig.sh $SCRIPT_DIR/load-data-job/load-data.sh - cp $(dirname "$0")/load-data-job/dockerfile.orig $(dirname "$0")/load-data-job/dockerfile + cp $SCRIPT_DIR/load-data-job/dockerfile.orig $SCRIPT_DIR/load-data-job/dockerfile else echo "Unknown version ${VERSION}" exit 1 @@ -100,7 +102,7 @@ else fi # build image for loading local data via NetBox API -cd "$(dirname "$0")/load-data-job" +cd "$SCRIPT_DIR/load-data-job" # Append image registry prefix only if defined PYTHON_IMAGE_NAME="python:3.12" if [ -n "$IMAGE_REGISTRY" ]; then @@ -150,7 +152,7 @@ ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ $REGISTRY_ARG # Deploy the database -${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" +${KUBECTL} apply --namespace="${NAMESPACE}" -f "$SCRIPT_DIR/netbox-db.yaml" ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db echo "loading demo-data into NetBox…" @@ -159,7 +161,7 @@ echo "loading demo-data into NetBox…" # that YAML into `${KUBECTL} apply` so it’s applied against the selected # target (Kind or vCluster) via our `${KUBECTL}` wrapper. kubectl create configmap netbox-demo-data-load-job-scripts \ - --from-file="$(dirname "$0")/load-data-job" \ + --from-file="$SCRIPT_DIR/load-data-job" \ --dry-run=client -o yaml \ | ${KUBECTL} apply -n "${NAMESPACE}" -f - @@ -167,7 +169,7 @@ kubectl create configmap netbox-demo-data-load-job-scripts \ SPILO_IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io}" SPILO_IMAGE="${SPILO_IMAGE_REGISTRY}/zalando/spilo-16:3.2-p3" -JOB_DIR="$(dirname "$0")/job" +JOB_DIR="$SCRIPT_DIR/job" cd "$JOB_DIR" kustomize edit set image ghcr.io/zalando/spilo-16="$SPILO_IMAGE" @@ -235,5 +237,5 @@ ${KUBECTL} create job netbox-load-local-data --namespace="${NAMESPACE}" --image= ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data # clean up -rm $(dirname "$0")/load-data-job/load-data.sh -rm $(dirname "$0")/load-data-job/dockerfile +rm $SCRIPT_DIR/load-data-job/load-data.sh +rm $SCRIPT_DIR/load-data-job/dockerfile From 41d25e59f5c35f8230efb898e9c334d549a9b9ec Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Wed, 7 May 2025 16:21:55 +0200 Subject: [PATCH 24/32] Using HELM_REPOS instead of HELM_CHARTS for env substitution --- kind/deploy-netbox.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 0e3455a1..c6db6704 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -8,7 +8,7 @@ set -e -o pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" # Allow override via environment variable, otherwise fallback to default -NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz}" +NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" if [[ $# -lt 3 || $# -gt 4 ]]; then echo "Usage: $0 [--vcluster]" @@ -47,7 +47,7 @@ if [[ "${VERSION}" == "3.7.8" ]] ;then "ghcr.io/zalando/spilo-16:3.2-p3" \ ) # Allow override via environment variable, otherwise fallback to default - NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz}" + NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # patch load-data.sh sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh @@ -65,7 +65,7 @@ elif [[ "${VERSION}" == "4.0.11" ]] ;then "ghcr.io/zalando/spilo-16:3.2-p3" \ ) # Allow override via environment variable, otherwise fallback to default - NETBOX_HELM_CHART="${NETBOX_HELM_CHART:-https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz}" + NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # patch load-data.sh sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh @@ -142,7 +142,7 @@ fi # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default -POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_CHART:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz}" +POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_REPO:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator}/postgres-operator-1.12.2.tgz" ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --namespace="${NAMESPACE}" \ --create-namespace \ @@ -211,7 +211,7 @@ NETBOX_IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" # Build optional set flag if registry is not defined REGISTRY_ARG="" if [ -n "$NETBOX_IMAGE_REGISTRY" ]; then - REGISTRY_ARG="--set image.registry=$NETBOX_IMAGE_REGISTRY" + REGISTRY_ARG="--set global.imageRegistry=$NETBOX_IMAGE_REGISTRY --set global.security.allowInsecureImages=true" fi # Install NetBox From 27a30c145b828ac3954310061dc8c27cb4bc61a9 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Thu, 8 May 2025 09:47:27 +0200 Subject: [PATCH 25/32] Adding a patch to the kind: postgresql spec.dockerImage --- kind/deploy-netbox.sh | 6 ++++-- kind/netbox-db/kustomization.yaml | 8 ++++++++ kind/netbox-db/netbox-db-patch.tmpl.yaml | 8 ++++++++ kind/netbox-db/netbox-db.yaml | 21 +++++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 kind/netbox-db/kustomization.yaml create mode 100644 kind/netbox-db/netbox-db-patch.tmpl.yaml create mode 100644 kind/netbox-db/netbox-db.yaml diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index c6db6704..aa058674 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -152,8 +152,10 @@ ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ $REGISTRY_ARG # Deploy the database -${KUBECTL} apply --namespace="${NAMESPACE}" -f "$SCRIPT_DIR/netbox-db.yaml" -${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db +export SPILO_IMAGE="${IMAGE_REGISTRY:-ghcr.io}/zalando/spilo-16:3.2-p3" +echo "spilo image is $SPILO_IMAGE" +envsubst < "$SCRIPT_DIR/netbox-db/netbox-db-patch.tmpl.yaml" > "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" +${KUBECTL} apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" echo "loading demo-data into NetBox…" # We use plain `kubectl create … --dry-run=client -o yaml` here to generate diff --git a/kind/netbox-db/kustomization.yaml b/kind/netbox-db/kustomization.yaml new file mode 100644 index 00000000..de417ced --- /dev/null +++ b/kind/netbox-db/kustomization.yaml @@ -0,0 +1,8 @@ +resources: + - netbox-db.yaml + +patches: + - path: netbox-db-patch.yaml + target: + kind: postgresql + name: netbox-db \ No newline at end of file diff --git a/kind/netbox-db/netbox-db-patch.tmpl.yaml b/kind/netbox-db/netbox-db-patch.tmpl.yaml new file mode 100644 index 00000000..de2cc4ea --- /dev/null +++ b/kind/netbox-db/netbox-db-patch.tmpl.yaml @@ -0,0 +1,8 @@ +# spilo-image-patch.yaml +# This is used to allow the patch of the spec.postgresql.image based on the env variable IMAGE_REGISTRY +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: netbox-db +spec: + dockerImage: ${SPILO_IMAGE} diff --git a/kind/netbox-db/netbox-db.yaml b/kind/netbox-db/netbox-db.yaml new file mode 100644 index 00000000..ead11131 --- /dev/null +++ b/kind/netbox-db/netbox-db.yaml @@ -0,0 +1,21 @@ +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: netbox-db +spec: + teamId: "netbox" + volume: + size: 100Mi + numberOfInstances: 1 + enableMasterLoadBalancer: true + users: + # database owner + netbox: + - superuser + - createdb + + #databases: name->owner + databases: + netbox: netbox + postgresql: + version: "16" From 0a0174fbfb8ae2ab8124d10b71aa85d2d5c9f8c8 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Thu, 15 May 2025 10:42:17 +0200 Subject: [PATCH 26/32] Remove Dockerfile and use a ConfigMap mounting the main.py to run the script to load the local data --- kind/deploy-netbox.sh | 57 ++++++++----------- kind/load-data-job/dockerfile.orig | 25 -------- .../main.py | 2 +- .../netbox-load-local-data-job.yaml | 38 +++++++++++++ 4 files changed, 62 insertions(+), 60 deletions(-) delete mode 100644 kind/load-data-job/dockerfile.orig rename kind/{load-data-job => load-local-data-job}/main.py (99%) create mode 100644 kind/load-local-data-job/netbox-load-local-data-job.yaml diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index aa058674..650bcc61 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -53,7 +53,6 @@ if [[ "${VERSION}" == "3.7.8" ]] ;then sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh # patch dockerfile (See README at https://github.com/netbox-community/pynetbox for the supported version matrix) - sed 's/RUN pip install -Iv pynetbox==7.4.1/RUN pip install -Iv pynetbox==7.3.4/g' $SCRIPT_DIR/load-data-job/dockerfile.orig > $SCRIPT_DIR/load-data-job/dockerfile elif [[ "${VERSION}" == "4.0.11" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -70,7 +69,6 @@ elif [[ "${VERSION}" == "4.0.11" ]] ;then # patch load-data.sh sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh - cp $SCRIPT_DIR/load-data-job/dockerfile.orig $SCRIPT_DIR/load-data-job/dockerfile elif [[ "${VERSION}" == "4.1.8" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -85,7 +83,6 @@ elif [[ "${VERSION}" == "4.1.8" ]] ;then # create load-data.sh cp $SCRIPT_DIR/load-data-job/load-data.orig.sh $SCRIPT_DIR/load-data-job/load-data.sh - cp $SCRIPT_DIR/load-data-job/dockerfile.orig $SCRIPT_DIR/load-data-job/dockerfile else echo "Unknown version ${VERSION}" exit 1 @@ -103,33 +100,6 @@ fi # build image for loading local data via NetBox API cd "$SCRIPT_DIR/load-data-job" -# Append image registry prefix only if defined -PYTHON_IMAGE_NAME="python:3.12" -if [ -n "$IMAGE_REGISTRY" ]; then - PYTHON_BASE_IMAGE="${IMAGE_REGISTRY}/${PYTHON_IMAGE_NAME}" -else - PYTHON_BASE_IMAGE="$PYTHON_IMAGE_NAME" -fi - -docker build -t netbox-load-local-data:1.0 \ - --load --no-cache --progress=plain \ - --build-arg PYTHON_BASE_IMAGE="$PYTHON_BASE_IMAGE" \ - --build-arg ARTIFACTORY_PYPI_URL="${ARTIFACTORY_PYPI_URL:-}" \ - --build-arg ARTIFACTORY_TRUSTED_HOST="${ARTIFACTORY_TRUSTED_HOST:-}" \ - -f ./dockerfile . -cd - - -if ! $IS_VCLUSTER; then - echo "Loading local images into kind cluster..." - declare -a Local_Images=( \ - "netbox-load-local-data:1.0" \ - ) - for img in "${Local_Images[@]}"; do - kind load docker-image "$img" --name "${CLUSTER}" - done -else - echo "Skipping local image loading into Kind (vCluster mode)." -fi # Assign IMAGE_REGISTRY from env if set, else empty POSTGRES_IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" @@ -233,11 +203,30 @@ ${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ ${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox -# Load local data +echo "Loading local data into NetBox via ConfigMap-based Job..." + +# Create ConfigMap for the python script +TMP_CONFIGMAP_YAML="$(mktemp)" +kubectl create configmap netbox-loader-script \ + --namespace="${NAMESPACE}" \ + --from-file=main.py="$SCRIPT_DIR/load-local-data-job/main.py" \ + --dry-run=client -o yaml > "$TMP_CONFIGMAP_YAML" + +vcluster connect "${NAMESPACE}" -n "${NAMESPACE}" -- kubectl apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" +rm "$TMP_CONFIGMAP_YAML" + +# Delete previous job if it exists ${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found -${KUBECTL} create job netbox-load-local-data --namespace="${NAMESPACE}" --image=netbox-load-local-data:1.0 + +# Apply pre-written job YAML from file +${KUBECTL} apply -n "${NAMESPACE}" -f "$SCRIPT_DIR/load-local-data-job/netbox-load-local-data-job.yaml" + +# Wait for job to complete ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data +# Load local data +${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" +${KUBECTL} delete configmap netbox-loader-script --namespace="${NAMESPACE}" + # clean up -rm $SCRIPT_DIR/load-data-job/load-data.sh -rm $SCRIPT_DIR/load-data-job/dockerfile +rm $SCRIPT_DIR/load-data-job/load-data.sh \ No newline at end of file diff --git a/kind/load-data-job/dockerfile.orig b/kind/load-data-job/dockerfile.orig deleted file mode 100644 index c1853744..00000000 --- a/kind/load-data-job/dockerfile.orig +++ /dev/null @@ -1,25 +0,0 @@ -ARG PYTHON_BASE_IMAGE=python:3.12 -FROM ${PYTHON_BASE_IMAGE} - -# Optional Artifactory pip index -ARG ARTIFACTORY_PYPI_URL="" -ARG ARTIFACTORY_TRUSTED_HOST="" - -# Set envs so they’re usable in RUN commands -ENV ARTIFACTORY_PYPI_URL=${ARTIFACTORY_PYPI_URL} -ENV ARTIFACTORY_TRUSTED_HOST=${ARTIFACTORY_TRUSTED_HOST} - -ADD main.py . - -# Conditional pip install using Artifactory if set -RUN if [ -n "$ARTIFACTORY_PYPI_URL" ]; then \ - echo "Using Artifactory pip index: $ARTIFACTORY_PYPI_URL"; \ - pip install -Iv pynetbox==7.4.1 \ - --index-url="$ARTIFACTORY_PYPI_URL" \ - --trusted-host="$ARTIFACTORY_TRUSTED_HOST"; \ - else \ - echo "Using public PyPI"; \ - pip install -Iv pynetbox==7.4.1; \ - fi - -CMD ["python", "./main.py"] diff --git a/kind/load-data-job/main.py b/kind/load-local-data-job/main.py similarity index 99% rename from kind/load-data-job/main.py rename to kind/load-local-data-job/main.py index b0793ffd..b7be177b 100644 --- a/kind/load-data-job/main.py +++ b/kind/load-local-data-job/main.py @@ -5,7 +5,7 @@ print("Starting to load data onto NetBox through API") try: nb = pynetbox.api( - 'http://netbox', + 'http://netbox.test', token='0123456789abcdef0123456789abcdef01234567' ) except pynetbox.RequestError as e: diff --git a/kind/load-local-data-job/netbox-load-local-data-job.yaml b/kind/load-local-data-job/netbox-load-local-data-job.yaml new file mode 100644 index 00000000..e2c6b87e --- /dev/null +++ b/kind/load-local-data-job/netbox-load-local-data-job.yaml @@ -0,0 +1,38 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: netbox-load-local-data +spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + containers: + - name: loader + image: python:3.12-slim + workingDir: /app + command: ["/bin/sh", "-c"] + args: + - | + if [ -n "$ARTIFACTORY_PYPI_URL" ]; then + echo "Using Artifactory pip index: $ARTIFACTORY_PYPI_URL"; + pip install pynetbox \ + --index-url="$ARTIFACTORY_PYPI_URL" \ + --trusted-host="$ARTIFACTORY_TRUSTED_HOST"; + else + echo "Using public PyPI"; + pip install pynetbox; + fi && + python main.py + env: + - name: ARTIFACTORY_PYPI_URL + value: "https://artifactory.swisscom.com/artifactory/api/pypi/pypi-remote/simple" + - name: ARTIFACTORY_TRUSTED_HOST + value: "artifactory.swisscom.com" + volumeMounts: + - name: script-volume + mountPath: /app + volumes: + - name: script-volume + configMap: + name: netbox-loader-script \ No newline at end of file From 66a12e2939b905c3281582912d779720896260e6 Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Tue, 20 May 2025 09:19:24 +0200 Subject: [PATCH 27/32] Using PYPI_REPOSITORY_URL instead of ARTIFACTORY_PYPI_URL --- kind/load-local-data-job/netbox-load-local-data-job.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kind/load-local-data-job/netbox-load-local-data-job.yaml b/kind/load-local-data-job/netbox-load-local-data-job.yaml index e2c6b87e..7b0829ad 100644 --- a/kind/load-local-data-job/netbox-load-local-data-job.yaml +++ b/kind/load-local-data-job/netbox-load-local-data-job.yaml @@ -14,10 +14,10 @@ spec: command: ["/bin/sh", "-c"] args: - | - if [ -n "$ARTIFACTORY_PYPI_URL" ]; then - echo "Using Artifactory pip index: $ARTIFACTORY_PYPI_URL"; + if [ -n "$PYPI_REPOSITORY_URL" ]; then + echo "Using Artifactory pip index: $PYPI_REPOSITORY_URL"; pip install pynetbox \ - --index-url="$ARTIFACTORY_PYPI_URL" \ + --index-url="$PYPI_REPOSITORY_URL" \ --trusted-host="$ARTIFACTORY_TRUSTED_HOST"; else echo "Using public PyPI"; @@ -25,7 +25,7 @@ spec: fi && python main.py env: - - name: ARTIFACTORY_PYPI_URL + - name: PYPI_REPOSITORY_URL value: "https://artifactory.swisscom.com/artifactory/api/pypi/pypi-remote/simple" - name: ARTIFACTORY_TRUSTED_HOST value: "artifactory.swisscom.com" From cddcc26412f23d9c18aa41c292accbaaeae6341f Mon Sep 17 00:00:00 2001 From: Pablo Garcia Miranda Date: Fri, 6 Jun 2025 10:46:17 +0200 Subject: [PATCH 28/32] Refactoring of scripts to inject dynamically env variables --- kind/deploy-netbox.sh | 75 ++++++++++++------- kind/load-local-data-job/main.py | 12 ++- .../netbox-load-local-data-job.yaml | 5 -- 3 files changed, 57 insertions(+), 35 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 650bcc61..1f4a8666 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -28,11 +28,7 @@ fi # Choose kubectl and helm commands depending if we run on vCluster if $IS_VCLUSTER; then - KUBECTL="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- kubectl" - HELM="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- helm" -else - KUBECTL="kubectl" - HELM="helm" + vcluster connect ${CLUSTER} -n ${NAMESPACE} fi # load remote images @@ -113,7 +109,7 @@ fi # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_REPO:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator}/postgres-operator-1.12.2.tgz" -${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ +helm upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --namespace="${NAMESPACE}" \ --create-namespace \ --set podPriorityClassName.create=false \ @@ -125,17 +121,13 @@ ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ export SPILO_IMAGE="${IMAGE_REGISTRY:-ghcr.io}/zalando/spilo-16:3.2-p3" echo "spilo image is $SPILO_IMAGE" envsubst < "$SCRIPT_DIR/netbox-db/netbox-db-patch.tmpl.yaml" > "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" -${KUBECTL} apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" +kubectl apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" echo "loading demo-data into NetBox…" -# We use plain `kubectl create … --dry-run=client -o yaml` here to generate -# the ConfigMap manifest locally (no cluster connection needed), then pipe -# that YAML into `${KUBECTL} apply` so it’s applied against the selected -# target (Kind or vCluster) via our `${KUBECTL}` wrapper. kubectl create configmap netbox-demo-data-load-job-scripts \ --from-file="$SCRIPT_DIR/load-data-job" \ --dry-run=client -o yaml \ -| ${KUBECTL} apply -n "${NAMESPACE}" -f - +| kubectl apply -n "${NAMESPACE}" -f - # Set the image of the kustomization.yaml to the one specified (from env or default) SPILO_IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io}" @@ -168,13 +160,13 @@ EOF kustomize edit add patch --path sql-env-patch.yaml # Apply the customized job -kustomize build . | ${KUBECTL} apply -n "${NAMESPACE}" -f - +kustomize build . | kubectl apply -n "${NAMESPACE}" -f - cd .. -${KUBECTL} wait \ +kubectl wait \ -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job -${KUBECTL} delete \ +kubectl delete \ -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts # Assign IMAGE_REGISTRY from env if set, else empty @@ -187,7 +179,7 @@ if [ -n "$NETBOX_IMAGE_REGISTRY" ]; then fi # Install NetBox -${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ +helm upgrade --install netbox ${NETBOX_HELM_CHART} \ --namespace="${NAMESPACE}" \ --create-namespace \ --set postgresql.enabled="false" \ @@ -201,32 +193,61 @@ ${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ --set resources.limits.memory="2Gi" \ $REGISTRY_ARG -${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox - -echo "Loading local data into NetBox via ConfigMap-based Job..." +kubectl rollout status --namespace="${NAMESPACE}" deployment netbox -# Create ConfigMap for the python script +# Create ConfigMap for the Python script TMP_CONFIGMAP_YAML="$(mktemp)" kubectl create configmap netbox-loader-script \ --namespace="${NAMESPACE}" \ --from-file=main.py="$SCRIPT_DIR/load-local-data-job/main.py" \ --dry-run=client -o yaml > "$TMP_CONFIGMAP_YAML" -vcluster connect "${NAMESPACE}" -n "${NAMESPACE}" -- kubectl apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" +kubectl apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" rm "$TMP_CONFIGMAP_YAML" +# Prepare Job YAML with optional environment variable injection +JOB_YAML="$SCRIPT_DIR/load-local-data-job/netbox-load-local-data-job.yaml" +TMP_JOB_YAML="$(mktemp)" +cp "$JOB_YAML" "$TMP_JOB_YAML" + +# Define internal NetBox service endpoint (used in Kind) +NETBOX_API_URL="http://netbox.${NAMESPACE}.svc.cluster.local" + +PATCHED_TMP_JOB_YAML="$(mktemp)" + +# Convert YAML to JSON and inject variables if containers exist +yq -o=json "$TMP_JOB_YAML" | jq \ + --arg netboxApi "$NETBOX_API_URL" \ + --arg pypiUrl "$PYPI_REPOSITORY_URL" \ + --arg artifactoryHost "$ARTIFACTORY_TRUSTED_HOST" ' + .spec.template.spec.containers[0].env //= [] | + .spec.template.spec.containers[0].env += + [{"name": "NETBOX_API", "value": $netboxApi}] + + ( + if $pypiUrl != "" and $artifactoryHost != "" then + [ + {"name": "PYPI_REPOSITORY_URL", "value": $pypiUrl}, + {"name": "ARTIFACTORY_TRUSTED_HOST", "value": $artifactoryHost} + ] + else [] end + ) +' | yq -P > "$PATCHED_TMP_JOB_YAML" + +mv "$PATCHED_TMP_JOB_YAML" "$TMP_JOB_YAML" + # Delete previous job if it exists -${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found +kubectl delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found -# Apply pre-written job YAML from file -${KUBECTL} apply -n "${NAMESPACE}" -f "$SCRIPT_DIR/load-local-data-job/netbox-load-local-data-job.yaml" +# Apply patched job +kubectl apply -n "${NAMESPACE}" -f "$TMP_JOB_YAML" +rm "$TMP_JOB_YAML" # Wait for job to complete -${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data +kubectl wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data # Load local data -${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" -${KUBECTL} delete configmap netbox-loader-script --namespace="${NAMESPACE}" +kubectl delete job netbox-load-local-data --namespace="${NAMESPACE}" +kubectl delete configmap netbox-loader-script --namespace="${NAMESPACE}" # clean up rm $SCRIPT_DIR/load-data-job/load-data.sh \ No newline at end of file diff --git a/kind/load-local-data-job/main.py b/kind/load-local-data-job/main.py index b7be177b..d9a0e87f 100644 --- a/kind/load-local-data-job/main.py +++ b/kind/load-local-data-job/main.py @@ -1,17 +1,23 @@ +import os import pynetbox from pprint import pprint from dataclasses import dataclass print("Starting to load data onto NetBox through API") + +NETBOX_API = os.getenv("NETBOX_API", "http://netbox") + try: nb = pynetbox.api( - 'http://netbox.test', + NETBOX_API, token='0123456789abcdef0123456789abcdef01234567' ) except pynetbox.RequestError as e: pprint(e.error) - -print("Connected to NetBoxAPI") + raise SystemExit(f"Failed to connect to NetBox at {NETBOX_API}") + +print(f"Connected to NetBoxAPI at {NETBOX_API}") + # insert Tenants @dataclass diff --git a/kind/load-local-data-job/netbox-load-local-data-job.yaml b/kind/load-local-data-job/netbox-load-local-data-job.yaml index 7b0829ad..1b12683a 100644 --- a/kind/load-local-data-job/netbox-load-local-data-job.yaml +++ b/kind/load-local-data-job/netbox-load-local-data-job.yaml @@ -24,11 +24,6 @@ spec: pip install pynetbox; fi && python main.py - env: - - name: PYPI_REPOSITORY_URL - value: "https://artifactory.swisscom.com/artifactory/api/pypi/pypi-remote/simple" - - name: ARTIFACTORY_TRUSTED_HOST - value: "artifactory.swisscom.com" volumeMounts: - name: script-volume mountPath: /app From 6b6107e7362f41ed95c14d7fa65866e4d9ab1017 Mon Sep 17 00:00:00 2001 From: bruelea <166021996+bruelea@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:41:19 +0200 Subject: [PATCH 29/32] minor fixes --- kind/deploy-netbox.sh | 57 +++++++++++++++++++++++-------------- kind/job/kustomization.yaml | 1 + 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 1f4a8666..d6d4e5af 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -28,7 +28,11 @@ fi # Choose kubectl and helm commands depending if we run on vCluster if $IS_VCLUSTER; then - vcluster connect ${CLUSTER} -n ${NAMESPACE} + KUBECTL="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- kubectl" + HELM="vcluster connect ${CLUSTER} -n ${NAMESPACE} -- helm" +else + KUBECTL="kubectl" + HELM="helm" fi # load remote images @@ -109,25 +113,29 @@ fi # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_REPO:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator}/postgres-operator-1.12.2.tgz" -helm upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ - --namespace="${NAMESPACE}" \ - --create-namespace \ - --set podPriorityClassName.create=false \ - --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ - --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ - $REGISTRY_ARG +echo $POSTGRES_OPERATOR_HELM_CHART +${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ + --namespace="${NAMESPACE}" \ + --create-namespace \ + --set podPriorityClassName.create=false \ + --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ + --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ + $REGISTRY_ARG + +echo "reached 1" # Deploy the database export SPILO_IMAGE="${IMAGE_REGISTRY:-ghcr.io}/zalando/spilo-16:3.2-p3" echo "spilo image is $SPILO_IMAGE" envsubst < "$SCRIPT_DIR/netbox-db/netbox-db-patch.tmpl.yaml" > "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" -kubectl apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" +${KUBECTL} apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" +rm "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" echo "loading demo-data into NetBox…" kubectl create configmap netbox-demo-data-load-job-scripts \ --from-file="$SCRIPT_DIR/load-data-job" \ --dry-run=client -o yaml \ -| kubectl apply -n "${NAMESPACE}" -f - +| ${KUBECTL} apply -n "${NAMESPACE}" -f - # Set the image of the kustomization.yaml to the one specified (from env or default) SPILO_IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io}" @@ -160,13 +168,16 @@ EOF kustomize edit add patch --path sql-env-patch.yaml # Apply the customized job -kustomize build . | kubectl apply -n "${NAMESPACE}" -f - +kustomize build . | ${KUBECTL} apply -n "${NAMESPACE}" -f - +# reset the kustomization to default value +rm sql-env-patch.yaml +kustomize edit set image ghcr.io/zalando/spilo-16="ghcr.io/zalando/spilo-16" cd .. -kubectl wait \ +${KUBECTL} wait \ -n "${NAMESPACE}" --for=condition=complete --timeout=600s job/netbox-demo-data-load-job -kubectl delete \ +${KUBECTL} delete \ -n "${NAMESPACE}" configmap/netbox-demo-data-load-job-scripts # Assign IMAGE_REGISTRY from env if set, else empty @@ -179,7 +190,7 @@ if [ -n "$NETBOX_IMAGE_REGISTRY" ]; then fi # Install NetBox -helm upgrade --install netbox ${NETBOX_HELM_CHART} \ +${HELM} upgrade --install netbox ${NETBOX_HELM_CHART} \ --namespace="${NAMESPACE}" \ --create-namespace \ --set postgresql.enabled="false" \ @@ -193,7 +204,7 @@ helm upgrade --install netbox ${NETBOX_HELM_CHART} \ --set resources.limits.memory="2Gi" \ $REGISTRY_ARG -kubectl rollout status --namespace="${NAMESPACE}" deployment netbox +${KUBECTL} rollout status --namespace="${NAMESPACE}" deployment netbox # Create ConfigMap for the Python script TMP_CONFIGMAP_YAML="$(mktemp)" @@ -202,7 +213,7 @@ kubectl create configmap netbox-loader-script \ --from-file=main.py="$SCRIPT_DIR/load-local-data-job/main.py" \ --dry-run=client -o yaml > "$TMP_CONFIGMAP_YAML" -kubectl apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" +${KUBECTL} apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" rm "$TMP_CONFIGMAP_YAML" # Prepare Job YAML with optional environment variable injection @@ -219,8 +230,10 @@ PATCHED_TMP_JOB_YAML="$(mktemp)" yq -o=json "$TMP_JOB_YAML" | jq \ --arg netboxApi "$NETBOX_API_URL" \ --arg pypiUrl "$PYPI_REPOSITORY_URL" \ - --arg artifactoryHost "$ARTIFACTORY_TRUSTED_HOST" ' + --arg artifactoryHost "$ARTIFACTORY_TRUSTED_HOST" \ + --arg imageRegistry "${IMAGE_REGISTRY:-ghcr.io}" ' .spec.template.spec.containers[0].env //= [] | + .spec.template.spec.containers[0].image = $imageRegistry+"/python:3.12-slim" | .spec.template.spec.containers[0].env += [{"name": "NETBOX_API", "value": $netboxApi}] + ( @@ -236,18 +249,18 @@ yq -o=json "$TMP_JOB_YAML" | jq \ mv "$PATCHED_TMP_JOB_YAML" "$TMP_JOB_YAML" # Delete previous job if it exists -kubectl delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found +${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found # Apply patched job -kubectl apply -n "${NAMESPACE}" -f "$TMP_JOB_YAML" +${KUBECTL} apply -n "${NAMESPACE}" -f "$TMP_JOB_YAML" rm "$TMP_JOB_YAML" # Wait for job to complete -kubectl wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data +${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data # Load local data -kubectl delete job netbox-load-local-data --namespace="${NAMESPACE}" -kubectl delete configmap netbox-loader-script --namespace="${NAMESPACE}" +${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" +${KUBECTL} delete configmap netbox-loader-script --namespace="${NAMESPACE}" # clean up rm $SCRIPT_DIR/load-data-job/load-data.sh \ No newline at end of file diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml index 34c4c561..68fcafcd 100644 --- a/kind/job/kustomization.yaml +++ b/kind/job/kustomization.yaml @@ -5,5 +5,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - name: ghcr.io/zalando/spilo-16 + newName: ghcr.io/zalando/spilo-16 patches: - path: sql-env-patch.yaml From 4b49011def972ba66604ef2bf7466f57b14d80ad Mon Sep 17 00:00:00 2001 From: bruelea <166021996+bruelea@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:47:51 +0200 Subject: [PATCH 30/32] add empty lines at end of file --- kind/deploy-netbox.sh | 2 +- kind/load-local-data-job/netbox-load-local-data-job.yaml | 2 +- kind/netbox-db/kustomization.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index d6d4e5af..4e085143 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -263,4 +263,4 @@ ${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" ${KUBECTL} delete configmap netbox-loader-script --namespace="${NAMESPACE}" # clean up -rm $SCRIPT_DIR/load-data-job/load-data.sh \ No newline at end of file +rm $SCRIPT_DIR/load-data-job/load-data.sh diff --git a/kind/load-local-data-job/netbox-load-local-data-job.yaml b/kind/load-local-data-job/netbox-load-local-data-job.yaml index 1b12683a..b9ce511b 100644 --- a/kind/load-local-data-job/netbox-load-local-data-job.yaml +++ b/kind/load-local-data-job/netbox-load-local-data-job.yaml @@ -30,4 +30,4 @@ spec: volumes: - name: script-volume configMap: - name: netbox-loader-script \ No newline at end of file + name: netbox-loader-script diff --git a/kind/netbox-db/kustomization.yaml b/kind/netbox-db/kustomization.yaml index de417ced..80910a9e 100644 --- a/kind/netbox-db/kustomization.yaml +++ b/kind/netbox-db/kustomization.yaml @@ -5,4 +5,4 @@ patches: - path: netbox-db-patch.yaml target: kind: postgresql - name: netbox-db \ No newline at end of file + name: netbox-db From c5dedf68f2fc9b9cf0840d2f0925e3ac4ac270e4 Mon Sep 17 00:00:00 2001 From: bruelea <166021996+bruelea@users.noreply.github.com> Date: Fri, 6 Jun 2025 15:52:46 +0200 Subject: [PATCH 31/32] remove debug comments --- kind/deploy-netbox.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 4e085143..4c91b87d 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -113,7 +113,6 @@ fi # Install Postgres Operator # Allow override via environment variable, otherwise fallback to default POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_REPO:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator}/postgres-operator-1.12.2.tgz" -echo $POSTGRES_OPERATOR_HELM_CHART ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --namespace="${NAMESPACE}" \ --create-namespace \ @@ -122,8 +121,6 @@ ${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ $REGISTRY_ARG -echo "reached 1" - # Deploy the database export SPILO_IMAGE="${IMAGE_REGISTRY:-ghcr.io}/zalando/spilo-16:3.2-p3" echo "spilo image is $SPILO_IMAGE" From 58c7e769d826cedd57681e6315d83a37d32d50f8 Mon Sep 17 00:00:00 2001 From: bruelea <166021996+bruelea@users.noreply.github.com> Date: Fri, 6 Jun 2025 16:11:19 +0200 Subject: [PATCH 32/32] fix default python image registry --- kind/deploy-netbox.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 4c91b87d..1d11d05f 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -228,7 +228,7 @@ yq -o=json "$TMP_JOB_YAML" | jq \ --arg netboxApi "$NETBOX_API_URL" \ --arg pypiUrl "$PYPI_REPOSITORY_URL" \ --arg artifactoryHost "$ARTIFACTORY_TRUSTED_HOST" \ - --arg imageRegistry "${IMAGE_REGISTRY:-ghcr.io}" ' + --arg imageRegistry "${IMAGE_REGISTRY:-docker.io}" ' .spec.template.spec.containers[0].env //= [] | .spec.template.spec.containers[0].image = $imageRegistry+"/python:3.12-slim" | .spec.template.spec.containers[0].env +=