diff --git a/kind/deploy-netbox.sh b/kind/deploy-netbox.sh index 417718d5..1d11d05f 100755 --- a/kind/deploy-netbox.sh +++ b/kind/deploy-netbox.sh @@ -1,11 +1,14 @@ #!/bin/bash -set -e -u -o pipefail +set -e -o pipefail # Deploy NetBox (with its PostgreSQL operator and demo data) into either: # • a local kind cluster (preloading images) # • a virtual cluster using vcluster: https://github.com/loft-sh/vcluster ( used for testing pipeline, loading of images not needed ) -NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # default value +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Allow override via environment variable, otherwise fallback to default +NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" if [[ $# -lt 3 || $# -gt 4 ]]; then echo "Usage: $0 [--vcluster]" @@ -43,13 +46,13 @@ if [[ "${VERSION}" == "3.7.8" ]] ;then "ghcr.io/zalando/postgres-operator:v1.12.2" \ "ghcr.io/zalando/spilo-16:3.2-p3" \ ) - NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta5/netbox-5.0.0-beta5.tgz" + # Allow override via environment variable, otherwise fallback to default + NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # patch load-data.sh - sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v3.7.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh # patch dockerfile (See README at https://github.com/netbox-community/pynetbox for the supported version matrix) - sed 's/RUN pip install -Iv pynetbox==7.4.1/RUN pip install -Iv pynetbox==7.3.4/g' $(dirname "$0")/load-data-job/dockerfile.orig > $(dirname "$0")/load-data-job/dockerfile elif [[ "${VERSION}" == "4.0.11" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -60,12 +63,12 @@ elif [[ "${VERSION}" == "4.0.11" ]] ;then "ghcr.io/zalando/postgres-operator:v1.12.2" \ "ghcr.io/zalando/spilo-16:3.2-p3" \ ) - NETBOX_HELM_CHART="https://github.com/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.84/netbox-5.0.0-beta.84.tgz" + # Allow override via environment variable, otherwise fallback to default + NETBOX_HELM_CHART="${NETBOX_HELM_REPO:-https://github.com}/netbox-community/netbox-chart/releases/download/netbox-5.0.0-beta.169/netbox-5.0.0-beta.169.tgz" # patch load-data.sh - sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $(dirname "$0")/load-data-job/load-data.orig.sh > $(dirname "$0")/load-data-job/load-data.sh && chmod +x $(dirname "$0")/load-data-job/load-data.sh + sed 's/netbox-demo-v4.1.sql/netbox-demo-v4.0.sql/g' $SCRIPT_DIR/load-data-job/load-data.orig.sh > $SCRIPT_DIR/load-data-job/load-data.sh && chmod +x $SCRIPT_DIR/load-data-job/load-data.sh - cp $(dirname "$0")/load-data-job/dockerfile.orig $(dirname "$0")/load-data-job/dockerfile elif [[ "${VERSION}" == "4.1.8" ]] ;then echo "Using version ${VERSION}" # need to align with netbox-chart otherwise the creation of the cluster will hang @@ -78,9 +81,8 @@ elif [[ "${VERSION}" == "4.1.8" ]] ;then ) # create load-data.sh - cp $(dirname "$0")/load-data-job/load-data.orig.sh $(dirname "$0")/load-data-job/load-data.sh + cp $SCRIPT_DIR/load-data-job/load-data.orig.sh $SCRIPT_DIR/load-data-job/load-data.sh - cp $(dirname "$0")/load-data-job/dockerfile.orig $(dirname "$0")/load-data-job/dockerfile else echo "Unknown version ${VERSION}" exit 1 @@ -97,47 +99,77 @@ else fi # build image for loading local data via NetBox API -cd "$(dirname "$0")/load-data-job" -docker build -t netbox-load-local-data:1.0 --load --no-cache --progress=plain -f ./dockerfile . -cd - - -if ! $IS_VCLUSTER; then - echo "Loading local images into kind cluster..." - declare -a Local_Images=( \ - "netbox-load-local-data:1.0" \ - ) - for img in "${Local_Images[@]}"; do - kind load docker-image "$img" --name "${CLUSTER}" - done -else - echo "Skipping local image loading into Kind (vCluster mode)." +cd "$SCRIPT_DIR/load-data-job" + +# Assign IMAGE_REGISTRY from env if set, else empty +POSTGRES_IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" + +# Build optional set flag if registry is not defined +REGISTRY_ARG="" +if [ -n "$POSTGRES_IMAGE_REGISTRY" ]; then + REGISTRY_ARG="--set image.registry=$POSTGRES_IMAGE_REGISTRY" fi # Install Postgres Operator -${HELM} upgrade --install postgres-operator \ - --namespace="${NAMESPACE}" \ - --create-namespace \ - --set podPriorityClassName.create=false \ - --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ - --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ - https://opensource.zalando.com/postgres-operator/charts/postgres-operator/postgres-operator-1.12.2.tgz +# Allow override via environment variable, otherwise fallback to default +POSTGRES_OPERATOR_HELM_CHART="${POSTGRES_OPERATOR_HELM_REPO:-https://opensource.zalando.com/postgres-operator/charts/postgres-operator}/postgres-operator-1.12.2.tgz" +${HELM} upgrade --install postgres-operator "$POSTGRES_OPERATOR_HELM_CHART" \ + --namespace="${NAMESPACE}" \ + --create-namespace \ + --set podPriorityClassName.create=false \ + --set podServiceAccount.name="postgres-pod-${NAMESPACE}" \ + --set serviceAccount.name="postgres-operator-${NAMESPACE}" \ + $REGISTRY_ARG # Deploy the database -${KUBECTL} apply --namespace="${NAMESPACE}" -f "$(dirname "$0")/netbox-db.yaml" -${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=jsonpath='{.status.PostgresClusterStatus}'=Running postgresql/netbox-db +export SPILO_IMAGE="${IMAGE_REGISTRY:-ghcr.io}/zalando/spilo-16:3.2-p3" +echo "spilo image is $SPILO_IMAGE" +envsubst < "$SCRIPT_DIR/netbox-db/netbox-db-patch.tmpl.yaml" > "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" +${KUBECTL} apply -n "$NAMESPACE" -k "$SCRIPT_DIR/netbox-db" +rm "$SCRIPT_DIR/netbox-db/netbox-db-patch.yaml" echo "loading demo-data into NetBox…" -# We use plain `kubectl create … --dry-run=client -o yaml` here to generate -# the ConfigMap manifest locally (no cluster connection needed), then pipe -# that YAML into `${KUBECTL} apply` so it’s applied against the selected -# target (Kind or vCluster) via our `${KUBECTL}` wrapper. kubectl create configmap netbox-demo-data-load-job-scripts \ - --from-file="$(dirname "$0")/load-data-job" \ + --from-file="$SCRIPT_DIR/load-data-job" \ --dry-run=client -o yaml \ | ${KUBECTL} apply -n "${NAMESPACE}" -f - -${KUBECTL} apply -n "${NAMESPACE}" \ - -f "$(dirname "$0")/load-data-job.yaml" +# Set the image of the kustomization.yaml to the one specified (from env or default) +SPILO_IMAGE_REGISTRY="${IMAGE_REGISTRY:-ghcr.io}" +SPILO_IMAGE="${SPILO_IMAGE_REGISTRY}/zalando/spilo-16:3.2-p3" + +JOB_DIR="$SCRIPT_DIR/job" +cd "$JOB_DIR" +kustomize edit set image ghcr.io/zalando/spilo-16="$SPILO_IMAGE" + +# Create a patch file to inject NETBOX_SQL_DUMP_URL (from env or default) +NETBOX_SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" + +# Create patch +cat > sql-env-patch.yaml < "$TMP_CONFIGMAP_YAML" + +${KUBECTL} apply -f "$TMP_CONFIGMAP_YAML" --namespace="${NAMESPACE}" +rm "$TMP_CONFIGMAP_YAML" + +# Prepare Job YAML with optional environment variable injection +JOB_YAML="$SCRIPT_DIR/load-local-data-job/netbox-load-local-data-job.yaml" +TMP_JOB_YAML="$(mktemp)" +cp "$JOB_YAML" "$TMP_JOB_YAML" + +# Define internal NetBox service endpoint (used in Kind) +NETBOX_API_URL="http://netbox.${NAMESPACE}.svc.cluster.local" + +PATCHED_TMP_JOB_YAML="$(mktemp)" + +# Convert YAML to JSON and inject variables if containers exist +yq -o=json "$TMP_JOB_YAML" | jq \ + --arg netboxApi "$NETBOX_API_URL" \ + --arg pypiUrl "$PYPI_REPOSITORY_URL" \ + --arg artifactoryHost "$ARTIFACTORY_TRUSTED_HOST" \ + --arg imageRegistry "${IMAGE_REGISTRY:-docker.io}" ' + .spec.template.spec.containers[0].env //= [] | + .spec.template.spec.containers[0].image = $imageRegistry+"/python:3.12-slim" | + .spec.template.spec.containers[0].env += + [{"name": "NETBOX_API", "value": $netboxApi}] + + ( + if $pypiUrl != "" and $artifactoryHost != "" then + [ + {"name": "PYPI_REPOSITORY_URL", "value": $pypiUrl}, + {"name": "ARTIFACTORY_TRUSTED_HOST", "value": $artifactoryHost} + ] + else [] end + ) +' | yq -P > "$PATCHED_TMP_JOB_YAML" + +mv "$PATCHED_TMP_JOB_YAML" "$TMP_JOB_YAML" + +# Delete previous job if it exists ${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" --ignore-not-found -${KUBECTL} create job netbox-load-local-data --namespace="${NAMESPACE}" --image=netbox-load-local-data:1.0 + +# Apply patched job +${KUBECTL} apply -n "${NAMESPACE}" -f "$TMP_JOB_YAML" +rm "$TMP_JOB_YAML" + +# Wait for job to complete ${KUBECTL} wait --namespace="${NAMESPACE}" --timeout=600s --for=condition=complete job/netbox-load-local-data +# Load local data +${KUBECTL} delete job netbox-load-local-data --namespace="${NAMESPACE}" +${KUBECTL} delete configmap netbox-loader-script --namespace="${NAMESPACE}" + # clean up -rm $(dirname "$0")/load-data-job/load-data.sh -rm $(dirname "$0")/load-data-job/dockerfile +rm $SCRIPT_DIR/load-data-job/load-data.sh diff --git a/kind/job/kustomization.yaml b/kind/job/kustomization.yaml new file mode 100644 index 00000000..68fcafcd --- /dev/null +++ b/kind/job/kustomization.yaml @@ -0,0 +1,10 @@ +resources: +- load-data-job.yaml + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: ghcr.io/zalando/spilo-16 + newName: ghcr.io/zalando/spilo-16 +patches: +- path: sql-env-patch.yaml diff --git a/kind/load-data-job.yaml b/kind/job/load-data-job.yaml similarity index 100% rename from kind/load-data-job.yaml rename to kind/job/load-data-job.yaml diff --git a/kind/load-data-job/dockerfile.orig b/kind/load-data-job/dockerfile.orig deleted file mode 100644 index 447d0eb9..00000000 --- a/kind/load-data-job/dockerfile.orig +++ /dev/null @@ -1,4 +0,0 @@ -FROM python:3.12 -ADD main.py . -RUN pip install -Iv pynetbox==7.4.1 -CMD ["python", "./main.py"] diff --git a/kind/load-data-job/load-data.orig.sh b/kind/load-data-job/load-data.orig.sh index a8cfe3bf..670c8d80 100755 --- a/kind/load-data-job/load-data.orig.sh +++ b/kind/load-data-job/load-data.orig.sh @@ -1,8 +1,17 @@ #!/bin/sh set -o errexit +# Allow override of demo SQL file URL +NETBOX_SQL_DUMP_URL="${NETBOX_SQL_DUMP_URL:-https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql}" + TMP_SQL_FILE=$(mktemp /tmp/netbox-data-dump.XXXXXXX.sql) || exit 1 -curl -k https://raw.githubusercontent.com/netbox-community/netbox-demo-data/master/sql/netbox-demo-v4.1.sql > "${TMP_SQL_FILE}" + +# Download the SQL dump +curl -k "${NETBOX_SQL_DUMP_URL}" > "${TMP_SQL_FILE}" + +# Load it into the database psql "user=netbox host=netbox-db.${NAMESPACE}.svc.cluster.local" netbox -q -f "${TMP_SQL_FILE}" rm "${TMP_SQL_FILE}" + +# Load additional local data psql "user=netbox host=netbox-db.${NAMESPACE}.svc.cluster.local" netbox -q -f /load-data-job/local-data-setup.sql diff --git a/kind/load-data-job/main.py b/kind/load-local-data-job/main.py similarity index 99% rename from kind/load-data-job/main.py rename to kind/load-local-data-job/main.py index b0793ffd..d9a0e87f 100644 --- a/kind/load-data-job/main.py +++ b/kind/load-local-data-job/main.py @@ -1,17 +1,23 @@ +import os import pynetbox from pprint import pprint from dataclasses import dataclass print("Starting to load data onto NetBox through API") + +NETBOX_API = os.getenv("NETBOX_API", "http://netbox") + try: nb = pynetbox.api( - 'http://netbox', + NETBOX_API, token='0123456789abcdef0123456789abcdef01234567' ) except pynetbox.RequestError as e: pprint(e.error) - -print("Connected to NetBoxAPI") + raise SystemExit(f"Failed to connect to NetBox at {NETBOX_API}") + +print(f"Connected to NetBoxAPI at {NETBOX_API}") + # insert Tenants @dataclass diff --git a/kind/load-local-data-job/netbox-load-local-data-job.yaml b/kind/load-local-data-job/netbox-load-local-data-job.yaml new file mode 100644 index 00000000..b9ce511b --- /dev/null +++ b/kind/load-local-data-job/netbox-load-local-data-job.yaml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: netbox-load-local-data +spec: + backoffLimit: 1 + template: + spec: + restartPolicy: Never + containers: + - name: loader + image: python:3.12-slim + workingDir: /app + command: ["/bin/sh", "-c"] + args: + - | + if [ -n "$PYPI_REPOSITORY_URL" ]; then + echo "Using Artifactory pip index: $PYPI_REPOSITORY_URL"; + pip install pynetbox \ + --index-url="$PYPI_REPOSITORY_URL" \ + --trusted-host="$ARTIFACTORY_TRUSTED_HOST"; + else + echo "Using public PyPI"; + pip install pynetbox; + fi && + python main.py + volumeMounts: + - name: script-volume + mountPath: /app + volumes: + - name: script-volume + configMap: + name: netbox-loader-script diff --git a/kind/netbox-db/kustomization.yaml b/kind/netbox-db/kustomization.yaml new file mode 100644 index 00000000..80910a9e --- /dev/null +++ b/kind/netbox-db/kustomization.yaml @@ -0,0 +1,8 @@ +resources: + - netbox-db.yaml + +patches: + - path: netbox-db-patch.yaml + target: + kind: postgresql + name: netbox-db diff --git a/kind/netbox-db/netbox-db-patch.tmpl.yaml b/kind/netbox-db/netbox-db-patch.tmpl.yaml new file mode 100644 index 00000000..de2cc4ea --- /dev/null +++ b/kind/netbox-db/netbox-db-patch.tmpl.yaml @@ -0,0 +1,8 @@ +# spilo-image-patch.yaml +# This is used to allow the patch of the spec.postgresql.image based on the env variable IMAGE_REGISTRY +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: netbox-db +spec: + dockerImage: ${SPILO_IMAGE} diff --git a/kind/netbox-db/netbox-db.yaml b/kind/netbox-db/netbox-db.yaml new file mode 100644 index 00000000..ead11131 --- /dev/null +++ b/kind/netbox-db/netbox-db.yaml @@ -0,0 +1,21 @@ +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: netbox-db +spec: + teamId: "netbox" + volume: + size: 100Mi + numberOfInstances: 1 + enableMasterLoadBalancer: true + users: + # database owner + netbox: + - superuser + - createdb + + #databases: name->owner + databases: + netbox: netbox + postgresql: + version: "16"