From e37affdf7c0259ca92eb2a2ff32dccdbe57ad00f Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Sun, 23 Feb 2025 00:05:32 -0600 Subject: [PATCH 1/2] fix: Basic Auth Fix --- .github/workflows/docker-build-scan.yml | 37 +++ .github/workflows/run-tests.yaml | 14 +- .gitignore | 5 + Dockerfile | 2 +- Makefile | 62 +++- Tiltfile | 4 +- controllers/weightsandbiases_controller.go | 6 +- .../weightsandbiases_controller_test.go | 1 + go.mod | 14 +- go.sum | 24 +- .../kind/local-registry-config.yaml | 14 + hack/testing-manifests/wandb/default.yaml | 3 +- .../wandb/local-registry.yaml | 122 +++++++ main.go | 4 +- pkg/wandb/spec/channel/deployer/deployer.go | 9 +- .../spec/channel/deployer/deployer_test.go | 198 +++++------- pkg/wandb/spec/charts/repo.go | 138 ++++++-- pkg/wandb/spec/charts/repo_test.go | 301 ++++++++++++++++++ pkg/wandb/spec/charts/repo_test_helpers.go | 83 +++++ tilt-settings.sample.json | 11 +- 20 files changed, 879 insertions(+), 173 deletions(-) create mode 100644 .github/workflows/docker-build-scan.yml create mode 100644 hack/testing-manifests/kind/local-registry-config.yaml create mode 100644 hack/testing-manifests/wandb/local-registry.yaml create mode 100644 pkg/wandb/spec/charts/repo_test.go create mode 100644 pkg/wandb/spec/charts/repo_test_helpers.go diff --git a/.github/workflows/docker-build-scan.yml b/.github/workflows/docker-build-scan.yml new file mode 100644 index 00000000..f11386cb --- /dev/null +++ b/.github/workflows/docker-build-scan.yml @@ -0,0 +1,37 @@ +name: Docker Build and Security Scan + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + build-and-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: false + load: true + tags: wandb/operator:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: wandb/operator:${{ github.sha }} + format: "table" + exit-code: "1" + ignore-unfixed: true + vuln-type: "os,library" + severity: "HIGH,CRITICAL" diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7cb1ec25..e38aca3b 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -19,7 +19,19 @@ jobs: - name: Install dependencies run: go get . - name: Tests - run: make test + run: make test-coverage + + build: + name: Build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.24 + - name: Build + run: make build dependency-check: name: Dependency Check diff --git a/.gitignore b/.gitignore index c519d4da..7570a6dc 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,11 @@ Dockerfile.cross # Output of the go coverage tool, specifically when used with LiteIDE *.out +# Ginkgo test results +coverage.html +report.json +junit.xml + # Kubernetes Generated files - skip generated files, except for vendored files !vendor/**/zz_generated.* diff --git a/Dockerfile b/Dockerfile index fc434703..fff77cfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go # Create a helm cache directory, set group ownership and permissions, and apply the sticky bit RUN mkdir -p /helm/.cache/helm /helm/.config/helm /helm/.local/share/helm && chmod -R 1777 /helm -FROM registry.access.redhat.com/ubi9/ubi +FROM registry.access.redhat.com/ubi9/ubi-micro COPY --from=manager-builder /workspace/manager . COPY --from=manager-builder /helm /helm diff --git a/Makefile b/Makefile index b8fc0abd..bbb8a1bd 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,12 @@ else GOBIN=$(shell go env GOBIN) endif +# Set environment variables to suppress linker warnings on macOS +ifeq ($(shell uname),Darwin) + export CGO_LDFLAGS=-Wl,-w + export LDFLAGS=-w +endif + # Setting SHELL to bash allows bash commands to be executed by recipes. # Options are set to exit when a recipe line exits non-zero or a piped command fails. SHELL = /usr/bin/env bash -o pipefail @@ -102,8 +108,54 @@ vet: ## Run go vet against code. go vet ./... .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out +test: manifests generate fmt vet envtest ginkgo ## Run tests. + @echo "Running tests..." + @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \ + ginkgo -p -r --compilers=4 --timeout=5m --fail-fast --race --trace --randomize-all \ + --output-interceptor-mode=none \ + --no-color=false \ + --show-node-events \ + --json-report=report.json \ + --junit-report=junit.xml || exit 1 + @echo "All tests passed!" + +.PHONY: test-verbose +test-verbose: manifests generate fmt vet envtest ginkgo ## Run tests with verbose output. + @echo "Running tests in verbose mode..." + @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \ + ginkgo -p -r -v --compilers=4 --timeout=5m --fail-fast --race --trace --randomize-all \ + --output-interceptor-mode=none \ + --no-color=false \ + --show-node-events \ + --json-report=report.json \ + --junit-report=junit.xml || exit 1 + @echo "All tests passed!" + +.PHONY: test-watch +test-watch: manifests generate fmt vet envtest ginkgo ## Run tests in watch mode. + @echo "Running tests in watch mode..." + @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \ + ginkgo watch -p -r --compilers=4 --timeout=5m --fail-fast --race --trace --randomize-all \ + --output-interceptor-mode=none \ + --no-color=false \ + --show-node-events \ + --json-report=report.json \ + --junit-report=junit.xml || exit 1 + +.PHONY: test-coverage +test-coverage: manifests generate fmt vet envtest ginkgo ## Run tests with coverage report. + @echo "Running tests with coverage..." + @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" \ + ginkgo -p -r --compilers=4 --timeout=5m --fail-fast --race --trace --randomize-all \ + --output-interceptor-mode=none \ + --no-color=false \ + --show-node-events \ + --json-report=report.json \ + --junit-report=junit.xml \ + --coverprofile=coverage.out || exit 1 + @echo "Generating coverage report..." + @go tool cover -html=coverage.out -o coverage.html + @echo "Coverage report generated at coverage.html" debug: controller-gen $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases -w @@ -179,6 +231,7 @@ $(LOCALBIN): KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest +GINKGO ?= $(LOCALBIN)/ginkgo ## Tool Versions KUSTOMIZE_VERSION ?= v4.2.0 @@ -260,3 +313,8 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) + +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + test -s $(LOCALBIN)/ginkgo || GOBIN=$(LOCALBIN) go install github.com/onsi/ginkgo/v2/ginkgo@latest diff --git a/Tiltfile b/Tiltfile index 9933be92..1398c08e 100644 --- a/Tiltfile +++ b/Tiltfile @@ -23,7 +23,7 @@ os.putenv('PATH', './bin:' + os.getenv('PATH')) load('ext://restart_process', 'docker_build_with_restart') DOCKERFILE = ''' -FROM registry.access.redhat.com/ubi9/ubi +FROM registry.access.redhat.com/ubi9/ubi-micro ADD tilt_bin/manager /manager @@ -104,7 +104,7 @@ local_resource('Watch&Compile', generate() + binary(), if settings.get("installWandb"): local_resource('Sample YAML', 'kubectl apply -f ./hack/testing-manifests/wandb/' + settings.get('wandbCRD') + - '.yaml', deps=["./hack/testing-manifests/wandb/default.yaml"], resource_deps=["controller-manager"]) + '.yaml', deps=["./hack/testing-manifests/wandb/" + settings.get('wandbCRD') + ".yaml"], resource_deps=["controller-manager"]) docker_build_with_restart(IMG, '.', dockerfile_contents=DOCKERFILE, diff --git a/controllers/weightsandbiases_controller.go b/controllers/weightsandbiases_controller.go index d3943ba2..55b2e549 100644 --- a/controllers/weightsandbiases_controller.go +++ b/controllers/weightsandbiases_controller.go @@ -18,9 +18,10 @@ package controllers import ( "context" - rbacv1 "k8s.io/api/rbac/v1" "reflect" + rbacv1 "k8s.io/api/rbac/v1" + "github.com/wandb/operator/pkg/wandb/spec/state" appsv1 "k8s.io/api/apps/v1" networkingv1 "k8s.io/api/networking/v1" @@ -147,6 +148,7 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req License: license, ActiveState: currentActiveSpec, ReleaseId: releaseID, + Debug: r.Debug, }) if err != nil { log.Info("Failed to get spec from deployer", "error", err) @@ -156,7 +158,7 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req // deployer release in the cache // Attempt to retrieve the cached release if deployerSpec, err = specManager.Get("latest-cached-release"); err != nil { - log.Error(err, "No cached release found for deployer spec", err, "error") + log.Info("No cached release found", "error", err.Error()) } if r.Debug { log.Info("Using cached deployer spec", "spec", deployerSpec.SensitiveValuesMasked()) diff --git a/controllers/weightsandbiases_controller_test.go b/controllers/weightsandbiases_controller_test.go index 049cf55f..4f2403ac 100644 --- a/controllers/weightsandbiases_controller_test.go +++ b/controllers/weightsandbiases_controller_test.go @@ -33,6 +33,7 @@ var deployerSpec = spec.Spec{ URL: "https://charts.wandb.ai", Name: "operator-wandb", Version: "0.14.3", + Debug: false, }, Values: spec.Values{ "app": map[string]interface{}{ diff --git a/go.mod b/go.mod index 01e882a1..a43a3113 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,11 @@ module github.com/wandb/operator go 1.24.0 require ( + github.com/go-logr/logr v1.4.2 github.com/go-playground/validator/v10 v10.22.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 + github.com/onsi/ginkgo/v2 v2.22.2 + github.com/onsi/gomega v1.36.2 github.com/pkg/errors v0.9.1 helm.sh/helm/v3 v3.16.2 k8s.io/api v0.31.2 @@ -54,7 +55,6 @@ require ( github.com/gabriel-vasile/mimetype v1.4.6 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -71,7 +71,7 @@ require ( github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect @@ -135,7 +135,7 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect - golang.org/x/mod v0.21.0 // indirect + golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.10.0 // indirect @@ -143,11 +143,11 @@ require ( golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.7.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/tools v0.28.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect google.golang.org/grpc v1.67.1 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 3c60c231..eece8712 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -279,10 +279,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -420,8 +420,8 @@ golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBn golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -463,8 +463,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -478,8 +478,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/hack/testing-manifests/kind/local-registry-config.yaml b/hack/testing-manifests/kind/local-registry-config.yaml new file mode 100644 index 00000000..6e918507 --- /dev/null +++ b/hack/testing-manifests/kind/local-registry-config.yaml @@ -0,0 +1,14 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry:5000"] + endpoint = ["http://registry:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."registry:5000".auth] + username = "admin" + password = "admin" + [plugins."io.containerd.grpc.v1.cri".registry.configs."registry:5000".tls] + insecure_skip_verify = true +networking: + ipFamily: dual + apiServerAddress: "127.0.0.1" diff --git a/hack/testing-manifests/wandb/default.yaml b/hack/testing-manifests/wandb/default.yaml index 175144ab..cfb7052d 100644 --- a/hack/testing-manifests/wandb/default.yaml +++ b/hack/testing-manifests/wandb/default.yaml @@ -57,6 +57,7 @@ spec: requests: cpu: "100m" memory: "128Mi" + redis: install: true auth: @@ -64,4 +65,4 @@ spec: resources: requests: cpu: "100m" - memory: "128Mi" \ No newline at end of file + memory: "128Mi" diff --git a/hack/testing-manifests/wandb/local-registry.yaml b/hack/testing-manifests/wandb/local-registry.yaml new file mode 100644 index 00000000..96725a5c --- /dev/null +++ b/hack/testing-manifests/wandb/local-registry.yaml @@ -0,0 +1,122 @@ +--- +apiVersion: apps.wandb.com/v1 +kind: WeightsAndBiases +metadata: + labels: + app.kubernetes.io/name: weightsandbiases + app.kubernetes.io/instance: weightsandbiases-sample + app.kubernetes.io/part-of: operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: operator + name: wandb-default +spec: + chart: + url: http://chartmuseum:8080 + name: "operator-wandb" + version: "0.26.9" + username: "admin" + password: "admin" + values: + global: + bucket: + provider: "s3" + name: "minio.minio.svc.cluster.local:9000/bucket" + region: "us-east-1" + accessKey: "minio" + secretKey: "minio123" + + app: + image: + repository: "registry:5000/local" + tag: "0.66.0" + resources: + requests: + cpu: "100m" + memory: "129Mi" + + parquet: + image: + repository: "registry:5000/local" + tag: "0.66.0" + resources: + requests: + cpu: "100m" + memory: "128Mi" + + weave: + image: + repository: "registry:5000/local" + tag: "0.66.0" + resources: + requests: + cpu: "100m" + memory: "128Mi" + + console: + image: + repository: "registry:5000/console" + tag: "2.15.2" + resources: + requests: + cpu: "100m" + memory: "128Mi" + + controller: + resources: + requests: + cpu: "100m" + memory: "128Mi" + + weaveTrace: + image: + repository: "registry:5000/weave-trace" + tag: "0.66.0" + resources: + requests: + cpu: "100m" + memory: "128Mi" + + opentelemetryCollector: + image: + repository: "registry:5000/opentelemetry-collector-contrib" + tag: "0.97.0" + resources: + requests: + cpu: "100m" + memory: "128Mi" + + prometheus: + image: + repository: "registry:5000/prometheus" + tag: "v2.47.0" + resources: + requests: + cpu: "100m" + memory: "128Mi" + configReloader: + image: + repository: "registry:5000/prometheus-config-reloader" + tag: "v0.67.0" + + ingress: + install: false + create: false + + mysql: + install: true + resources: + requests: + cpu: "100m" + memory: "128Mi" + + redis: + image: + repository: "registry:5000/redis" + tag: "7.2.4-debian-12-r9" + install: true + auth: + enabled: true + resources: + requests: + cpu: "100m" + memory: "128Mi" diff --git a/main.go b/main.go index 6af82af3..fadc01c7 100644 --- a/main.go +++ b/main.go @@ -20,16 +20,14 @@ import ( "flag" "fmt" "os" + "strings" "github.com/wandb/operator/pkg/wandb/spec/channel/deployer" - "strings" - "sigs.k8s.io/controller-runtime/pkg/cache" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/wandb/spec/channel/deployer/deployer.go b/pkg/wandb/spec/channel/deployer/deployer.go index 918cef9e..8344c5bd 100644 --- a/pkg/wandb/spec/channel/deployer/deployer.go +++ b/pkg/wandb/spec/channel/deployer/deployer.go @@ -25,6 +25,8 @@ type GetSpecOptions struct { License string ActiveState *spec.Spec ReleaseId string + Debug bool + RetryDelay time.Duration } //counterfeiter:generate . DeployerInterface @@ -63,6 +65,11 @@ func (c *DeployerClient) GetSpec(opts GetSpecOptions) (*spec.Spec, error) { client := &http.Client{} + retryDelay := opts.RetryDelay + if retryDelay == 0 { + retryDelay = 2 * time.Second + } + maxRetries := 5 for i := 0; i < maxRetries; i++ { req, err := http.NewRequest(http.MethodGet, url, nil) @@ -77,7 +84,7 @@ func (c *DeployerClient) GetSpec(opts GetSpecOptions) (*spec.Spec, error) { resp, err := client.Do(req) if err != nil || resp.StatusCode != http.StatusOK { - time.Sleep(time.Second * 2) + time.Sleep(retryDelay) continue } defer resp.Body.Close() diff --git a/pkg/wandb/spec/channel/deployer/deployer_test.go b/pkg/wandb/spec/channel/deployer/deployer_test.go index f69a9964..f630cee3 100644 --- a/pkg/wandb/spec/channel/deployer/deployer_test.go +++ b/pkg/wandb/spec/channel/deployer/deployer_test.go @@ -3,131 +3,97 @@ package deployer import ( "net/http" "net/http/httptest" - "reflect" "strings" "testing" + "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/wandb/operator/pkg/wandb/spec" ) -func TestDeployerClient_GetSpec(t *testing.T) { - type fields struct { - testServer func(license string) *httptest.Server - } - type args struct { - license string - activeState *spec.Spec - } - tests := []struct { - name string - fields fields - args args - want *spec.Spec - wantErr bool - }{ - { - "Test the HTTP request has expected headers and returns 200", - fields{ - testServer: func(license string) *httptest.Server { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("Content-Type") != "application/json" { - t.Errorf("Expected Content-Type: application/json header, got: %s", r.Header.Get("Accept")) - } - if username, _, _ := r.BasicAuth(); username != license { - t.Errorf("Expected BasicAuth to match %s, got: %s", license, username) - } - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{}`)) - })) - return server - }, - }, - args{license: "license", activeState: &spec.Spec{}}, - &spec.Spec{}, - false, - }, { - "Test the HTTP request fails repeatedly", - fields{ - testServer: func(license string) *httptest.Server { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusBadGateway) - _, _ = w.Write([]byte(`{}`)) - })) - return server - }, - }, - args{license: "license", activeState: &spec.Spec{}}, - nil, - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - server := tt.fields.testServer(tt.args.license) - defer server.Close() - c := &DeployerClient{ - DeployerAPI: server.URL, +func TestDeployer(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Deployer Suite") +} + +var _ = Describe("DeployerClient", func() { + Describe("GetSpec", func() { + var server *httptest.Server + var client *DeployerClient + + AfterEach(func() { + if server != nil { + server.Close() } - got, err := c.GetSpec(GetSpecOptions{ - License: tt.args.license, - ActiveState: tt.args.activeState, + }) + + Context("when the HTTP request is successful", func() { + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.Header.Get("Content-Type")).To(Equal("application/json"), "Expected Content-Type: application/json header") + username, _, _ := r.BasicAuth() + Expect(username).To(Equal("license"), "Expected BasicAuth to match license") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{}`)) + })) + client = &DeployerClient{ + DeployerAPI: server.URL, + } + }) + + It("should make a request with correct headers and return successfully", func() { + got, err := client.GetSpec(GetSpecOptions{ + License: "license", + ActiveState: &spec.Spec{}, + RetryDelay: time.Millisecond, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(got).To(Equal(&spec.Spec{})) }) - if (err != nil) != tt.wantErr { - t.Errorf("GetSpec() error = %v, wantErr %v", err, tt.wantErr) - return - } - if tt.wantErr && err.Error() != "all retries failed" { - t.Errorf("GetSpec() error = %v, expected %v", err, "all retries failed") - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetSpec() got = %v, want %v", got, tt.want) - } }) - } -} -func TestDeployerClient_getDeployerURL(t *testing.T) { - tests := []struct { - name string - deployerChannelUrl string - deployerReleaseURL string - releaseId string - want string - }{ - { - name: "No releaseId, default channel URL", - releaseId: "", - want: DeployerAPI + DeployerChannelPath, - }, - { - name: "No releaseId, custom channel URL", - deployerChannelUrl: "https://custom-channel.example.com", - releaseId: "", - want: "https://custom-channel.example.com" + DeployerChannelPath, - }, - { - name: "With releaseId, default release URL", - releaseId: "123", - want: DeployerAPI + strings.Replace(DeployerReleaseAPIPath, ":versionId", "123", 1), - }, - { - name: "With releaseId, custom release URL", - deployerChannelUrl: "https://custom-release.example.com", - releaseId: "456", - want: "https://custom-release.example.com/api/v1/operator/channel/release/456", - }, - } + Context("when the HTTP request fails repeatedly", func() { + BeforeEach(func() { + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte(`{}`)) + })) + client = &DeployerClient{ + DeployerAPI: server.URL, + } + }) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := &DeployerClient{ - DeployerAPI: tt.deployerChannelUrl, - } - got := c.getDeployerURL(GetSpecOptions{ReleaseId: tt.releaseId}) - if got != tt.want { - t.Errorf("getDeployerURL() = %v, want %v", got, tt.want) - } + It("should return an error after all retries fail", func() { + got, err := client.GetSpec(GetSpecOptions{ + License: "license", + ActiveState: &spec.Spec{}, + RetryDelay: 10 * time.Millisecond, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("all retries failed")) + Expect(got).To(BeNil()) + }) }) - } -} + }) + + Describe("getDeployerURL", func() { + DescribeTable("should return the correct URL based on inputs", + func(deployerChannelUrl, releaseId, expected string) { + client := &DeployerClient{ + DeployerAPI: deployerChannelUrl, + } + got := client.getDeployerURL(GetSpecOptions{ReleaseId: releaseId}) + Expect(got).To(Equal(expected)) + }, + Entry("No releaseId, default channel URL", + "", "", DeployerAPI+DeployerChannelPath), + Entry("No releaseId, custom channel URL", + "https://custom-channel.example.com", "", "https://custom-channel.example.com"+DeployerChannelPath), + Entry("With releaseId, default release URL", + "", "123", DeployerAPI+strings.Replace(DeployerReleaseAPIPath, ":versionId", "123", 1)), + Entry("With releaseId, custom release URL", + "https://custom-release.example.com", "456", "https://custom-release.example.com/api/v1/operator/channel/release/456"), + ) + }) +}) diff --git a/pkg/wandb/spec/charts/repo.go b/pkg/wandb/spec/charts/repo.go index 32948cc5..550f78e1 100644 --- a/pkg/wandb/spec/charts/repo.go +++ b/pkg/wandb/spec/charts/repo.go @@ -2,8 +2,11 @@ package charts import ( "context" + "fmt" + "net/url" "os" "path/filepath" + "strings" "github.com/go-playground/validator/v10" v1 "github.com/wandb/operator/api/v1" @@ -16,6 +19,7 @@ import ( "helm.sh/helm/v3/pkg/repo" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" ) type RepoRelease struct { @@ -25,8 +29,32 @@ type RepoRelease struct { // If version is not set, download latest. Version string `json:"version"` + // Optional repository name override. If not set, will be derived from URL. + RepoName string `json:"repoName,omitempty"` + Password string `json:"password"` Username string `json:"username"` + Debug bool `json:"debug"` +} + +// deriveRepoName generates a repository name from the URL if one isn't explicitly set +func (r RepoRelease) deriveRepoName() (string, error) { + if r.RepoName != "" { + return r.RepoName, nil + } + + parsedURL, err := url.Parse(r.URL) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + // Use hostname without dots as the repo name + repoName := strings.ReplaceAll(parsedURL.Hostname(), ".", "-") + if repoName == "" { + return "", fmt.Errorf("could not derive repository name from URL: %s", r.URL) + } + + return repoName, nil } func (r RepoRelease) Chart() (*chart.Chart, error) { @@ -81,72 +109,146 @@ func (r RepoRelease) Prune( } func (r RepoRelease) downloadChart() (string, error) { + log := ctrllog.Log.WithName("chart-repo") + + repoName, err := r.deriveRepoName() + if err != nil { + log.Error(err, "Failed to derive repository name") + return "", err + } + entry := new(repo.Entry) entry.URL = r.URL - entry.Name = r.Name + entry.Name = repoName entry.Username = r.Username entry.Password = r.Password + // Skip TLS verification for all URLs in development/testing environments + entry.InsecureSkipTLSverify = true + if r.Debug { + log.Info("TLS verification disabled", "url", r.URL) + } + + if r.Debug { + log.Info("Setting up chart repository", + "url", entry.URL, + "name", r.Name, + "repoName", entry.Name, + "username", entry.Username) + } + file := repo.NewFile() file.Update(entry) - // Initialize Helm CLI settings respecting environment variables settings := cli.New() if helmCache := os.Getenv("HELM_CACHE_HOME"); helmCache != "" { settings.RepositoryCache = filepath.Join(helmCache, "repository") + if r.Debug { + log.Info("Using HELM_CACHE_HOME", "path", helmCache) + } } if helmConfig := os.Getenv("HELM_CONFIG_HOME"); helmConfig != "" { settings.RepositoryConfig = filepath.Join(helmConfig, "repositories.yaml") + if r.Debug { + log.Info("Using HELM_CONFIG_HOME", "path", helmConfig) + } + } + + getterOpts := []getter.Option{ + getter.WithBasicAuth(r.Username, r.Password), + getter.WithInsecureSkipVerifyTLS(true), } providers := getter.All(settings) + if r.Debug { + log.Info("Created providers") + } + chartRepo, err := repo.NewChartRepository(entry, providers) if err != nil { + log.Error(err, "Failed to create chart repository") return "", err } + + if r.Debug { + log.Info("Attempting to download index file", + "url", entry.URL, + "username", entry.Username) + } _, err = chartRepo.DownloadIndexFile() if err != nil { + log.Error(err, "Failed to download index file") + return "", fmt.Errorf("failed to download index file from %s: %w", entry.URL, err) + } + + if r.Debug { + log.Info("Successfully downloaded index file", + "chart", r.Name, + "version", r.Version) + } + + if chartRepo.IndexFile == nil { + log.Error(nil, "Index file is nil") + return "", fmt.Errorf("index file is nil") + } + + indexPath := filepath.Join(settings.RepositoryCache, fmt.Sprintf("%s-index.yaml", entry.Name)) + indexFile, err := repo.LoadIndexFile(indexPath) + if err != nil { + log.Error(err, "Failed to load index file") return "", err } - chartURL, err := repo.FindChartInRepoURL( - entry.URL, entry.Name, r.Version, - "", "", "", - providers, - ) + + cv, err := indexFile.Get(r.Name, r.Version) if err != nil { + log.Error(err, "Failed to find chart version") return "", err } + if len(cv.URLs) == 0 { + return "", fmt.Errorf("chart %s version %s has no downloadable URLs", r.Name, r.Version) + } + + chartURL := cv.URLs[0] + if !strings.HasPrefix(chartURL, "http://") && !strings.HasPrefix(chartURL, "https://") { + chartURL = fmt.Sprintf("%s/%s", strings.TrimSuffix(r.URL, "/"), chartURL) + } + if r.Debug { + log.Info("Found chart URL", "url", chartURL) + } + _, cfg, err := helm.InitConfig("") if err != nil { + log.Error(err, "Failed to init helm config") return "", err } + if r.Debug { + log.Info("Setting up chart downloader with auth", "username", r.Username) + } client := downloader.ChartDownloader{ - Verify: downloader.VerifyNever, - Getters: getter.All(settings), - Options: []getter.Option{ - getter.WithBasicAuth(r.Username, r.Password), - // TODO: Add support for other auth methods - // getter.WithPassCredentialsAll(r.PassCredentialsAll), - // getter.WithTLSClientConfig(r.CertFile, r.KeyFile, r.CaFile), - // getter.WithInsecureSkipVerifyTLS(r.InsecureSkipTLSverify), - }, + Verify: downloader.VerifyNever, + Getters: providers, + Options: getterOpts, RegistryClient: cfg.RegistryClient, RepositoryConfig: settings.RepositoryConfig, RepositoryCache: settings.RepositoryCache, } - // Use HELM_DATA_HOME as the destination directory if set, or fallback to a default location dest := filepath.Join(os.Getenv("HELM_DATA_HOME"), "charts") if dest == "" { dest = "./charts" } os.MkdirAll(dest, 0755) + + if r.Debug { + log.Info("Attempting to download chart", "destination", dest) + } saved, _, err := client.DownloadTo(chartURL, r.Version, dest) if err != nil { + log.Error(err, "Failed to download chart") return "", err } - return saved, err + return saved, nil } diff --git a/pkg/wandb/spec/charts/repo_test.go b/pkg/wandb/spec/charts/repo_test.go new file mode 100644 index 00000000..1bb42887 --- /dev/null +++ b/pkg/wandb/spec/charts/repo_test.go @@ -0,0 +1,301 @@ +package charts + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/repo" +) + +func TestRepo(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Repo Suite", Label("charts")) +} + +var _ = SynchronizedBeforeSuite(func() []byte { + // Suite setup code (runs once) + return nil +}, func(data []byte) { + // Node setup code (runs on each parallel node) +}) + +var _ = SynchronizedAfterSuite(func() { + // Node cleanup code +}, func() { + // Suite cleanup code (runs once) +}) + +var _ = Describe("RepoRelease", func() { + var tempDir string + var repoRelease *RepoRelease + var logCap *logCapture + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "repo-test-*") + Expect(err).NotTo(HaveOccurred()) + + repoRelease = &RepoRelease{ + URL: "https://charts.example.com", + Name: "test-chart", + Version: "1.0.0", + Username: "test-user", + Password: "test-pass", + Debug: false, + } + + // Set up mock responses + chartMetadata := &chart.Metadata{ + Version: "1.0.0", + Name: "test-chart", + } + + chartVersion := repo.ChartVersion{ + Metadata: chartMetadata, + URLs: []string{"https://charts.example.com/test-chart-1.0.0.tgz"}, + } + + indexFile := &repo.IndexFile{ + APIVersion: "v1", + Entries: map[string]repo.ChartVersions{ + "test-chart": {&chartVersion}, + }, + } + + // Create required directories + Expect(os.MkdirAll(filepath.Join(tempDir, "cache"), 0755)).To(Succeed()) + Expect(os.MkdirAll(filepath.Join(tempDir, "config"), 0755)).To(Succeed()) + Expect(os.MkdirAll(filepath.Join(tempDir, "data"), 0755)).To(Succeed()) + + // Write index files for different registries + registries := []string{"charts-example-com", "custom-registry"} + for _, registry := range registries { + indexPath := filepath.Join(tempDir, "cache", registry+"-index.yaml") + err = indexFile.WriteFile(indexPath, 0644) + Expect(err).NotTo(HaveOccurred()) + } + + logCap = newLogCapture() + logCap.setup() + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + logCap.teardown() + }) + + Describe("deriveRepoName", func() { + Context("with explicit repo name", func() { + BeforeEach(func() { + repoRelease.RepoName = "explicit-name" + }) + + It("should use the explicit name", func() { + name, err := repoRelease.deriveRepoName() + Expect(err).NotTo(HaveOccurred()) + Expect(name).To(Equal("explicit-name")) + }) + }) + + Context("with various URLs", func() { + DescribeTable("should derive correct repo names", + func(url, expected string) { + repoRelease.URL = url + name, err := repoRelease.deriveRepoName() + Expect(err).NotTo(HaveOccurred()) + Expect(name).To(Equal(expected)) + }, + Entry("simple hostname", "https://example.com", "example-com"), + Entry("subdomain", "https://charts.example.com", "charts-example-com"), + Entry("with port", "https://example.com:8080", "example-com"), + Entry("with path", "https://example.com/charts", "example-com"), + ) + + It("should fail with invalid URL", func() { + repoRelease.URL = "not-a-url" + _, err := repoRelease.deriveRepoName() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not derive repository name from URL")) + }) + }) + }) + + Describe("Chart", func() { + It("should return error when ToLocalRelease fails", func() { + repoRelease.URL = "invalid-url" + chart, err := repoRelease.Chart() + Expect(err).To(HaveOccurred()) + Expect(chart).To(BeNil()) + }) + }) + + Describe("Validate", func() { + Context("with valid configuration", func() { + It("should validate successfully", func() { + err := repoRelease.Validate() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("with missing required fields", func() { + It("should fail validation when URL is missing", func() { + repoRelease.URL = "" + err := repoRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("URL")) + }) + + It("should fail validation when Name is missing", func() { + repoRelease.Name = "" + err := repoRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Name")) + }) + }) + + Context("with invalid URL", func() { + It("should fail validation", func() { + repoRelease.URL = "not-a-url" + err := repoRelease.Validate() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("URL")) + }) + }) + }) + + Describe("ToLocalRelease", func() { + It("should convert to LocalRelease", func() { + local, err := repoRelease.ToLocalRelease() + Expect(err).To(HaveOccurred()) // Expected since we're not fully mocking helm + Expect(local).To(BeNil()) + }) + + It("should fail when downloadChart fails", func() { + repoRelease.URL = "invalid-url" + local, err := repoRelease.ToLocalRelease() + Expect(err).To(HaveOccurred()) + Expect(local).To(BeNil()) + }) + }) + + Describe("Apply", func() { + It("should return error when ToLocalRelease fails", func() { + repoRelease.URL = "invalid-url" + err := repoRelease.Apply(nil, nil, nil, nil, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("Prune", func() { + It("should return error when ToLocalRelease fails", func() { + repoRelease.URL = "invalid-url" + err := repoRelease.Prune(nil, nil, nil, nil, nil) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("downloadChart", func() { + Context("with environment variables", func() { + BeforeEach(func() { + os.Setenv("HELM_CACHE_HOME", filepath.Join(tempDir, "cache")) + os.Setenv("HELM_CONFIG_HOME", filepath.Join(tempDir, "config")) + os.Setenv("HELM_DATA_HOME", filepath.Join(tempDir, "data")) + }) + + AfterEach(func() { + os.Unsetenv("HELM_CACHE_HOME") + os.Unsetenv("HELM_CONFIG_HOME") + os.Unsetenv("HELM_DATA_HOME") + }) + + Context("TLS verification settings", func() { + DescribeTable("should handle TLS verification appropriately", + func(url string, expectSkipTLS bool) { + repoRelease.URL = url + // Create a mock server that immediately returns 404 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + repoRelease.URL = server.URL // Use mock server URL + + _, err := repoRelease.downloadChart() + Expect(err).To(HaveOccurred()) // Expected since we're returning 404 + + // Verify that the error is not related to TLS verification + Expect(err.Error()).NotTo(ContainSubstring("x509: certificate")) + }, + Entry("HTTP URL should skip TLS", "http://example.com", true), + Entry("HTTPS URL should not skip TLS", "https://example.com", false), + Entry("HTTP URL with uppercase should skip TLS", "HTTP://example.com", true), + Entry("HTTPS URL with uppercase should not skip TLS", "HTTPS://example.com", false), + ) + }) + + Context("with different repository URLs", func() { + DescribeTable("should use correct index file", + func(url string) { + repoRelease.URL = url + // Create a mock server that immediately returns 404 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + repoRelease.URL = server.URL // Use mock server URL + + _, err := repoRelease.downloadChart() + Expect(err).To(HaveOccurred()) // Expected since we're returning 404 + // The error should be about HTTP, not about missing index file + Expect(err.Error()).NotTo(ContainSubstring("Failed to load index file")) + }, + Entry("simple hostname", "https://example.com"), + Entry("subdomain", "https://charts.example.com"), + Entry("with port", "https://example.com:8080"), + Entry("with path", "https://example.com/charts"), + ) + }) + + It("should attempt to download chart and log debug info", func() { + // Create a mock server that immediately returns 404 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + repoRelease.URL = server.URL // Use mock server URL + + _, err := repoRelease.downloadChart() + Expect(err).To(HaveOccurred()) + }) + + Context("with invalid chart URL", func() { + BeforeEach(func() { + repoRelease.URL = "invalid-url" + }) + + It("should return error", func() { + path, err := repoRelease.downloadChart() + Expect(err).To(HaveOccurred()) + Expect(path).To(BeEmpty()) + }) + }) + + Context("with non-existent chart version", func() { + BeforeEach(func() { + repoRelease.Version = "9.9.9" + }) + + It("should return error", func() { + path, err := repoRelease.downloadChart() + Expect(err).To(HaveOccurred()) + Expect(path).To(BeEmpty()) + }) + }) + }) + }) +}) diff --git a/pkg/wandb/spec/charts/repo_test_helpers.go b/pkg/wandb/spec/charts/repo_test_helpers.go new file mode 100644 index 00000000..099f1e6f --- /dev/null +++ b/pkg/wandb/spec/charts/repo_test_helpers.go @@ -0,0 +1,83 @@ +package charts + +import ( + "net/http" + "net/http/httptest" + "time" + + "github.com/go-logr/logr" + "github.com/onsi/ginkgo/v2" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" +) + +type mockTransport struct { + responses map[string]*http.Response + timeout time.Duration +} + +func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.timeout > 0 { + time.Sleep(t.timeout) + } + resp := t.responses[req.URL.String()] + if resp == nil { + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: http.NoBody, + }, nil + } + return resp, nil +} + +func setupMockHTTPClient(responses map[string]*http.Response) *http.Client { + return &http.Client{ + Transport: &mockTransport{ + responses: responses, + timeout: 1 * time.Millisecond, + }, + Timeout: 10 * time.Millisecond, + } +} + +func setupTestServer(handler http.HandlerFunc) *httptest.Server { + return httptest.NewServer(handler) +} + +type logCapture struct{} + +func newLogCapture() *logCapture { + return &logCapture{} +} + +func (lc *logCapture) setup() { + logger := logr.New(&ginkgoLogSink{}) + ctrllog.SetLogger(logger) +} + +func (lc *logCapture) teardown() { + ctrllog.SetLogger(logr.Discard()) +} + +type ginkgoLogSink struct{} + +func (g *ginkgoLogSink) Init(info logr.RuntimeInfo) {} + +func (g *ginkgoLogSink) Enabled(level int) bool { + return true +} + +func (g *ginkgoLogSink) Info(level int, msg string, keysAndValues ...interface{}) { + ginkgo.GinkgoWriter.Printf("INFO: %s\n", msg) +} + +func (g *ginkgoLogSink) Error(err error, msg string, keysAndValues ...interface{}) { + ginkgo.GinkgoWriter.Printf("ERROR: %s: %v\n", msg, err) +} + +func (g *ginkgoLogSink) WithValues(keysAndValues ...interface{}) logr.LogSink { + return g +} + +func (g *ginkgoLogSink) WithName(name string) logr.LogSink { + return g +} diff --git a/tilt-settings.sample.json b/tilt-settings.sample.json index b6ee68a1..16d0d012 100644 --- a/tilt-settings.sample.json +++ b/tilt-settings.sample.json @@ -1,9 +1,6 @@ { - "allowedContexts": [ - "docker-desktop", - "minikube", - "kind-kind" - ], + "allowedContexts": ["docker-desktop", "minikube", "kind-kind"], "installMinio": true, - "installWandb": true -} \ No newline at end of file + "installWandb": true, + "wandbCRD": "default" +} From 0ad36df1c9e3254c94967f442d770386f0db2315 Mon Sep 17 00:00:00 2001 From: Zachary Blasczyk Date: Sun, 23 Feb 2025 00:13:28 -0600 Subject: [PATCH 2/2] fix new test --- .github/workflows/run-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index e38aca3b..98f09abc 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -18,6 +18,8 @@ jobs: go-version: 1.24 - name: Install dependencies run: go get . + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest - name: Tests run: make test-coverage