diff --git a/operator/Makefile b/operator/Makefile index 456233b0..e281b07b 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -9,7 +9,7 @@ WITH_RELEASE_REPO = KO_DOCKER_REPO=$(RELEASE_REPO) KIT_NAMESPACE ?= kit CONTAINER_IMAGE_REGISTRY ?= ## Extra helm options -HELM_OPTS ?= +HELM_OPTS ?= --set serviceAccount.create=false help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -47,14 +47,16 @@ licenses: ## Verifies dependency licenses and requires GITHUB_TOKEN to be set golicense hack/license-config.hcl bin/operator apply: ## Apply the controller into your ~/.kube/config cluster - KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/ - KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/controller - KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit ko apply --bare -f config/webhook + KO_DOCKER_REPO=$(CONTAINER_IMAGE_REGISTRY)/kit helm template --include-crds kit charts/kit-operator --namespace $(KIT_NAMESPACE) \ + $(HELM_OPTS) \ + --set controller.image=ko://github.com/awslabs/kit/operator/cmd/controller \ + --set webhook.image=ko://github.com/awslabs/kit/operator/cmd/webhook \ + | $(WITH_GOFLAGS) ko apply -B -f - delete: ## Delete the controller from your ~/.kube/config cluster - kubectl delete -f config - kubectl delete -f config/webhook - kubectl delete -f config/controller + helm template kit charts/kit-operator --namespace $(KIT_NAMESPACE) \ + $(HELM_OPTS) \ + | kubectl delete -f - codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/... hack/codegen.sh diff --git a/operator/README.md b/operator/README.md index 6746e35e..a1070813 100644 --- a/operator/README.md +++ b/operator/README.md @@ -71,7 +71,7 @@ EOF 2. Get the admin KUBECONFIG for the guest cluster from the substrate cluster ```bash - kubectl get secret example-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > /tmp/kubeconfig + kubectl get secret ${GUEST_CLUSTER_NAME}-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > /tmp/kubeconfig ``` > NOTE: It takes about 3-4 minutes for the cluster control plane to be available and healthy @@ -108,4 +108,4 @@ EOF --namespace kit \ --cluster ${SUBSTRATE_CLUSTER_NAME} \ --region=$AWS_REGION -``` \ No newline at end of file +``` diff --git a/operator/charts/kit-operator/templates/control-plane-crd.yaml b/operator/charts/kit-operator/crds/control-plane-crd.yaml similarity index 100% rename from operator/charts/kit-operator/templates/control-plane-crd.yaml rename to operator/charts/kit-operator/crds/control-plane-crd.yaml diff --git a/operator/charts/kit-operator/templates/data-plane-crd.yaml b/operator/charts/kit-operator/crds/data-plane-crd.yaml similarity index 100% rename from operator/charts/kit-operator/templates/data-plane-crd.yaml rename to operator/charts/kit-operator/crds/data-plane-crd.yaml diff --git a/operator/charts/kit-operator/templates/webhook/webhook.yaml b/operator/charts/kit-operator/templates/webhook/webhook.yaml index 12665a95..8310c0fd 100644 --- a/operator/charts/kit-operator/templates/webhook/webhook.yaml +++ b/operator/charts/kit-operator/templates/webhook/webhook.yaml @@ -1,7 +1,7 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: - name: defaulting.webhook.controlplane.kit.k8s.sh + name: defaulting.webhook.kit.k8s.sh webhooks: - admissionReviewVersions: ["v1"] clientConfig: @@ -10,7 +10,7 @@ webhooks: namespace: {{ .Release.Namespace }} failurePolicy: Fail sideEffects: None - name: defaulting.webhook.controlplane.kit.k8s.sh + name: defaulting.webhook.kit.k8s.sh rules: - apiGroups: - kit.k8s.sh @@ -19,6 +19,8 @@ webhooks: resources: - controlplanes controlplanes/status + - dataplanes + dataplanes/status operations: - CREATE - UPDATE @@ -28,7 +30,7 @@ webhooks: apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: - name: validation.webhook.controlplane.kit.k8s.sh + name: validation.webhook.kit.k8s.sh webhooks: - admissionReviewVersions: ["v1"] clientConfig: @@ -37,7 +39,7 @@ webhooks: namespace: {{ .Release.Namespace }} failurePolicy: Fail sideEffects: None - name: validation.webhook.controlplane.kit.k8s.sh + name: validation.webhook.kit.k8s.sh rules: - apiGroups: - kit.k8s.sh @@ -46,6 +48,8 @@ webhooks: resources: - controlplanes controlplanes/status + - dataplanes + dataplanes/status operations: - CREATE - UPDATE diff --git a/operator/cmd/webhook/main.go b/operator/cmd/webhook/main.go index 1bd9e476..bedff179 100644 --- a/operator/cmd/webhook/main.go +++ b/operator/cmd/webhook/main.go @@ -18,7 +18,9 @@ import ( "context" "flag" - "github.com/awslabs/kit/operator/pkg/apis/controlplane/v1alpha1" + cpv1alpha1 "github.com/awslabs/kit/operator/pkg/apis/controlplane/v1alpha1" + dpv1alpha1 "github.com/awslabs/kit/operator/pkg/apis/dataplane/v1alpha1" + "k8s.io/apimachinery/pkg/runtime/schema" "knative.dev/pkg/configmap" "knative.dev/pkg/controller" @@ -28,12 +30,14 @@ import ( "knative.dev/pkg/system" "knative.dev/pkg/webhook" "knative.dev/pkg/webhook/certificates" + "knative.dev/pkg/webhook/resourcesemantics" "knative.dev/pkg/webhook/resourcesemantics/defaulting" "knative.dev/pkg/webhook/resourcesemantics/validation" ) var ( - options = Options{} + options = Options{} + kitResources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{} ) type Options struct { @@ -46,6 +50,12 @@ func main() { config := injection.ParseAndGetRESTConfigOrDie() + // merge all kit resource handlers + kitResources = cpv1alpha1.Resources + for gvk, resource := range dpv1alpha1.Resources { + kitResources[gvk] = resource + } + // Controllers and webhook sharedmain.MainWithConfig( webhook.WithOptions(injection.WithNamespaceScope(signals.NewContext(), system.Namespace()), webhook.Options{ @@ -63,9 +73,9 @@ func main() { func NewCRDDefaultingWebhook(ctx context.Context, w configmap.Watcher) *controller.Impl { return defaulting.NewAdmissionController(ctx, - "defaulting.webhook.controlplane.kit.k8s.sh", + "defaulting.webhook.kit.k8s.sh", "/default-resource", - v1alpha1.Resources, + kitResources, InjectContext, true, ) @@ -73,9 +83,9 @@ func NewCRDDefaultingWebhook(ctx context.Context, w configmap.Watcher) *controll func NewCRDValidationWebhook(ctx context.Context, w configmap.Watcher) *controller.Impl { return validation.NewAdmissionController(ctx, - "validation.webhook.controlplane.kit.k8s.sh", + "validation.webhook.kit.k8s.sh", "/validate-resource", - v1alpha1.Resources, + kitResources, InjectContext, true, ) diff --git a/operator/pkg/awsprovider/instances/reconciler.go b/operator/pkg/awsprovider/instances/reconciler.go index 0d168d60..8ee11686 100644 --- a/operator/pkg/awsprovider/instances/reconciler.go +++ b/operator/pkg/awsprovider/instances/reconciler.go @@ -67,6 +67,14 @@ func (c *Controller) Reconcile(ctx context.Context, dataplane *v1alpha1.DataPlan } func (c *Controller) Finalize(ctx context.Context, dataplane *v1alpha1.DataPlane) error { + asg, err := c.getAutoScalingGroup(ctx, AutoScalingGroupNameFor(dataplane)) + if err != nil { + return err + } + // ASG is already gone + if asg == nil { + return nil + } if _, err := c.autoscaling.DeleteAutoScalingGroupWithContext(ctx, &autoscaling.DeleteAutoScalingGroupInput{ AutoScalingGroupName: ptr.String(AutoScalingGroupNameFor(dataplane)), ForceDelete: ptr.Bool(true), // terminate all the nodes in the ASG