-
Notifications
You must be signed in to change notification settings - Fork 9
Implement executionhook controller #4
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
60c1dd2
0e6ae7b
b72a53a
307d1f1
20a04ec
f05b3ab
84f28da
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
|
|
||
| # Binaries for programs and plugins | ||
| *.exe | ||
| *.exe~ | ||
| *.dll | ||
| *.so | ||
| *.dylib | ||
| bin | ||
|
|
||
| # Test binary, build with `go test -c` | ||
| *.test | ||
|
|
||
| # Output of the go coverage tool, specifically when used with LiteIDE | ||
| *.out | ||
|
|
||
| # Kubernetes Generated files - skip generated files, except for vendored files | ||
|
|
||
| !vendor/**/zz_generated.* | ||
|
|
||
| # editor and IDE paraphernalia | ||
| .idea | ||
| *.swp | ||
| *.swo | ||
| *~ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| # Build the manager binary | ||
| FROM golang:1.13 as builder | ||
|
|
||
| WORKDIR /workspace | ||
| # Copy the Go Modules manifests | ||
| COPY go.mod go.mod | ||
| COPY go.sum go.sum | ||
| # cache deps before building and copying source so that we don't need to re-download as much | ||
| # and so that source changes don't invalidate our downloaded layer | ||
| RUN go mod download | ||
|
|
||
| # Copy the go source | ||
| COPY main.go main.go | ||
| COPY ./ ./ | ||
|
|
||
| # Build | ||
| RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go | ||
|
|
||
| # Use distroless as minimal base image to package the manager binary | ||
| # Refer to https://github.com/GoogleContainerTools/distroless for more details | ||
| FROM gcr.io/distroless/static:nonroot | ||
| WORKDIR / | ||
| COPY --from=builder /workspace/manager . | ||
| USER nonroot:nonroot | ||
|
|
||
| ENTRYPOINT ["/manager"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| # Copyright 2018 The Kubernetes Authors. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| .DEFAULT_GOAL:=help | ||
|
|
||
| REGISTRY ?= quay.io/ashish_amarnath | ||
| IMAGE_NAME ?= executionhook-controller | ||
| TAG ?= dev | ||
| # Image URL to use all building/pushing image targets | ||
| CONTROLLER_IMAGE ?= $(REGISTRY)/$(IMAGE_NAME):$(TAG) | ||
| # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) | ||
| CRD_OPTIONS ?= "crd:trivialVersions=true" | ||
|
|
||
| # Directories. | ||
| TOOLS_DIR := hack/tools | ||
| TOOLS_BIN_DIR := $(TOOLS_DIR)/bin | ||
|
|
||
| # Tool binaries. | ||
| KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize | ||
| CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen | ||
|
|
||
| $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod | ||
| cd $(TOOLS_DIR); go build -tags=tools -o ./bin/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen | ||
|
|
||
| $(KUSTOMIZE): $(TOOLS_DIR)/go.mod | ||
| cd $(TOOLS_DIR); go build -tags=tools -o ./bin/kustomize sigs.k8s.io/kustomize/kustomize/v3 | ||
|
|
||
| .PHONY: help | ||
| help: ## Display this help | ||
| @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) | ||
|
|
||
| .PHONY: clean-bin | ||
| clean-bin: ## Remove all generated binaries | ||
| rm -rf bin | ||
| rm -rf hack/tools/bin | ||
|
|
||
| .PHONY: all | ||
| all: manager | ||
|
|
||
| .PHONY: test | ||
| test: generate fmt vet manifests ## Run tests | ||
| go test -v ./... -coverprofile cover.out | ||
|
|
||
| .PHONY: manager | ||
| manager: generate fmt vet test ## Build manager binary | ||
| go build -o bin/manager main.go | ||
|
|
||
| .PHONY: install | ||
| install: manifests $(KUSTOMIZE) ## Install CRDs into a cluster in the current context at ~/.kube/config | ||
| $(KUSTOMIZE) build config/crd | kubectl apply -f - | ||
|
|
||
| .PHONY: uninstall | ||
| uninstall: manifests $(KUSTOMIZE) ## Uninstall latest version of CRDs from a cluster in the current context at ~/.kube/config | ||
| $(KUSTOMIZE) build config/crd | kubectl delete -f - | ||
|
|
||
| .PHONY: deploy | ||
| # hacky, works for now. TODO: ashish-amarnath make this better | ||
| deploy: manifests $(KUSTOMIZE) ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config | ||
| cd config/manager && ../../$(KUSTOMIZE) edit set image controller=${CONTROLLER_IMAGE} | ||
| $(KUSTOMIZE) build config/default | kubectl apply -f - | ||
|
|
||
| .PHONY: manifests | ||
| manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. | ||
| $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases | ||
|
|
||
| fmt: ## Run go fmt against code | ||
| go fmt ./... | ||
|
|
||
| vet: ## Run go vet against code | ||
| go vet ./... | ||
|
|
||
| .PHONY: modules | ||
| modules: ## Runs go mod to ensure modules are up-to-date. | ||
| go mod tidy | ||
| cd $(TOOLS_DIR); go mod tidy | ||
|
|
||
| .PHONY: verify-modules | ||
| verify-modules: modules | ||
| @if !(git diff --quiet HEAD -- go.sum go.mod hack/tools/go.mod hack/tools/go.sum); then \ | ||
| echo "go module files are out of date"; exit 1; \ | ||
| fi | ||
|
|
||
| generate: $(CONTROLLER_GEN) ## Generate code | ||
| $(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..." | ||
|
|
||
|
|
||
| .PHONY: verify-gen | ||
| verify-gen: generate manifests | ||
| @if !(git diff --quiet HEAD); then \ | ||
| echo "generated code and manifest files are out of date, run make generate manifests"; exit 1; \ | ||
| fi | ||
|
|
||
| .PHONY: docker-build | ||
| docker-build: test # Build the controller image | ||
| docker build . -t ${CONTROLLER_IMAGE} | ||
|
|
||
| .PHONY: docker-push | ||
| docker-push: docker-build # Push the controller image | ||
| docker push ${CONTROLLER_IMAGE} | ||
|
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| domain: k8s.io | ||
| repo: sigs.k8s.io/execution-hook | ||
| resources: | ||
| - group: apps | ||
| kind: ExecutionHook | ||
| version: v1alpha1 | ||
| - group: apps | ||
| kind: HookAction | ||
| version: v1alpha1 | ||
| version: "2" |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,180 @@ | ||
| /* | ||
| Copyright 2020 The Kubernetes Authors. | ||
|
|
||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package v1alpha1 | ||
|
|
||
| import ( | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| const ( | ||
| ExecutionHookFinalizer = "executionhook.apps.x-k8s.io" | ||
| ) | ||
|
|
||
| // ExecutionHookSpec defines the desired state of ExecutionHook | ||
| // HookActionName is copied to ExecutionHookSpec by the controller such as | ||
| // the Snapshot Controller. | ||
| type ExecutionHookSpec struct { | ||
| // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster | ||
| // Important: Run "make" to regenerate code after modifying this file | ||
|
|
||
| // PodSelection defines how to select pods and containers to run | ||
| // the executionhook. If multiple pod/containers are selected, the action will executed on them | ||
| // asynchronously. If execution ordering is required, caller has to implement the logic and create | ||
| // different hooks in order. | ||
| // This field is required. | ||
| PodSelection PodSelection `json:"podSelection" protobuf:"bytes,1,opt,name=podSelection"` | ||
|
|
||
| // Name of the HookAction. This is required. | ||
| ActionName string `json:"actionName" protobuf:"bytes,2,opt,name=actionName"` | ||
| } | ||
|
|
||
| // PodSelection contains two fields, PodContainerNamesList and PodContainerSelector, | ||
| // where one of them must be defined so that the hook controller knows where to | ||
| // run the hook. | ||
| type PodSelection struct { | ||
| // PodContainerNamesList lists the pods/containers on which the ExecutionHook | ||
| // should be executed. If not specified, the ExecutionHook controller will find | ||
| // all pods and containers based on PodContainerSelector. | ||
| // If both PodContainerNamesList and PodContainerSelector are not | ||
| // specified, the ExecutionHook cannot be executed and it will fail. | ||
| // +optional | ||
| PodContainerNamesList []PodContainerNames `json:"podContainerNamesList,omitempty" protobuf:"bytes,1,rep,name=podContainerNamesList"` | ||
|
|
||
| // PodContainerSelector is for hook controller to find pods and containers | ||
| // based on the pod label selector and container names | ||
| // If PodContainerNamesList is specified, this field will not be used. | ||
| // +optional | ||
| PodContainerSelector *PodContainerSelector `json:"podContainerSelector,omitempty" protobuf:"bytes,2,opt,name=podContainerSelector"` | ||
| } | ||
|
|
||
| // PodContainerNames lists the containers the ExecutionHook should be executed | ||
| // on in a Pod. | ||
| type PodContainerNames struct { | ||
| // This field is required | ||
| PodName string `json:"podName" protobuf:"bytes,1,opt,name=podName"` | ||
|
|
||
| // +optional | ||
| ContainerNames []string `json:"containerNames,omitempty" protobuf:"bytes,2,rep,name=containerNames"` | ||
| } | ||
|
|
||
| // PodContainerSelector defines the selector and containers the ExecutionHook | ||
| // should be executed on. | ||
| type PodContainerSelector struct { | ||
| // PodSelector specifies a label query over a set of pods. | ||
| // +optional | ||
| PodSelector *metav1.LabelSelector `json:"podSelector,omitempty" protobuf:"bytes,1,opt,name=podSelector"` | ||
|
|
||
| // If specified, controller only select the containers that are listed from the selected pods based on PodSelector. | ||
| // Otherwise, all containers of the pods will be selected | ||
| // +optional | ||
| ContainerList []string `json:"containerList,omitempty" protobuf:"bytes,2,rep,name=containerList"` | ||
| } | ||
|
|
||
| // ExecutionHookStatus defines the observed state of ExecutionHook | ||
| type ExecutionHookStatus struct { | ||
| // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster | ||
| // Important: Run "make" to regenerate code after modifying this file | ||
|
|
||
| // This is a list of ContainerExecutionHookStatus, with each status representing | ||
| // information about how hook is executed in a container, including pod name, | ||
| // container name, ActionTimestamp, ActionSucceed, etc. | ||
| // +optional | ||
| HookStatuses []ContainerExecutionHookStatus `json:"hookStatuses,omitempty" protobuf:"bytes,1,rep,name=hookStatuses"` | ||
| } | ||
|
|
||
| // ContainerExecutionHookStatus represents the current state of a hook for a specific | ||
| // container in a pod | ||
| type ContainerExecutionHookStatus struct { | ||
| // This field is required | ||
| PodName string `json:"podName" protobuf:"bytes,1,opt,name=podName"` | ||
|
|
||
| // This field is required | ||
| ContainerName string `json:"containerName" protobuf:"bytes,2,opt,name=containerName"` | ||
|
|
||
| // If not set, it is nil, indicating Action has not started | ||
| // If set, it means Action has started at the specified time | ||
| // +optional | ||
| Timestamp *metav1.Time `json:"actionTimestamp,omitempty" protobuf:"bytes,3,opt,name=actionTimestamp"` | ||
|
|
||
| // Succeed is set to true when the action is executed in the container successfully. | ||
| // It will be set to false if the action cannot be executed successfully after | ||
| // ActionTimeoutSeconds passes. | ||
| // +optional | ||
| Succeed *bool `json:"actionSucceed,omitempty" protobuf:"varint,4,opt,name=actionSucceed"` | ||
|
|
||
| // The last error encountered when executing the action. The hook controller might | ||
| // update this field each time it retries the execution. | ||
| // +optional | ||
| Error *HookError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error"` | ||
| } | ||
|
|
||
| // HookError describes the error occurred from hook execution. | ||
| type HookError struct { | ||
| // Type of the error | ||
| // This is required | ||
| ErrorType ErrorType `json:"errorType" protobuf:"bytes,1,opt,name=errorType"` | ||
|
|
||
| // Error message | ||
| // +optional | ||
| Message *string `json:"message,omitempty" protobuf:"bytes,2,opt,name=message"` | ||
|
|
||
| // More detailed reason why error happens | ||
| // +optional | ||
| Reason *string `json:"reason,omitempty" protobuf:"bytes,3,opt,name=reason"` | ||
|
|
||
| // It indicates when the error occurred | ||
| // +optional | ||
| Timestamp *metav1.Time `json:"timestamp,omitempty" protobuf:"bytes,4,opt,name=timestamp"` | ||
| } | ||
|
|
||
| // ErrorType defines the type of error occurred from hook execution. | ||
| type ErrorType string | ||
|
|
||
| // More error types could be added, e.g., Forbidden, Unauthorized, AlreadyInProgress, etc. | ||
| const ( | ||
| // The execution hook times out | ||
| Timeout ErrorType = "Timeout" | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Another reason to consider using the Conditions pattern. This proposal might encourage end users to build a state machine off the error conditions (which as you point out in the comment) are likely to be extended and change. Across the rest of the API we generally avoid this as we've learned that its brittle.
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the suggestion. I will revisit this as I iterate on the implementation. WDYT? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @kow3ns If Conditions pattern is used, end user can still add more condition types? |
||
|
|
||
| // The execution hook fails with an error | ||
| Error ErrorType = "Error" | ||
| ) | ||
|
|
||
| // +kubebuilder:object:root=true | ||
| // +kubebuilder:resource:path=executionhook,shortName=eh,scope=Namespaced,categories=executionhook | ||
| // +kubebuilder:subresource:status | ||
|
|
||
| // ExecutionHook is the Schema for the executionhook API | ||
| type ExecutionHook struct { | ||
| metav1.TypeMeta `json:",inline"` | ||
| metav1.ObjectMeta `json:"metadata,omitempty"` | ||
|
|
||
| Spec ExecutionHookSpec `json:"spec,omitempty"` | ||
| Status ExecutionHookStatus `json:"status,omitempty"` | ||
| } | ||
|
|
||
| // +kubebuilder:object:root=true | ||
|
|
||
| // ExecutionHookList contains a list of ExecutionHook | ||
| type ExecutionHookList struct { | ||
| metav1.TypeMeta `json:",inline"` | ||
| metav1.ListMeta `json:"metadata,omitempty"` | ||
| Items []ExecutionHook `json:"items"` | ||
| } | ||
|
|
||
| func init() { | ||
| SchemeBuilder.Register(&ExecutionHook{}, &ExecutionHookList{}) | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There can be only one error? That is not an unreasonable assumption. However, this pattern seems to be similar to the Conditions pattern implemented by most built in types. Is there a reason to go with a singular error over Status.Conditions?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The expectation is that the error represents the error condition encountered executing the hook action on a specific container in a pod. So there is a 1:1 mapping between where the hook action was executed (Pod,Container) and an error condition, if any. For that reason, using a singular error was thought of as a better fit. However, using
Status.Conditionswould be more appropriate if/ when we aggregate the errors into the status of theExecutionHookobject.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think we were given recommendations not to use Conditions during the API review, although I couldn't remember the reason behind that. @thockin @jingxu97 do you want to comment on this?