diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a59d7a2b..48fa9cfeb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,10 +48,12 @@ jobs: key: repo-{{ .Environment.CIRCLE_SHA1 }} - run: DEP_RELEASE_TAG=v0.5.0 curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - run: make machine-controller + - run: make webhook - save_cache: key: machine-controller-{{ .Revision }} paths: - /go/src/github.com/kubermatic/machine-controller + - /go/src/github.com/kubermatic/webhook end-to-end: <<: *defaults steps: diff --git a/.gitignore b/.gitignore index e16f95f27..b5adba2bb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,8 @@ test/tools/verify/verify terraform terraform-provider-hcloud .kubeconfig +examples/*.pem +examples/*.csr +examples/*.srl +webhook +!cmd/webhook diff --git a/Dockerfile b/Dockerfile index 000754e69..eab1fca76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,6 @@ FROM alpine:3.7 RUN apk add --no-cache ca-certificates cdrkit COPY machine-controller /usr/local/bin +COPY webhook /usr/local/bin USER nobody diff --git a/Gopkg.lock b/Gopkg.lock index 2e4ae2ab6..0c553abfb 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -552,6 +552,14 @@ pruneopts = "NUT" revision = "03f2033d19d5860aef995fe360ac7d395cd8ce65" +[[projects]] + branch = "master" + digest = "1:0e9bfc47ab9941ecc3344e580baca5deb4091177e84dd9773b48b38ec26b93d5" + name = "github.com/mattbaird/jsonpatch" + packages = ["."] + pruneopts = "NUT" + revision = "81af80346b1a01caae0cbc27fd3c1ba5b11e189f" + [[projects]] digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6" name = "github.com/matttproud/golang_protobuf_extensions" @@ -1395,6 +1403,7 @@ "github.com/gophercloud/gophercloud/pagination", "github.com/heptiolabs/healthcheck", "github.com/hetznercloud/hcloud-go/hcloud", + "github.com/mattbaird/jsonpatch", "github.com/oklog/run", "github.com/pborman/uuid", "github.com/pmezard/go-difflib/difflib", @@ -1411,6 +1420,7 @@ "golang.org/x/oauth2", "gopkg.in/gcfg.v1", "gopkg.in/ini.v1", + "k8s.io/api/admission/v1beta1", "k8s.io/api/core/v1", "k8s.io/api/policy/v1beta1", "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", @@ -1418,6 +1428,7 @@ "k8s.io/apimachinery/pkg/api/equality", "k8s.io/apimachinery/pkg/api/errors", "k8s.io/apimachinery/pkg/apis/meta/v1", + "k8s.io/apimachinery/pkg/apis/meta/v1/validation", "k8s.io/apimachinery/pkg/fields", "k8s.io/apimachinery/pkg/labels", "k8s.io/apimachinery/pkg/runtime", @@ -1426,10 +1437,13 @@ "k8s.io/apimachinery/pkg/selection", "k8s.io/apimachinery/pkg/types", "k8s.io/apimachinery/pkg/util/errors", + "k8s.io/apimachinery/pkg/util/intstr", "k8s.io/apimachinery/pkg/util/rand", "k8s.io/apimachinery/pkg/util/runtime", "k8s.io/apimachinery/pkg/util/sets", "k8s.io/apimachinery/pkg/util/uuid", + "k8s.io/apimachinery/pkg/util/validation", + "k8s.io/apimachinery/pkg/util/validation/field", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/util/yaml", "k8s.io/apimachinery/pkg/watch", diff --git a/Makefile b/Makefile index 97aa6ab16..a847b089c 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,13 @@ machine-controller: $(shell find cmd pkg -name '*.go') vendor -o machine-controller \ github.com/kubermatic/machine-controller/cmd/controller -docker-image: machine-controller docker-image-nodep +webhook: $(shell find cmd pkg -name '*.go') vendor + go build -v \ + -ldflags '-s -w' \ + -o webhook \ + github.com/kubermatic/machine-controller/cmd/webhook + +docker-image: machine-controller admission-webhook docker-image-nodep # This target exists because in our CI # we do not want to restore the vendor @@ -68,3 +74,31 @@ e2e-cluster: e2e-destroy: ./test/tools/integration/cleanup_machines.sh make -C test/tools/integration destroy + +examples/ca-key.pem: + openssl genrsa -out examples/ca-key.pem 4096 + +examples/ca-cert.pem: examples/ca-key.pem + openssl req -x509 -new -nodes -key examples/ca-key.pem \ + -subj "/C=US/ST=CA/O=Acme/CN=k8s-machine-controller-ca" \ + -sha256 -days 10000 -out examples/ca-cert.pem + +examples/admission-key.pem: examples/ca-cert.pem + openssl genrsa -out examples/admission-key.pem 2048 + chmod 0600 examples/admission-key.pem + +examples/admission-cert.pem: examples/admission-key.pem + openssl req -new -sha256 \ + -key examples/admission-key.pem \ + -subj "/C=US/ST=CA/O=Acme/CN=machine-controller-webhook.kube-system.svc" \ + -out examples/admission.csr + openssl x509 -req -in examples/admission.csr -CA examples/ca-cert.pem \ + -CAkey examples/ca-key.pem -CAcreateserial \ + -out examples/admission-cert.pem -days 10000 -sha256 + +deploy: examples/admission-cert.pem + @cat examples/machine-controller.yaml \ + |sed "s/__admission_ca_cert__/$(shell cat examples/ca-cert.pem|base64 -w0)/g" \ + |sed "s/__admission_cert__/$(shell cat examples/admission-cert.pem|base64 -w0)/g" \ + |sed "s/__admission_key__/$(shell cat examples/admission-key.pem|base64 -w0)/g" \ + |kubectl apply -f - diff --git a/README.md b/README.md index ee9badf8d..8ceb4dd7e 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ ## Deploy the machine-controller -`kubectl apply -f examples/machine-controller.yaml` +`make deploy` ## Creating a machineDeployment ```bash diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go new file mode 100644 index 000000000..7760ae0fb --- /dev/null +++ b/cmd/webhook/main.go @@ -0,0 +1,34 @@ +package main + +import ( + "flag" + + "github.com/golang/glog" + + "github.com/kubermatic/machine-controller/pkg/admission" +) + +var ( + admissionListenAddress string + admissionTLSCertPath string + admissionTLSKeyPath string +) + +func main() { + flag.StringVar(&admissionListenAddress, "listen-address", ":9876", "The address on which the MutatingWebhook will listen on") + flag.StringVar(&admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook") + flag.StringVar(&admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook") + flag.Parse() + + s := admission.New(admissionListenAddress) + if err := s.ListenAndServeTLS(admissionTLSCertPath, admissionTLSKeyPath); err != nil { + glog.Fatalf("Failed to start server: %v", err) + } + defer func() { + if err := s.Close(); err != nil { + glog.Fatalf("Failed to shutdown server: %v", err) + } + }() + glog.Infof("Listening on %s", admissionListenAddress) + select {} +} diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index edb7c51e8..b9bdc8ad6 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -189,6 +189,75 @@ spec: port: 8085 periodSeconds: 5 --- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: machine-controller-webhook + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: machine-controller-webhook + template: + metadata: + labels: + app: machine-controller-webhook + spec: + containers: + - image: kubermatic/machine-controller:latest + imagePullPolicy: IfNotPresent + name: webhook + command: + - /usr/local/bin/webhook + - -logtostderr + - -v=6 + - -listen-address=0.0.0.0:9876 + volumeMounts: + - name: machine-controller-admission-cert + mountPath: /tmp/cert + livenessProbe: + httpGet: + path: /healthz + port: 9876 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 9876 + scheme: HTTPS + periodSeconds: 5 + volumes: + - name: machine-controller-admission-cert + secret: + secretName: machine-controller-admission-cert +--- +apiVersion: v1 +kind: Secret +metadata: + name: machine-controller-admission-cert + namespace: kube-system +data: + "cert.pem": __admission_cert__ + "key.pem": __admission_key__ +--- +apiVersion: v1 +kind: Service +metadata: + name: machine-controller-webhook + namespace: kube-system +spec: + ports: + - name: 443-9876 + port: 443 + protocol: TCP + targetPort: 9876 + selector: + app: machine-controller-webhook + type: ClusterIP +--- apiVersion: v1 kind: ServiceAccount metadata: @@ -404,3 +473,27 @@ subjects: - kind: ServiceAccount name: machine-controller namespace: kube-system +--- +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: machinedeployments.machine-controller.kubermatic.io +webhooks: +- name: machinedeployments.machine-controller.kubermatic.io + failurePolicy: Fail + rules: + - apiGroups: + - "cluster.k8s.io" + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - machinedeployments + clientConfig: + service: + namespace: kube-system + name: machine-controller-webhook + path: /machinedeployments + caBundle: __admission_ca_cert__ diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go new file mode 100644 index 000000000..2b350f0ac --- /dev/null +++ b/pkg/admission/admission.go @@ -0,0 +1,49 @@ +package admission + +import ( + "encoding/json" + "fmt" + "net/http" + "reflect" + "time" + + "github.com/golang/glog" + "github.com/mattbaird/jsonpatch" + + "k8s.io/apimachinery/pkg/runtime" +) + +func New(listenAddress string) *http.Server { + m := http.NewServeMux() + m.HandleFunc("/machinedeployments", handleFuncFactory(mutateMachineDeployments)) + m.HandleFunc("/healthz", healthZHandler) + return &http.Server{ + Addr: listenAddress, + Handler: m, + ReadTimeout: 5 * time.Second, + WriteTimeout: 10 * time.Second, + } +} + +func healthZHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func newJSONPatch(original, current runtime.Object) ([]jsonpatch.JsonPatchOperation, error) { + originalGVK := original.GetObjectKind().GroupVersionKind() + currentGVK := current.GetObjectKind().GroupVersionKind() + if !reflect.DeepEqual(originalGVK, currentGVK) { + return nil, fmt.Errorf("GroupVersionKind %#v is expected to match %#v", originalGVK, currentGVK) + } + ori, err := json.Marshal(original) + if err != nil { + return nil, err + } + glog.V(4).Infof("jsonpatch: Marshaled original: %s", string(ori)) + cur, err := json.Marshal(current) + if err != nil { + return nil, err + } + glog.V(4).Infof("jsonpatch: Marshaled target: %s", string(cur)) + return jsonpatch.CreatePatch(ori, cur) +} diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go new file mode 100644 index 000000000..470aa8b69 --- /dev/null +++ b/pkg/admission/machinedeployments.go @@ -0,0 +1,113 @@ +package admission + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/golang/glog" + + admissionv1beta1 "k8s.io/api/admission/v1beta1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + + clusterv1alpha1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +var ( + codecs = serializer.NewCodecFactory(runtime.NewScheme()) + jsonPatch = admissionv1beta1.PatchTypeJSONPatch +) + +func mutateMachineDeployments(ar admissionv1beta1.AdmissionReview) (*admissionv1beta1.AdmissionResponse, error) { + + machineDeployment := clusterv1alpha1.MachineDeployment{} + if err := json.Unmarshal(ar.Request.Object.Raw, &machineDeployment); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %v", err) + } + machineDeploymentOriginal := machineDeployment.DeepCopy() + + machineDeploymentDefaultingFunction(&machineDeployment) + if errs := validateMachineDeployment(machineDeployment); len(errs) > 0 { + return nil, fmt.Errorf("validation failed: %v", errs) + } + + response := &admissionv1beta1.AdmissionResponse{} + response.Allowed = true + if !apiequality.Semantic.DeepEqual(*machineDeploymentOriginal, machineDeployment) { + patchOpts, err := newJSONPatch(machineDeploymentOriginal, &machineDeployment) + if err != nil { + return nil, fmt.Errorf("failed to create json patch: %v", err) + } + + patchRaw, err := json.Marshal(patchOpts) + if err != nil { + return nil, fmt.Errorf("failed to marshal json patch: %v", err) + } + glog.V(6).Infof("Produced jsonpatch: %s", string(patchRaw)) + + response.Patch = patchRaw + response.PatchType = &jsonPatch + } + + return response, nil +} + +func handleFuncFactory(mutate func(admissionv1beta1.AdmissionReview) (*admissionv1beta1.AdmissionResponse, error)) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + var body []byte + if r.Body != nil { + if data, err := ioutil.ReadAll(r.Body); err == nil { + body = data + } + } + + // verify the content type is accurate + contentType := r.Header.Get("Content-Type") + if contentType != "application/json" { + glog.Errorf("contentType=%s, expect application/json", contentType) + w.WriteHeader(http.StatusBadRequest) + return + } + + var reviewResponse *admissionv1beta1.AdmissionResponse + ar := admissionv1beta1.AdmissionReview{} + deserializer := codecs.UniversalDeserializer() + if _, _, err := deserializer.Decode(body, nil, &ar); err != nil { + glog.Error(err) + reviewResponse.Result = &metav1.Status{Message: err.Error()} + } else { + reviewResponse, err = mutate(ar) + if err != nil { + glog.Errorf("Error mutating: %v", err) + } + reviewResponse.Result = &metav1.Status{Message: fmt.Sprintf("Error mutating: %v", err)} + } + + response := admissionv1beta1.AdmissionReview{} + if reviewResponse != nil { + response.Response = reviewResponse + response.Response.UID = ar.Request.UID + } else { + // Required to not have the apiserver crash with an NPE on older versions + // https://github.com/kubernetes/apiserver/commit/584fe98b6432033007b686f1b8063e05d20d328d + response.Response = &admissionv1beta1.AdmissionResponse{} + } + + // reset the Object and OldObject, they are not needed in a response. + ar.Request.Object = runtime.RawExtension{} + ar.Request.OldObject = runtime.RawExtension{} + + resp, err := json.Marshal(response) + if err != nil { + glog.Errorf("failed to marshal response: %v", err) + return + } + if _, err := w.Write(resp); err != nil { + glog.Errorf("failed to write response: %v", err) + } + } +} diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go new file mode 100644 index 000000000..1c2cf5732 --- /dev/null +++ b/pkg/admission/machinedeployments_validation.go @@ -0,0 +1,132 @@ +package admission + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/intstr" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/validation/field" + "log" + "sigs.k8s.io/cluster-api/pkg/apis/cluster/common" + "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { + log.Printf("Validating fields for MachineDeployment %s\n", md.Name) + allErrs := field.ErrorList{} + allErrs = append(allErrs, validateMachineDeploymentSpec(&md.Spec, field.NewPath("spec"))...) + return allErrs +} + +func validateMachineDeploymentSpec(spec *v1alpha1.MachineDeploymentSpec, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, fldPath.Child("selector"))...) + if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for MachineSet.")) + } + selector, err := metav1.LabelSelectorAsSelector(&spec.Selector) + if err != nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "invalid label selector.")) + } else { + labels := labels.Set(spec.Template.Labels) + if !selector.Matches(labels) { + allErrs = append(allErrs, field.Invalid(fldPath.Child("template", "metadata", "labels"), spec.Template.Labels, "`selector` does not match template `labels`")) + } + } + if spec.Replicas == nil || *spec.Replicas < 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("replicas"), *spec.Replicas, "replicas must be specified and can not be negative")) + } + allErrs = append(allErrs, validateMachineDeploymentStrategy(&spec.Strategy, fldPath.Child("strategy"))...) + return allErrs +} + +func validateMachineDeploymentStrategy(strategy *v1alpha1.MachineDeploymentStrategy, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + switch strategy.Type { + case common.RollingUpdateMachineDeploymentStrategyType: + if strategy.RollingUpdate != nil { + allErrs = append(allErrs, validateMachineRollingUpdateDeployment(strategy.RollingUpdate, fldPath.Child("rollingUpdate"))...) + } + default: + allErrs = append(allErrs, field.Invalid(fldPath.Child("Type"), strategy.Type, "is an invalid type")) + } + return allErrs +} + +func validateMachineRollingUpdateDeployment(rollingUpdate *v1alpha1.MachineRollingUpdateDeployment, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + var maxUnavailable int + var maxSurge int + if rollingUpdate.MaxUnavailable != nil { + allErrs = append(allErrs, validatePositiveIntOrPercent(rollingUpdate.MaxUnavailable, fldPath.Child("maxUnavailable"))...) + maxUnavailable, _ = getIntOrPercent(rollingUpdate.MaxUnavailable, false) + // Validate that MaxUnavailable is not more than 100%. + if len(utilvalidation.IsValidPercent(rollingUpdate.MaxUnavailable.StrVal)) == 0 && maxUnavailable > 100 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("maxUnavailable"), rollingUpdate.MaxUnavailable, "should not be more than 100%")) + } + } + if rollingUpdate.MaxSurge != nil { + allErrs = append(allErrs, validatePositiveIntOrPercent(rollingUpdate.MaxSurge, fldPath.Child("maxSurge"))...) + maxSurge, _ = getIntOrPercent(rollingUpdate.MaxSurge, true) + } + if rollingUpdate.MaxUnavailable != nil && rollingUpdate.MaxSurge != nil && maxUnavailable == 0 && maxSurge == 0 { + // Both MaxSurge and MaxUnavailable cannot be zero. + allErrs = append(allErrs, field.Invalid(fldPath.Child("maxUnavailable"), rollingUpdate.MaxUnavailable, "may not be 0 when `maxSurge` is 0")) + } + return allErrs +} + +func validatePositiveIntOrPercent(s *intstr.IntOrString, fldPath *field.Path) field.ErrorList { + allErrs := field.ErrorList{} + if x, err := getIntOrPercent(s, false); err != nil { + allErrs = append(allErrs, field.Invalid(fldPath, s.StrVal, "value should be int(5) or percentage(5%)")) + } else if x < 0 { + allErrs = append(allErrs, field.Invalid(fldPath, x, "value should not be negative")) + } + return allErrs +} +func getIntOrPercent(s *intstr.IntOrString, roundUp bool) (int, error) { + return intstr.GetValueFromIntOrPercent(s, 100, roundUp) +} + +func machineDeploymentDefaultingFunction(obj *v1alpha1.MachineDeployment) { + // set default field values here + log.Printf("Defaulting fields for MachineDeployment %s\n", obj.Name) + if obj.Spec.Replicas == nil { + obj.Spec.Replicas = new(int32) + *obj.Spec.Replicas = 1 + } + if obj.Spec.MinReadySeconds == nil { + obj.Spec.MinReadySeconds = new(int32) + *obj.Spec.MinReadySeconds = 0 + } + if obj.Spec.RevisionHistoryLimit == nil { + obj.Spec.RevisionHistoryLimit = new(int32) + *obj.Spec.RevisionHistoryLimit = 1 + } + if obj.Spec.ProgressDeadlineSeconds == nil { + obj.Spec.ProgressDeadlineSeconds = new(int32) + *obj.Spec.ProgressDeadlineSeconds = 600 + } + if obj.Spec.Strategy.Type == "" { + obj.Spec.Strategy.Type = common.RollingUpdateMachineDeploymentStrategyType + } + // Default RollingUpdate strategy only if strategy type is RollingUpdate. + if obj.Spec.Strategy.Type == common.RollingUpdateMachineDeploymentStrategyType { + if obj.Spec.Strategy.RollingUpdate == nil { + obj.Spec.Strategy.RollingUpdate = &v1alpha1.MachineRollingUpdateDeployment{} + } + if obj.Spec.Strategy.RollingUpdate.MaxSurge == nil { + x := intstr.FromInt(1) + obj.Spec.Strategy.RollingUpdate.MaxSurge = &x + } + if obj.Spec.Strategy.RollingUpdate.MaxUnavailable == nil { + x := intstr.FromInt(0) + obj.Spec.Strategy.RollingUpdate.MaxUnavailable = &x + } + } + if len(obj.Namespace) == 0 { + obj.Namespace = metav1.NamespaceDefault + } +} diff --git a/test/tools/integration/provision_master.sh b/test/tools/integration/provision_master.sh index 16ec72f72..56e77ece4 100755 --- a/test/tools/integration/provision_master.sh +++ b/test/tools/integration/provision_master.sh @@ -15,9 +15,8 @@ for try in {1..100}; do sleep 1; done - -rsync -av -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ - ../../../{examples/machine-controller.yaml,machine-controller,Dockerfile} \ +rsync -avR -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ + ../../.././{Makefile,examples/machine-controller.yaml,machine-controller,Dockerfile,webhook} \ root@$ADDR:/root/ cat <