diff --git a/CHANGELOG.md b/CHANGELOG.md index 6afee72e8..8ee84d4a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) - (Feature) Improve Kubernetes clientsets management - Migrate storage-operator CustomResourceDefinition apiVersion to apiextensions.k8s.io/v1 +- (Feature) Add CRD Installer ## [1.2.8](https://github.com/arangodb/kube-arangodb/tree/1.2.8) (2022-02-24) - Do not check License V2 on Community images diff --git a/chart/kube-arangodb/templates/crd/cluster-role-binding.yaml b/chart/kube-arangodb/templates/crd/cluster-role-binding.yaml new file mode 100644 index 000000000..a0355a66f --- /dev/null +++ b/chart/kube-arangodb/templates/crd/cluster-role-binding.yaml @@ -0,0 +1,26 @@ +{{ if .Values.rbac.enabled -}} +{{ if not (eq .Values.operator.scope "namespaced") -}} +{{ if .Values.operator.enableCRDManagement -}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "kube-arangodb.rbac-cluster" . }}-crd + labels: + app.kubernetes.io/name: {{ template "kube-arangodb.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "kube-arangodb.rbac-cluster" . }}-crd +subjects: + - kind: ServiceAccount + name: {{ template "kube-arangodb.operatorName" . }} + namespace: {{ .Release.Namespace }} + +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/chart/kube-arangodb/templates/crd/cluster-role.yaml b/chart/kube-arangodb/templates/crd/cluster-role.yaml new file mode 100644 index 000000000..48c8e58bb --- /dev/null +++ b/chart/kube-arangodb/templates/crd/cluster-role.yaml @@ -0,0 +1,24 @@ +{{ if .Values.rbac.enabled -}} +{{ if not (eq .Values.operator.scope "namespaced") -}} +{{ if .Values.operator.enableCRDManagement -}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "kube-arangodb.rbac-cluster" . }}-crd + labels: + app.kubernetes.io/name: {{ template "kube-arangodb.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + release: {{ .Release.Name }} +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch", "update", "delete"] + resourceNames: + - "arangoclustersynchronizations.database.arangodb.com" + +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/chart/kube-arangodb/values.yaml b/chart/kube-arangodb/values.yaml index 711afc602..3b5f6100a 100644 --- a/chart/kube-arangodb/values.yaml +++ b/chart/kube-arangodb/values.yaml @@ -35,6 +35,8 @@ operator: allowChaos: false nodeSelector: {} + + enableCRDManagement: true features: deployment: true diff --git a/cmd/main.go b/cmd/main.go index 876d075a5..a9e9b8708 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -61,6 +61,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" + "github.com/arangodb/kube-arangodb/pkg/crd" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme" "github.com/arangodb/kube-arangodb/pkg/logging" "github.com/arangodb/kube-arangodb/pkg/operator" @@ -120,6 +121,9 @@ var ( singleMode bool scope string } + crdOptions struct { + install bool + } operatorKubernetesOptions struct { maxBatchSize int64 @@ -178,6 +182,7 @@ func init() { f.Int64Var(&operatorKubernetesOptions.maxBatchSize, "kubernetes.max-batch-size", globals.DefaultKubernetesRequestBatchSize, "Size of batch during objects read") f.Float32Var(&operatorKubernetesOptions.qps, "kubernetes.qps", kclient.DefaultQPS, "Number of queries per second for k8s API") f.IntVar(&operatorKubernetesOptions.burst, "kubernetes.burst", kclient.DefaultBurst, "Burst for the k8s API") + f.BoolVar(&crdOptions.install, "crd.install", true, "Install missing CRD if access is possible") f.IntVar(&operatorBackup.concurrentUploads, "backup-concurrent-uploads", globals.DefaultBackupConcurrentUploads, "Number of concurrent uploads per deployment") features.Init(&cmdMain) } @@ -275,6 +280,13 @@ func executeMain(cmd *cobra.Command, args []string) { cliLog.Fatal().Msg("Failed to get client") } + if crdOptions.install { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + crd.EnsureCRD(ctx, logService.MustGetLogger("crd"), client) + } + secrets := client.Kubernetes().CoreV1().Secrets(namespace) // Create operator diff --git a/pkg/crd/apply.go b/pkg/crd/apply.go new file mode 100644 index 000000000..8a858f778 --- /dev/null +++ b/pkg/crd/apply.go @@ -0,0 +1,139 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "context" + "fmt" + + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/util/kclient" + "github.com/rs/zerolog" + authorization "k8s.io/api/authorization/v1" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func EnsureCRD(ctx context.Context, log zerolog.Logger, client kclient.Client) { + crdsLock.Lock() + defer crdsLock.Unlock() + + for crd, spec := range crds { + getAccess := verifyCRDAccess(ctx, client, crd, "get") + + if !getAccess.Allowed { + log.Info().Str("crd", crd).Msgf("Get Operations is not allowed. Continue") + continue + } + + c, err := client.KubernetesExtensions().ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd, meta.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + log.Warn().Err(err).Str("crd", crd).Msgf("Get Operations is not allowed due to error. Continue") + continue + } + + createAccess := verifyCRDAccess(ctx, client, crd, "create") + + if !createAccess.Allowed { + log.Info().Str("crd", crd).Msgf("Create Operations is not allowed but CRD is missing. Continue") + continue + } + + c = &apiextensions.CustomResourceDefinition{ + ObjectMeta: meta.ObjectMeta{ + Name: crd, + Labels: map[string]string{ + Version: string(spec.version), + }, + }, + Spec: spec.spec, + } + + if _, err := client.KubernetesExtensions().ApiextensionsV1().CustomResourceDefinitions().Create(ctx, c, meta.CreateOptions{}); err != nil { + log.Warn().Err(err).Str("crd", crd).Msgf("Create Operations is not allowed due to error. Continue") + continue + } + + log.Info().Str("crd", crd).Msgf("CRD Created") + continue + } + + updateAccess := verifyCRDAccess(ctx, client, crd, "update") + + if !updateAccess.Allowed { + log.Info().Str("crd", crd).Msgf("Update Operations is not allowed. Continue") + continue + } + + if c.ObjectMeta.Labels == nil { + c.ObjectMeta.Labels = map[string]string{} + } + + if v, ok := c.ObjectMeta.Labels[Version]; ok { + if v != "" { + if !isUpdateRequired(spec.version, driver.Version(v)) { + log.Info().Str("crd", crd).Msgf("CRD Update not required") + continue + } + } + } + + c.ObjectMeta.Labels[Version] = string(spec.version) + + c.Spec = spec.spec + + if _, err := client.KubernetesExtensions().ApiextensionsV1().CustomResourceDefinitions().Update(ctx, c, meta.UpdateOptions{}); err != nil { + log.Warn().Err(err).Str("crd", crd).Msgf("Create Operations is not allowed due to error. Continue") + continue + } + log.Info().Str("crd", crd).Msgf("CRD Updated") + } +} + +func verifyCRDAccess(ctx context.Context, client kclient.Client, crd string, verb string) authorization.SubjectAccessReviewStatus { + r, err := verifyCRDAccessRequest(ctx, client, crd, verb) + if err != nil { + return authorization.SubjectAccessReviewStatus{ + Allowed: false, + Reason: fmt.Sprintf("Unable to check access: %s", err.Error()), + } + } + + return r.Status +} + +func verifyCRDAccessRequest(ctx context.Context, client kclient.Client, crd string, verb string) (*authorization.SelfSubjectAccessReview, error) { + review := authorization.SelfSubjectAccessReview{ + Spec: authorization.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorization.ResourceAttributes{ + Verb: verb, + Group: "apiextensions.k8s.io", + Version: "v1", + Resource: "customresourcedefinitions", + Name: crd, + }, + }, + } + + return client.Kubernetes().AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, &review, meta.CreateOptions{}) +} diff --git a/pkg/crd/apply_test.go b/pkg/crd/apply_test.go new file mode 100644 index 000000000..40bd62ee9 --- /dev/null +++ b/pkg/crd/apply_test.go @@ -0,0 +1,39 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "context" + "testing" + + "github.com/arangodb/kube-arangodb/pkg/util/kclient" + "github.com/rs/zerolog/log" + "github.com/stretchr/testify/require" +) + +func Test_Apply(t *testing.T) { + t.Run("Ensure CRD exists", func(t *testing.T) { + c, ok := kclient.GetDefaultFactory().Client() + require.True(t, ok) + + EnsureCRD(context.Background(), log.Logger, c) + }) +} diff --git a/pkg/crd/arangoclustersynchronizations.go b/pkg/crd/arangoclustersynchronizations.go new file mode 100644 index 000000000..ea85ddd05 --- /dev/null +++ b/pkg/crd/arangoclustersynchronizations.go @@ -0,0 +1,75 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +func init() { + registerCRDWithPanic("arangoclustersynchronizations.database.arangodb.com", crd{ + version: "1.0.1", + spec: apiextensions.CustomResourceDefinitionSpec{ + Group: "database.arangodb.com", + Names: apiextensions.CustomResourceDefinitionNames{ + Plural: "arangoclustersynchronizations", + Singular: "arangoclustersynchronization", + ShortNames: []string{ + "arangoclustersync", + }, + Kind: "ArangoClusterSynchronization", + ListKind: "ArangoClusterSynchronizationList", + }, + Scope: apiextensions.NamespaceScoped, + Versions: []apiextensions.CustomResourceDefinitionVersion{ + { + Name: "v1", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + XPreserveUnknownFields: util.NewBool(true), + }, + }, + Served: true, + Storage: true, + Subresources: &apiextensions.CustomResourceSubresources{ + Status: &apiextensions.CustomResourceSubresourceStatus{}, + }, + }, + { + Name: "v2alpha1", + Schema: &apiextensions.CustomResourceValidation{ + OpenAPIV3Schema: &apiextensions.JSONSchemaProps{ + Type: "object", + XPreserveUnknownFields: util.NewBool(true), + }, + }, + Served: true, + Storage: false, + Subresources: &apiextensions.CustomResourceSubresources{ + Status: &apiextensions.CustomResourceSubresourceStatus{}, + }, + }, + }, + }, + }) +} diff --git a/pkg/crd/definitions.go b/pkg/crd/definitions.go new file mode 100644 index 000000000..62edf50d7 --- /dev/null +++ b/pkg/crd/definitions.go @@ -0,0 +1,61 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "sync" + + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/util/errors" + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" +) + +const Version = "arangodb.com/version" + +var ( + crds = map[string]crd{} + + crdsLock sync.Mutex +) + +type crd struct { + version driver.Version + spec apiextensions.CustomResourceDefinitionSpec +} + +func registerCRDWithPanic(name string, crd crd) { + if err := registerCRD(name, crd); err != nil { + panic(err) + } +} + +func registerCRD(name string, crd crd) error { + crdsLock.Lock() + defer crdsLock.Unlock() + + if _, ok := crds[name]; ok { + return errors.Newf("CRD %s already exists", name) + } + + crds[name] = crd + + return nil +} diff --git a/pkg/crd/version.go b/pkg/crd/version.go new file mode 100644 index 000000000..45dfd03a4 --- /dev/null +++ b/pkg/crd/version.go @@ -0,0 +1,57 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "strconv" + "strings" + + "github.com/arangodb/go-driver" +) + +func isVersionValid(a driver.Version) bool { + q := strings.SplitN(string(a), ".", 3) + + if len(q) < 2 { + // We do not have 2 parts + return false + } + + _, err := strconv.Atoi(q[0]) + if err != nil { + return false + } + + _, err = strconv.Atoi(q[1]) + return err == nil +} + +func isUpdateRequired(a, b driver.Version) bool { + if a == b { + return false + } + + if !isVersionValid(b) { + return true + } + + return a.CompareTo(b) > 0 +} diff --git a/pkg/crd/version_test.go b/pkg/crd/version_test.go new file mode 100644 index 000000000..1324f7d76 --- /dev/null +++ b/pkg/crd/version_test.go @@ -0,0 +1,72 @@ +// +// DISCLAIMER +// +// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package crd + +import ( + "testing" + + "github.com/arangodb/go-driver" + "github.com/stretchr/testify/require" +) + +func Test_Versions(t *testing.T) { + type c struct { + update bool + a, b driver.Version + } + + cases := map[string]c{ + "Empty": { + update: false, + }, + "Local to empty": { + update: true, + a: "1.0.0", + b: "", + }, + "Local to nonparsable": { + update: true, + a: "1.0.0", + b: "ffdfdasfdasdfsa", + }, + "Same": { + update: false, + a: "1.0.0", + b: "1.0.0", + }, + "Below": { + update: true, + a: "1.1.0", + b: "1.0.0", + }, + "Above": { + update: false, + a: "1.0.0", + b: "1.1.0", + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + require.Equal(t, c.update, isUpdateRequired(c.a, c.b)) + }) + } +} diff --git a/pkg/util/kclient/client_factory.go b/pkg/util/kclient/client_factory.go index e3a810087..0f3c79bb9 100644 --- a/pkg/util/kclient/client_factory.go +++ b/pkg/util/kclient/client_factory.go @@ -24,11 +24,15 @@ import ( "sync" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + versionedFake "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/fake" "github.com/dchest/uniuri" "github.com/pkg/errors" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned" + monitoringFake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apiextensionsclientFake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" "k8s.io/client-go/kubernetes" + kubernetesFake "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/rest" ) @@ -166,6 +170,10 @@ type Client interface { Monitoring() monitoring.Interface } +func NewFakeClient() Client { + return NewStaticClient(kubernetesFake.NewSimpleClientset(), apiextensionsclientFake.NewSimpleClientset(), versionedFake.NewSimpleClientset(), monitoringFake.NewSimpleClientset()) +} + func NewStaticClient(kubernetes kubernetes.Interface, kubernetesExtensions apiextensionsclient.Interface, arango versioned.Interface, monitoring monitoring.Interface) Client { return &client{ kubernetes: kubernetes,