From 5f3bc00ff43e9b549e80561f10e29761f8822e86 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Thu, 5 Jul 2018 12:40:08 -0700 Subject: [PATCH 1/8] adding vck-initializer --- vck-initializer/README.md | 3 + .../helloworld-with-annotation.yaml | 35 +++ vck-initializer/deployments/helloworld.yaml | 20 ++ .../deployments/vck-initializer.yaml | 23 ++ vck-initializer/docs/cleanup.md | 11 + .../docs/deploy-vck-initializer.md | 25 ++ .../docs/initializing-deployments-vck.md | 120 ++++++++ .../initializer-configurations/vck.yaml | 13 + vck-initializer/vck-initializer/Dockerfile | 3 + vck-initializer/vck-initializer/README.md | 22 ++ vck-initializer/vck-initializer/build | 1 + .../vck-initializer/build-container | 5 + vck-initializer/vck-initializer/main.go | 256 ++++++++++++++++++ 13 files changed, 537 insertions(+) create mode 100644 vck-initializer/README.md create mode 100644 vck-initializer/deployments/helloworld-with-annotation.yaml create mode 100644 vck-initializer/deployments/helloworld.yaml create mode 100644 vck-initializer/deployments/vck-initializer.yaml create mode 100644 vck-initializer/docs/cleanup.md create mode 100644 vck-initializer/docs/deploy-vck-initializer.md create mode 100644 vck-initializer/docs/initializing-deployments-vck.md create mode 100644 vck-initializer/initializer-configurations/vck.yaml create mode 100644 vck-initializer/vck-initializer/Dockerfile create mode 100644 vck-initializer/vck-initializer/README.md create mode 100755 vck-initializer/vck-initializer/build create mode 100755 vck-initializer/vck-initializer/build-container create mode 100644 vck-initializer/vck-initializer/main.go diff --git a/vck-initializer/README.md b/vck-initializer/README.md new file mode 100644 index 00000000..bb069e29 --- /dev/null +++ b/vck-initializer/README.md @@ -0,0 +1,3 @@ +# VCK Initializer + +The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that appends VCK volumes to deployments based on annotation. \ No newline at end of file diff --git a/vck-initializer/deployments/helloworld-with-annotation.yaml b/vck-initializer/deployments/helloworld-with-annotation.yaml new file mode 100644 index 00000000..840b4d47 --- /dev/null +++ b/vck-initializer/deployments/helloworld-with-annotation.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + annotations: + "initializer.kubernetes.io/vck": '{ + "name": "vck-example3", + "id": "vol2", + "containers": ["helloworld-1"], + "mount-path": "/var/datasets" + }' + + labels: + app: helloworld + envoy: "true" + name: helloworld-with-annotation +spec: + replicas: 1 + template: + metadata: + labels: + app: helloworld + envoy: "true" + name: helloworld-with-annotation + spec: + containers: + - name: helloworld-1 + image: ubuntu:latest + imagePullPolicy: Always + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] + - name: helloworld-2 + image: ubuntu:latest + imagePullPolicy: Always + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/vck-initializer/deployments/helloworld.yaml b/vck-initializer/deployments/helloworld.yaml new file mode 100644 index 00000000..15ed2299 --- /dev/null +++ b/vck-initializer/deployments/helloworld.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + labels: + app: helloworld + name: helloworld +spec: + replicas: 1 + template: + metadata: + labels: + app: helloworld + name: helloworld + spec: + containers: + - name: helloworld + image: ubuntu:latest + imagePullPolicy: Always + command: [ "/bin/bash", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] diff --git a/vck-initializer/deployments/vck-initializer.yaml b/vck-initializer/deployments/vck-initializer.yaml new file mode 100644 index 00000000..03ba4628 --- /dev/null +++ b/vck-initializer/deployments/vck-initializer.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + initializers: + pending: [] + labels: + app: vck-initializer + name: vck-initializer +spec: + replicas: 1 + template: + metadata: + labels: + app: vck-initializer + name: vck-initializer + spec: + containers: + - name: vck-initializer + image: gcr.io/constant-cubist-173123/vck-initializer:0.0.1 + imagePullPolicy: Always + args: + - -logtostderr=true + - "-namespace=vck" diff --git a/vck-initializer/docs/cleanup.md b/vck-initializer/docs/cleanup.md new file mode 100644 index 00000000..f758b7f0 --- /dev/null +++ b/vck-initializer/docs/cleanup.md @@ -0,0 +1,11 @@ +# Cleaning Up + +The following commands will delete the Kubernetes objects associated with this initializer. + +``` +kubectl delete initializerconfiguration vck +``` + +``` +kubectl delete deployment vck-initializer helloworld helloworld-with-annotation +``` \ No newline at end of file diff --git a/vck-initializer/docs/deploy-vck-initializer.md b/vck-initializer/docs/deploy-vck-initializer.md new file mode 100644 index 00000000..d6a82d00 --- /dev/null +++ b/vck-initializer/docs/deploy-vck-initializer.md @@ -0,0 +1,25 @@ +# Deploy The VCK Initializer + +The VCK Initializer is a [Kubernetes Initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects an [Envoy](https://lyft.github.io/envoy) proxy into Deployments based on containers and volumes defined in a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap). + +## Install + +### Create the VCK Initializer Deployment + +Deploy the `VCK-initializer` controller: + +``` +kubectl apply -f deployments/vck-initializer.yaml +``` + +The `vck-initializer` Deployment sets pending initializers to an empty list which bypasses initialization. This prevents the VCK Initializer from getting stuck waiting for initialization, which can happen if the `vck` [Initialization Configuration](initializing-deployments.md#create-the-vck-initializer-InitializerConfiguration) is created before the `vck-initializer` Deployment. + +``` +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + initializers: + pending: [] +``` + +At this point the VCK Initializer is ready to initialize new Deployments. diff --git a/vck-initializer/docs/initializing-deployments-vck.md b/vck-initializer/docs/initializing-deployments-vck.md new file mode 100644 index 00000000..de007df6 --- /dev/null +++ b/vck-initializer/docs/initializing-deployments-vck.md @@ -0,0 +1,120 @@ +# Initializing Deployments Based On Metadata + +It's possible to select which objects are initialized using metadata. The VCK Initializer is configured to only initialize Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. + + +## Deploy the VCK Initializer + +Deploy the VCK Initializer with the `-require-annotation` flag set. This will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value as given below. +``` +"initializer.kubernetes.io/vck": '{ + "name": "vck-example3", \\created volumemanger name + "id": "vol2", \\id of the volumenamger to be injected to the dpeloyment + "containers": ["helloworld-1"], \\ list of container to append the volumemount,can be set to empty to append to all containers + "mount-path": "/var/datasets" \\ location of volumemount + }' +``` +``` +kubectl apply -f deployments/vck-initializer.yaml +``` + +Create the `helloworld` Deployment: + +``` +kubectl apply -f deployments/helloworld.yaml +``` + +Notice the `helloworld` Deployment has been initialized without injecting the VCK mounts: + +``` +kubectl describe deployment helloworld +``` +``` +Name: helloworld +Namespace: vck +CreationTimestamp: Thu, 05 Jul 2018 11:19:13 -0700 +Labels: app=helloworld +Annotations: deployment.kubernetes.io/revision=1 +Selector: app=helloworld +Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 25% max unavailable, 25% max surge +Pod Template: + Labels: app=helloworld + Containers: + helloworld: + Image: ubuntu:latest + Port: + Host Port: + Command: + /bin/bash + -c + -- + Args: + while true; do sleep 30; done; + Environment: + Mounts: + Volumes: +``` + +### Create the helloworld-with-annotation Deployment + +``` +kubectl apply -f deployments/helloworld-with-annotation.yaml +``` + +Notice the `helloworld-with-annotation` Deployment has been initialized with the VCK volumemounts: + +``` +kubectl describe deployment helloworld-with-annotation +``` +``` + +Name: helloworld-with-annotation +Namespace: vck +CreationTimestamp: Thu, 05 Jul 2018 11:20:46 -0700 +Labels: app=helloworld + envoy=true +Annotations: deployment.kubernetes.io/revision=1 + initializer.kubernetes.io/vck={ "name": "vck-example3", "id": "vol2", "containers": ["helloworld-1"], "mount-path": "/var/datasets" } +Selector: app=helloworld,envoy=true +Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable +StrategyType: RollingUpdate +MinReadySeconds: 0 +RollingUpdateStrategy: 25% max unavailable, 25% max surge +Pod Template: + Labels: app=helloworld + envoy=true + Containers: + helloworld-1: + Image: ubuntu:latest + Port: + Host Port: + Command: + /bin/bash + -c + -- + Args: + while true; do sleep 30; done; + Environment: + Mounts: + /var/datasets from dataset-claim (rw) + helloworld-2: + Image: ubuntu:latest + Port: + Host Port: + Command: + /bin/bash + -c + -- + Args: + while true; do sleep 30; done; + Environment: + Mounts: + Volumes: + dataset-claim: + Type: HostPath (bare host directory volume) + Path: /var/datasets/vck-resource-bb444014-7e4c-11e8-9b69-0a580a50012e + HostPathType: +``` diff --git a/vck-initializer/initializer-configurations/vck.yaml b/vck-initializer/initializer-configurations/vck.yaml new file mode 100644 index 00000000..323d0215 --- /dev/null +++ b/vck-initializer/initializer-configurations/vck.yaml @@ -0,0 +1,13 @@ +apiVersion: admissionregistration.k8s.io/v1alpha1 +kind: InitializerConfiguration +metadata: + name: envvckoy +initializers: + - name: vck.initializer.kubernetes.io + rules: + - apiGroups: + - "*" + apiVersions: + - "*" + resources: + - deployments diff --git a/vck-initializer/vck-initializer/Dockerfile b/vck-initializer/vck-initializer/Dockerfile new file mode 100644 index 00000000..5c33f952 --- /dev/null +++ b/vck-initializer/vck-initializer/Dockerfile @@ -0,0 +1,3 @@ +FROM scratch +ADD vck-initializer /vck-initializer +ENTRYPOINT ["/vck-initializer"] diff --git a/vck-initializer/vck-initializer/README.md b/vck-initializer/vck-initializer/README.md new file mode 100644 index 00000000..ead76169 --- /dev/null +++ b/vck-initializer/vck-initializer/README.md @@ -0,0 +1,22 @@ +# VCK Initializer + +The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects the [envoy](https://lyft.github.io/envoy) proxy into a pod based on policy. + +## Usage + +``` +envoy-initializer -h +``` +``` +Usage of envoy-initializer: + -annotation string + The annotation to trigger initialization (default "initializer.kubernetes.io/envoy") + -configmap string + The envoy initializer configuration configmap (default "envoy-initializer") + -initializer-name string + The initializer name (default "envoy.initializer.kubernetes.io") + -namespace string + The configuration namespace (default "default") + -require-annotation + Require annotation for initialization +``` diff --git a/vck-initializer/vck-initializer/build b/vck-initializer/vck-initializer/build new file mode 100755 index 00000000..c34c2ee2 --- /dev/null +++ b/vck-initializer/vck-initializer/build @@ -0,0 +1 @@ +GOOS=linux go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o vck-initializer . diff --git a/vck-initializer/vck-initializer/build-container b/vck-initializer/vck-initializer/build-container new file mode 100755 index 00000000..71f90963 --- /dev/null +++ b/vck-initializer/vck-initializer/build-container @@ -0,0 +1,5 @@ +#!/bin/bash +./build +docker build -t gcr.io/constant-cubist-173123/vck-initializer:0.0.1 . +gcloud docker -- push gcr.io/constant-cubist-173123/vck-initializer:0.0.1 +rm vck-initializer \ No newline at end of file diff --git a/vck-initializer/vck-initializer/main.go b/vck-initializer/vck-initializer/main.go new file mode 100644 index 00000000..b53fb761 --- /dev/null +++ b/vck-initializer/vck-initializer/main.go @@ -0,0 +1,256 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "os/signal" + "syscall" + "time" + + vckv1 "github.com/IntelAI/vck/pkg/apis/vck/v1" + vckv1_client "github.com/IntelAI/vck/pkg/client/clientset/versioned" + "k8s.io/api/apps/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" +) + +const ( + defaultAnnotation = "initializer.kubernetes.io/vck" + defaultInitializerName = "vck.initializer.kubernetes.io" + defaultNamespace = "vck" +) + +var ( + annotation string + initializerName string + namespace string + requireAnnotation bool +) + +type data struct { + Name string `json:"name"` + ID string `json:"id"` + Containers []string `json:"containers,omitempty"` + MountPath string `json:"mount-path"` +} + +type config struct { + Containers []corev1.Container + Volumes []corev1.Volume +} + +func main() { + flag.StringVar(&annotation, "annotation", defaultAnnotation, "The annotation to trigger initialization") + flag.StringVar(&initializerName, "initializer-name", defaultInitializerName, "The initializer name") + flag.StringVar(&namespace, "namespace", defaultNamespace, "The configuration namespace") + flag.BoolVar(&requireAnnotation, "require-annotation", true, "Require annotation for initialization") + flag.Parse() + + log.Println("Starting the Kubernetes initializer...") + log.Printf("Initializer name set to: %s", initializerName) + + clusterConfig, err := rest.InClusterConfig() + if err != nil { + log.Fatal(err.Error()) + } + + clientset, err := kubernetes.NewForConfig(clusterConfig) + if err != nil { + log.Fatal(err) + } + crdClient, err := vckv1_client.NewForConfig(clusterConfig) + if err != nil { + panic(err) + } + + // Watch uninitialized Deployments in all namespaces. + restClient := clientset.AppsV1beta1().RESTClient() + watchlist := cache.NewListWatchFromClient(restClient, "deployments", corev1.NamespaceAll, fields.Everything()) + + // Wrap the returned watchlist to workaround the inability to include + // the `IncludeUninitialized` list option when setting up watch clients. + includeUninitializedWatchlist := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.IncludeUninitialized = true + return watchlist.List(options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.IncludeUninitialized = true + return watchlist.Watch(options) + }, + } + + resyncPeriod := 30 * time.Second + + _, controller := cache.NewInformer(includeUninitializedWatchlist, &v1beta1.Deployment{}, resyncPeriod, + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + o := obj.(*v1beta1.Deployment) + err := initializeDeployment(o, clientset, crdClient) + if err != nil { + log.Println(err) + log.Println("Deleteting Deployment " + o.Name) + deletePolicy := metav1.DeletePropagationBackground + err := clientset.AppsV1().Deployments(o.Namespace).Delete(o.Name, &metav1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }) + if err != nil { + panic(err) + } + log.Println("Deleted Deployment.") + + } + }, + }, + ) + + stop := make(chan struct{}) + go controller.Run(stop) + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + <-signalChan + + log.Println("Shutdown signal received, exiting...") + close(stop) +} + +func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes.Clientset, crdClient *vckv1_client.Clientset) error { + if deployment.ObjectMeta.GetInitializers() != nil { + pendingInitializers := deployment.ObjectMeta.GetInitializers().Pending + + if initializerName == pendingInitializers[0].Name { + log.Printf("Initializing deployment: %s", deployment.Name) + + o := deployment.DeepCopyObject() + initializedDeployment := o.(*v1beta1.Deployment) + + // Remove self from the list of pending Initializers while preserving ordering. + if len(pendingInitializers) == 1 { + initializedDeployment.ObjectMeta.Initializers = nil + } else { + initializedDeployment.ObjectMeta.Initializers.Pending = append(pendingInitializers[:0], pendingInitializers[1:]...) + } + if requireAnnotation { + a := deployment.ObjectMeta.GetAnnotations() + _, ok := a[annotation] + if !ok { + log.Printf("Required '%s' annotation missing; skipping vck initializing", annotation) + _, err := clientset.AppsV1beta1().Deployments(deployment.Namespace).Update(initializedDeployment) + if err != nil { + return err + } + return nil + } + //log.Print("annotation: ", a[annotation]) + info := &data{} + err := json.Unmarshal([]byte(a[annotation]), info) + if err != nil { + return err + } + fmt.Println("Unmarshal:", info.MountPath) + vckVM, err := crdClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) + if err != nil { + return err + } + volumeVCK, affinityVCK, err := addVolumesAffinity(vckVM, info) + if err != nil { + return err + } + if info.Containers == nil { + for _, container := range deployment.Spec.Template.Spec.Containers { + info.Containers = append(info.Containers, container.Name) + } + } + + for _, container := range info.Containers { + volumeMount, containerID, err := addVolumeMount(deployment, vckVM, container, info.MountPath) + if err != nil { + return err + } + initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[containerID].VolumeMounts, *volumeMount) + } + initializedDeployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, *volumeVCK) + initializedDeployment.Spec.Template.Spec.Affinity = affinityVCK + + } + oldData, err := json.Marshal(deployment) + if err != nil { + return err + } + + newData, err := json.Marshal(initializedDeployment) + if err != nil { + return err + } + + patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1beta1.Deployment{}) + if err != nil { + return err + } + + _, err = clientset.AppsV1beta1().Deployments(deployment.Namespace).Patch(deployment.Name, types.StrategicMergePatchType, patchBytes) + if err != nil { + return err + } + } + } + return nil +} + +func addVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, error) { + for _, item := range vckVM.Status.Volumes { + if info.ID == item.ID { + volumeVCK := corev1.Volume{ + Name: "dataset-claim", + VolumeSource: corev1.VolumeSource{ + HostPath: item.VolumeSource.HostPath, + }, + } + affinityVCK := corev1.Affinity{ + NodeAffinity: &item.NodeAffinity, + } + return &volumeVCK, &affinityVCK, nil + } + } + return nil, nil, errors.New("given id for vck does not exists") + +} + +func addVolumeMount(deployment *v1beta1.Deployment, vckVM *vckv1.VolumeManager, container string, mountPath string) (*corev1.VolumeMount, int, error) { + containerID := -1 + for id, item := range deployment.Spec.Template.Spec.Containers { + if container == item.Name { + containerID = id + volumeMount := corev1.VolumeMount{ + MountPath: mountPath, + Name: "dataset-claim", + } + return &volumeMount, containerID, nil + } + } + return nil, -1, errors.New("given container name for vck does not exists ") +} From aed19cdaad00e20a540594a3a480066bff322c1a Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Thu, 5 Jul 2018 12:44:11 -0700 Subject: [PATCH 2/8] updating README.md --- vck-initializer/vck-initializer/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/vck-initializer/vck-initializer/README.md b/vck-initializer/vck-initializer/README.md index ead76169..d3fa78d4 100644 --- a/vck-initializer/vck-initializer/README.md +++ b/vck-initializer/vck-initializer/README.md @@ -1,22 +1,20 @@ # VCK Initializer -The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects the [envoy](https://lyft.github.io/envoy) proxy into a pod based on policy. +The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects the vck volume mounts into the deployment ## Usage ``` -envoy-initializer -h +vck-initializer -h ``` ``` -Usage of envoy-initializer: +Usage of vck-initializer: -annotation string - The annotation to trigger initialization (default "initializer.kubernetes.io/envoy") - -configmap string - The envoy initializer configuration configmap (default "envoy-initializer") + The annotation to trigger initialization (default "initializer.kubernetes.io/vck") -initializer-name string - The initializer name (default "envoy.initializer.kubernetes.io") + The initializer name (default "vck.initializer.kubernetes.io") -namespace string - The configuration namespace (default "default") + The configuration namespace (default "vck") -require-annotation Require annotation for initialization ``` From 69b16c782919477bbc63340f56f2e9e51e89b358 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Tue, 17 Jul 2018 10:47:52 -0700 Subject: [PATCH 3/8] Integrating initializer with vck --- docs/user.md | 26 ++++ .../templates/initializer-configuration.yaml | 4 +- main.go | 7 + .../main.go => pkg/initializer/initializer.go | 133 +++++++++--------- vck-initializer/README.md | 3 - .../helloworld-with-annotation.yaml | 35 ----- vck-initializer/deployments/helloworld.yaml | 20 --- .../deployments/vck-initializer.yaml | 23 --- vck-initializer/docs/cleanup.md | 11 -- .../docs/deploy-vck-initializer.md | 25 ---- .../docs/initializing-deployments-vck.md | 120 ---------------- vck-initializer/vck-initializer/Dockerfile | 3 - vck-initializer/vck-initializer/README.md | 20 --- vck-initializer/vck-initializer/build | 1 - .../vck-initializer/build-container | 5 - 15 files changed, 100 insertions(+), 336 deletions(-) rename vck-initializer/initializer-configurations/vck.yaml => helm-charts/kube-volume-controller/templates/initializer-configuration.yaml (85%) rename vck-initializer/vck-initializer/main.go => pkg/initializer/initializer.go (60%) delete mode 100644 vck-initializer/README.md delete mode 100644 vck-initializer/deployments/helloworld-with-annotation.yaml delete mode 100644 vck-initializer/deployments/helloworld.yaml delete mode 100644 vck-initializer/deployments/vck-initializer.yaml delete mode 100644 vck-initializer/docs/cleanup.md delete mode 100644 vck-initializer/docs/deploy-vck-initializer.md delete mode 100644 vck-initializer/docs/initializing-deployments-vck.md delete mode 100644 vck-initializer/vck-initializer/Dockerfile delete mode 100644 vck-initializer/vck-initializer/README.md delete mode 100755 vck-initializer/vck-initializer/build delete mode 100755 vck-initializer/vck-initializer/build-container diff --git a/docs/user.md b/docs/user.md index 9348767a..a33fe653 100644 --- a/docs/user.md +++ b/docs/user.md @@ -169,6 +169,32 @@ $ kubectl create -f resources/deployments/vck-deployment.yaml deployment "vck-example-deployment" created ``` +## Create a Deployment using the VCK initializer +The VCK Initializer will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. +```yaml + +"initializer.kubernetes.io/vck": '{ + "name": "", + "id": "", + "containers": [ + { + "name": "", + "mount-path" : "" + } + ], + }' +``` + +| Key | Required | Description | Default | +|:----------------------|:---------:|:------------------------------------------------|:--------------| +| name | yes | The VCK name to append volumes to containers | | +| id | no | The id of the volume to append to container | first volume | +| containers | no | Name and MountPath of container | all | +| container.name | yes | Name of the container to append the VCK volume | | +| container.mount-path | no | Path for the VCK to mount the volume | /var/dataset | + +The id key is optional it picks the first volume by default, similarly the container object is optional it picks all containers by default and appends it to default mount path "/var/datasets". If the container object just contains the name,vck is appended to default mount path "/var/datasets". + ## Types of Sources The following source types are currently implemented: * S3: Files present in an S3 bucket and provided as `volumeConfig.sourceURL` in the CR are downloaded/synced onto the number of nodes equal to `volumeConfig.replicas` and made available as a hostPath volume. Node affinity details are provided through `volume.nodeAffinity` to guide the scheduling of pods. diff --git a/vck-initializer/initializer-configurations/vck.yaml b/helm-charts/kube-volume-controller/templates/initializer-configuration.yaml similarity index 85% rename from vck-initializer/initializer-configurations/vck.yaml rename to helm-charts/kube-volume-controller/templates/initializer-configuration.yaml index 323d0215..57726e53 100644 --- a/vck-initializer/initializer-configurations/vck.yaml +++ b/helm-charts/kube-volume-controller/templates/initializer-configuration.yaml @@ -1,7 +1,7 @@ apiVersion: admissionregistration.k8s.io/v1alpha1 kind: InitializerConfiguration metadata: - name: envvckoy + name: vck initializers: - name: vck.initializer.kubernetes.io rules: @@ -10,4 +10,4 @@ initializers: apiVersions: - "*" resources: - - deployments + - deployments \ No newline at end of file diff --git a/main.go b/main.go index fb1cb5ed..76a33a9c 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "context" "flag" + "github.com/IntelAI/vck/pkg/resource/reify" apiv1 "k8s.io/api/core/v1" @@ -13,6 +14,7 @@ import ( "github.com/IntelAI/vck/pkg/controller" "github.com/IntelAI/vck/pkg/handlers" "github.com/IntelAI/vck/pkg/hooks" + initializer "github.com/IntelAI/vck/pkg/initializer" "github.com/IntelAI/vck/pkg/resource" "github.com/IntelAI/vck/pkg/util" corev1 "k8s.io/api/core/v1" @@ -110,6 +112,11 @@ func main() { // Start a controller for instances of our custom resource. controller := controller.New(hooks, crdClient) + + // Start initializer for vck + initializer := initializer.New(k8sClientset, crdClient) + go initializer.RunIntializer() + go controller.Run(ctx, *namespace) <-ctx.Done() diff --git a/vck-initializer/vck-initializer/main.go b/pkg/initializer/initializer.go similarity index 60% rename from vck-initializer/vck-initializer/main.go rename to pkg/initializer/initializer.go index b53fb761..9a9e563d 100644 --- a/vck-initializer/vck-initializer/main.go +++ b/pkg/initializer/initializer.go @@ -1,20 +1,8 @@ -// Copyright 2017 Google Inc. All Rights Reserved. -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main +package initialzer import ( "encoding/json" "errors" - "flag" "fmt" "log" "os" @@ -33,28 +21,30 @@ import ( "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ) const ( - defaultAnnotation = "initializer.kubernetes.io/vck" - defaultInitializerName = "vck.initializer.kubernetes.io" - defaultNamespace = "vck" + defaultAnnotation = "initializer.kubernetes.io/vck" + defaultInitializerName = "vck.initializer.kubernetes.io" + defaultRequireAnnotation = true ) var ( annotation string initializerName string - namespace string requireAnnotation bool ) type data struct { - Name string `json:"name"` - ID string `json:"id"` - Containers []string `json:"containers,omitempty"` - MountPath string `json:"mount-path"` + Name string `json:"name"` + ID string `json:"id,omitempty"` + Containers []containers `json:"containers,omitempty"` +} + +type containers struct { + Name string `json:"name"` + MountPath string `json:"mount-path,omitempty"` } type config struct { @@ -62,36 +52,25 @@ type config struct { Volumes []corev1.Volume } -func main() { - flag.StringVar(&annotation, "annotation", defaultAnnotation, "The annotation to trigger initialization") - flag.StringVar(&initializerName, "initializer-name", defaultInitializerName, "The initializer name") - flag.StringVar(&namespace, "namespace", defaultNamespace, "The configuration namespace") - flag.BoolVar(&requireAnnotation, "require-annotation", true, "Require annotation for initialization") - flag.Parse() - - log.Println("Starting the Kubernetes initializer...") - log.Printf("Initializer name set to: %s", initializerName) - - clusterConfig, err := rest.InClusterConfig() - if err != nil { - log.Fatal(err.Error()) - } +// Initializer watches a deployment and delegates create events +// to a set of supplied callback functions. +type Initializer struct { + ClientSet *kubernetes.Clientset + CRDClient *vckv1_client.Clientset +} - clientset, err := kubernetes.NewForConfig(clusterConfig) - if err != nil { - log.Fatal(err) - } - crdClient, err := vckv1_client.NewForConfig(clusterConfig) - if err != nil { - panic(err) +// New returns a new Initializer. +func New(clientset *kubernetes.Clientset, crdClient *vckv1_client.Clientset) *Initializer { + return &Initializer{ + ClientSet: clientset, + CRDClient: crdClient, } +} - // Watch uninitialized Deployments in all namespaces. - restClient := clientset.AppsV1beta1().RESTClient() +// RunIntializer starts a vck inistializer +func (i *Initializer) RunIntializer() { + restClient := i.ClientSet.AppsV1beta1().RESTClient() watchlist := cache.NewListWatchFromClient(restClient, "deployments", corev1.NamespaceAll, fields.Everything()) - - // Wrap the returned watchlist to workaround the inability to include - // the `IncludeUninitialized` list option when setting up watch clients. includeUninitializedWatchlist := &cache.ListWatch{ ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { options.IncludeUninitialized = true @@ -104,17 +83,16 @@ func main() { } resyncPeriod := 30 * time.Second - _, controller := cache.NewInformer(includeUninitializedWatchlist, &v1beta1.Deployment{}, resyncPeriod, cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { o := obj.(*v1beta1.Deployment) - err := initializeDeployment(o, clientset, crdClient) + err := initializeDeployment(o, i) if err != nil { log.Println(err) log.Println("Deleteting Deployment " + o.Name) deletePolicy := metav1.DeletePropagationBackground - err := clientset.AppsV1().Deployments(o.Namespace).Delete(o.Name, &metav1.DeleteOptions{ + err := i.ClientSet.AppsV1().Deployments(o.Namespace).Delete(o.Name, &metav1.DeleteOptions{ PropagationPolicy: &deletePolicy, }) if err != nil { @@ -136,9 +114,9 @@ func main() { log.Println("Shutdown signal received, exiting...") close(stop) -} -func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes.Clientset, crdClient *vckv1_client.Clientset) error { +} +func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializer) error { if deployment.ObjectMeta.GetInitializers() != nil { pendingInitializers := deployment.ObjectMeta.GetInitializers().Pending @@ -159,7 +137,7 @@ func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes. _, ok := a[annotation] if !ok { log.Printf("Required '%s' annotation missing; skipping vck initializing", annotation) - _, err := clientset.AppsV1beta1().Deployments(deployment.Namespace).Update(initializedDeployment) + _, err := initializer.ClientSet.AppsV1beta1().Deployments(deployment.Namespace).Update(initializedDeployment) if err != nil { return err } @@ -171,23 +149,26 @@ func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes. if err != nil { return err } - fmt.Println("Unmarshal:", info.MountPath) - vckVM, err := crdClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) + fmt.Println("Unmarshal:", info.Name) + vckVM, err := initializer.CRDClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) if err != nil { return err } - volumeVCK, affinityVCK, err := addVolumesAffinity(vckVM, info) + volumeVCK, affinityVCK, err := getVolumesAffinity(vckVM, info) if err != nil { return err } - if info.Containers == nil { + if len(info.Containers) == 0 { for _, container := range deployment.Spec.Template.Spec.Containers { - info.Containers = append(info.Containers, container.Name) + tempContainer := containers{ + Name: container.Name, + MountPath: "/var/datasets", + } + info.Containers = append(info.Containers, tempContainer) } } - for _, container := range info.Containers { - volumeMount, containerID, err := addVolumeMount(deployment, vckVM, container, info.MountPath) + volumeMount, containerID, err := addVolumeMount(deployment, vckVM.Name, container) if err != nil { return err } @@ -212,7 +193,7 @@ func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes. return err } - _, err = clientset.AppsV1beta1().Deployments(deployment.Namespace).Patch(deployment.Name, types.StrategicMergePatchType, patchBytes) + _, err = initializer.ClientSet.AppsV1beta1().Deployments(deployment.Namespace).Patch(deployment.Name, types.StrategicMergePatchType, patchBytes) if err != nil { return err } @@ -221,11 +202,24 @@ func initializeDeployment(deployment *v1beta1.Deployment, clientset *kubernetes. return nil } -func addVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, error) { +func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, error) { + if len(info.ID) == 0 { + item := vckVM.Status.Volumes[0] + volumeVCK := corev1.Volume{ + Name: vckVM.Name, + VolumeSource: corev1.VolumeSource{ + HostPath: item.VolumeSource.HostPath, + }, + } + affinityVCK := corev1.Affinity{ + NodeAffinity: &item.NodeAffinity, + } + return &volumeVCK, &affinityVCK, nil + } for _, item := range vckVM.Status.Volumes { if info.ID == item.ID { volumeVCK := corev1.Volume{ - Name: "dataset-claim", + Name: vckVM.Name, VolumeSource: corev1.VolumeSource{ HostPath: item.VolumeSource.HostPath, }, @@ -240,14 +234,17 @@ func addVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, } -func addVolumeMount(deployment *v1beta1.Deployment, vckVM *vckv1.VolumeManager, container string, mountPath string) (*corev1.VolumeMount, int, error) { +func addVolumeMount(deployment *v1beta1.Deployment, name string, container containers) (*corev1.VolumeMount, int, error) { containerID := -1 + if len(container.MountPath) == 0 { + container.MountPath = "/var/datasets" + } for id, item := range deployment.Spec.Template.Spec.Containers { - if container == item.Name { + if container.Name == item.Name { containerID = id volumeMount := corev1.VolumeMount{ - MountPath: mountPath, - Name: "dataset-claim", + MountPath: container.MountPath, + Name: name, } return &volumeMount, containerID, nil } diff --git a/vck-initializer/README.md b/vck-initializer/README.md deleted file mode 100644 index bb069e29..00000000 --- a/vck-initializer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# VCK Initializer - -The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that appends VCK volumes to deployments based on annotation. \ No newline at end of file diff --git a/vck-initializer/deployments/helloworld-with-annotation.yaml b/vck-initializer/deployments/helloworld-with-annotation.yaml deleted file mode 100644 index 840b4d47..00000000 --- a/vck-initializer/deployments/helloworld-with-annotation.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - annotations: - "initializer.kubernetes.io/vck": '{ - "name": "vck-example3", - "id": "vol2", - "containers": ["helloworld-1"], - "mount-path": "/var/datasets" - }' - - labels: - app: helloworld - envoy: "true" - name: helloworld-with-annotation -spec: - replicas: 1 - template: - metadata: - labels: - app: helloworld - envoy: "true" - name: helloworld-with-annotation - spec: - containers: - - name: helloworld-1 - image: ubuntu:latest - imagePullPolicy: Always - command: [ "/bin/bash", "-c", "--" ] - args: [ "while true; do sleep 30; done;" ] - - name: helloworld-2 - image: ubuntu:latest - imagePullPolicy: Always - command: [ "/bin/bash", "-c", "--" ] - args: [ "while true; do sleep 30; done;" ] diff --git a/vck-initializer/deployments/helloworld.yaml b/vck-initializer/deployments/helloworld.yaml deleted file mode 100644 index 15ed2299..00000000 --- a/vck-initializer/deployments/helloworld.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - labels: - app: helloworld - name: helloworld -spec: - replicas: 1 - template: - metadata: - labels: - app: helloworld - name: helloworld - spec: - containers: - - name: helloworld - image: ubuntu:latest - imagePullPolicy: Always - command: [ "/bin/bash", "-c", "--" ] - args: [ "while true; do sleep 30; done;" ] diff --git a/vck-initializer/deployments/vck-initializer.yaml b/vck-initializer/deployments/vck-initializer.yaml deleted file mode 100644 index 03ba4628..00000000 --- a/vck-initializer/deployments/vck-initializer.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - initializers: - pending: [] - labels: - app: vck-initializer - name: vck-initializer -spec: - replicas: 1 - template: - metadata: - labels: - app: vck-initializer - name: vck-initializer - spec: - containers: - - name: vck-initializer - image: gcr.io/constant-cubist-173123/vck-initializer:0.0.1 - imagePullPolicy: Always - args: - - -logtostderr=true - - "-namespace=vck" diff --git a/vck-initializer/docs/cleanup.md b/vck-initializer/docs/cleanup.md deleted file mode 100644 index f758b7f0..00000000 --- a/vck-initializer/docs/cleanup.md +++ /dev/null @@ -1,11 +0,0 @@ -# Cleaning Up - -The following commands will delete the Kubernetes objects associated with this initializer. - -``` -kubectl delete initializerconfiguration vck -``` - -``` -kubectl delete deployment vck-initializer helloworld helloworld-with-annotation -``` \ No newline at end of file diff --git a/vck-initializer/docs/deploy-vck-initializer.md b/vck-initializer/docs/deploy-vck-initializer.md deleted file mode 100644 index d6a82d00..00000000 --- a/vck-initializer/docs/deploy-vck-initializer.md +++ /dev/null @@ -1,25 +0,0 @@ -# Deploy The VCK Initializer - -The VCK Initializer is a [Kubernetes Initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects an [Envoy](https://lyft.github.io/envoy) proxy into Deployments based on containers and volumes defined in a [ConfigMap](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap). - -## Install - -### Create the VCK Initializer Deployment - -Deploy the `VCK-initializer` controller: - -``` -kubectl apply -f deployments/vck-initializer.yaml -``` - -The `vck-initializer` Deployment sets pending initializers to an empty list which bypasses initialization. This prevents the VCK Initializer from getting stuck waiting for initialization, which can happen if the `vck` [Initialization Configuration](initializing-deployments.md#create-the-vck-initializer-InitializerConfiguration) is created before the `vck-initializer` Deployment. - -``` -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - initializers: - pending: [] -``` - -At this point the VCK Initializer is ready to initialize new Deployments. diff --git a/vck-initializer/docs/initializing-deployments-vck.md b/vck-initializer/docs/initializing-deployments-vck.md deleted file mode 100644 index de007df6..00000000 --- a/vck-initializer/docs/initializing-deployments-vck.md +++ /dev/null @@ -1,120 +0,0 @@ -# Initializing Deployments Based On Metadata - -It's possible to select which objects are initialized using metadata. The VCK Initializer is configured to only initialize Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. - - -## Deploy the VCK Initializer - -Deploy the VCK Initializer with the `-require-annotation` flag set. This will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value as given below. -``` -"initializer.kubernetes.io/vck": '{ - "name": "vck-example3", \\created volumemanger name - "id": "vol2", \\id of the volumenamger to be injected to the dpeloyment - "containers": ["helloworld-1"], \\ list of container to append the volumemount,can be set to empty to append to all containers - "mount-path": "/var/datasets" \\ location of volumemount - }' -``` -``` -kubectl apply -f deployments/vck-initializer.yaml -``` - -Create the `helloworld` Deployment: - -``` -kubectl apply -f deployments/helloworld.yaml -``` - -Notice the `helloworld` Deployment has been initialized without injecting the VCK mounts: - -``` -kubectl describe deployment helloworld -``` -``` -Name: helloworld -Namespace: vck -CreationTimestamp: Thu, 05 Jul 2018 11:19:13 -0700 -Labels: app=helloworld -Annotations: deployment.kubernetes.io/revision=1 -Selector: app=helloworld -Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable -StrategyType: RollingUpdate -MinReadySeconds: 0 -RollingUpdateStrategy: 25% max unavailable, 25% max surge -Pod Template: - Labels: app=helloworld - Containers: - helloworld: - Image: ubuntu:latest - Port: - Host Port: - Command: - /bin/bash - -c - -- - Args: - while true; do sleep 30; done; - Environment: - Mounts: - Volumes: -``` - -### Create the helloworld-with-annotation Deployment - -``` -kubectl apply -f deployments/helloworld-with-annotation.yaml -``` - -Notice the `helloworld-with-annotation` Deployment has been initialized with the VCK volumemounts: - -``` -kubectl describe deployment helloworld-with-annotation -``` -``` - -Name: helloworld-with-annotation -Namespace: vck -CreationTimestamp: Thu, 05 Jul 2018 11:20:46 -0700 -Labels: app=helloworld - envoy=true -Annotations: deployment.kubernetes.io/revision=1 - initializer.kubernetes.io/vck={ "name": "vck-example3", "id": "vol2", "containers": ["helloworld-1"], "mount-path": "/var/datasets" } -Selector: app=helloworld,envoy=true -Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable -StrategyType: RollingUpdate -MinReadySeconds: 0 -RollingUpdateStrategy: 25% max unavailable, 25% max surge -Pod Template: - Labels: app=helloworld - envoy=true - Containers: - helloworld-1: - Image: ubuntu:latest - Port: - Host Port: - Command: - /bin/bash - -c - -- - Args: - while true; do sleep 30; done; - Environment: - Mounts: - /var/datasets from dataset-claim (rw) - helloworld-2: - Image: ubuntu:latest - Port: - Host Port: - Command: - /bin/bash - -c - -- - Args: - while true; do sleep 30; done; - Environment: - Mounts: - Volumes: - dataset-claim: - Type: HostPath (bare host directory volume) - Path: /var/datasets/vck-resource-bb444014-7e4c-11e8-9b69-0a580a50012e - HostPathType: -``` diff --git a/vck-initializer/vck-initializer/Dockerfile b/vck-initializer/vck-initializer/Dockerfile deleted file mode 100644 index 5c33f952..00000000 --- a/vck-initializer/vck-initializer/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM scratch -ADD vck-initializer /vck-initializer -ENTRYPOINT ["/vck-initializer"] diff --git a/vck-initializer/vck-initializer/README.md b/vck-initializer/vck-initializer/README.md deleted file mode 100644 index d3fa78d4..00000000 --- a/vck-initializer/vck-initializer/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# VCK Initializer - -The VCK Initializer is a [Kubernetes initializer](https://kubernetes.io/docs/admin/extensible-admission-controllers/#what-are-initializers) that injects the vck volume mounts into the deployment - -## Usage - -``` -vck-initializer -h -``` -``` -Usage of vck-initializer: - -annotation string - The annotation to trigger initialization (default "initializer.kubernetes.io/vck") - -initializer-name string - The initializer name (default "vck.initializer.kubernetes.io") - -namespace string - The configuration namespace (default "vck") - -require-annotation - Require annotation for initialization -``` diff --git a/vck-initializer/vck-initializer/build b/vck-initializer/vck-initializer/build deleted file mode 100755 index c34c2ee2..00000000 --- a/vck-initializer/vck-initializer/build +++ /dev/null @@ -1 +0,0 @@ -GOOS=linux go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o vck-initializer . diff --git a/vck-initializer/vck-initializer/build-container b/vck-initializer/vck-initializer/build-container deleted file mode 100755 index 71f90963..00000000 --- a/vck-initializer/vck-initializer/build-container +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -./build -docker build -t gcr.io/constant-cubist-173123/vck-initializer:0.0.1 . -gcloud docker -- push gcr.io/constant-cubist-173123/vck-initializer:0.0.1 -rm vck-initializer \ No newline at end of file From 3d236461c4db69967af4a2d68987f4d3d2304a94 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Tue, 17 Jul 2018 15:46:03 -0700 Subject: [PATCH 4/8] PR changes to docs and variables undefined --- docs/user.md | 34 ++++++++++++- pkg/initializer/initializer.go | 89 ++++++++++++++++------------------ 2 files changed, 75 insertions(+), 48 deletions(-) diff --git a/docs/user.md b/docs/user.md index a33fe653..0bec1442 100644 --- a/docs/user.md +++ b/docs/user.md @@ -6,6 +6,7 @@ * [Create a Volume Manager Custom Resource](#create-a-volume-manager-custom-resource) * [Create a Pod using the Custom Resource Status](#create-a-pod-using-the-custom-resource-status) * [Create a Deployment using the Custom Resource Status](#create-a-deployment-using-the-custom-resource-status) + * [Create a Deployment using the VCK Initializer](#create-a-deployment-using-the-vck-initializer) * [Types of Sources](#types-of-sources) * [Data distribution] (#data-distribution) @@ -168,7 +169,6 @@ command below. $ kubectl create -f resources/deployments/vck-deployment.yaml deployment "vck-example-deployment" created ``` - ## Create a Deployment using the VCK initializer The VCK Initializer will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. ```yaml @@ -194,6 +194,38 @@ The VCK Initializer will ensure the volume manager data is only injected into D | container.mount-path | no | Path for the VCK to mount the volume | /var/dataset | The id key is optional it picks the first volume by default, similarly the container object is optional it picks all containers by default and appends it to default mount path "/var/datasets". If the container object just contains the name,vck is appended to default mount path "/var/datasets". +* Annotation with only name +```yaml + +"initializer.kubernetes.io/vck": '{ + "name": "", + }' +``` + +* Annotation with no containers +```yaml +"initializer.kubernetes.io/vck": '{ + "name": "", + "id": "", + }' +``` + +* Annotation with no container.mount-path +```yaml +"initializer.kubernetes.io/vck": '{ + "name": "", + "id": "", + "containers": [ + { + "name": "", + } + ], + }' +``` + + + + ## Types of Sources The following source types are currently implemented: diff --git a/pkg/initializer/initializer.go b/pkg/initializer/initializer.go index 9a9e563d..b2b379af 100644 --- a/pkg/initializer/initializer.go +++ b/pkg/initializer/initializer.go @@ -12,6 +12,7 @@ import ( vckv1 "github.com/IntelAI/vck/pkg/apis/vck/v1" vckv1_client "github.com/IntelAI/vck/pkg/client/clientset/versioned" + state "github.com/IntelAI/vck/pkg/states" "k8s.io/api/apps/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -25,15 +26,8 @@ import ( ) const ( - defaultAnnotation = "initializer.kubernetes.io/vck" - defaultInitializerName = "vck.initializer.kubernetes.io" - defaultRequireAnnotation = true -) - -var ( - annotation string - initializerName string - requireAnnotation bool + annotation = "initializer.kubernetes.io/vck" + initializerName = "vck.initializer.kubernetes.io" ) type data struct { @@ -132,52 +126,53 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ } else { initializedDeployment.ObjectMeta.Initializers.Pending = append(pendingInitializers[:0], pendingInitializers[1:]...) } - if requireAnnotation { - a := deployment.ObjectMeta.GetAnnotations() - _, ok := a[annotation] - if !ok { - log.Printf("Required '%s' annotation missing; skipping vck initializing", annotation) - _, err := initializer.ClientSet.AppsV1beta1().Deployments(deployment.Namespace).Update(initializedDeployment) - if err != nil { - return err - } - return nil - } - //log.Print("annotation: ", a[annotation]) - info := &data{} - err := json.Unmarshal([]byte(a[annotation]), info) + a := deployment.ObjectMeta.GetAnnotations() + _, ok := a[annotation] + if !ok { + log.Printf("Required '%s' annotation missing; skipping vck initializing", annotation) + _, err := initializer.ClientSet.AppsV1beta1().Deployments(deployment.Namespace).Update(initializedDeployment) if err != nil { return err } - fmt.Println("Unmarshal:", info.Name) - vckVM, err := initializer.CRDClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) - if err != nil { - return err + return nil + } + //log.Print("annotation: ", a[annotation]) + info := &data{} + err := json.Unmarshal([]byte(a[annotation]), info) + if err != nil { + return err + } + fmt.Println("Unmarshal:", info.Name) + vckVM, err := initializer.CRDClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if vckVM.Status.State == state.Running && len(vckVM.Status.Volumes) > 0 { + return errors.New("given vck is not in usable state") + } + volumeVCK, affinityVCK, err := getVolumesAffinity(vckVM, info) + if err != nil { + return err + } + if len(info.Containers) == 0 { + for _, container := range deployment.Spec.Template.Spec.Containers { + tempContainer := containers{ + Name: container.Name, + MountPath: "/var/datasets", + } + info.Containers = append(info.Containers, tempContainer) } - volumeVCK, affinityVCK, err := getVolumesAffinity(vckVM, info) + } + for _, container := range info.Containers { + volumeMount, containerID, err := addVolumeMount(deployment, vckVM.Name, container) if err != nil { return err } - if len(info.Containers) == 0 { - for _, container := range deployment.Spec.Template.Spec.Containers { - tempContainer := containers{ - Name: container.Name, - MountPath: "/var/datasets", - } - info.Containers = append(info.Containers, tempContainer) - } - } - for _, container := range info.Containers { - volumeMount, containerID, err := addVolumeMount(deployment, vckVM.Name, container) - if err != nil { - return err - } - initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[containerID].VolumeMounts, *volumeMount) - } - initializedDeployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, *volumeVCK) - initializedDeployment.Spec.Template.Spec.Affinity = affinityVCK - + initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[containerID].VolumeMounts, *volumeMount) } + initializedDeployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, *volumeVCK) + initializedDeployment.Spec.Template.Spec.Affinity = affinityVCK + oldData, err := json.Marshal(deployment) if err != nil { return err From f1486e203f822ee979cb978723ecfff1672e2880 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Tue, 17 Jul 2018 15:49:52 -0700 Subject: [PATCH 5/8] doc nav changes --- docs/user.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/user.md b/docs/user.md index 0bec1442..28c1431d 100644 --- a/docs/user.md +++ b/docs/user.md @@ -169,8 +169,11 @@ command below. $ kubectl create -f resources/deployments/vck-deployment.yaml deployment "vck-example-deployment" created ``` -## Create a Deployment using the VCK initializer + +## Create a Deployment using the VCK initializer + The VCK Initializer will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. + ```yaml "initializer.kubernetes.io/vck": '{ @@ -194,7 +197,9 @@ The VCK Initializer will ensure the volume manager data is only injected into D | container.mount-path | no | Path for the VCK to mount the volume | /var/dataset | The id key is optional it picks the first volume by default, similarly the container object is optional it picks all containers by default and appends it to default mount path "/var/datasets". If the container object just contains the name,vck is appended to default mount path "/var/datasets". + * Annotation with only name + ```yaml "initializer.kubernetes.io/vck": '{ @@ -203,6 +208,7 @@ The id key is optional it picks the first volume by default, similarly the conta ``` * Annotation with no containers + ```yaml "initializer.kubernetes.io/vck": '{ "name": "", @@ -211,6 +217,7 @@ The id key is optional it picks the first volume by default, similarly the conta ``` * Annotation with no container.mount-path + ```yaml "initializer.kubernetes.io/vck": '{ "name": "", @@ -223,11 +230,8 @@ The id key is optional it picks the first volume by default, similarly the conta }' ``` - - - - ## Types of Sources + The following source types are currently implemented: * S3: Files present in an S3 bucket and provided as `volumeConfig.sourceURL` in the CR are downloaded/synced onto the number of nodes equal to `volumeConfig.replicas` and made available as a hostPath volume. Node affinity details are provided through `volume.nodeAffinity` to guide the scheduling of pods. * NFS: The path exported by an NFS server is mounted and made available as a PVC. From 6559eaa9ed0abf3be965e0d9810d28c661b5e4b8 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Tue, 17 Jul 2018 15:51:21 -0700 Subject: [PATCH 6/8] doc nav changes --- docs/user.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user.md b/docs/user.md index 28c1431d..e4add27e 100644 --- a/docs/user.md +++ b/docs/user.md @@ -170,7 +170,7 @@ $ kubectl create -f resources/deployments/vck-deployment.yaml deployment "vck-example-deployment" created ``` -## Create a Deployment using the VCK initializer +## Create a Deployment using the VCK Initializer The VCK Initializer will ensure the volume manager data is only injected into Deployments with an `initializer.kubernetes.io/vck` annotation set to a non-empty value. From c32dbcfdb2f6a95bd627de7f557fbc1a6ceb837f Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Wed, 18 Jul 2018 13:36:12 -0700 Subject: [PATCH 7/8] error handling for no vck running --- docs/user.md | 6 +++--- pkg/initializer/initializer.go | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/user.md b/docs/user.md index e4add27e..220ea9f8 100644 --- a/docs/user.md +++ b/docs/user.md @@ -203,7 +203,7 @@ The id key is optional it picks the first volume by default, similarly the conta ```yaml "initializer.kubernetes.io/vck": '{ - "name": "", + "name": "" }' ``` @@ -212,7 +212,7 @@ The id key is optional it picks the first volume by default, similarly the conta ```yaml "initializer.kubernetes.io/vck": '{ "name": "", - "id": "", + "id": "" }' ``` @@ -224,7 +224,7 @@ The id key is optional it picks the first volume by default, similarly the conta "id": "", "containers": [ { - "name": "", + "name": "" } ], }' diff --git a/pkg/initializer/initializer.go b/pkg/initializer/initializer.go index b2b379af..fca6a26b 100644 --- a/pkg/initializer/initializer.go +++ b/pkg/initializer/initializer.go @@ -147,10 +147,10 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ if err != nil { return err } - if vckVM.Status.State == state.Running && len(vckVM.Status.Volumes) > 0 { - return errors.New("given vck is not in usable state") + if vckVM.Status.State != state.Running && len(vckVM.Status.Volumes) == 0 { + return errors.New("given vck is not in usable state " + string(vckVM.Status.State)) } - volumeVCK, affinityVCK, err := getVolumesAffinity(vckVM, info) + volumeVCK, affinityVCK, mountName, err := getVolumesAffinity(vckVM, info) if err != nil { return err } @@ -164,7 +164,7 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ } } for _, container := range info.Containers { - volumeMount, containerID, err := addVolumeMount(deployment, vckVM.Name, container) + volumeMount, containerID, err := addVolumeMount(deployment, mountName, container) if err != nil { return err } @@ -197,7 +197,7 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ return nil } -func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, error) { +func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, string, error) { if len(info.ID) == 0 { item := vckVM.Status.Volumes[0] volumeVCK := corev1.Volume{ @@ -209,12 +209,12 @@ func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, affinityVCK := corev1.Affinity{ NodeAffinity: &item.NodeAffinity, } - return &volumeVCK, &affinityVCK, nil + return &volumeVCK, &affinityVCK, (vckVM.Name + vckVM.Status.Volumes[0].ID), nil } - for _, item := range vckVM.Status.Volumes { + for i, item := range vckVM.Status.Volumes { if info.ID == item.ID { volumeVCK := corev1.Volume{ - Name: vckVM.Name, + Name: vckVM.Name + vckVM.Status.Volumes[i].ID, VolumeSource: corev1.VolumeSource{ HostPath: item.VolumeSource.HostPath, }, @@ -222,10 +222,10 @@ func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, affinityVCK := corev1.Affinity{ NodeAffinity: &item.NodeAffinity, } - return &volumeVCK, &affinityVCK, nil + return &volumeVCK, &affinityVCK, (vckVM.Name + vckVM.Status.Volumes[i].ID), nil } } - return nil, nil, errors.New("given id for vck does not exists") + return nil, nil, "", errors.New("given id for vck does not exists") } From 837bd166230b3f0275bb4a32f8a482ce81e9d430 Mon Sep 17 00:00:00 2001 From: Sudesh Shinde Date: Thu, 26 Jul 2018 09:42:30 -0700 Subject: [PATCH 8/8] support multiple vck mount --- pkg/initializer/initializer.go | 119 ++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 39 deletions(-) diff --git a/pkg/initializer/initializer.go b/pkg/initializer/initializer.go index fca6a26b..79c5c7ff 100644 --- a/pkg/initializer/initializer.go +++ b/pkg/initializer/initializer.go @@ -137,41 +137,83 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ return nil } //log.Print("annotation: ", a[annotation]) - info := &data{} - err := json.Unmarshal([]byte(a[annotation]), info) + infoArray := make([]data, 0) + nodeSelectorTermArr := make([]corev1.NodeSelectorTerm, 0) + err := json.Unmarshal([]byte(a[annotation]), &infoArray) if err != nil { return err } - fmt.Println("Unmarshal:", info.Name) - vckVM, err := initializer.CRDClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) - if err != nil { - return err - } - if vckVM.Status.State != state.Running && len(vckVM.Status.Volumes) == 0 { - return errors.New("given vck is not in usable state " + string(vckVM.Status.State)) - } - volumeVCK, affinityVCK, mountName, err := getVolumesAffinity(vckVM, info) - if err != nil { - return err - } - if len(info.Containers) == 0 { - for _, container := range deployment.Spec.Template.Spec.Containers { - tempContainer := containers{ - Name: container.Name, - MountPath: "/var/datasets", - } - info.Containers = append(info.Containers, tempContainer) + fmt.Println("Unmarshal:", infoArray[0].Name) + for _, info := range infoArray { + + vckVM, err := initializer.CRDClient.VckV1().VolumeManagers(deployment.GetNamespace()).Get(info.Name, metav1.GetOptions{}) + if err != nil { + return err } - } - for _, container := range info.Containers { - volumeMount, containerID, err := addVolumeMount(deployment, mountName, container) + if vckVM.Status.State != state.Running && len(vckVM.Status.Volumes) == 0 { + return errors.New("given vck is not in usable state " + string(vckVM.Status.State)) + } + volumeVCK, nodeSelectorTerm, err := getVolumesAffinity(vckVM, &info) if err != nil { return err } - initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts = append(deployment.Spec.Template.Spec.Containers[containerID].VolumeMounts, *volumeMount) + if len(info.Containers) == 0 { + for _, container := range deployment.Spec.Template.Spec.Containers { + tempContainer := containers{ + Name: container.Name, + MountPath: "/var/datasets", + } + info.Containers = append(info.Containers, tempContainer) + } + } + for _, container := range info.Containers { + if info.ID == "" { + info.ID = vckVM.Status.Volumes[0].ID + } + volumeMount, containerID, err := addVolumeMount(deployment, info.Name+info.ID, container) + if err != nil { + return err + } + + initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts = append(initializedDeployment.Spec.Template.Spec.Containers[containerID].VolumeMounts, *volumeMount) + } + found := false + for _, item := range initializedDeployment.Spec.Template.Spec.Volumes { + if item.Name == volumeVCK.Name { + found = true + break + } + } + if !found { + initializedDeployment.Spec.Template.Spec.Volumes = append(initializedDeployment.Spec.Template.Spec.Volumes, *volumeVCK) + } + found = false + for _, item1 := range nodeSelectorTermArr { + for _, item2 := range *nodeSelectorTerm { + if item1.MatchExpressions[0].Key == item2.MatchExpressions[0].Key { + found = true + break + } + } + if found { + break + } + } + if !found { + nodeSelectorTermArr = append(nodeSelectorTermArr, *nodeSelectorTerm...) + } + + } + + vckAffinity := corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: nodeSelectorTermArr, + }, + }, } - initializedDeployment.Spec.Template.Spec.Volumes = append(deployment.Spec.Template.Spec.Volumes, *volumeVCK) - initializedDeployment.Spec.Template.Spec.Affinity = affinityVCK + + initializedDeployment.Spec.Template.Spec.Affinity = &vckAffinity oldData, err := json.Marshal(deployment) if err != nil { @@ -192,40 +234,39 @@ func initializeDeployment(deployment *v1beta1.Deployment, initializer *Initializ if err != nil { return err } + } } return nil } -func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *corev1.Affinity, string, error) { +func getVolumesAffinity(vckVM *vckv1.VolumeManager, info *data) (*corev1.Volume, *[]corev1.NodeSelectorTerm, error) { if len(info.ID) == 0 { item := vckVM.Status.Volumes[0] volumeVCK := corev1.Volume{ - Name: vckVM.Name, + Name: info.Name + vckVM.Status.Volumes[0].ID, VolumeSource: corev1.VolumeSource{ HostPath: item.VolumeSource.HostPath, }, } - affinityVCK := corev1.Affinity{ - NodeAffinity: &item.NodeAffinity, - } - return &volumeVCK, &affinityVCK, (vckVM.Name + vckVM.Status.Volumes[0].ID), nil + nodeSelectorTerm := vckVM.Status.Volumes[0].NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + log.Print("matchExpression: ", nodeSelectorTerm) + return &volumeVCK, &nodeSelectorTerm, nil } for i, item := range vckVM.Status.Volumes { if info.ID == item.ID { volumeVCK := corev1.Volume{ - Name: vckVM.Name + vckVM.Status.Volumes[i].ID, + Name: info.Name + info.ID, VolumeSource: corev1.VolumeSource{ HostPath: item.VolumeSource.HostPath, }, } - affinityVCK := corev1.Affinity{ - NodeAffinity: &item.NodeAffinity, - } - return &volumeVCK, &affinityVCK, (vckVM.Name + vckVM.Status.Volumes[i].ID), nil + nodeSelectorTerm := vckVM.Status.Volumes[i].NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + log.Print("matchExpression: ", nodeSelectorTerm) + return &volumeVCK, &nodeSelectorTerm, nil } } - return nil, nil, "", errors.New("given id for vck does not exists") + return nil, nil, errors.New("given id for vck does not exists") }