Implementation of coscheduling based on CRD [Part 1]

Makefile

@@ -17,33 +17,39 @@ BUILDENVVAR=CGO_ENABLED=0
 
 LOCAL_REGISTRY=localhost:5000/scheduler-plugins
 LOCAL_IMAGE=kube-scheduler:latest
+LOCAL_CONTROLLER_IMAGE=controller:latest
 
 # RELEASE_REGISTRY is the container registry to push
 # into. The default is to push to the staging
 # registry, not production(k8s.gcr.io).
 RELEASE_REGISTRY?=gcr.io/k8s-staging-scheduler-plugins
 RELEASE_VERSION?=$(shell git describe --tags --match "v*")
 RELEASE_IMAGE:=kube-scheduler:$(RELEASE_VERSION)
+RELEASE_CONTROLLER_IMAGE:=controller:$(RELEASE_VERSION)
 
 .PHONY: all
 all: build
 
 .PHONY: build
 build: autogen
-	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/kube-scheduler cmd/main.go
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/kube-scheduler cmd/scheduler/main.go
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/controller cmd/controller/controller.go
 
 .PHONY: local-image
 local-image: clean
-	docker build -t $(LOCAL_REGISTRY)/$(LOCAL_IMAGE) .
+	docker build -f ./build/scheduler/Dockerfile -t $(LOCAL_REGISTRY)/$(LOCAL_IMAGE) .
+	docker build -f ./build/controller/Dockerfile -t $(LOCAL_REGISTRY)/$(LOCAL_CONTROLLER_IMAGE) .
 
 .PHONY: release-image
 release-image: clean
-	docker build -t $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) .
+	docker build -f ./build/scheduler/Dockerfile -t $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) .
+	docker build -f ./build/controller/Dockerfile -t $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE) .
 
 .PHONY: push-release-image
 push-release-image: release-image
 	gcloud auth configure-docker
 	docker push $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)
+	docker push $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE)
 
 .PHONY: update-vendor
 update-vendor:

build/controller/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM golang:1.15.0
+
+WORKDIR /go/src/sigs.k8s.io/scheduler-plugins
+COPY . .
+RUN make
+
+FROM alpine:3.12
+
+COPY --from=0 /go/src/sigs.k8s.io/scheduler-plugins/bin/controller /bin/controller
+
+WORKDIR /bin
+CMD ["controller"]

cmd/controller/app/options.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"github.com/spf13/pflag"
+)
+
+type ServerRunOptions struct {
+	KubeConfig           string
+	MasterUrl            string
+	InCluster            bool
+	ApiServerQPS         int
+	ApiServerBurst       int
+	Workers              int
+	EnableLeaderElection bool
+}
+
+func NewServerRunOptions() *ServerRunOptions {
+	options := &ServerRunOptions{}
+	options.addAllFlags()
+	return options
+}
+
+func (s *ServerRunOptions) addAllFlags() {
+	pflag.BoolVar(&s.InCluster, "incluster", s.InCluster, "If controller run incluster.")
+	pflag.StringVar(&s.MasterUrl, "kubeConfig", s.MasterUrl, "Kube Config path if not run in cluster.")
+	pflag.StringVar(&s.MasterUrl, "masterUrl", s.MasterUrl, "Master Url if not run in cluster.")
+	pflag.IntVar(&s.ApiServerQPS, "qps", 5, "qps of query apiserver.")
+	pflag.IntVar(&s.ApiServerBurst, "burst", 10, "burst of query apiserver.")
+	pflag.IntVar(&s.Workers, "workers", 1, "workers of scheduler-plugin-controllers.")
+	pflag.BoolVar(&s.EnableLeaderElection, "enableLeaderElection", s.EnableLeaderElection, "If EnableLeaderElection for controller.")
+}

cmd/controller/app/server.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"context"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/klog/v2"
+	"os"
+
+	"sigs.k8s.io/scheduler-plugins/pkg/controller"
+	pgclientset "sigs.k8s.io/scheduler-plugins/pkg/generated/clientset/versioned"
+	pgformers "sigs.k8s.io/scheduler-plugins/pkg/generated/informers/externalversions"
+	"sigs.k8s.io/scheduler-plugins/pkg/util"
+)
+
+func newConfig(kubeconfig, master string, inCluster bool) (*restclient.Config, error) {
+	var (
+		config *rest.Config
+		err    error
+	)
+	if inCluster {
+		config, err = rest.InClusterConfig()
+	} else {
+		config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return config, nil
+}
+
+func Run(s *ServerRunOptions) error {
+	ctx := context.Background()
+	config, err := newConfig(s.KubeConfig, s.MasterUrl, s.InCluster)
+	if err != nil {
+		klog.Fatal(err)
+	}
+	config.QPS = float32(s.ApiServerQPS)
+	config.Burst = s.ApiServerBurst
+	stopCh := server.SetupSignalHandler()
+
+	pgClient := pgclientset.NewForConfigOrDie(config)
+	kubeClient := kubernetes.NewForConfigOrDie(config)
+
+	pgInformerFactory := pgformers.NewSharedInformerFactory(pgClient, 0)
+	pgInformer := pgInformerFactory.Scheduling().V1alpha1().PodGroups()
+
+	informerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, 0, informers.WithTweakListOptions(func(opt *metav1.ListOptions) {
+		opt.LabelSelector = util.PodGroupLabel
+	}))
+	podInformer := informerFactory.Core().V1().Pods()
+	ctrl := controller.NewPodGroupController(kubeClient, pgInformer, podInformer, pgClient)
+	pgInformerFactory.Start(stopCh)
+	informerFactory.Start(stopCh)
+	run := func(ctx context.Context) {
+		ctrl.Run(s.Workers, ctx.Done())
+	}
+
+	if !s.EnableLeaderElection {
+		run(ctx)
+	} else {
+		id, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+
+		// add a uniquifier so that two processes on the same host don't accidentally both become active
+		id = id + "_" + string(uuid.NewUUID())
+
+		rl, err := resourcelock.New("endpoints",
+			"kube-system",
+			"sched-plugins-controller",
+			kubeClient.CoreV1(),
+			kubeClient.CoordinationV1(),
+			resourcelock.ResourceLockConfig{
+				Identity: id,
+			})
+		if err != nil {
+			klog.Fatalf("error creating lock: %v", err)
+		}
+
+		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
+			Lock: rl,
+			Callbacks: leaderelection.LeaderCallbacks{
+				OnStartedLeading: run,
+				OnStoppedLeading: func() {
+					klog.Fatalf("leaderelection lost")
+				},
+			},
+			Name: "scheduler-plugins controller",
+		})
+	}
+
+	<-stopCh
+	return nil
+}

cmd/controller/controller.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+	"sigs.k8s.io/scheduler-plugins/cmd/controller/app"
+)
+
+func main() {
+	options := app.NewServerRunOptions()
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+
+	if err := app.Run(options); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

go.mod

@@ -3,6 +3,7 @@ module sigs.k8s.io/scheduler-plugins
 go 1.15
 
 require (
+	github.com/evanphx/json-patch v4.9.0+incompatible
 	github.com/google/go-cmp v0.4.0
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.19.0
@@ -35,6 +36,7 @@ replace (
 	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.0
 	k8s.io/kubectl => k8s.io/kubectl v0.19.0
 	k8s.io/kubelet => k8s.io/kubelet v0.19.0
+	k8s.io/kubernetes => k8s.io/kubernetes v1.19.0
 	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.0
 	k8s.io/metrics => k8s.io/metrics v0.19.0
 	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.0