Add controller

Makefile

@@ -23,6 +23,15 @@ all: build
 build: autogen
 	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/kube-scheduler cmd/main.go
 
+.PHONY: build-coscheduling
+build-coscheduling: autogen build-controller build-coscheduler
+
+build-coscheduler:
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/kube-scheduler cmd/coscheduling/scheduler/main.go
+
+build-controller:
+	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/controller cmd/coscheduling/controller/controller.go
+
 .PHONY: local-image
 local-image: autogen
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags '-w' -o ./build/kube-scheduler cmd/main.go

build/coscheduling/controller/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.12
+
+COPY controller /bin/controller
+CMD ["controller"]

build/coscheduling/scheduler/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.12
+
+COPY kube-scheduler /bin/kube-scheduler
+CMD ["kube-scheduler"]

cmd/coscheduling/controller/app/options.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"github.com/spf13/pflag"
+)
+
+type ServerRunOptions struct {
+	KubeConfig           string
+	MasterUrl            string
+	InCluster            bool
+	ApiServerQPS         uint32
+	ApiServerBurst       uint32
+	EnableLeaderElection bool
+}
+
+func NewServerRunOptions() *ServerRunOptions {
+	options := &ServerRunOptions{}
+	options.addAllFlags()
+	return options
+}
+
+func (s *ServerRunOptions) addAllFlags() {
+	pflag.BoolVar(&s.InCluster, "incluster", s.InCluster, "If controller run incluster.")
+	pflag.StringVar(&s.MasterUrl, "kubeConfig", s.MasterUrl, "Kube Config path if not run in cluster.")
+	pflag.StringVar(&s.MasterUrl, "masterUrl", s.MasterUrl, "Master Url if not run in cluster.")
+	pflag.Uint32Var(&s.ApiServerBurst, "qps", 5, "qps of query apiserver.")
+	pflag.Uint32Var(&s.ApiServerQPS, "burst", 10, "burst of query apiserver.")
+	pflag.BoolVar(&s.EnableLeaderElection, "enableLeaderElection", s.EnableLeaderElection, "If EnableLeaderElection for controller.")
+}

cmd/coscheduling/controller/app/server.go

@@ -0,0 +1,123 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"context"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"os"
+	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling/util"
+
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/klog/v2"
+
+	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling/controller"
+	pgclientset "sigs.k8s.io/scheduler-plugins/pkg/coscheduling/generated/clientset/versioned"
+	pgformers "sigs.k8s.io/scheduler-plugins/pkg/coscheduling/generated/informers/externalversions"
+)
+
+func newConfig(kubeconfig, master string, inCluster bool) (*restclient.Config, error) {
+	var (
+		config *rest.Config
+		err    error
+	)
+	if inCluster {
+		config, err = rest.InClusterConfig()
+	} else {
+		config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig)
+	}
+	if err != nil {
+		return nil, err
+	}
+	config.Burst = 200
+	config.QPS = 100
+	return config, nil
+}
+
+func Run(s *ServerRunOptions) error {
+	ctx := context.Background()
+	config, err := newConfig(s.KubeConfig, s.MasterUrl, s.InCluster)
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	stopCh := server.SetupSignalHandler()
+
+	pgClient := pgclientset.NewForConfigOrDie(config)
+	kubeClient := kubernetes.NewForConfigOrDie(config)
+
+	scheduleInformer := pgformers.NewSharedInformerFactory(pgClient, 0)
+	pgInformer := scheduleInformer.Scheduling().V1alpha1().PodGroups()
+
+	informerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, 0, informers.WithTweakListOptions(func(opt *metav1.ListOptions) {
+		opt.LabelSelector = util.PodGroupLabel
+	}))
+	podInformer := informerFactory.Core().V1().Pods()
+	ctrl := controller.NewPodGroupController(kubeClient, pgInformer, podInformer, pgClient)
+	informerFactory.Start(ctx.Done())
+	scheduleInformer.Start(ctx.Done())
+	run := func(ctx context.Context) {
+		ctrl.Run(10, ctx.Done())
+	}
+
+	if !s.EnableLeaderElection {
+		run(ctx)
+
+	} else {
+		id, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+
+		// add a uniquifier so that two processes on the same host don't accidentally both become active
+		id = id + "_" + string(uuid.NewUUID())
+
+		rl, err := resourcelock.New("endpoints",
+			"kube-system",
+			"kube-scheduler",
+			kubeClient.CoreV1(),
+			kubeClient.CoordinationV1(),
+			resourcelock.ResourceLockConfig{
+				Identity: id,
+			})
+		if err != nil {
+			klog.Fatalf("error creating lock: %v", err)
+		}
+
+		leaderelection.RunOrDie(context.TODO(), leaderelection.LeaderElectionConfig{
+			Lock: rl,
+			Callbacks: leaderelection.LeaderCallbacks{
+				OnStartedLeading: run,
+				OnStoppedLeading: func() {
+					klog.Fatalf("leaderelection lost")
+				},
+			},
+			Name: "podgroup controller",
+		})
+	}
+
+	<-stopCh
+	return nil
+}

cmd/coscheduling/controller/controller.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/spf13/pflag"
+	"sigs.k8s.io/scheduler-plugins/cmd/coscheduling/controller/app"
+)
+
+func main() {
+	options := app.NewServerRunOptions()
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+
+	if err := app.Run(options); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

cmd/coscheduling/scheduler/main.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"math/rand"
+	"os"
+	"time"
+
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+
+	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling"
+	// Ensure scheme package is initialized.
+	_ "sigs.k8s.io/scheduler-plugins/pkg/apis/config/scheme"
+)
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+	// Register custom plugins to the scheduler framework.
+	// Later they can consist of scheduler profile(s) and hence
+	// used by various kinds of workloads.
+	command := app.NewSchedulerCommand(
+		app.WithPlugin(coscheduling.Name, coscheduling.New),
+	)
+	if err := command.Execute(); err != nil {
+		os.Exit(1)
+	}
+}