project-codeflare · Jan 2, 2021 · Jan 2, 2021 · Jan 2, 2021 · Jan 2, 2021 · Feb 4, 2021
Showing 6,366 changed files with 3,259 additions and 1,626,864 deletions.
diff --git a/.gitignore b/.gitignore
@@ -109,7 +109,9 @@ kubernetes.tar.gz
 
 # generated files in any directory
 # TODO(thockin): uncomment this when we stop committing the generated files.
+
 #zz_generated.*
+
 #zz_generated.openapi.go
 
 # make-related metadata

diff --git a/.travis.yml b/.travis.yml
@@ -9,7 +9,7 @@ services:
   - docker
 
 go:
-  - "1.11"
+  - "1.16.3"
 
 go_import_path: github.com/IBM/multi-cluster-app-dispatcher
 

diff --git a/CONTROLLER_VERSION b/CONTROLLER_VERSION
@@ -1 +1 @@
-1.29.4
+1.29.37
diff --git a/cmd/deepcopy-gen/main.go b/cmd/deepcopy-gen/main.go
@@ -13,7 +13,21 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+/*
+Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 // deepcopy-gen is a tool for auto-generating DeepCopy functions.
 //
 // Given a list of input directories, it will generate functions that
@@ -48,20 +62,22 @@ limitations under the License.
 package main
 
 import (
+	"path/filepath"
+
 	"k8s.io/gengo/args"
 	"k8s.io/gengo/examples/deepcopy-gen/generators"
-	"path/filepath"
 
-	"github.com/golang/glog"
 	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
 )
 
 func main() {
+	klog.InitFlags(nil)
 	arguments := args.Default()
 
 	// Override defaults.
 	arguments.OutputFileBaseName = "deepcopy_generated"
-	current_loc, _:= filepath.Abs(".")
+	current_loc, _ := filepath.Abs(".")
 	arguments.GoHeaderFilePath = filepath.Join(current_loc, "hack/boilerplate/boilerplate.go.txt")
 
 	// Custom args.
@@ -76,7 +92,7 @@ func main() {
 		generators.DefaultNameSystem(),
 		generators.Packages,
 	); err != nil {
-		glog.Fatalf("Error: %v", err)
+		klog.Fatalf("Error: %v", err)
 	}
-	glog.V(2).Info("Completed successfully.")
+	klog.V(2).Info("Completed successfully.")
 }
diff --git a/cmd/kar-controllers/app/options/options.go b/cmd/kar-controllers/app/options/options.go
@@ -18,25 +18,27 @@ package options
 
 import (
 	"github.com/spf13/pflag"
+	"k8s.io/klog"
 	"os"
 	"strconv"
 	"strings"
 )
 
 // ServerOption is the main context object for the controller manager.
 type ServerOption struct {
-	Master			string
-	Kubeconfig		string
-	SchedulerName 		string
-	Dispatcher		bool
-	AgentConfigs 		string
-	SecurePort		int
-	DynamicPriority		bool  // If DynamicPriority=true then no preemption is allowed by program logic
-	Preemption 		bool  // Preemption is not allowed under DynamicPriority
-	BackoffTime		int   // Number of seconds a job will go away for, if it can not be scheduled.  Default is 20.
+	Master          string
+	Kubeconfig      string
+	SchedulerName   string
+	Dispatcher      bool
+	AgentConfigs    string
+	SecurePort      int
+	DynamicPriority bool // If DynamicPriority=true then no preemption is allowed by program logic
+	Preemption      bool // Preemption is not allowed under DynamicPriority
+	BackoffTime     int  // Number of seconds a job will go away for, if it can not be scheduled.  Default is 20.
 	// Head of line job will not be bumped away for at least HeadOfLineHoldingTime seconds by higher priority jobs.
 	// Default setting to 0 disables this mechanism.
-	HeadOfLineHoldingTime	int
+	HeadOfLineHoldingTime int
+	HealthProbeListenAddr string
 }
 
 // NewServerOption creates a new CMServer with a default config.
@@ -53,13 +55,14 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.Master, "scheduler", s.SchedulerName, "scheduler name for placing pods")
 	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.BoolVar(&s.Dispatcher,"dispatcher",s.Dispatcher,"set dispather mode(true) or agent mode(false)")
+	fs.BoolVar(&s.Dispatcher, "dispatcher", s.Dispatcher, "set dispather mode(true) or agent mode(false)")
 	fs.StringVar(&s.AgentConfigs, "agentconfigs", s.AgentConfigs, "Paths to agent config file:deploymentName separted by commas(,)")
-	fs.BoolVar(&s.DynamicPriority,"dynamicpriority", s.DynamicPriority,"If true, set controller to use dynamic priority. If false, set controller to use static priority.  Default is false.")
-	fs.BoolVar(&s.Preemption,"preemption", s.Preemption,"Set controller to allow preemption if set to true. Note: when set to true, the Kubernetes Scheduler must be configured to enable preemption.  Default is false.")
-	fs.IntVar(&s.BackoffTime,"backofftime", s.BackoffTime,"Number of seconds a job will go away for, if it can not be scheduled.  Default is 20.")
-	fs.IntVar(&s.HeadOfLineHoldingTime,"headoflineholdingtime", s.HeadOfLineHoldingTime,"Number of seconds a job can stay at the Head Of Line without being bumped.  Default is 0.")
-//	fs.IntVar(&s.SecurePort, "secure-port", 6443, "The port on which to serve secured, uthenticated access for metrics.")
+	fs.BoolVar(&s.DynamicPriority, "dynamicpriority", s.DynamicPriority, "If true, set controller to use dynamic priority. If false, set controller to use static priority.  Default is false.")
+	fs.BoolVar(&s.Preemption, "preemption", s.Preemption, "Set controller to allow preemption if set to true. Note: when set to true, the Kubernetes Scheduler must be configured to enable preemption.  Default is false.")
+	fs.IntVar(&s.BackoffTime, "backofftime", s.BackoffTime, "Number of seconds a job will go away for, if it can not be scheduled.  Default is 20.")
+	fs.IntVar(&s.HeadOfLineHoldingTime, "headoflineholdingtime", s.HeadOfLineHoldingTime, "Number of seconds a job can stay at the Head Of Line without being bumped.  Default is 0.")
+	fs.StringVar(&s.HealthProbeListenAddr, "healthProbeListenAddr", ":8081", "Listen address for health probes. Defaults to ':8081'")
+	klog.V(4).Infof("[AddFlags] Controller configuration: %+v", s)
 }
 
 func (s *ServerOption) loadDefaultsFromEnvVars() {

diff --git a/cmd/kar-controllers/app/server.go b/cmd/kar-controllers/app/server.go
@@ -13,15 +13,31 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+/*
+Copyright 2019, 2021 The Multi-Cluster App Dispatcher Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 package app
 
 import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"net/http"
 
 	"github.com/IBM/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejob"
+	"github.com/IBM/multi-cluster-app-dispatcher/pkg/health"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 )
@@ -41,16 +57,32 @@ func Run(opt *options.ServerOption) error {
 
 	neverStop := make(chan struct{})
 
-	config.QPS   = 100.0
+	config.QPS = 100.0
 	config.Burst = 200.0
 
 	jobctrl := queuejob.NewJobController(config, opt)
-	if jobctrl ==nil {
+	if jobctrl == nil {
 		return nil
 	}
 	jobctrl.Run(neverStop)
 
-	<-neverStop
+	// This call is blocking (unless an error occurs) which equates to <-neverStop
+	err = listenHealthProbe(opt)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Starts the health probe listener
+func listenHealthProbe(opt *options.ServerOption) error {
+	handler := http.NewServeMux()
+	handler.Handle("/healthz", &health.Handler{})
+	err := http.ListenAndServe(opt.HealthProbeListenAddr, handler)
+	if err != nil {
+		return err
+	}
 
 	return nil
 }
diff --git a/cmd/kar-controllers/main.go b/cmd/kar-controllers/main.go
@@ -16,20 +16,40 @@ limitations under the License.
 package main
 
 import (
+	"flag"
 	"fmt"
+
 	"github.com/IBM/multi-cluster-app-dispatcher/cmd/kar-controllers/app"
 	"github.com/IBM/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
+
+	"k8s.io/klog"
+
 	"github.com/spf13/pflag"
-	"k8s.io/apiserver/pkg/util/flag"
+
 	"os"
 )
 
-
 func main() {
+	// based on the tips here: https://github.com/kubernetes/klog/blob/main/examples/coexist_glog/coexist_glog.go
+	flag.Parse()
+
+	klogFlags := flag.NewFlagSet("klog", flag.ContinueOnError)
+	klog.InitFlags(klogFlags)
+
+	// Sync the glog and klog flags.
+	flag.CommandLine.VisitAll(func(f1 *flag.Flag) {
+		f2 := klogFlags.Lookup(f1.Name)
+
+		if f2 != nil {
+			value := f1.Value.String()
+			f2.Value.Set(value)
+		}
+	})
+
 	s := options.NewServerOption()
 	s.AddFlags(pflag.CommandLine)
 
-	flag.InitFlags()
+	//	flag.InitFlags()
 	s.CheckOptionOrDie()
 
 	if err := app.Run(s); err != nil {

diff --git a/deployment/Dockerfile.both b/deployment/Dockerfile.both
@@ -1,11 +1,13 @@
-From ubuntu:18.04
+FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 
 ADD mcad-controller /usr/local/bin
 
-RUN apt-get update
-RUN apt-get -y upgrade
-
-RUN apt-get install -y curl
+RUN true \
+    && microdnf update \
+    && microdnf --nodocs install \
+        curl \
+    && microdnf clean all \
+    && true
 
 RUN cd /usr/local/bin && curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl
 

diff --git a/deployment/build.sh b/deployment/build.sh
@@ -4,6 +4,6 @@ set -x
 project_root=$(cd ..; pwd)
 git_path=github.com/IBM/multi-cluster-app-dispatcher
 
-container_id=$(docker run  --rm -v "$project_root":/go/src/$git_path -d -w /go/src/$git_path/deployment golang:alpine ./build-inside-container.sh)
+container_id=$(docker run  --rm -v "$project_root":/go/src/$git_path -d -w /go/src/$git_path/deployment golang:1.16.3-alpine3.13 ./build-inside-container.sh)
 
 docker logs -f $container_id
diff --git a/deployment/mcad-controller/templates/deployment.yaml b/deployment/mcad-controller/templates/deployment.yaml
@@ -14,6 +14,7 @@ spec:
   selector:
     app: custom-metrics-apiserver
 ---
+#{{ if .Values.configMap.multiCluster }}
 apiVersion: apiregistration.k8s.io/v1beta1
 kind: APIService
 metadata:
@@ -41,6 +42,7 @@ spec:
   insecureSkipTLSVerify: true
   groupPriorityMinimum: 100
   versionPriority: 100
+ #{{ end }} 
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -285,7 +287,7 @@ subjects:
   namespace: kube-system
 #{{ end }}
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ .Values.deploymentName }}

diff --git a/deployment/mcad-controller/values.yaml b/deployment/mcad-controller/values.yaml
@@ -1,7 +1,7 @@
 # Default values for MCAD-Controller.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
-deploymentName: xqueuejob-controller
+deploymentName: mcad-controller
 namespace: kube-system
 replicaCount: 1
 loglevel: 4
@@ -23,13 +23,14 @@ imagePullSecret:
   registry: registry.stage1.ng.bluemix.net
   password: dummyvalue
 
-serviceAccount: xqueuejob-controller
+serviceAccount: mcad-controller
 
 nodeSelector:
   hostname:
 
 configMap:
   name:
+  multiCluster: false
   dispatcherMode: "false"
   agentConfigs:
 

diff --git a/doc/usage/examples/aw-1-k8s-job.yaml b/doc/usage/examples/aw-1-k8s-job.yaml
@@ -0,0 +1,24 @@
+apiVersion: mcad.ibm.com/v1alpha1
+kind: AppWrapper
+metadata:
+  labels:
+  name: aw-pi
+  namespace: default
+spec:
+  resources:
+    GenericItems:
+    - replicas: 1
+      generictemplate:
+        apiVersion: batch/v1
+        kind: Job
+        metadata:
+          name: pi
+        spec:
+          template:
+            spec:
+              containers:
+              - name: pi
+                image: perl
+                command: ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
+              restartPolicy: Never
+          backoffLimit: 4
diff --git a/doc/usage/examples/aw-1-k8s-job1.yaml b/doc/usage/examples/aw-1-k8s-job1.yaml
@@ -0,0 +1,38 @@
+apiVersion: mcad.ibm.com/v1alpha1
+kind: AppWrapper
+metadata:
+  name: aw-generic-statefulset-2
+  namespace: test1
+spec:
+  schedulingSpec:
+    minAvailable: 2
+  resources:
+    GenericItems:
+      replicas: 1
+      metadata:
+        name: aw-generic-statefulset-2
+        namespace: test1
+      generictemplate:
+        apiVersion: apps/v1
+        kind: StatefulSet
+        metadata:
+          name: aw-generic-statefulset-2
+          namespace: test1
+          labels:
+            app: aw-generic-statefulset-2
+        spec:
+          replicas: 2
+          selector:
+            matchLabels:
+              app: aw-generic-statefulset-2
+          template:
+            metadata:
+              labels:
+                app: aw-generic-statefulset-2
+            spec:
+              containers:
+              - name: aw-generic-statefulset-2
+                image: k8s.gcr.io/echoserver:1.4
+                imagePullPolicy: Never
+                ports:
+                - containerPort: 80
diff --git a/doc/usage/examples/aw-1-k8s-job2.yaml b/doc/usage/examples/aw-1-k8s-job2.yaml
@@ -0,0 +1,36 @@
+apiVersion: mcad.ibm.com/v1alpha1
+kind: AppWrapper
+metadata:
+  name: aw-generic-statefulset-2
+  namespace: test1
+spec:
+  schedulingSpec:
+    minAvailable: 2
+  resources:
+    Items:
+    - replicas: 1
+      type: StatefulSet
+      template:
+        apiVersion: apps/v1
+        kind: StatefulSet
+        metadata:
+          name: aw-statefulset-2
+          namespace: test1
+          labels:
+            app: aw-statefulset-2
+        spec:
+          replicas: 2
+          selector:
+            matchLabels:
+              app: aw-statefulset-2
+          template:
+            metadata:
+              labels:
+                app: aw-statefulset-2
+            spec:
+              containers:
+              - name: aw-statefulset-2
+                image: k8s.gcr.io/echoserver:1.4
+                imagePullPolicy: Never
+                ports:
+                - containerPort: 80
diff --git a/doc/usage/examples/aw-1-k8s-job3.yaml b/doc/usage/examples/aw-1-k8s-job3.yaml
@@ -0,0 +1,32 @@
+apiVersion: mcad.ibm.com/v1alpha1
+kind: AppWrapper
+metadata:
+  name: aw-generic-statefulset-2
+spec:
+  resources:
+    Items:
+    - replicas: 1
+      type: StatefulSet
+      template:
+        apiVersion: apps/v1
+        kind: StatefulSet
+        metadata:
+          name: aw-generic-statefulset-2
+          labels:
+            app: aw-generic-statefulset-2
+        spec:
+          replicas: 2
+          selector:
+            matchLabels:
+              app: aw-generic-statefulset-2
+          template:
+            metadata:
+              labels:
+                app: aw-generic-statefulset-2
+            spec:
+              containers:
+              - name: aw-generic-statefulset-2
+                image: k8s.gcr.io/echoserver:1.4
+                imagePullPolicy: Never
+                ports:
+                - containerPort: 80
diff --git a/go.mod b/go.mod
@@ -0,0 +1,36 @@
+module github.com/IBM/multi-cluster-app-dispatcher
+
+go 1.16
+
+require (
+	github.com/emicklei/go-restful v2.14.3+incompatible
+	github.com/emicklei/go-restful-swagger12 v0.0.0-20201014110547-68ccff494617
+	github.com/googleapis/gnostic v0.4.1
+	github.com/kubernetes-sigs/custom-metrics-apiserver v0.0.0-20210311094424-0ca2b1909cdc
+	github.com/onsi/ginkgo v1.11.0
+	github.com/onsi/gomega v1.7.0
+	github.com/spf13/pflag v1.0.5
+	github.com/stretchr/testify v1.6.1
+	golang.org/x/tools v0.1.1 // indirect
+	k8s.io/api v0.20.0
+	k8s.io/apiextensions-apiserver v0.20.0
+	k8s.io/apimachinery v0.20.0
+	k8s.io/apiserver v0.20.0
+	k8s.io/client-go v0.20.0
+	k8s.io/component-base v0.20.0
+	k8s.io/gengo v0.0.0-20210203185629-de9496dff47b
+	k8s.io/klog v1.0.0
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7
+	k8s.io/metrics v0.20.0
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
+)
+
+replace (
+	golang.org/x/tools => golang.org/x/tools v0.1.1 // indirect
+	k8s.io/api => k8s.io/api v0.20.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.20.0
+	k8s.io/client-go => k8s.io/client-go v0.20.0
+	k8s.io/code-generator => k8s.io/code-generator v0.20.0
+	k8s.io/metrics => k8s.io/metrics v0.20.0
+)
diff --git a/go.sum b/go.sum
diff --git a/hack/e2e-kind-config.yaml b/hack/e2e-kind-config.yaml
@@ -1,25 +1,13 @@
 # this config file contains all config fields with comments
 kind: Cluster
-apiVersion: kind.sigs.k8s.io/v1alpha3
-# 1 control plane node and 3 workers
+apiVersion: kind.x-k8s.io/v1alpha4
+# 1 control plane node and 1 worker
 nodes:
   # the control plane node config
   - role: control-plane
-    # kubernetes version 1.11.10
-    image: kindest/node:v1.11.10@sha256:abd0275ead5ddfd477b7bc491f71957d7dd75408a346834ffe8a9bee5fbdc15b
+    # kubernetes version 1.20.7 from kind v0.11.0
+    image: kindest/node:v1.20.7@sha256:e645428988191fc824529fd0bb5c94244c12401cf5f5ea3bd875eb0a787f0fe9
   # the three workers
   - role: worker
-    # replicas specifies the number of nodes to create with this configuration
-    replicas: 1
-    # kubernetes version 1.11.10
-    image: kindest/node:v1.11.10@sha256:abd0275ead5ddfd477b7bc491f71957d7dd75408a346834ffe8a9bee5fbdc15b
-#  - role: worker
-#    # replicas specifies the number of nodes to create with this configuration
-#    replicas: 1
-#    # kubernetes version 1.11.10
-#    image: kindest/node:v1.11.10@sha256:abd0275ead5ddfd477b7bc491f71957d7dd75408a346834ffe8a9bee5fbdc15b
-#  - role: worker
-#    # replicas specifies the number of nodes to create with this configuration
-#    replicas: 1
-#    # kubernetes version 1.11.10
-#    image: kindest/node:v1.11.10@sha256:abd0275ead5ddfd477b7bc491f71957d7dd75408a346834ffe8a9bee5fbdc15b
+    # kubernetes version 1.20.7 from kind v0.11.0
+    image: kindest/node:v1.20.7@sha256:e645428988191fc824529fd0bb5c94244c12401cf5f5ea3bd875eb0a787f0fe9
diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh
@@ -210,8 +210,8 @@ runTests() {
     # the build artifacts but doesn't run the tests.  The two together provide
     # a large speedup for tests that do not need to be rebuilt.
     go test -i "${goflags[@]:+${goflags[@]}}" \
-      ${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
-     "${testargs[@]:+${testargs[@]}}"
+    ${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
+    "${testargs[@]:+${testargs[@]}}"
     go test "${goflags[@]:+${goflags[@]}}" \
       ${KUBE_RACE} ${KUBE_TIMEOUT} "${@}" \
      "${testargs[@]:+${testargs[@]}}" \

diff --git a/hack/run-e2e-kind.sh b/hack/run-e2e-kind.sh
@@ -20,10 +20,10 @@ curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add
 echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
 sudo apt-get update
 # Using older version due to older version of kubernetes cluster"
-sudo apt-get install -y  kubectl=1.11.10-00
+sudo apt-get install -y  kubectl=1.16.3-00
 
-# Download kind binary (0.2.0)
-sudo curl -o /usr/local/bin/kind -L https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64
+# Download kind binary (0.6.1)
+sudo curl -o /usr/local/bin/kind -L https://github.com/kubernetes-sigs/kind/releases/download/v0.11.0/kind-linux-amd64
 sudo chmod +x /usr/local/bin/kind
 
 # check if kind installed
@@ -288,7 +288,7 @@ function kube-test-env-up {
     sleep 10
     echo "Listing MCAD Controller Helm Chart and Pod YAML..."
     helm list
-    mcad_pod=$(kubectl get pods -n kube-system | grep xqueuejob | awk '{print $1}')
+    mcad-controller=$(kubectl get pods -n kube-system | grep mcad-controller | awk '{print $1}')
     if [[ "$mcad_pod" != "" ]]
     then
         kubectl get pod ${mcad_pod} -n kube-system -o yaml

diff --git a/pkg/apis/controller/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/controller/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/client/clientset/controller-versioned/clients/appwrapper.go b/pkg/client/clientset/controller-versioned/clients/appwrapper.go
@@ -17,18 +17,19 @@ limitations under the License.
 package clients
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"time"
 
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 
-	"github.com/golang/glog"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
 )
 
 const appWrapperKindName = arbv1.AppWrapperPlural + "." + arbv1.GroupName
@@ -48,15 +49,15 @@ func CreateAppWrapperKind(clientset apiextensionsclient.Interface) (*apiextensio
 			},
 		},
 	}
-	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.Background(), crd, metav1.CreateOptions{})
 
 	if err != nil {
 		return nil, err
 	}
 
 	// wait for CRD being established
 	err = wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
-		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(appWrapperKindName, metav1.GetOptions{})
+		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.Background(), appWrapperKindName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -75,14 +76,14 @@ func CreateAppWrapperKind(clientset apiextensionsclient.Interface) (*apiextensio
 		return false, err
 	})
 	if err != nil {
-		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(appWrapperKindName, nil)
+		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(context.Background(), appWrapperKindName, metav1.DeleteOptions{})
 		if deleteErr != nil {
 			return nil, errors.NewAggregate([]error{err, deleteErr})
 		}
 		return nil, err
 	}
 
-	glog.V(4).Infof("AppWrapper CRD was created.")
+	klog.V(4).Infof("AppWrapper CRD was created.")
 
 	return crd, nil
 }
diff --git a/pkg/client/clientset/controller-versioned/clients/factory.go b/pkg/client/clientset/controller-versioned/clients/factory.go
@@ -34,7 +34,7 @@ func NewClient(cfg *rest.Config) (*rest.RESTClient, *runtime.Scheme, error) {
 	config.GroupVersion = &arbv1.SchemeGroupVersion
 	config.APIPath = "/apis"
 	config.ContentType = runtime.ContentTypeJSON
-	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
+	config.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
 
 	client, err := rest.RESTClientFor(&config)
 	if err != nil {

diff --git a/pkg/client/clientset/controller-versioned/clients/queuejob.go b/pkg/client/clientset/controller-versioned/clients/queuejob.go
@@ -17,18 +17,19 @@ limitations under the License.
 package clients
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"time"
 
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 
-	"github.com/golang/glog"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
 )
 
 const queueJobKindName = arbv1.QueueJobPlural + "." + arbv1.GroupName
@@ -48,15 +49,15 @@ func CreateQueueJobKind(clientset apiextensionsclient.Interface) (*apiextensions
 			},
 		},
 	}
-	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.Background(), crd, metav1.CreateOptions{})
 
 	if err != nil {
 		return nil, err
 	}
 
 	// wait for CRD being established
 	err = wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
-		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(queueJobKindName, metav1.GetOptions{})
+		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.Background(), queueJobKindName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -75,14 +76,14 @@ func CreateQueueJobKind(clientset apiextensionsclient.Interface) (*apiextensions
 		return false, err
 	})
 	if err != nil {
-		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(queueJobKindName, nil)
+		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(context.Background(), queueJobKindName, metav1.DeleteOptions{})
 		if deleteErr != nil {
 			return nil, errors.NewAggregate([]error{err, deleteErr})
 		}
 		return nil, err
 	}
 
-	glog.V(4).Infof("QueueJob CRD was created.")
+	klog.V(4).Infof("QueueJob CRD was created.")
 
 	return crd, nil
 }
diff --git a/pkg/client/clientset/controller-versioned/clients/schedulingspec.go b/pkg/client/clientset/controller-versioned/clients/schedulingspec.go
@@ -17,18 +17,19 @@ limitations under the License.
 package clients
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"time"
 
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 
-	"github.com/golang/glog"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
 )
 
 const schedulingSpecKindName = arbv1.SchedulingSpecPlural + "." + arbv1.GroupName
@@ -48,16 +49,15 @@ func CreateSchedulingSpecKind(clientset apiextensionsclient.Interface) (*apiexte
 			},
 		},
 	}
-	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+	_, err := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(context.Background(), crd, metav1.CreateOptions{})
 
 	if err != nil {
 		return nil, err
 	}
 
 	// wait for CRD being established
 	err = wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) {
-		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(
-			schedulingSpecKindName, metav1.GetOptions{})
+		crd, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Get(context.Background(), schedulingSpecKindName, metav1.GetOptions{})
 		if err != nil {
 			return false, err
 		}
@@ -76,15 +76,14 @@ func CreateSchedulingSpecKind(clientset apiextensionsclient.Interface) (*apiexte
 		return false, err
 	})
 	if err != nil {
-		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(
-			schedulingSpecKindName, nil)
+		deleteErr := clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(context.Background(), schedulingSpecKindName, metav1.DeleteOptions{})
 		if deleteErr != nil {
 			return nil, errors.NewAggregate([]error{err, deleteErr})
 		}
 		return nil, err
 	}
 
-	glog.V(3).Infof("SchedulingSpec CRD was created.")
+	klog.V(3).Infof("SchedulingSpec CRD was created.")
 
 	return crd, nil
 }
diff --git a/pkg/client/clientset/controller-versioned/typed/v1/appwrapper.go b/pkg/client/clientset/controller-versioned/typed/v1/appwrapper.go
@@ -17,6 +17,8 @@ limitations under the License.
 package v1
 
 import (
+	"context"
+
 	v1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned/scheme"
 
@@ -58,7 +60,7 @@ func (c *appwrappers) Create(appwrapper *v1.AppWrapper) (result *v1.AppWrapper,
 		Namespace(c.ns).
 		Resource(v1.AppWrapperPlural).
 		Body(appwrapper).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -71,7 +73,7 @@ func (c *appwrappers) Update(appwrapper *v1.AppWrapper) (result *v1.AppWrapper,
 		Resource(v1.AppWrapperPlural).
 		Name(appwrapper.Name).
 		Body(appwrapper).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -87,7 +89,7 @@ func (c *appwrappers) UpdateStatus(appwrapper *v1.AppWrapper) (result *v1.AppWra
 		Name(appwrapper.Name).
 		SubResource("status").
 		Body(appwrapper).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -99,7 +101,7 @@ func (c *appwrappers) Delete(name string, options *meta_v1.DeleteOptions) error
 		Resource(v1.AppWrapperPlural).
 		Name(name).
 		Body(options).
-		Do().
+		Do(context.Background()).
 		Error()
 }
 
@@ -111,7 +113,7 @@ func (c *appwrappers) Get(name string, options meta_v1.GetOptions) (result *v1.A
 		Resource(v1.AppWrapperPlural).
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -123,7 +125,7 @@ func (c *appwrappers) List(opts meta_v1.ListOptions) (result *v1.AppWrapperList,
 		Namespace(c.ns).
 		Resource(v1.AppWrapperPlural).
 		VersionedParams(&opts, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
diff --git a/pkg/client/clientset/controller-versioned/typed/v1/client.go b/pkg/client/clientset/controller-versioned/typed/v1/client.go
@@ -85,7 +85,7 @@ func setConfigDefaults(config *rest.Config) error {
 	config.GroupVersion = &v1.SchemeGroupVersion
 	config.APIPath = "/apis"
 	config.ContentType = runtime.ContentTypeJSON
-	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
+	config.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)} // serializer.DirectCodecFactory{CodecFactory: serializer.NewCodecFactory(scheme)}
 
 	return nil
 }

diff --git a/pkg/client/clientset/controller-versioned/typed/v1/queuejob.go b/pkg/client/clientset/controller-versioned/typed/v1/queuejob.go
@@ -17,6 +17,8 @@ limitations under the License.
 package v1
 
 import (
+	"context"
+
 	v1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned/scheme"
 
@@ -58,7 +60,7 @@ func (c *queuejobs) Create(queuejob *v1.QueueJob) (result *v1.QueueJob, err erro
 		Namespace(c.ns).
 		Resource(v1.QueueJobPlural).
 		Body(queuejob).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -71,7 +73,7 @@ func (c *queuejobs) Update(queuejob *v1.QueueJob) (result *v1.QueueJob, err erro
 		Resource(v1.QueueJobPlural).
 		Name(queuejob.Name).
 		Body(queuejob).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -87,7 +89,7 @@ func (c *queuejobs) UpdateStatus(queuejob *v1.QueueJob) (result *v1.QueueJob, er
 		Name(queuejob.Name).
 		SubResource("status").
 		Body(queuejob).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -99,7 +101,7 @@ func (c *queuejobs) Delete(name string, options *meta_v1.DeleteOptions) error {
 		Resource(v1.QueueJobPlural).
 		Name(name).
 		Body(options).
-		Do().
+		Do(context.Background()).
 		Error()
 }
 
@@ -111,7 +113,7 @@ func (c *queuejobs) Get(name string, options meta_v1.GetOptions) (result *v1.Que
 		Resource(v1.QueueJobPlural).
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -123,7 +125,7 @@ func (c *queuejobs) List(opts meta_v1.ListOptions) (result *v1.QueueJobList, err
 		Namespace(c.ns).
 		Resource(v1.QueueJobPlural).
 		VersionedParams(&opts, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
diff --git a/pkg/client/clientset/controller-versioned/typed/v1/schedulingspec.go b/pkg/client/clientset/controller-versioned/typed/v1/schedulingspec.go
@@ -17,6 +17,8 @@ limitations under the License.
 package v1
 
 import (
+	"context"
+
 	v1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned/scheme"
 
@@ -58,7 +60,7 @@ func (c *schedulingSpecs) Create(queue *v1.SchedulingSpec) (result *v1.Schedulin
 		Namespace(c.ns).
 		Resource(v1.SchedulingSpecPlural).
 		Body(queue).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -71,7 +73,7 @@ func (c *schedulingSpecs) Update(queue *v1.SchedulingSpec) (result *v1.Schedulin
 		Resource(v1.SchedulingSpecPlural).
 		Name(queue.Name).
 		Body(queue).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -87,7 +89,7 @@ func (c *schedulingSpecs) UpdateStatus(queue *v1.SchedulingSpec) (result *v1.Sch
 		Name(queue.Name).
 		SubResource("status").
 		Body(queue).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -99,7 +101,7 @@ func (c *schedulingSpecs) Delete(name string, options *meta_v1.DeleteOptions) er
 		Resource(v1.SchedulingSpecPlural).
 		Name(name).
 		Body(options).
-		Do().
+		Do(context.Background()).
 		Error()
 }
 
@@ -111,7 +113,7 @@ func (c *schedulingSpecs) Get(name string, options meta_v1.GetOptions) (result *
 		Resource(v1.SchedulingSpecPlural).
 		Name(name).
 		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
@@ -123,7 +125,7 @@ func (c *schedulingSpecs) List(opts meta_v1.ListOptions) (result *v1.SchedulingS
 		Namespace(c.ns).
 		Resource(v1.SchedulingSpecPlural).
 		VersionedParams(&opts, scheme.ParameterCodec).
-		Do().
+		Do(context.Background()).
 		Into(result)
 	return
 }
diff --git a/pkg/controller/clusterstate/api/helpers.go b/pkg/controller/clusterstate/api/helpers.go
@@ -18,10 +18,10 @@ package api
 
 import (
 	"fmt"
-	"github.com/golang/glog"
 
 	"k8s.io/api/core/v1"
 	clientcache "k8s.io/client-go/tools/cache"
+	"k8s.io/klog"
 )
 
 // PodKey returns the string key of a pod.
@@ -100,10 +100,10 @@ func MergeErrors(errs ...error) error {
 // JobTerminated checkes whether job was terminated.
 func JobTerminated(job *JobInfo) bool {
 	if job.SchedSpec == nil && job.PDB == nil && len(job.Tasks) == 0 {
-		glog.V(9).Infof("Job: %v is terminated.", job.UID)
+		klog.V(9).Infof("Job: %v is terminated.", job.UID)
 		return true
 	} else {
-		glog.V(10).Infof("Job: %v not terminated, scheduleSpec: %v, PDB; %v, tasks: %v.",
+		klog.V(10).Infof("Job: %v not terminated, scheduleSpec: %v, PDB; %v, tasks: %v.",
 			job.UID, job.SchedSpec, job.PDB, job.Tasks)
 		return false
 	}

diff --git a/pkg/controller/clusterstate/api/node_info.go b/pkg/controller/clusterstate/api/node_info.go
@@ -18,9 +18,9 @@ package api
 
 import (
 	"fmt"
-	"github.com/golang/glog"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/klog"
 )
 
 // NodeInfo is node level aggregated information.
@@ -145,7 +145,7 @@ func (ni *NodeInfo) AddTask(task *TaskInfo) error {
 }
 
 func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error {
-	glog.V(10).Infof("Attempting to remove task: %s on node: %s", ti.Name,  ni.Name)
+	klog.V(10).Infof("Attempting to remove task: %s on node: %s", ti.Name,  ni.Name)
 
 	key := PodKey(ti.Pod)
 
@@ -156,15 +156,15 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error {
 	}
 
 	if ni.Node != nil {
-		glog.V(10).Infof("Found node for task: %s, node: %s, task status: %v", task.Name,  ni.Name, task.Status)
+		klog.V(10).Infof("Found node for task: %s, node: %s, task status: %v", task.Name,  ni.Name, task.Status)
 		if task.Status == Releasing {
 			ni.Releasing.Sub(task.Resreq)
 		}
 
 		ni.Idle.Add(task.Resreq)
 		ni.Used.Sub(task.Resreq)
 	} else {
-		glog.V(10).Infof("No node info found for task: %s, node: %s", task.Name,  ni.Name)
+		klog.V(10).Infof("No node info found for task: %s, node: %s", task.Name,  ni.Name)
 	}
 
 	delete(ni.Tasks, key)
@@ -173,7 +173,7 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error {
 }
 
 func (ni *NodeInfo) UpdateTask(ti *TaskInfo) error {
-	glog.V(10).Infof("Attempting to update task: %s on node: %s", ti.Name,  ni.Name)
+	klog.V(10).Infof("Attempting to update task: %s on node: %s", ti.Name,  ni.Name)
 	if err := ni.RemoveTask(ti); err != nil {
 		return err
 	}

diff --git a/pkg/controller/clusterstate/cache/cache.go b/pkg/controller/clusterstate/cache/cache.go
@@ -17,19 +17,19 @@ limitations under the License.
 package cache
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/golang/glog"
-
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/informers"
 	clientv1 "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog"
 
 	client "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned/clients"
 	informerfactory "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/informers/controller-externalversion"
@@ -58,9 +58,9 @@ type ClusterStateCache struct {
 	resourceCapacities *api.Resource
 	deletedJobs        *cache.FIFO
 
-	errTasks    *cache.FIFO
-
+	errTasks *cache.FIFO
 }
+
 func taskKey(obj interface{}) (string, error) {
 	if obj == nil {
 		return "", fmt.Errorf("the object is nil")
@@ -95,7 +95,6 @@ func newClusterStateCache(config *rest.Config) *ClusterStateCache {
 		Nodes:       make(map[string]*api.NodeInfo),
 		errTasks:    cache.NewFIFO(taskKey),
 		deletedJobs: cache.NewFIFO(jobKey),
-
 	}
 
 	sc.kubeclient = kubernetes.NewForConfigOrDie(config)
@@ -160,7 +159,7 @@ func newClusterStateCache(config *rest.Config) *ClusterStateCache {
 }
 
 func (sc *ClusterStateCache) Run(stopCh <-chan struct{}) {
-	glog.V(8).Infof("Cluster State Cache started.")
+	klog.V(8).Infof("Cluster State Cache started.")
 
 	go sc.podInformer.Informer().Run(stopCh)
 	go sc.nodeInformer.Informer().Run(stopCh)
@@ -184,7 +183,6 @@ func (sc *ClusterStateCache) WaitForCacheSync(stopCh <-chan struct{}) bool {
 		sc.nodeInformer.Informer().HasSynced)
 }
 
-
 // Gets available free resoures.
 func (sc *ClusterStateCache) GetUnallocatedResources() *api.Resource {
 	sc.Mutex.Lock()
@@ -205,19 +203,19 @@ func (sc *ClusterStateCache) GetResourceCapacities() *api.Resource {
 
 // Save the cluster state.
 func (sc *ClusterStateCache) saveState(available *api.Resource, capacity *api.Resource) error {
-	glog.V(12).Infof("Saving Cluster State")
+	klog.V(12).Infof("Saving Cluster State")
 
 	sc.Mutex.Lock()
 	defer sc.Mutex.Unlock()
 	sc.availableResources.Replace(available)
 	sc.resourceCapacities.Replace(capacity)
-	glog.V(12).Infof("Updated Cluster State completed.")
+	klog.V(12).Infof("Updated Cluster State completed.")
 	return nil
 }
 
 // Gets available free resoures.
 func (sc *ClusterStateCache) updateState() error {
-	glog.V(11).Infof("Calculating Cluster State")
+	klog.V(11).Infof("Calculating Cluster State")
 
 	cluster := sc.Snapshot()
 	total := api.EmptyResource()
@@ -229,15 +227,14 @@ func (sc *ClusterStateCache) updateState() error {
 		used = used.Add(value.Used)
 		idle = idle.Add(value.Idle)
 	}
-	glog.V(8).Infof("Total capacity %+v, used %+v, free space %+v", total, used, idle)
+	klog.V(8).Infof("Total capacity %+v, used %+v, free space %+v", total, used, idle)
 
 	err := sc.saveState(idle, total)
 	return err
 }
 
-
 func (sc *ClusterStateCache) deleteJob(job *api.JobInfo) {
-	glog.V(4).Infof("[deleteJob] Attempting to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name)
+	klog.V(4).Infof("[deleteJob] Attempting to delete Job <%v:%v/%v>", job.UID, job.Namespace, job.Name)
 
 	time.AfterFunc(5*time.Second, func() {
 		sc.deletedJobs.AddIfNotPresent(job)
@@ -257,7 +254,7 @@ func (sc *ClusterStateCache) processCleanupJob() error {
 
 			if api.JobTerminated(job) {
 				delete(sc.Jobs, job.UID)
-				glog.V(3).Infof("[processCleanupJob] Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name)
+				klog.V(3).Infof("[processCleanupJob] Job <%v:%v/%v> was deleted.", job.UID, job.Namespace, job.Name)
 			} else {
 				// Retry
 				sc.deleteJob(job)
@@ -274,18 +271,18 @@ func (sc *ClusterStateCache) cleanupJobs() {
 	for {
 		err := sc.processCleanupJob()
 		if err != nil {
-			glog.Errorf("Failed to process job clean up: %v", err)
+			klog.Errorf("Failed to process job clean up: %v", err)
 		}
 	}
 }
 
 func (sc *ClusterStateCache) updateCache() {
-	glog.V(9).Infof("Starting to update Cluster State Cache")
+	klog.V(9).Infof("Starting to update Cluster State Cache")
 
 	for {
 		err := sc.updateState()
 		if err != nil {
-			glog.Errorf("Failed update state: %v", err)
+			klog.Errorf("Failed update state: %v", err)
 		}
 
 		time.Sleep(1 * time.Second)
@@ -296,7 +293,7 @@ func (sc *ClusterStateCache) resync() {
 	for {
 		err := sc.processResyncTask()
 		if err != nil {
-			glog.Errorf("Failed to process resync: %v", err)
+			klog.Errorf("Failed to process resync: %v", err)
 		}
 	}
 }
@@ -309,7 +306,7 @@ func (sc *ClusterStateCache) processResyncTask() error {
 		}
 
 		if err := sc.syncTask(task); err != nil {
-			glog.Errorf("Failed to sync pod <%v/%v>", task.Namespace, task.Name)
+			klog.Errorf("Failed to sync pod <%v/%v>", task.Namespace, task.Name)
 			return err
 		}
 		return nil
@@ -335,7 +332,7 @@ func (sc *ClusterStateCache) Snapshot() *api.ClusterInfo {
 		// If no scheduling spec, does not handle it.
 		if value.SchedSpec == nil && value.PDB == nil {
 			// Jobs.Tasks are more recognizable than Jobs.UID
-			glog.V(5).Infof("The scheduling spec of Job <%v> with tasks <%+v> is nil, ignore it.", value.UID, value.Tasks)
+			klog.V(5).Infof("The scheduling spec of Job <%v> with tasks <%+v> is nil, ignore it.", value.UID, value.Tasks)
 			continue
 		}
 
@@ -351,7 +348,7 @@ func (sc *ClusterStateCache) LoadConf(path string) (map[string]string, error) {
 		return nil, err
 	}
 
-	confMap, err := sc.kubeclient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{})
+	confMap, err := sc.kubeclient.CoreV1().ConfigMaps(ns).Get(context.Background(), name, metav1.GetOptions{})
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/controller/clusterstate/cache/event_handlers.go b/pkg/controller/clusterstate/cache/event_handlers.go
diff --git a/pkg/controller/clusterstate/util.go b/pkg/controller/clusterstate/util.go
diff --git a/pkg/controller/clusterstate/util/priority_queue.go b/pkg/controller/clusterstate/util/priority_queue.go
@@ -19,7 +19,7 @@ package util
 import (
 	"container/heap"
 
-	"github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
 )
 
 type PriorityQueue struct {

diff --git a/pkg/controller/metrics/adapter/adapter.go b/pkg/controller/metrics/adapter/adapter.go
@@ -22,9 +22,9 @@ import (
 	"os"
 
 	"github.com/emicklei/go-restful"
-	"github.com/golang/glog"
-	"k8s.io/client-go/rest"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/rest"
+	"k8s.io/klog"
 
 	adapterprov "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/adapter/provider"
 	basecmd "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/cmd"
@@ -46,46 +46,45 @@ type MetricsAdpater struct {
 }
 
 func (a *MetricsAdpater) makeProviderOrDie(clusterStateCache clusterstatecache.Cache) (provider.MetricsProvider, *restful.WebService) {
-	glog.Infof("Entered makeProviderOrDie()")
+	klog.Infof("Entered makeProviderOrDie()")
 	client, err := a.DynamicClient()
 	if err != nil {
-		glog.Fatalf("unable to construct dynamic client: %v", err)
+		klog.Fatalf("unable to construct dynamic client: %v", err)
 	}
 
 	mapper, err := a.RESTMapper()
 	if err != nil {
-		glog.Fatalf("unable to construct discovery REST mapper: %v", err)
+		klog.Fatalf("unable to construct discovery REST mapper: %v", err)
 	}
 
 	return adapterprov.NewFakeProvider(client, mapper, clusterStateCache)
 }
 
 func newMetricsAdpater(config *rest.Config, clusterStateCache clusterstatecache.Cache) *MetricsAdpater {
-	glog.V(10).Infof("Entered main()")
+	klog.V(10).Infof("Entered main()")
 
 	cmd := &MetricsAdpater{}
 	cmd.Flags().StringVar(&cmd.Message, "msg", "starting adapter...", "startup message")
-	glog.Infof("")
+	klog.Infof("")
 	cmd.Flags().AddGoFlagSet(flag.CommandLine) // make sure we get the glog flags
-	glog.V(9).Infof("commandline: %v", flag.CommandLine)
+	klog.V(9).Infof("commandline: %v", flag.CommandLine)
 	cmd.Flags().Args()
 	cmd.Flags().Parse(os.Args)
 
 	testProvider, webService := cmd.makeProviderOrDie(clusterStateCache)
 	cmd.WithCustomMetrics(testProvider)
 	cmd.WithExternalMetrics(testProvider)
 
-	glog.Infof(cmd.Message)
+	klog.Infof(cmd.Message)
 	// Set up POST endpoint for writing fake metric values
 	restful.DefaultContainer.Add(webService)
 	go func() {
 		// Open port for POSTing fake metrics
-		glog.Fatal(http.ListenAndServe(":8080", nil))
+		klog.Fatal(http.ListenAndServe(":8080", nil))
 	}()
-//	if err := cmd.Run(wait.NeverStop); err != nil {
-//		glog.Fatalf("unable to run custom metrics adapter: %v", err)
-//	}
+	//	if err := cmd.Run(wait.NeverStop); err != nil {
+	//		klog.Fatalf("unable to run custom metrics adapter: %v", err)
+	//	}
 	go cmd.Run(wait.NeverStop)
 	return cmd
 }
-
diff --git a/pkg/controller/metrics/adapter/provider/provider.go b/pkg/controller/metrics/adapter/provider/provider.go
@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/emicklei/go-restful"
-	"github.com/golang/glog"
+	"k8s.io/klog/v2"
 
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -124,27 +124,31 @@ var (
 	}
 )
 
+type metricValue struct {
+	labels labels.Set
+	value  resource.Quantity
+}
+
 // testingProvider is a sample implementation of provider.MetricsProvider which stores a map of fake metrics
 type testingProvider struct {
 	client dynamic.Interface
 	mapper apimeta.RESTMapper
 
 	valuesLock      sync.RWMutex
-	values          map[CustomMetricResource]resource.Quantity
+	values          map[CustomMetricResource]metricValue
 	externalMetrics []ExternalMetric
-	cache2			clusterstatecache.Cache
-
+	cache2          clusterstatecache.Cache
 }
 
 // NewFakeProvider returns an instance of testingProvider, along with its restful.WebService that opens endpoints to post new fake metrics
 func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper, clusterStateCache clusterstatecache.Cache) (provider.MetricsProvider, *restful.WebService) {
-	glog.V(10).Infof("Entered NewFakeProvider()")
+	klog.V(10).Infof("Entered NewFakeProvider()")
 	provider := &testingProvider{
 		client:          client,
 		mapper:          mapper,
-		values:          make(map[CustomMetricResource]resource.Quantity),
+		values:          make(map[CustomMetricResource]metricValue),
 		externalMetrics: testingExternalMetrics,
-		cache2:			 clusterStateCache,
+		cache2:          clusterStateCache,
 	}
 	return provider, provider.webService()
 }
@@ -154,7 +158,7 @@ func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper, cluste
 // There are 3 metric types available: namespaced, root-scoped, and namespaces.
 // (Note: Namespaces, we're assuming, are themselves namespaced resources, but for consistency with how metrics are retreived they have a separate route)
 func (p *testingProvider) webService() *restful.WebService {
-	glog.V(10).Infof("Entered webService()")
+	klog.V(10).Infof("Entered webService()")
 	ws := new(restful.WebService)
 
 	ws.Path("/write-metrics")
@@ -175,35 +179,37 @@ func (p *testingProvider) webService() *restful.WebService {
 
 // updateMetric writes the metric provided by a restful request and stores it in memory
 func (p *testingProvider) updateMetric(request *restful.Request, response *restful.Response) {
-	glog.V(10).Infof("Entered updateMetric()")
 	p.valuesLock.Lock()
 	defer p.valuesLock.Unlock()
 
 	namespace := request.PathParameter("namespace")
-	glog.V(9).Infof("Namespace=%s", namespace)
 	resourceType := request.PathParameter("resourceType")
-	glog.V(9).Infof("Resource type=%s", resourceType)
 	namespaced := false
 	if len(namespace) > 0 || resourceType == "namespaces" {
 		namespaced = true
-		glog.V(9).Infof("Namespaced=true")
 	}
 	name := request.PathParameter("name")
-	glog.V(9).Infof("Name=%s", name)
 	metricName := request.PathParameter("metric")
-	glog.V(9).Infof("MetricName=%s", metricName)
 
 	value := new(resource.Quantity)
 	err := request.ReadEntity(value)
-	glog.V(9).Infof("Value=%v", value)
 	if err != nil {
 		response.WriteErrorString(http.StatusBadRequest, err.Error())
-		glog.V(10).Infof("Bad Value: %v", value)
 		return
 	}
 
 	groupResource := schema.ParseGroupResource(resourceType)
 
+	metricLabels := labels.Set{}
+	sel := request.QueryParameter("labels")
+	if len(sel) > 0 {
+		metricLabels, err = labels.ConvertSelectorToLabelsMap(sel)
+		if err != nil {
+			response.WriteErrorString(http.StatusBadRequest, err.Error())
+			return
+		}
+	}
+
 	info := provider.CustomMetricInfo{
 		GroupResource: groupResource,
 		Metric:        metricName,
@@ -212,7 +218,7 @@ func (p *testingProvider) updateMetric(request *restful.Request, response *restf
 
 	info, _, err = info.Normalized(p.mapper)
 	if err != nil {
-		glog.Errorf("Error normalizing info: %s", err)
+		klog.Errorf("Error normalizing info: %s", err)
 	}
 	namespacedName := types.NamespacedName{
 		Name:      name,
@@ -223,12 +229,14 @@ func (p *testingProvider) updateMetric(request *restful.Request, response *restf
 		CustomMetricInfo: info,
 		NamespacedName:   namespacedName,
 	}
-	p.values[metricInfo] = *value
+	p.values[metricInfo] = metricValue{
+		labels: metricLabels,
+		value:  *value,
+	}
 }
 
-// valueFor is a helper function to get just the Value of a specific metric
-func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName) (resource.Quantity, error) {
-	glog.V(10).Infof("Entered valueFor()")
+// valueFor is a helper function to get just the value of a specific metric
+func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName, metricSelector labels.Selector) (resource.Quantity, error) {
 	info, _, err := info.Normalized(p.mapper)
 	if err != nil {
 		return resource.Quantity{}, err
@@ -243,13 +251,15 @@ func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.Na
 		return resource.Quantity{}, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name)
 	}
 
-	glog.Infof("valueFor(): metricInfo=%v, Value=%v", metricInfo, value)
-	return value, nil
+	if !metricSelector.Matches(value.labels) {
+		return resource.Quantity{}, provider.NewMetricNotFoundForSelectorError(info.GroupResource, info.Metric, name.Name, metricSelector)
+	}
+
+	return value.value, nil
 }
 
-// metricFor is a helper function which formats a Value, metric, and object info into a MetricValue which can be returned by the metrics API
-func (p *testingProvider) metricFor(value resource.Quantity, name types.NamespacedName, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
-	glog.V(10).Infof("Entered metricFor()")
+// metricFor is a helper function which formats a value, metric, and object info into a MetricValue which can be returned by the metrics API
+func (p *testingProvider) metricFor(value resource.Quantity, name types.NamespacedName, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
 	objRef, err := helpers.ReferenceFor(p.mapper, name, info)
 	if err != nil {
 		return nil, err
@@ -264,20 +274,19 @@ func (p *testingProvider) metricFor(value resource.Quantity, name types.Namespac
 		Value:     value,
 	}
 
-	if len(selector.String()) > 0 {
-		labelSelector, err := metav1.ParseToLabelSelector(selector.String())
+	if len(metricSelector.String()) > 0 {
+		sel, err := metav1.ParseToLabelSelector(metricSelector.String())
 		if err != nil {
 			return nil, err
 		}
-		metric.Metric.Selector = labelSelector
+		metric.Metric.Selector = sel
 	}
-	glog.V(9).Infof("Metric Value=%v", metric)
+
 	return metric, nil
 }
 
 // metricsFor is a wrapper used by GetMetricBySelector to format several metrics which match a resource selector
-func (p *testingProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
-	glog.V(10).Infof("Entered metricFor()")
+func (p *testingProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	names, err := helpers.ListObjectNames(p.mapper, p.client, namespace, selector, info)
 	if err != nil {
 		return nil, err
@@ -286,47 +295,42 @@ func (p *testingProvider) metricsFor(namespace string, selector labels.Selector,
 	res := make([]custom_metrics.MetricValue, 0, len(names))
 	for _, name := range names {
 		namespacedName := types.NamespacedName{Name: name, Namespace: namespace}
-		value, err := p.valueFor(info, namespacedName)
+		value, err := p.valueFor(info, namespacedName, metricSelector)
 		if err != nil {
 			if apierr.IsNotFound(err) {
 				continue
 			}
 			return nil, err
 		}
 
-		metric, err := p.metricFor(value, namespacedName, selector, info)
+		metric, err := p.metricFor(value, namespacedName, selector, info, metricSelector)
 		if err != nil {
 			return nil, err
 		}
 		res = append(res, *metric)
 	}
-	glog.V(9).Infof("Metric Value: res=%v", res)
 
 	return &custom_metrics.MetricValueList{
 		Items: res,
 	}, nil
 }
 
-func (p *testingProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
-	glog.V(10).Infof("Entered GetMetricByName()")
+func (p *testingProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
-	value, err := p.valueFor(info, name)
-	glog.Infof("GetMetricByName(): info=%v, name=%v, value=%v", info, name, value)
-
+	value, err := p.valueFor(info, name, metricSelector)
 	if err != nil {
 		return nil, err
 	}
-	return p.metricFor(value, name, labels.Everything(), info)
+	return p.metricFor(value, name, labels.Everything(), info, metricSelector)
 }
 
-func (p *testingProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
-	glog.V(10).Infof("Entered GetMetricBySelector()")
+func (p *testingProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
-	return p.metricsFor(namespace, selector, info)
+	return p.metricsFor(namespace, selector, info, metricSelector)
 }
 
 func (p *testingProvider) ListAllMetrics() []provider.CustomMetricInfo {
@@ -349,37 +353,37 @@ func (p *testingProvider) ListAllMetrics() []provider.CustomMetricInfo {
 }
 
 func (p *testingProvider) GetExternalMetric(namespace string, metricSelector labels.Selector,
-				info provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) {
-	glog.V(10).Infof("Entered GetExternalMetric()")
-	glog.V(9).Infof("metricsSelector: %s, metricsInfo: %s", metricSelector.String(), info.Metric)
+	info provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) {
+	klog.V(10).Infof("Entered GetExternalMetric()")
+	klog.V(9).Infof("metricsSelector: %s, metricsInfo: %s", metricSelector.String(), info.Metric)
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
 	matchingMetrics := []external_metrics.ExternalMetricValue{}
 	for _, metric := range p.externalMetrics {
-		glog.V(9).Infof("externalMetricsInfo: %s, externalMetricValue: %v, externalMetricLabels: %v ",
-									metric.info.Metric, metric.Value, metric.labels)
+		klog.V(9).Infof("externalMetricsInfo: %s, externalMetricValue: %v, externalMetricLabels: %v ",
+			metric.info.Metric, metric.Value, metric.labels)
 		if metric.info.Metric == info.Metric && metricSelector.Matches(labels.Set(metric.labels)) {
 			metricValue := metric.Value
 			labelVal := metric.labels["cluster"]
-			glog.V(9).Infof("'cluster label value: %s, ", labelVal)
+			klog.V(9).Infof("'cluster label value: %s, ", labelVal)
 			// Set memory Value
-			if strings.Compare(labelVal, "memory") == 0  {
+			if strings.Compare(labelVal, "memory") == 0 {
 				resources := p.cache2.GetUnallocatedResources()
-				glog.V(9).Infof("Cache resources: %v", resources)
+				klog.V(9).Infof("Cache resources: %v", resources)
 
-				glog.V(10).Infof("Setting memory metric Value: %f.", resources.Memory)
+				klog.V(10).Infof("Setting memory metric Value: %f.", resources.Memory)
 				metricValue.Value = *resource.NewQuantity(int64(resources.Memory), resource.DecimalSI)
 				//metricValue.Value = *resource.NewQuantity(4500000000, resource.DecimalSI)
 			} else if strings.Compare(labelVal, "cpu") == 0 {
 				// Set cpu Value
 				resources := p.cache2.GetUnallocatedResources()
-				glog.V(9).Infof("Cache resources: %f", resources)
+				klog.V(9).Infof("Cache resources: %f", resources)
 
-				glog.V(10).Infof("Setting cpu metric Value: %v.", resources.MilliCPU)
+				klog.V(10).Infof("Setting cpu metric Value: %v.", resources.MilliCPU)
 				metricValue.Value = *resource.NewQuantity(int64(resources.MilliCPU), resource.DecimalSI)
 			} else {
-				glog.V(10).Infof("Not setting cpu/memory metric Value")
+				klog.V(10).Infof("Not setting cpu/memory metric Value")
 			}
 
 			metricValue.Timestamp = metav1.Now()
@@ -392,15 +396,15 @@ func (p *testingProvider) GetExternalMetric(namespace string, metricSelector lab
 }
 
 func (p *testingProvider) ListAllExternalMetrics() []provider.ExternalMetricInfo {
-	glog.V(10).Infof("Entered ListAllExternalMetrics()")
+	klog.V(10).Infof("Entered ListAllExternalMetrics()")
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
 	externalMetricsInfo := []provider.ExternalMetricInfo{}
 	for _, metric := range p.externalMetrics {
 		externalMetricsInfo = append(externalMetricsInfo, metric.info)
-		glog.V(9).Infof("Add metric=%v to externalMetricsInfo", metric)
+		klog.V(9).Infof("Add metric=%v to externalMetricsInfo", metric)
 	}
-	glog.V(9).Infof("ExternalMetricsInfo=%v", externalMetricsInfo)
+	klog.V(9).Infof("ExternalMetricsInfo=%v", externalMetricsInfo)
 	return externalMetricsInfo
 }
diff --git a/pkg/controller/metrics/apiserver/apiserver.go b/pkg/controller/metrics/apiserver/apiserver.go
@@ -25,6 +25,7 @@ import (
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/informers"
 
+	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver/installer"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/provider"
 	cminstall "k8s.io/metrics/pkg/apis/custom_metrics/install"
 	eminstall "k8s.io/metrics/pkg/apis/external_metrics/install"
@@ -39,6 +40,9 @@ func init() {
 	cminstall.Install(Scheme)
 	eminstall.Install(Scheme)
 
+	// we need custom conversion functions to list resources with options
+	installer.RegisterConversions(Scheme)
+
 	// we need to add the options to empty v1
 	// TODO fix the server code to avoid this
 	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})

diff --git a/pkg/controller/metrics/apiserver/cmapis.go b/pkg/controller/metrics/apiserver/cmapis.go
@@ -32,31 +32,35 @@ import (
 )
 
 func (s *CustomMetricsAdapterServer) InstallCustomMetricsAPI() error {
-	groupInfo := genericapiserver.NewDefaultAPIGroupInfo(custom_metrics.GroupName, Scheme, metav1.ParameterCodec, Codecs)
+	groupInfo := genericapiserver.NewDefaultAPIGroupInfo(custom_metrics.GroupName, Scheme, runtime.NewParameterCodec(Scheme), Codecs)
+	container := s.GenericAPIServer.Handler.GoRestfulContainer
 
-	mainGroupVer := groupInfo.PrioritizedVersions[0]
-	preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{
-		GroupVersion: mainGroupVer.String(),
-		Version:      mainGroupVer.Version,
-	}
-	groupVersion := metav1.GroupVersionForDiscovery{
-		GroupVersion: mainGroupVer.String(),
-		Version:      mainGroupVer.Version,
-	}
-	apiGroup := metav1.APIGroup{
-		Name:             mainGroupVer.Group,
-		Versions:         []metav1.GroupVersionForDiscovery{groupVersion},
-		PreferredVersion: preferredVersionForDiscovery,
-	}
+	// Register custom metrics REST handler for all supported API versions.
+	for versionIndex, mainGroupVer := range groupInfo.PrioritizedVersions {
+		preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{
+			GroupVersion: mainGroupVer.String(),
+			Version:      mainGroupVer.Version,
+		}
+		groupVersion := metav1.GroupVersionForDiscovery{
+			GroupVersion: mainGroupVer.String(),
+			Version:      mainGroupVer.Version,
+		}
+		apiGroup := metav1.APIGroup{
+			Name:             mainGroupVer.Group,
+			Versions:         []metav1.GroupVersionForDiscovery{groupVersion},
+			PreferredVersion: preferredVersionForDiscovery,
+		}
 
-	cmAPI := s.cmAPI(&groupInfo, mainGroupVer)
-	if err := cmAPI.InstallREST(s.GenericAPIServer.Handler.GoRestfulContainer); err != nil {
-		return err
-	}
-
-	s.GenericAPIServer.DiscoveryGroupManager.AddGroup(apiGroup)
-	s.GenericAPIServer.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.GenericAPIServer.Serializer, apiGroup).WebService())
+		cmAPI := s.cmAPI(&groupInfo, mainGroupVer)
+		if err := cmAPI.InstallREST(container); err != nil {
+			return err
+		}
 
+		if versionIndex == 0 {
+			s.GenericAPIServer.DiscoveryGroupManager.AddGroup(apiGroup)
+			container.Add(discovery.NewAPIGroupHandler(s.GenericAPIServer.Serializer, apiGroup).WebService())
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/controller/metrics/apiserver/endpoints/handlers/get.go b/pkg/controller/metrics/apiserver/endpoints/handlers/get.go
@@ -0,0 +1,176 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/handlers"
+	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	utiltrace "k8s.io/utils/trace"
+
+	cm_rest "github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/apiserver/registry/rest"
+)
+
+func ListResourceWithOptions(r cm_rest.ListerWithOptions, scope handlers.RequestScope) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		// For performance tracking purposes.
+		trace := utiltrace.New("List " + req.URL.Path)
+
+		namespace, err := scope.Namer.Namespace(req)
+		if err != nil {
+			writeError(&scope, err, w, req)
+			return
+		}
+
+		// Watches for single objects are routed to this function.
+		// Treat a name parameter the same as a field selector entry.
+		hasName := true
+		_, name, err := scope.Namer.Name(req)
+		if err != nil {
+			hasName = false
+		}
+
+		ctx := req.Context()
+		ctx = request.WithNamespace(ctx, namespace)
+
+		opts := metainternalversion.ListOptions{}
+		if err := metainternalversionscheme.ParameterCodec.DecodeParameters(req.URL.Query(), scope.MetaGroupVersion, &opts); err != nil {
+			err = errors.NewBadRequest(err.Error())
+			writeError(&scope, err, w, req)
+			return
+		}
+
+		// transform fields
+		// TODO: DecodeParametersInto should do this.
+		if opts.FieldSelector != nil {
+			fn := func(label, value string) (newLabel, newValue string, err error) {
+				return scope.Convertor.ConvertFieldLabel(scope.Kind, label, value)
+			}
+			if opts.FieldSelector, err = opts.FieldSelector.Transform(fn); err != nil {
+				// TODO: allow bad request to set field causes based on query parameters
+				err = errors.NewBadRequest(err.Error())
+				writeError(&scope, err, w, req)
+				return
+			}
+		}
+
+		if hasName {
+			// metadata.name is the canonical internal name.
+			// SelectionPredicate will notice that this is a request for
+			// a single object and optimize the storage query accordingly.
+			nameSelector := fields.OneTermEqualSelector("metadata.name", name)
+
+			// Note that fieldSelector setting explicitly the "metadata.name"
+			// will result in reaching this branch (as the value of that field
+			// is propagated to requestInfo as the name parameter.
+			// That said, the allowed field selectors in this branch are:
+			// nil, fields.Everything and field selector matching metadata.name
+			// for our name.
+			if opts.FieldSelector != nil && !opts.FieldSelector.Empty() {
+				selectedName, ok := opts.FieldSelector.RequiresExactMatch("metadata.name")
+				if !ok || name != selectedName {
+					writeError(&scope, errors.NewBadRequest("fieldSelector metadata.name doesn't match requested name"), w, req)
+					return
+				}
+			} else {
+				opts.FieldSelector = nameSelector
+			}
+		}
+
+		// Log only long List requests (ignore Watch).
+		defer trace.LogIfLong(500 * time.Millisecond)
+		trace.Step("About to List from storage")
+		extraOpts, hasSubpath, subpathKey := r.NewListOptions()
+		if err := getRequestOptions(req, scope, extraOpts, hasSubpath, subpathKey, false); err != nil {
+			err = errors.NewBadRequest(err.Error())
+			writeError(&scope, err, w, req)
+			return
+		}
+		result, err := r.List(ctx, &opts, extraOpts)
+		if err != nil {
+			writeError(&scope, err, w, req)
+			return
+		}
+		trace.Step("Listing from storage done")
+		numberOfItems, err := setListSelfLink(result, ctx, req, scope.Namer)
+		if err != nil {
+			writeError(&scope, err, w, req)
+			return
+		}
+		trace.Step("Self-linking done")
+		// Ensure empty lists return a non-nil items slice
+		if numberOfItems == 0 && meta.IsListType(result) {
+			if err := meta.SetList(result, []runtime.Object{}); err != nil {
+				writeError(&scope, err, w, req)
+				return
+			}
+		}
+		responsewriters.WriteObjectNegotiated(scope.Serializer, negotiation.DefaultEndpointRestrictions, scope.Kind.GroupVersion(), w, req, http.StatusOK, result)
+		trace.Step(fmt.Sprintf("Writing http response done (%d items)", numberOfItems))
+	}
+}
+
+// getRequestOptions parses out options and can include path information.  The path information shouldn't include the subresource.
+func getRequestOptions(req *http.Request, scope handlers.RequestScope, into runtime.Object, hasSubpath bool, subpathKey string, isSubresource bool) error {
+	if into == nil {
+		return nil
+	}
+
+	query := req.URL.Query()
+	if hasSubpath {
+		newQuery := make(url.Values)
+		for k, v := range query {
+			newQuery[k] = v
+		}
+
+		ctx := req.Context()
+		requestInfo, _ := request.RequestInfoFrom(ctx)
+		startingIndex := 2
+		if isSubresource {
+			startingIndex = 3
+		}
+
+		p := strings.Join(requestInfo.Parts[startingIndex:], "/")
+
+		// ensure non-empty subpaths correctly reflect a leading slash
+		if len(p) > 0 && !strings.HasPrefix(p, "/") {
+			p = "/" + p
+		}
+
+		// ensure subpaths correctly reflect the presence of a trailing slash on the original request
+		if strings.HasSuffix(requestInfo.Path, "/") && !strings.HasSuffix(p, "/") {
+			p += "/"
+		}
+
+		newQuery[subpathKey] = []string{p}
+		query = newQuery
+	}
+	return scope.ParameterCodec.DecodeParameters(query, scope.Kind.GroupVersion(), into)
+}
diff --git a/pkg/controller/metrics/apiserver/endpoints/handlers/rest.go b/pkg/controller/metrics/apiserver/endpoints/handlers/rest.go
@@ -0,0 +1,73 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handlers
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/handlers"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog/v2"
+)
+
+func writeError(scope *handlers.RequestScope, err error, w http.ResponseWriter, req *http.Request) {
+	responsewriters.ErrorNegotiated(err, scope.Serializer, scope.Kind.GroupVersion(), w, req)
+}
+
+// setSelfLink sets the self link of an object (or the child items in a list) to the base URL of the request
+// plus the path and query generated by the provided linkFunc
+func setSelfLink(obj runtime.Object, requestInfo *request.RequestInfo, namer handlers.ScopeNamer) error {
+	// TODO: SelfLink generation should return a full URL?
+	uri, err := namer.GenerateLink(requestInfo, obj)
+	if err != nil {
+		return nil
+	}
+
+	return namer.SetSelfLink(obj, uri)
+}
+
+// setListSelfLink sets the self link of a list to the base URL, then sets the self links
+// on all child objects returned. Returns the number of items in the list.
+func setListSelfLink(obj runtime.Object, ctx context.Context, req *http.Request, namer handlers.ScopeNamer) (int, error) {
+	if !meta.IsListType(obj) {
+		return 0, nil
+	}
+
+	uri, err := namer.GenerateListLink(req)
+	if err != nil {
+		return 0, err
+	}
+	if err := namer.SetSelfLink(obj, uri); err != nil {
+		klog.V(4).Infof("Unable to set self link on object: %v", err)
+	}
+	requestInfo, ok := request.RequestInfoFrom(ctx)
+	if !ok {
+		return 0, fmt.Errorf("missing requestInfo")
+	}
+
+	count := 0
+	err = meta.EachListItem(obj, func(obj runtime.Object) error {
+		count++
+		return setSelfLink(obj, requestInfo, namer)
+	})
+	return count, err
+}
diff --git a/pkg/controller/metrics/apiserver/installer/apiserver_test.go b/pkg/controller/metrics/apiserver/installer/apiserver_test.go
@@ -71,6 +71,9 @@ func init() {
 	installcm.Install(Scheme)
 	installem.Install(Scheme)
 
+	// we need custom conversion functions to list resources with options
+	RegisterConversions(Scheme)
+
 	// we need to add the options to empty v1
 	// TODO fix the server code to avoid this
 	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
@@ -85,9 +88,9 @@ func init() {
 		&metav1.APIResourceList{},
 	)
 
-	customMetricsGroupInfo = genericapiserver.NewDefaultAPIGroupInfo(custom_metrics.GroupName, Scheme, metav1.ParameterCodec, Codecs)
+	customMetricsGroupInfo = genericapiserver.NewDefaultAPIGroupInfo(custom_metrics.GroupName, Scheme, runtime.NewParameterCodec(Scheme), Codecs)
 	customMetricsGroupVersion = customMetricsGroupInfo.PrioritizedVersions[0]
-	externalMetricsGroupInfo = genericapiserver.NewDefaultAPIGroupInfo(external_metrics.GroupName, Scheme, metav1.ParameterCodec, Codecs)
+	externalMetricsGroupInfo = genericapiserver.NewDefaultAPIGroupInfo(external_metrics.GroupName, Scheme, runtime.NewParameterCodec(Scheme), Codecs)
 	externalMetricsGroupVersion = externalMetricsGroupInfo.PrioritizedVersions[0]
 }
 
@@ -165,7 +168,7 @@ func handleExternalMetrics(prov provider.ExternalMetricsProvider) http.Handler {
 			Linker:          runtime.SelfLinker(meta.NewAccessor()),
 		},
 		ResourceLister: provider.NewExternalMetricResourceLister(prov),
-		Handlers:       &CMHandlers{},
+		Handlers:       &EMHandlers{},
 	}
 
 	if err := group.InstallREST(container); err != nil {
@@ -186,7 +189,7 @@ type fakeCMProvider struct {
 	metrics                []provider.CustomMetricInfo
 }
 
-func (p *fakeCMProvider) valuesFor(name types.NamespacedName, info provider.CustomMetricInfo) (string, []custom_metrics.MetricValue, bool) {
+func (p *fakeCMProvider) valuesFor(name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (string, []custom_metrics.MetricValue, bool) {
 	if info.Namespaced {
 		metricId := name.Namespace + "/" + info.GroupResource.String() + "/" + name.Name + "/" + info.Metric
 		values, ok := p.namespacedValues[metricId]
@@ -198,17 +201,17 @@ func (p *fakeCMProvider) valuesFor(name types.NamespacedName, info provider.Cust
 	}
 }
 
-func (p *fakeCMProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
-	metricId, values, ok := p.valuesFor(name, info)
+func (p *fakeCMProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
+	metricId, values, ok := p.valuesFor(name, info, metricSelector)
 	if !ok {
 		return nil, fmt.Errorf("non-existent metric requested (id: %s)", metricId)
 	}
 
 	return &values[0], nil
 }
 
-func (p *fakeCMProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
-	metricId, values, ok := p.valuesFor(types.NamespacedName{Namespace: namespace, Name: "*"}, info)
+func (p *fakeCMProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
+	metricId, values, ok := p.valuesFor(types.NamespacedName{Namespace: namespace, Name: "*"}, info, metricSelector)
 	if !ok {
 		return nil, fmt.Errorf("non-existent metric requested (id: %s)", metricId)
 	}

diff --git a/pkg/controller/metrics/apiserver/installer/cmhandlers.go b/pkg/controller/metrics/apiserver/installer/cmhandlers.go
@@ -17,17 +17,15 @@ limitations under the License.
 package installer
 
 import (
-	"github.com/golang/glog"
 	"net/http"
 	gpath "path"
 
+	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver/registry/rest"
+	"github.com/emicklei/go-restful"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/endpoints/handlers"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
-	"k8s.io/apiserver/pkg/registry/rest"
-
-	"github.com/emicklei/go-restful"
 )
 
 type CMHandlers struct{}
@@ -36,7 +34,6 @@ type CMHandlers struct{}
 // Compared to the normal installer, this plays fast and loose a bit, but should still
 // follow the API conventions.
 func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restful.WebService) error {
-	glog.Infof("Entered CMHandlers registerResourceHandlers()")
 	optionsExternalVersion := a.group.GroupVersion
 	if a.group.OptionsExternalVersion != nil {
 		optionsExternalVersion = *a.group.OptionsExternalVersion
@@ -49,7 +46,7 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 
 	kind := fqKindToRegister.Kind
 
-	lister := a.group.DynamicStorage.(rest.Lister)
+	lister := a.group.DynamicStorage.(rest.ListerWithOptions)
 	list := lister.NewList()
 	listGVKs, _, err := a.group.Typer.ObjectKinds(list)
 	if err != nil {
@@ -66,6 +63,20 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 		return err
 	}
 
+	listOptions, _, _ := lister.NewListOptions()
+	listOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(listOptions)
+	if err != nil {
+		return err
+	}
+	listOptionsInternalKind := listOptionsInternalKinds[0]
+	versionedListExtraOptions, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listOptionsInternalKind.Kind))
+	if err != nil {
+		versionedListExtraOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(listOptionsInternalKind.Kind))
+		if err != nil {
+			return err
+		}
+	}
+
 	nameParam := ws.PathParameter("name", "name of the described resource").DataType("string")
 	resourceParam := ws.PathParameter("resource", "the name of the resource").DataType("string")
 	subresourceParam := ws.PathParameter("subresource", "the name of the subresource").DataType("string")
@@ -77,7 +88,6 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 		subresourceParam,
 	}
 	rootScopedPath := "{resource}/{name}/{subresource}"
-	glog.Infof("CMHandlers registerResourceHandlers(): rootScopedPath=%s", rootScopedPath)
 
 	// metrics describing namespaced objects (e.g. pods)
 	namespaceParam := ws.PathParameter("namespace", "object name and auth scope, such as for teams and projects").DataType("string")
@@ -88,10 +98,8 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 		subresourceParam,
 	}
 	namespacedPath := "namespaces/{namespace}/{resource}/{name}/{subresource}"
-	glog.Infof("CMHandlers registerResourceHandlers(): namespacedPath=%s", namespacedPath)
 
 	namespaceSpecificPath := "namespaces/{namespace}/metrics/{name}"
-	glog.Infof("CMHandlers registerResourceHandlers(): namespaceSpecificPath=%s", namespaceSpecificPath)
 	namespaceSpecificParams := []*restful.Parameter{
 		namespaceParam,
 		nameParam,
@@ -132,7 +140,18 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 		},
 	}
 
-	rootScopedHandler := metrics.InstrumentRouteFunc("LIST", "custom-metrics", "", "cluster", restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout))
+	rootScopedHandler := metrics.InstrumentRouteFunc(
+		"LIST",
+		a.group.GroupVersion.Group,
+		a.group.GroupVersion.Version,
+		reqScope.Resource.Resource,
+		reqScope.Subresource,
+		"cluster",
+		"custom-metrics",
+		false,
+		"",
+		restfulListResourceWithOptions(lister, reqScope),
+	)
 
 	// install the root-scoped route
 	rootScopedRoute := ws.GET(rootScopedPath).To(rootScopedHandler).
@@ -145,6 +164,9 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 	if err := addObjectParams(ws, rootScopedRoute, versionedListOptions); err != nil {
 		return err
 	}
+	if err := addObjectParams(ws, rootScopedRoute, versionedListExtraOptions); err != nil {
+		return err
+	}
 	addParams(rootScopedRoute, rootScopedParams)
 	ws.Route(rootScopedRoute)
 
@@ -156,7 +178,19 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 			SelfLinkPathPrefix: gpath.Join(a.prefix, "namespaces") + "/",
 		},
 	}
-	namespacedHandler := metrics.InstrumentRouteFunc("LIST", "custom-metrics-namespaced", "", "namespace", restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout))
+	namespacedHandler := metrics.InstrumentRouteFunc(
+		"LIST",
+		a.group.GroupVersion.Group,
+		a.group.GroupVersion.Version,
+		reqScope.Resource.Resource,
+		reqScope.Subresource,
+		"resource",
+		"custom-metrics",
+		false,
+		"",
+		restfulListResourceWithOptions(lister, reqScope),
+	)
+
 	namespacedRoute := ws.GET(namespacedPath).To(namespacedHandler).
 		Doc(doc).
 		Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -167,6 +201,9 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 	if err := addObjectParams(ws, namespacedRoute, versionedListOptions); err != nil {
 		return err
 	}
+	if err := addObjectParams(ws, namespacedRoute, versionedListExtraOptions); err != nil {
+		return err
+	}
 	addParams(namespacedRoute, namespacedParams)
 	ws.Route(namespacedRoute)
 
@@ -178,7 +215,20 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 			SelfLinkPathPrefix: gpath.Join(a.prefix, "namespaces") + "/",
 		},
 	}
-	namespaceSpecificHandler := metrics.InstrumentRouteFunc("LIST", "custom-metrics-for-namespace", "", "cluster", restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout))
+
+	namespaceSpecificHandler := metrics.InstrumentRouteFunc(
+		"LIST",
+		a.group.GroupVersion.Group,
+		a.group.GroupVersion.Version,
+		reqScope.Resource.Resource,
+		reqScope.Subresource,
+		"resource",
+		"custom-metrics",
+		false,
+		"",
+		restfulListResourceWithOptions(lister, reqScope),
+	)
+
 	namespaceSpecificRoute := ws.GET(namespaceSpecificPath).To(namespaceSpecificHandler).
 		Doc(doc).
 		Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -189,6 +239,9 @@ func (ch *CMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 	if err := addObjectParams(ws, namespaceSpecificRoute, versionedListOptions); err != nil {
 		return err
 	}
+	if err := addObjectParams(ws, namespaceSpecificRoute, versionedListExtraOptions); err != nil {
+		return err
+	}
 	addParams(namespaceSpecificRoute, namespaceSpecificParams)
 	ws.Route(namespaceSpecificRoute)
 

diff --git a/pkg/controller/metrics/apiserver/installer/conversion.go b/pkg/controller/metrics/apiserver/installer/conversion.go
@@ -0,0 +1,77 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package installer
+
+import (
+	"net/url"
+
+	"k8s.io/apimachinery/pkg/conversion"
+	"k8s.io/apimachinery/pkg/runtime"
+	cmv1beta1 "k8s.io/metrics/pkg/apis/custom_metrics/v1beta1"
+	cmv1beta2 "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2"
+)
+
+func Convert_url_Values_To_v1beta1_MetricListOptions(in *url.Values, out *cmv1beta1.MetricListOptions, s conversion.Scope) error {
+	if values, ok := map[string][]string(*in)["labelSelector"]; ok && len(values) > 0 {
+		if err := runtime.Convert_Slice_string_To_string(&values, &out.LabelSelector, s); err != nil {
+			return err
+		}
+	} else {
+		out.LabelSelector = ""
+	}
+	if values, ok := map[string][]string(*in)["metricLabelSelector"]; ok && len(values) > 0 {
+		if err := runtime.Convert_Slice_string_To_string(&values, &out.MetricLabelSelector, s); err != nil {
+			return err
+		}
+	} else {
+		out.MetricLabelSelector = ""
+	}
+	return nil
+}
+
+func Convert_url_Values_To_v1beta2_MetricListOptions(in *url.Values, out *cmv1beta2.MetricListOptions, s conversion.Scope) error {
+	if values, ok := map[string][]string(*in)["labelSelector"]; ok && len(values) > 0 {
+		if err := runtime.Convert_Slice_string_To_string(&values, &out.LabelSelector, s); err != nil {
+			return err
+		}
+	} else {
+		out.LabelSelector = ""
+	}
+	if values, ok := map[string][]string(*in)["metricLabelSelector"]; ok && len(values) > 0 {
+		if err := runtime.Convert_Slice_string_To_string(&values, &out.MetricLabelSelector, s); err != nil {
+			return err
+		}
+	} else {
+		out.MetricLabelSelector = ""
+	}
+	return nil
+}
+
+// RegisterConversions adds conversion functions to the given scheme.
+func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddConversionFunc((*url.Values)(nil), (*cmv1beta1.MetricListOptions)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_url_Values_To_v1beta1_MetricListOptions(a.(*url.Values), b.(*cmv1beta1.MetricListOptions), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddConversionFunc((*url.Values)(nil), (*cmv1beta2.MetricListOptions)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_url_Values_To_v1beta2_MetricListOptions(a.(*url.Values), b.(*cmv1beta2.MetricListOptions), scope)
+	}); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/pkg/controller/metrics/apiserver/installer/emhandlers.go b/pkg/controller/metrics/apiserver/installer/emhandlers.go
@@ -17,7 +17,6 @@ limitations under the License.
 package installer
 
 import (
-	"github.com/golang/glog"
 	"net/http"
 	gpath "path"
 
@@ -35,7 +34,6 @@ type EMHandlers struct{}
 // registerResourceHandlers registers the resource handlers for external metrics.
 // The implementation is based on corresponding registerResourceHandlers for Custom Metrics API
 func (ch *EMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restful.WebService) error {
-	glog.Infof("Entered EMHandlers registerResourceHandlers()")
 	optionsExternalVersion := a.group.GroupVersion
 	if a.group.OptionsExternalVersion != nil {
 		optionsExternalVersion = *a.group.OptionsExternalVersion
@@ -74,7 +72,6 @@ func (ch *EMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 	}
 	externalMetricPath := "namespaces" + "/{namespace}/{resource}"
 
-	glog.Infof("EMHandlers registerResourceHandlers(): externalMetricPath=%s", externalMetricPath)
 	mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
 	allMediaTypes := append(mediaTypes, streamMediaTypes...)
 	ws.Produces(allMediaTypes...)
@@ -109,7 +106,18 @@ func (ch *EMHandlers) registerResourceHandlers(a *MetricsAPIInstaller, ws *restf
 		},
 	}
 
-	externalMetricHandler := metrics.InstrumentRouteFunc("LIST", "external-metrics", "", "", restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout))
+	externalMetricHandler := metrics.InstrumentRouteFunc(
+		"LIST",
+		a.group.GroupVersion.Group,
+		a.group.GroupVersion.Version,
+		reqScope.Resource.Resource,
+		reqScope.Subresource,
+		"cluster",
+		"external-metrics",
+		false,
+		"",
+		restfulListResource(lister, nil, reqScope, false, a.minRequestTimeout),
+	)
 
 	externalMetricRoute := ws.GET(externalMetricPath).To(externalMetricHandler).
 		Doc(doc).

diff --git a/pkg/controller/metrics/apiserver/installer/installer.go b/pkg/controller/metrics/apiserver/installer/installer.go
@@ -18,12 +18,14 @@ package installer
 
 import (
 	"fmt"
-	"github.com/golang/glog"
 	gpath "path"
 	"reflect"
 	"strings"
 	"time"
 
+	cm_rest "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver/registry/rest"
+	"github.com/emicklei/go-restful"
+	cm_handlers "github.com/kubernetes-sigs/custom-metrics-apiserver/pkg/apiserver/endpoints/handlers"
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -34,8 +36,6 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
-
-	"github.com/emicklei/go-restful"
 )
 
 // NB: the contents of this file should mostly be a subset of the functionality
@@ -220,7 +220,10 @@ func addObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj in
 				if docable, ok := obj.(documentable); ok {
 					desc = docable.SwaggerDoc()[jsonName]
 				}
-				route.Param(ws.QueryParameter(jsonName, desc).DataType(typeToJSON(sf.Type.String())))
+
+				if route.ParameterNamed(jsonName) == nil {
+					route.Param(ws.QueryParameter(jsonName, desc).DataType(typeToJSON(sf.Type.String())))
+				}
 			}
 		}
 	}
@@ -241,7 +244,7 @@ func typeToJSON(typeName string) string {
 		return "string"
 	case "byte", "*byte":
 		return "string"
-	case "v1.DeletionPropagation", "*v1.DeletionPropagation":
+	case "v1.DeletionPropagation", "*v1.DeletionPropagation", "v1.ResourceVersionMatch":
 		return "string"
 
 	// TODO: Fix these when go-restful supports a way to specify an array query param:
@@ -268,11 +271,10 @@ type MetricsNaming struct {
 }
 
 func (n MetricsNaming) GenerateLink(requestInfo *request.RequestInfo, obj runtime.Object) (uri string, err error) {
-	glog.Infof("Entered GenerateLink()")
 	if requestInfo.Resource != "metrics" {
 		n.SelfLinkPathSuffix += "/" + requestInfo.Subresource
 	}
-	glog.Infof("GenerateLink(): SelfLinkPathSuffix=%v, requestInfo=%v", n.SelfLinkPathSuffix, requestInfo)
+
 	// since this is not a pointer receiver, it's ok to modify it here
 	// (since we copy around every method call)
 	if n.ClusterScoped {
@@ -284,9 +286,13 @@ func (n MetricsNaming) GenerateLink(requestInfo *request.RequestInfo, obj runtim
 }
 
 func restfulListResource(r rest.Lister, rw rest.Watcher, scope handlers.RequestScope, forceWatch bool, minRequestTimeout time.Duration) restful.RouteFunction {
-	glog.Infof("Entered restfulListResource()")
-	glog.Infof("restfulListResource(): restLister=%v, scope=%v", r, scope)
 	return func(req *restful.Request, res *restful.Response) {
-		handlers.ListResource(r, rw, scope, forceWatch, minRequestTimeout)(res.ResponseWriter, req.Request)
+		handlers.ListResource(r, rw, &scope, forceWatch, minRequestTimeout)(res.ResponseWriter, req.Request)
+	}
+}
+
+func restfulListResourceWithOptions(r cm_rest.ListerWithOptions, scope handlers.RequestScope) restful.RouteFunction {
+	return func(req *restful.Request, res *restful.Response) {
+		cm_handlers.ListResourceWithOptions(r, scope)(res.ResponseWriter, req.Request)
 	}
 }
diff --git a/pkg/controller/metrics/apiserver/registry/rest/rest.go b/pkg/controller/metrics/apiserver/registry/rest/rest.go
@@ -0,0 +1,45 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rest
+
+import (
+	"context"
+
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// ListerWithOptions is an object that can retrieve resources that match the provided field
+// and label criteria and takes additional options on the list request.
+type ListerWithOptions interface {
+	// NewList returns an empty object that can be used with the List call.
+	// This object must be a pointer type for use with Codec.DecodeInto([]byte, runtime.Object)
+	NewList() runtime.Object
+
+	// List selects resources in the storage which match to the selector. 'options' can be nil.
+	// The extraOptions object passed to it is of the same type returned by the NewListOptions
+	// method.
+	List(ctx context.Context, options *metainternalversion.ListOptions, extraOptions runtime.Object) (runtime.Object, error)
+
+	// NewListOptions returns an empty options object that will be used to pass extra options
+	// to the List method. It may return a bool and a string, if true, the
+	// value of the request path below the list will be included as the named
+	// string in the serialization of the runtime object. E.g., returning "path"
+	// will convert the trailing request scheme value to "path" in the map[string][]string
+	// passed to the converter.
+	NewListOptions() (runtime.Object, bool, string)
+}
diff --git a/pkg/controller/metrics/cmd/builder.go b/pkg/controller/metrics/cmd/builder.go
@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/golang/glog"
 	"sync"
 	"time"
 
@@ -30,6 +29,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
 
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/cmd/server"
@@ -67,6 +67,9 @@ type AdapterBase struct {
 	// if not explicitly set.
 	FlagSet *pflag.FlagSet
 
+	// OpenAPIConfig
+	OpenAPIConfig *openapicommon.Config
+
 	// flagOnce controls initialization of the flags.
 	flagOnce sync.Once
 
@@ -85,12 +88,11 @@ type AdapterBase struct {
 
 // InstallFlags installs the minimum required set of flags into the flagset.
 func (b *AdapterBase) InstallFlags() {
-	glog.Infof("Entered InstallFlags()")
-
 	b.initFlagSet()
 	b.flagOnce.Do(func() {
 		if b.CustomMetricsAdapterServerOptions == nil {
 			b.CustomMetricsAdapterServerOptions = server.NewCustomMetricsAdapterServerOptions()
+			b.CustomMetricsAdapterServerOptions.OpenAPIConfig = b.OpenAPIConfig
 		}
 
 		b.SecureServing.AddFlags(b.FlagSet)
@@ -108,28 +110,16 @@ func (b *AdapterBase) InstallFlags() {
 
 // initFlagSet populates the flagset to the CommandLine flags if it's not already set.
 func (b *AdapterBase) initFlagSet() {
-	glog.Infof("Entered initFlagSet()")
-
 	if b.FlagSet == nil {
 		// default to the normal commandline flags
-		glog.Infof("initFlagSet(): FlagSet was nil.")
 		b.FlagSet = pflag.CommandLine
 	}
-	p, err := b.FlagSet.GetInt("secure-port")
-	if err != nil {
-		glog.Infof("initFlagSet(): Flagset secure-port = %v", p)
-	} else {
-		glog.Infof("initFlagSet(): Error getting secure-port: %v", err)
-		b.FlagSet.Set("secure-port", "6443")
-		glog.Infof("initFlagSet(): Flagset secure-port = %v", p)
-	}
 }
 
 // Flags returns the flagset used by this adapter.
 // It will initialize the flagset with the minimum required set
 // of flags as well.
 func (b *AdapterBase) Flags() *pflag.FlagSet {
-	glog.Infof("Entered Flags()")
 	b.initFlagSet()
 	b.InstallFlags()
 
@@ -289,20 +279,10 @@ func (b *AdapterBase) Informers() (informers.SharedInformerFactory, error) {
 
 // Run runs this custom metrics adapter until the given stop channel is closed.
 func (b *AdapterBase) Run(stopCh <-chan struct{}) error {
-	glog.Infof("Entered Run()")
 	server, err := b.Server()
 	if err != nil {
-		glog.Infof("Error creating server: %v", err)
 		return err
 	}
 
-	glog.Infof("Successfully created server.")
-//	return server.GenericAPIServer.PrepareRun().Run(stopCh)
-	runErr := server.GenericAPIServer.PrepareRun().Run(stopCh)
-	if runErr != nil {
-		glog.Infof("Error running API Server: %v", runErr)
-
-	}
-	glog.Infof("Exited Run()")
-	return runErr
+	return server.GenericAPIServer.PrepareRun().Run(stopCh)
 }
diff --git a/pkg/controller/metrics/cmd/server/start.go b/pkg/controller/metrics/cmd/server/start.go
@@ -23,6 +23,7 @@ import (
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
 
 type CustomMetricsAdapterServerOptions struct {
@@ -31,6 +32,9 @@ type CustomMetricsAdapterServerOptions struct {
 	Authentication *genericoptions.DelegatingAuthenticationOptions
 	Authorization  *genericoptions.DelegatingAuthorizationOptions
 	Features       *genericoptions.FeatureOptions
+
+	// OpenAPIConfig
+	OpenAPIConfig *openapicommon.Config
 }
 
 func NewCustomMetricsAdapterServerOptions() *CustomMetricsAdapterServerOptions {
@@ -63,15 +67,17 @@ func (o CustomMetricsAdapterServerOptions) Config() (*apiserver.Config, error) {
 		return nil, err
 	}
 
-	if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, serverConfig.OpenAPIConfig); err != nil {
+	if err := o.Authentication.ApplyTo(&serverConfig.Authentication, serverConfig.SecureServing, nil); err != nil {
 		return nil, err
 	}
 	if err := o.Authorization.ApplyTo(&serverConfig.Authorization); err != nil {
 		return nil, err
 	}
 
-	// TODO: we can't currently serve swagger because we don't have a good way to dynamically update it
-	// serverConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
+	// enable OpenAPI schemas
+	if o.OpenAPIConfig != nil {
+		serverConfig.OpenAPIConfig = o.OpenAPIConfig
+	}
 
 	config := &apiserver.Config{
 		GenericConfig: serverConfig,

diff --git a/pkg/controller/metrics/dynamicmapper/fake_discovery.go b/pkg/controller/metrics/dynamicmapper/fake_discovery.go
@@ -19,10 +19,10 @@ package dynamicmapper
 import (
 	"fmt"
 
-	"github.com/emicklei/go-restful-swagger12"
-	"github.com/googleapis/gnostic/OpenAPIv2"
+	swagger "github.com/emicklei/go-restful-swagger12"
+	openapi_v2 "github.com/googleapis/gnostic/openapiv2"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/version"
@@ -61,6 +61,24 @@ func (c *FakeDiscovery) ServerResources() ([]*metav1.APIResourceList, error) {
 	return c.Resources, nil
 }
 
+func (c *FakeDiscovery) ServerGroupsAndResources() ([]*metav1.APIGroup, []*metav1.APIResourceList, error) {
+	sgs, err := c.ServerGroups()
+	if err != nil {
+		return nil, nil, err
+	}
+	resultGroups := []*metav1.APIGroup{}
+	for i := range sgs.Groups {
+		resultGroups = append(resultGroups, &sgs.Groups[i])
+	}
+
+	action := testing.ActionImpl{
+		Verb:     "get",
+		Resource: schema.GroupVersionResource{Resource: "resource"},
+	}
+	c.Invokes(action, nil)
+	return resultGroups, c.Resources, nil
+}
+
 func (c *FakeDiscovery) ServerPreferredResources() ([]*metav1.APIResourceList, error) {
 	return nil, nil
 }

diff --git a/pkg/controller/metrics/dynamicmapper/mapper.go b/pkg/controller/metrics/dynamicmapper/mapper.go
@@ -5,13 +5,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/golang/glog"
-
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/restmapper"
+	"k8s.io/klog/v2"
 )
 
 // RengeneratingDiscoveryRESTMapper is a RESTMapper which Regenerates its cache of mappings periodically.
@@ -44,7 +43,7 @@ func NewRESTMapper(discoveryClient discovery.DiscoveryInterface, refreshInterval
 func (m *RegeneratingDiscoveryRESTMapper) RunUntil(stop <-chan struct{}) {
 	go wait.Until(func() {
 		if err := m.RegenerateMappings(); err != nil {
-			glog.Errorf("error regenerating REST mappings from discovery: %v", err)
+			klog.Errorf("error regenerating REST mappings from discovery: %v", err)
 		}
 	}, m.refreshInterval, stop)
 }

diff --git a/pkg/controller/metrics/provider/errors.go b/pkg/controller/metrics/provider/errors.go
@@ -23,6 +23,7 @@ import (
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
 
@@ -47,3 +48,14 @@ func NewMetricNotFoundForError(resource schema.GroupResource, metricName string,
 		Message: fmt.Sprintf("the server could not find the metric %s for %s %s", metricName, resource.String(), resourceName),
 	}}
 }
+
+// NewMetricNotFoundForError returns a StatusError indicating the given metric could not be found for
+// the given named object. It is similar to NewNotFound, but more specialized
+func NewMetricNotFoundForSelectorError(resource schema.GroupResource, metricName string, resourceName string, selector labels.Selector) *apierr.StatusError {
+	return &apierr.StatusError{metav1.Status{
+		Status:  metav1.StatusFailure,
+		Code:    int32(http.StatusNotFound),
+		Reason:  metav1.StatusReasonNotFound,
+		Message: fmt.Sprintf("the server could not find the metric %s for %s %s with selector %s", metricName, resource.String(), resourceName, selector.String()),
+	}}
+}
diff --git a/pkg/controller/metrics/provider/helpers/helpers.go b/pkg/controller/metrics/provider/helpers/helpers.go
@@ -17,6 +17,7 @@ limitations under the License.
 package helpers
 
 import (
+	"context"
 	"fmt"
 
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -83,7 +84,7 @@ func ListObjectNames(mapper apimeta.RESTMapper, client dynamic.Interface, namesp
 		resClient = client.Resource(res)
 	}
 
-	matchingObjectsRaw, err := resClient.List(metav1.ListOptions{LabelSelector: selector.String()})
+	matchingObjectsRaw, err := resClient.List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()})
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/controller/metrics/provider/interfaces.go b/pkg/controller/metrics/provider/interfaces.go
@@ -84,11 +84,11 @@ func (i CustomMetricInfo) Normalized(mapper apimeta.RESTMapper) (normalizedInfo
 type CustomMetricsProvider interface {
 	// GetMetricByName fetches a particular metric for a particular object.
 	// The namespace will be empty if the metric is root-scoped.
-	GetMetricByName(name types.NamespacedName, info CustomMetricInfo) (*custom_metrics.MetricValue, error)
+	GetMetricByName(name types.NamespacedName, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error)
 
 	// GetMetricBySelector fetches a particular metric for a set of objects matching
 	// the given label selector.  The namespace will be empty if the metric is root-scoped.
-	GetMetricBySelector(namespace string, selector labels.Selector, info CustomMetricInfo) (*custom_metrics.MetricValueList, error)
+	GetMetricBySelector(namespace string, selector labels.Selector, info CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error)
 
 	// ListAllMetrics provides a list of all available metrics at
 	// the current time.  Note that this is not allowed to return
@@ -98,7 +98,7 @@ type CustomMetricsProvider interface {
 }
 
 // ExternalMetricsProvider is a source of external metrics.
-// Metric is normally idendified by a name and a set of labels/tags. It is up to a specific
+// Metric is normally identified by a name and a set of labels/tags. It is up to a specific
 // implementation how to translate metricSelector to a filter for metric values.
 // Namespace can be used by the implemetation for metric identification, access control or ignored.
 type ExternalMetricsProvider interface {

diff --git a/pkg/controller/metrics/registry/custom_metrics/reststorage.go b/pkg/controller/metrics/registry/custom_metrics/reststorage.go
@@ -19,8 +19,8 @@ package apiserver
 import (
 	"context"
 	"fmt"
-	"github.com/golang/glog"
 
+	cm_rest "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/apiserver/registry/rest"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/provider"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	"k8s.io/apimachinery/pkg/labels"
@@ -38,7 +38,7 @@ type REST struct {
 }
 
 var _ rest.Storage = &REST{}
-var _ rest.Lister = &REST{}
+var _ cm_rest.ListerWithOptions = &REST{}
 
 func NewREST(cmProvider provider.CustomMetricsProvider) *REST {
 	return &REST{
@@ -52,23 +52,37 @@ func (r *REST) New() runtime.Object {
 	return &custom_metrics.MetricValue{}
 }
 
-// Implement Lister
+// Implement ListerWithOptions
 
 func (r *REST) NewList() runtime.Object {
 	return &custom_metrics.MetricValueList{}
 }
 
-func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
-	glog.Infof("Entered List()")
+func (r *REST) NewListOptions() (runtime.Object, bool, string) {
+	return &custom_metrics.MetricListOptions{}, true, "metricName"
+}
+
+func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions, metricOpts runtime.Object) (runtime.Object, error) {
+	metricOptions, ok := metricOpts.(*custom_metrics.MetricListOptions)
+	if !ok {
+		return nil, fmt.Errorf("invalid options object: %#v", options)
+	}
 
 	// populate the label selector, defaulting to all
 	selector := labels.Everything()
-	glog.Infof("List(): labels=%v", labels.Everything())
 	if options != nil && options.LabelSelector != nil {
 		selector = options.LabelSelector
 	}
 
-	glog.Infof("List(): selector=%v", selector)
+	metricLabelSelector := labels.Everything()
+	if metricOptions != nil && len(metricOptions.MetricLabelSelector) > 0 {
+		sel, err := labels.Parse(metricOptions.MetricLabelSelector)
+		if err != nil {
+			return nil, err
+		}
+		metricLabelSelector = sel
+	}
+
 	// grab the name, if present, from the field selector list options
 	// (this is how the list handler logic injects it)
 	// (otherwise we'd have to write a custom list handler)
@@ -78,13 +92,10 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
 			name = nameMatch
 		}
 	}
-	glog.Infof("List(): name=%s", name)
 
 	namespace := genericapirequest.NamespaceValue(ctx)
-	glog.Infof("List(): namespace=%s", namespace)
 
 	requestInfo, ok := request.RequestInfoFrom(ctx)
-	glog.Infof("List(): requestInfo=%v", requestInfo)
 	if !ok {
 		return nil, fmt.Errorf("unable to get resource and metric name from request")
 	}
@@ -94,7 +105,6 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
 
 	groupResource := schema.ParseGroupResource(resourceRaw)
 
-	glog.Infof("List(): resourceRaw=%v, metricName=%v, groupResource=%v", resourceRaw, metricName, groupResource)
 	// handle metrics describing namespaces
 	if namespace != "" && resourceRaw == "metrics" {
 		// namespace-describing metrics have a path of /namespaces/$NS/metrics/$metric,
@@ -106,19 +116,18 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
 
 	// handle namespaced and root metrics
 	if name == "*" {
-		return r.handleWildcardOp(namespace, groupResource, selector, metricName)
+		return r.handleWildcardOp(namespace, groupResource, selector, metricName, metricLabelSelector)
 	} else {
-		return r.handleIndividualOp(namespace, groupResource, name, metricName)
+		return r.handleIndividualOp(namespace, groupResource, name, metricName, metricLabelSelector)
 	}
 }
 
-func (r *REST) handleIndividualOp(namespace string, groupResource schema.GroupResource, name string, metricName string) (*custom_metrics.MetricValueList, error) {
+func (r *REST) handleIndividualOp(namespace string, groupResource schema.GroupResource, name string, metricName string, metricLabelSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	singleRes, err := r.cmProvider.GetMetricByName(types.NamespacedName{Namespace: namespace, Name: name}, provider.CustomMetricInfo{
 		GroupResource: groupResource,
 		Metric:        metricName,
 		Namespaced:    namespace != "",
-	})
-	glog.Infof("Entered handleIndividualOp()")
+	}, metricLabelSelector)
 	if err != nil {
 		return nil, err
 	}
@@ -128,11 +137,10 @@ func (r *REST) handleIndividualOp(namespace string, groupResource schema.GroupRe
 	}, nil
 }
 
-func (r *REST) handleWildcardOp(namespace string, groupResource schema.GroupResource, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error) {
-	glog.Infof("Entered handleWildcardOp()")
+func (r *REST) handleWildcardOp(namespace string, groupResource schema.GroupResource, selector labels.Selector, metricName string, metricLabelSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	return r.cmProvider.GetMetricBySelector(namespace, selector, provider.CustomMetricInfo{
 		GroupResource: groupResource,
 		Metric:        metricName,
 		Namespaced:    namespace != "",
-	})
+	}, metricLabelSelector)
 }
diff --git a/pkg/controller/metrics/registry/external_metrics/reststorage.go b/pkg/controller/metrics/registry/external_metrics/reststorage.go
@@ -19,7 +19,7 @@ package apiserver
 import (
 	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/provider"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	"k8s.io/apimachinery/pkg/labels"
@@ -34,15 +34,14 @@ import (
 // interfaces.
 type REST struct {
 	emProvider provider.ExternalMetricsProvider
+	rest.TableConvertor
 }
 
 var _ rest.Storage = &REST{}
 var _ rest.Lister = &REST{}
 
 // NewREST returns new REST object for provided CustomMetricsProvider.
 func NewREST(emProvider provider.ExternalMetricsProvider) *REST {
-	glog.V(10).Infof("Entered NewREST()")
-
 	return &REST{
 		emProvider: emProvider,
 	}
@@ -64,16 +63,11 @@ func (r *REST) NewList() runtime.Object {
 
 // List selects resources in the storage which match to the selector.
 func (r *REST) List(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) {
-	glog.V(10).Infof("Entered List()")
-
-	glog.V(9).Infof("List(): labels=%v", labels.Everything())
 	// populate the label selector, defaulting to all
 	metricSelector := labels.Everything()
-
 	if options != nil && options.LabelSelector != nil {
 		metricSelector = options.LabelSelector
 	}
-	glog.V(9).Infof("List(): metricSelector=%v", metricSelector)
 
 	namespace := genericapirequest.NamespaceValue(ctx)
 
@@ -83,6 +77,5 @@ func (r *REST) List(ctx context.Context, options *metainternalversion.ListOption
 	}
 	metricName := requestInfo.Resource
 
-	glog.V(9).Infof("List(): namespace=%v, requestInfo=%v, metricName=%v", namespace, requestInfo, metricName)
 	return r.emProvider.GetExternalMetric(namespace, metricSelector, provider.ExternalMetricInfo{Metric: metricName})
 }
diff --git a/pkg/controller/metrics/test-adapter/main.go b/pkg/controller/metrics/test-adapter/main.go
@@ -22,9 +22,11 @@ import (
 	"os"
 
 	"github.com/emicklei/go-restful"
-	"github.com/golang/glog"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog"
+
+	// "k8s.io/apiserver/pkg/util/logs"
 
 	basecmd "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/cmd"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/metrics/provider"
@@ -39,15 +41,15 @@ type SampleAdapter struct {
 }
 
 func (a *SampleAdapter) makeProviderOrDie() (provider.MetricsProvider, *restful.WebService) {
-	glog.Infof("Entered makeProviderOrDie()")
+	klog.Infof("Entered makeProviderOrDie()")
 	client, err := a.DynamicClient()
 	if err != nil {
-		glog.Fatalf("unable to construct dynamic client: %v", err)
+		klog.Fatalf("unable to construct dynamic client: %v", err)
 	}
 
 	mapper, err := a.RESTMapper()
 	if err != nil {
-		glog.Fatalf("unable to construct discovery REST mapper: %v", err)
+		klog.Fatalf("unable to construct discovery REST mapper: %v", err)
 	}
 
 	return fakeprov.NewFakeProvider(client, mapper)
@@ -57,7 +59,7 @@ func main() {
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	glog.Infof("Entered main()")
+	klog.Infof("Entered main()")
 	cmd := &SampleAdapter{}
 	cmd.Flags().StringVar(&cmd.Message, "msg", "starting adapter...", "startup message")
 	cmd.Flags().AddGoFlagSet(flag.CommandLine) // make sure we get the glog flags
@@ -67,14 +69,14 @@ func main() {
 	cmd.WithCustomMetrics(testProvider)
 	cmd.WithExternalMetrics(testProvider)
 
-	glog.Infof(cmd.Message)
+	klog.Infof(cmd.Message)
 	// Set up POST endpoint for writing fake metric values
 	restful.DefaultContainer.Add(webService)
 	go func() {
 		// Open port for POSTing fake metrics
-		glog.Fatal(http.ListenAndServe(":8080", nil))
+		klog.Fatal(http.ListenAndServe(":8080", nil))
 	}()
 	if err := cmd.Run(wait.NeverStop); err != nil {
-		glog.Fatalf("unable to run custom metrics adapter: %v", err)
+		klog.Fatalf("unable to run custom metrics adapter: %v", err)
 	}
 }
diff --git a/pkg/controller/metrics/test-adapter/provider/provider.go b/pkg/controller/metrics/test-adapter/provider/provider.go
@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/emicklei/go-restful"
-	"github.com/golang/glog"
 
 	apierr "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -32,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/klog"
 	"k8s.io/metrics/pkg/apis/custom_metrics"
 	"k8s.io/metrics/pkg/apis/external_metrics"
 
@@ -96,23 +96,27 @@ var (
 	}
 )
 
+type metricValue struct {
+	labels labels.Set
+	value  resource.Quantity
+}
+
 // testingProvider is a sample implementation of provider.MetricsProvider which stores a map of fake metrics
 type testingProvider struct {
 	client dynamic.Interface
 	mapper apimeta.RESTMapper
 
 	valuesLock      sync.RWMutex
-	values          map[CustomMetricResource]resource.Quantity
+	values          map[CustomMetricResource]metricValue
 	externalMetrics []externalMetric
 }
 
 // NewFakeProvider returns an instance of testingProvider, along with its restful.WebService that opens endpoints to post new fake metrics
 func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper) (provider.MetricsProvider, *restful.WebService) {
-	glog.Infof("Entered NewFakeProvider()")
 	provider := &testingProvider{
 		client:          client,
 		mapper:          mapper,
-		values:          make(map[CustomMetricResource]resource.Quantity),
+		values:          make(map[CustomMetricResource]metricValue),
 		externalMetrics: testingExternalMetrics,
 	}
 	return provider, provider.webService()
@@ -123,7 +127,6 @@ func NewFakeProvider(client dynamic.Interface, mapper apimeta.RESTMapper) (provi
 // There are 3 metric types available: namespaced, root-scoped, and namespaces.
 // (Note: Namespaces, we're assuming, are themselves namespaced resources, but for consistency with how metrics are retreived they have a separate route)
 func (p *testingProvider) webService() *restful.WebService {
-	glog.Infof("Entered webService()")
 	ws := new(restful.WebService)
 
 	ws.Path("/write-metrics")
@@ -144,35 +147,37 @@ func (p *testingProvider) webService() *restful.WebService {
 
 // updateMetric writes the metric provided by a restful request and stores it in memory
 func (p *testingProvider) updateMetric(request *restful.Request, response *restful.Response) {
-	glog.Infof("Entered updateMetric()")
 	p.valuesLock.Lock()
 	defer p.valuesLock.Unlock()
 
 	namespace := request.PathParameter("namespace")
-	glog.Infof("updateMetric() namespace=%s", namespace)
 	resourceType := request.PathParameter("resourceType")
-	glog.Infof("updateMetric() resourceType=%s", resourceType)
 	namespaced := false
 	if len(namespace) > 0 || resourceType == "namespaces" {
 		namespaced = true
-		glog.Infof("updateMetric() namespaced=true")
 	}
 	name := request.PathParameter("name")
-	glog.Infof("updateMetric() name=%s", name)
 	metricName := request.PathParameter("metric")
-	glog.Infof("updateMetric() metricName=%s", metricName)
 
 	value := new(resource.Quantity)
 	err := request.ReadEntity(value)
-	glog.Infof("updateMetric() value=%v", value)
 	if err != nil {
 		response.WriteErrorString(http.StatusBadRequest, err.Error())
-		glog.Infof("updateMetric() bad value: %v", value)
 		return
 	}
 
 	groupResource := schema.ParseGroupResource(resourceType)
 
+	metricLabels := labels.Set{}
+	sel := request.QueryParameter("labels")
+	if len(sel) > 0 {
+		metricLabels, err = labels.ConvertSelectorToLabelsMap(sel)
+		if err != nil {
+			response.WriteErrorString(http.StatusBadRequest, err.Error())
+			return
+		}
+	}
+
 	info := provider.CustomMetricInfo{
 		GroupResource: groupResource,
 		Metric:        metricName,
@@ -181,7 +186,7 @@ func (p *testingProvider) updateMetric(request *restful.Request, response *restf
 
 	info, _, err = info.Normalized(p.mapper)
 	if err != nil {
-		glog.Errorf("Error normalizing info: %s", err)
+		klog.Errorf("Error normalizing info: %s", err)
 	}
 	namespacedName := types.NamespacedName{
 		Name:      name,
@@ -192,12 +197,14 @@ func (p *testingProvider) updateMetric(request *restful.Request, response *restf
 		CustomMetricInfo: info,
 		NamespacedName:   namespacedName,
 	}
-	p.values[metricInfo] = *value
+	p.values[metricInfo] = metricValue{
+		labels: metricLabels,
+		value:  *value,
+	}
 }
 
 // valueFor is a helper function to get just the value of a specific metric
-func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName) (resource.Quantity, error) {
-	glog.Infof("Entered valueFor(4)")
+func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.NamespacedName, metricSelector labels.Selector) (resource.Quantity, error) {
 	info, _, err := info.Normalized(p.mapper)
 	if err != nil {
 		return resource.Quantity{}, err
@@ -212,13 +219,15 @@ func (p *testingProvider) valueFor(info provider.CustomMetricInfo, name types.Na
 		return resource.Quantity{}, provider.NewMetricNotFoundForError(info.GroupResource, info.Metric, name.Name)
 	}
 
-	glog.Infof("valueFor(): metricInfo=%v, value=%v", metricInfo, value)
-	return value, nil
+	if !metricSelector.Matches(value.labels) {
+		return resource.Quantity{}, provider.NewMetricNotFoundForSelectorError(info.GroupResource, info.Metric, name.Name, metricSelector)
+	}
+
+	return value.value, nil
 }
 
 // metricFor is a helper function which formats a value, metric, and object info into a MetricValue which can be returned by the metrics API
-func (p *testingProvider) metricFor(value resource.Quantity, name types.NamespacedName, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
-	glog.Infof("Entered metricFor(4)")
+func (p *testingProvider) metricFor(value resource.Quantity, name types.NamespacedName, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
 	objRef, err := helpers.ReferenceFor(p.mapper, name, info)
 	if err != nil {
 		return nil, err
@@ -233,20 +242,19 @@ func (p *testingProvider) metricFor(value resource.Quantity, name types.Namespac
 		Value:     value,
 	}
 
-	if len(selector.String()) > 0 {
-		labelSelector, err := metav1.ParseToLabelSelector(selector.String())
+	if len(metricSelector.String()) > 0 {
+		sel, err := metav1.ParseToLabelSelector(metricSelector.String())
 		if err != nil {
 			return nil, err
 		}
-		metric.Metric.Selector = labelSelector
+		metric.Metric.Selector = sel
 	}
-	glog.Infof("metricFor(4): metric=%v", metric)
+
 	return metric, nil
 }
 
 // metricsFor is a wrapper used by GetMetricBySelector to format several metrics which match a resource selector
-func (p *testingProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
-	glog.Infof("Entered metricFor(3)")
+func (p *testingProvider) metricsFor(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	names, err := helpers.ListObjectNames(p.mapper, p.client, namespace, selector, info)
 	if err != nil {
 		return nil, err
@@ -255,49 +263,42 @@ func (p *testingProvider) metricsFor(namespace string, selector labels.Selector,
 	res := make([]custom_metrics.MetricValue, 0, len(names))
 	for _, name := range names {
 		namespacedName := types.NamespacedName{Name: name, Namespace: namespace}
-		value, err := p.valueFor(info, namespacedName)
+		value, err := p.valueFor(info, namespacedName, metricSelector)
 		if err != nil {
 			if apierr.IsNotFound(err) {
 				continue
 			}
 			return nil, err
 		}
 
-		metric, err := p.metricFor(value, namespacedName, selector, info)
+		metric, err := p.metricFor(value, namespacedName, selector, info, metricSelector)
 		if err != nil {
 			return nil, err
 		}
 		res = append(res, *metric)
 	}
-	glog.Infof("metricFor(3): res=%v", res)
 
 	return &custom_metrics.MetricValueList{
 		Items: res,
 	}, nil
 }
 
-func (p *testingProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo) (*custom_metrics.MetricValue, error) {
-	glog.Infof("Entered GetMetricByName()")
-	//debug.PrintStack()
+func (p *testingProvider) GetMetricByName(name types.NamespacedName, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
-	value, err := p.valueFor(info, name)
-	glog.Infof("GetMetricByName(): info=%v, name=%v, value=%v", info, name, value)
-
+	value, err := p.valueFor(info, name, metricSelector)
 	if err != nil {
 		return nil, err
 	}
-	return p.metricFor(value, name, labels.Everything(), info)
+	return p.metricFor(value, name, labels.Everything(), info, metricSelector)
 }
 
-func (p *testingProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo) (*custom_metrics.MetricValueList, error) {
-	glog.Infof("Entered GetMetricBySelector()")
-	//debug.PrintStack()
+func (p *testingProvider) GetMetricBySelector(namespace string, selector labels.Selector, info provider.CustomMetricInfo, metricSelector labels.Selector) (*custom_metrics.MetricValueList, error) {
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
-	return p.metricsFor(namespace, selector, info)
+	return p.metricsFor(namespace, selector, info, metricSelector)
 }
 
 func (p *testingProvider) ListAllMetrics() []provider.CustomMetricInfo {
@@ -320,8 +321,6 @@ func (p *testingProvider) ListAllMetrics() []provider.CustomMetricInfo {
 }
 
 func (p *testingProvider) GetExternalMetric(namespace string, metricSelector labels.Selector, info provider.ExternalMetricInfo) (*external_metrics.ExternalMetricValueList, error) {
-	glog.Infof("Entered GetExternalMetric()")
-	//debug.PrintStack()
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
@@ -340,16 +339,12 @@ func (p *testingProvider) GetExternalMetric(namespace string, metricSelector lab
 }
 
 func (p *testingProvider) ListAllExternalMetrics() []provider.ExternalMetricInfo {
-	glog.Infof("Entered ListAllExternalMetrics()")
-	//debug.PrintStack()
 	p.valuesLock.RLock()
 	defer p.valuesLock.RUnlock()
 
 	externalMetricsInfo := []provider.ExternalMetricInfo{}
 	for _, metric := range p.externalMetrics {
 		externalMetricsInfo = append(externalMetricsInfo, metric.info)
-		glog.Infof("ListAllExternalMetrics(): add metric=%v to externalMetricsInfo", metric)
 	}
-	glog.Infof("ListAllExternalMetrics(): externalMetricsInfo=%v", externalMetricsInfo)
 	return externalMetricsInfo
 }
@@ -17,13 +17,12 @@ limitations under the License.
 package queuejob
 
 import (
+	"context"
 	"fmt"
 	"sync"
 	"time"
 
-	"github.com/golang/glog"
-
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +34,7 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog"
 
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/utils"
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
@@ -122,7 +122,7 @@ func (cc *Controller) Run(stopCh chan struct{}) {
 func (cc *Controller) addQueueJob(obj interface{}) {
 	qj, ok := obj.(*arbv1.QueueJob)
 	if !ok {
-		glog.Errorf("obj is not QueueJob")
+		klog.Errorf("obj is not QueueJob")
 		return
 	}
 
@@ -132,7 +132,7 @@ func (cc *Controller) addQueueJob(obj interface{}) {
 func (cc *Controller) updateQueueJob(oldObj, newObj interface{}) {
 	newQJ, ok := newObj.(*arbv1.QueueJob)
 	if !ok {
-		glog.Errorf("newObj is not QueueJob")
+		klog.Errorf("newObj is not QueueJob")
 		return
 	}
 
@@ -142,7 +142,7 @@ func (cc *Controller) updateQueueJob(oldObj, newObj interface{}) {
 func (cc *Controller) deleteQueueJob(obj interface{}) {
 	qj, ok := obj.(*arbv1.QueueJob)
 	if !ok {
-		glog.Errorf("obj is not QueueJob")
+		klog.Errorf("obj is not QueueJob")
 		return
 	}
 
@@ -152,7 +152,7 @@ func (cc *Controller) deleteQueueJob(obj interface{}) {
 func (cc *Controller) addPod(obj interface{}) {
 	pod, ok := obj.(*v1.Pod)
 	if !ok {
-		glog.Error("Failed to convert %v to v1.Pod", obj)
+		klog.Error("Failed to convert %v to v1.Pod", obj)
 		return
 	}
 
@@ -162,7 +162,7 @@ func (cc *Controller) addPod(obj interface{}) {
 func (cc *Controller) updatePod(oldObj, newObj interface{}) {
 	pod, ok := newObj.(*v1.Pod)
 	if !ok {
-		glog.Error("Failed to convert %v to v1.Pod", newObj)
+		klog.Error("Failed to convert %v to v1.Pod", newObj)
 		return
 	}
 
@@ -178,17 +178,17 @@ func (cc *Controller) deletePod(obj interface{}) {
 		var ok bool
 		pod, ok = t.Obj.(*v1.Pod)
 		if !ok {
-			glog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
+			klog.Errorf("Cannot convert to *v1.Pod: %v", t.Obj)
 			return
 		}
 	default:
-		glog.Errorf("Cannot convert to *v1.Pod: %v", t)
+		klog.Errorf("Cannot convert to *v1.Pod: %v", t)
 		return
 	}
 
 	queuejobs, err := cc.queueJobLister.List(labels.Everything())
 	if err != nil {
-		glog.Errorf("Failed to list QueueJobs for Pod %v/%v", pod.Namespace, pod.Name)
+		klog.Errorf("Failed to list QueueJobs for Pod %v/%v", pod.Namespace, pod.Name)
 	}
 
 	ctl := utils.GetController(pod)
@@ -203,7 +203,7 @@ func (cc *Controller) deletePod(obj interface{}) {
 func (cc *Controller) enqueue(obj interface{}) {
 	err := cc.eventQueue.Add(obj)
 	if err != nil {
-		glog.Errorf("Fail to enqueue QueueJob to updateQueue, err %#v", err)
+		klog.Errorf("Fail to enqueue QueueJob to updateQueue, err %#v", err)
 	}
 }
 
@@ -216,7 +216,7 @@ func (cc *Controller) worker() {
 		case *v1.Pod:
 			queuejobs, err := cc.queueJobLister.List(labels.Everything())
 			if err != nil {
-				glog.Errorf("Failed to list QueueJobs for Pod %v/%v", v.Namespace, v.Name)
+				klog.Errorf("Failed to list QueueJobs for Pod %v/%v", v.Namespace, v.Name)
 			}
 
 			ctl := utils.GetController(v)
@@ -228,28 +228,28 @@ func (cc *Controller) worker() {
 			}
 
 		default:
-			glog.Errorf("Un-supported type of %v", obj)
+			klog.Errorf("Un-supported type of %v", obj)
 			return nil
 		}
 
 		if queuejob == nil {
 			if acc, err := meta.Accessor(obj); err != nil {
-				glog.Warningf("Failed to get QueueJob for %v/%v", acc.GetNamespace(), acc.GetName())
+				klog.Warningf("Failed to get QueueJob for %v/%v", acc.GetNamespace(), acc.GetName())
 			}
 
 			return nil
 		}
 
 		// sync Pods for a QueueJob
 		if err := cc.syncQueueJob(queuejob); err != nil {
-			glog.Errorf("Failed to sync QueueJob %s, err %#v", queuejob.Name, err)
+			klog.Errorf("Failed to sync QueueJob %s, err %#v", queuejob.Name, err)
 			// If any error, requeue it.
 			return err
 		}
 
 		return nil
 	}); err != nil {
-		glog.Errorf("Fail to pop item from updateQueue, err %#v", err)
+		klog.Errorf("Fail to pop item from updateQueue, err %#v", err)
 		return
 	}
 }
@@ -261,7 +261,7 @@ func filterActivePods(pods []*v1.Pod) []*v1.Pod {
 		if isPodActive(p) {
 			result = append(result, p)
 		} else {
-			glog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v",
+			klog.V(4).Infof("Ignoring inactive pod %v/%v in state %v, deletion time %v",
 				p.Namespace, p.Name, p.Status.Phase, p.DeletionTimestamp)
 		}
 	}
@@ -278,7 +278,7 @@ func (cc *Controller) syncQueueJob(qj *arbv1.QueueJob) error {
 	queueJob, err := cc.queueJobLister.QueueJobs(qj.Namespace).Get(qj.Name)
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			glog.V(3).Infof("Job has been deleted: %v", qj.Name)
+			klog.V(3).Infof("Job has been deleted: %v", qj.Name)
 			return nil
 		}
 		return err
@@ -338,11 +338,11 @@ func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods map[string][]*v1.P
 		schedSpc := createQueueJobSchedulingSpec(qj)
 		_, err := cc.arbclients.ArbV1().SchedulingSpecs(qj.Namespace).Create(schedSpc)
 		if err != nil {
-			glog.Errorf("Failed to create SchedulingSpec for QueueJob %v/%v: %v",
+			klog.Errorf("Failed to create SchedulingSpec for QueueJob %v/%v: %v",
 				qj.Namespace, qj.Name, err)
 		}
 	} else {
-		glog.V(3).Infof("There's %v SchedulingSpec for QueueJob %v/%v",
+		klog.V(3).Infof("There's %v SchedulingSpec for QueueJob %v/%v",
 			len(ss.Items), qj.Namespace, qj.Name)
 	}
 
@@ -360,12 +360,12 @@ func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods map[string][]*v1.P
 		succeededSum += succeeded
 		failedSum += failed
 
-		glog.V(3).Infof("There are %d pods of QueueJob %s (%s): replicas %d, pending %d, running %d, succeeded %d, failed %d",
+		klog.V(3).Infof("There are %d pods of QueueJob %s (%s): replicas %d, pending %d, running %d, succeeded %d, failed %d",
 			len(pods), qj.Name, name, replicas, pending, running, succeeded, failed)
 
 		// Create pod if necessary
 		if diff := replicas - pending - running - succeeded; diff > 0 {
-			glog.V(3).Infof("Try to create %v Pods for QueueJob %v/%v", diff, qj.Namespace, qj.Name)
+			klog.V(3).Infof("Try to create %v Pods for QueueJob %v/%v", diff, qj.Namespace, qj.Name)
 
 			var errs []error
 			wait := sync.WaitGroup{}
@@ -374,12 +374,16 @@ func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods map[string][]*v1.P
 				go func(ix int32) {
 					defer wait.Done()
 					newPod := createQueueJobPod(qj, &ts.Template, ix)
-					_, err := cc.clients.Core().Pods(newPod.Namespace).Create(newPod)
+					_, err := cc.clients.CoreV1().Pods(newPod.Namespace).Create(context.Background(), newPod, metav1.CreateOptions{
+						TypeMeta:     metav1.TypeMeta{},
+						DryRun:       []string{},
+						FieldManager: "",
+					})
 					if err != nil {
 						// Failed to create Pod, wait a moment and then create it again
 						// This is to ensure all pods under the same QueueJob created
 						// So gang-scheduling could schedule the QueueJob successfully
-						glog.Errorf("Failed to create pod %s for QueueJob %s, err %#v",
+						klog.Errorf("Failed to create pod %s for QueueJob %s, err %#v",
 							newPod.Name, qj.Name, err)
 						errs = append(errs, err)
 					}
@@ -403,7 +407,7 @@ func (cc *Controller) manageQueueJob(qj *arbv1.QueueJob, pods map[string][]*v1.P
 
 	// TODO(k82cn): replaced it with `UpdateStatus`
 	if _, err := cc.arbclients.ArbV1().QueueJobs(qj.Namespace).Update(qj); err != nil {
-		glog.Errorf("Failed to update status of QueueJob %v/%v: %v",
+		klog.Errorf("Failed to update status of QueueJob %v/%v: %v",
 			qj.Namespace, qj.Name, err)
 		return err
 	}

@@ -28,11 +28,12 @@ package queuejob
 
 import (
 	"fmt"
-	qjobv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
-	"github.com/golang/glog"
-	"k8s.io/client-go/tools/cache"
 	"reflect"
 	"sync"
+
+	qjobv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog"
 )
 
 // SchedulingQueue is an interface for a queue to store pods waiting to be scheduled.
@@ -144,7 +145,7 @@ func (p *PriorityQueue) MoveToActiveQueueIfExists(aw *qjobv1.AppWrapper) error {
 		p.unschedulableQ.Delete(aw)
 		err := p.activeQ.AddIfNotPresent(aw)
 		if err != nil {
-			glog.Errorf("[MoveToActiveQueueIfExists] Error adding AW %v to the scheduling queue: %v\n", aw.Name, err)
+			klog.Errorf("[MoveToActiveQueueIfExists] Error adding AW %v to the scheduling queue: %v\n", aw.Name, err)
 		}
 		p.cond.Broadcast()
 		return err
@@ -162,10 +163,10 @@ func (p *PriorityQueue) Add(qj *qjobv1.AppWrapper) error {
 	defer p.lock.Unlock()
 	err := p.activeQ.Add(qj)
 	if err != nil {
-		glog.Errorf("Error adding QJ %v to the scheduling queue: %v", qj.Name, err)
+		klog.Errorf("Error adding QJ %v to the scheduling queue: %v", qj.Name, err)
 	} else {
 		if p.unschedulableQ.Get(qj) != nil {
-			glog.Errorf("Error: QJ %v is already in the unschedulable queue.", qj.Name)
+			klog.Errorf("Error: QJ %v is already in the unschedulable queue.", qj.Name)
 			p.unschedulableQ.Delete(qj)
 		}
 		p.cond.Broadcast()
@@ -186,7 +187,7 @@ func (p *PriorityQueue) AddIfNotPresent(qj *qjobv1.AppWrapper) error {
 	}
 	err := p.activeQ.Add(qj)
 	if err != nil {
-		glog.Errorf("Error adding pod %v to the scheduling queue: %v", qj.Name, err)
+		klog.Errorf("Error adding pod %v to the scheduling queue: %v", qj.Name, err)
 	} else {
 		p.cond.Broadcast()
 	}

@@ -17,19 +17,21 @@ limitations under the License.
 package queuejobdispatch
 
 import (
+	"context"
 	"encoding/json"
-	"github.com/golang/glog"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
-	"math"
-	"strconv"
-	"strings"
-	"time"
+	"k8s.io/klog"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -42,31 +44,30 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 )
 
-type JobClusterAgent struct{
-		AgentId         string
-		DeploymentName  string
-		queuejobclients *clientset.Clientset
-		k8sClients     	*kubernetes.Clientset // for the update of aggr resouces
-		AggrResources  	*clusterstateapi.Resource
+type JobClusterAgent struct {
+	AgentId         string
+	DeploymentName  string
+	queuejobclients *clientset.Clientset
+	k8sClients      *kubernetes.Clientset // for the update of aggr resouces
+	AggrResources   *clusterstateapi.Resource
 
-		jobInformer 	informersv1.AppWrapperInformer
-		jobLister   	listersv1.AppWrapperLister
-		jobSynced   	func() bool
+	jobInformer informersv1.AppWrapperInformer
+	jobLister   listersv1.AppWrapperLister
+	jobSynced   func() bool
 
-		agentEventQueue	*cache.FIFO
+	agentEventQueue *cache.FIFO
 }
 
 func NewJobClusterAgent(config string, agentEventQueue *cache.FIFO) *JobClusterAgent {
-	configStrings:=strings.Split(config, ":")
-	if len(configStrings)<2 {
+	configStrings := strings.Split(config, ":")
+	if len(configStrings) < 2 {
 		return nil
 	}
-	glog.V(2).Infof("[Dispatcher: Agent] Creation: %s\n", "/root/kubernetes/" + configStrings[0])
+	klog.V(2).Infof("[Dispatcher: Agent] Creation: %s\n", "/root/kubernetes/"+configStrings[0])
 
-	agent_config, err:=clientcmd.BuildConfigFromFlags("", "/root/kubernetes/" + configStrings[0])
-	// agent_config, err:=clientcmd.BuildConfigFromFlags("", "/root/agent101config")
-	if err!=nil {
-		glog.V(2).Infof("[Dispatcher: Agent] Cannot crate client\n")
+	agent_config, err := clientcmd.BuildConfigFromFlags("", "/root/kubernetes/"+configStrings[0])
+	if err != nil {
+		klog.V(2).Infof("[Dispatcher: Agent] Cannot crate client\n")
 		return nil
 	}
 	qa := &JobClusterAgent{
@@ -76,12 +77,12 @@ func NewJobClusterAgent(config string, agentEventQueue *cache.FIFO) *JobClusterA
 		k8sClients:      kubernetes.NewForConfigOrDie(agent_config),
 		AggrResources:   clusterstateapi.EmptyResource(),
 	}
-	qa.agentEventQueue=agentEventQueue
+	qa.agentEventQueue = agentEventQueue
 
-	if qa.queuejobclients==nil {
-		glog.V(2).Infof("[Dispatcher: Agent] Cannot Create Client\n")
+	if qa.queuejobclients == nil {
+		klog.V(2).Infof("[Dispatcher: Agent] Cannot Create Client\n")
 	} else {
-		glog.V(2).Infof("[Dispatcher: Agent] %s: Create Clients Suceessfully\n", qa.AgentId)
+		klog.V(2).Infof("[Dispatcher: Agent] %s: Create Clients Suceessfully\n", qa.AgentId)
 	}
 
 	queueJobClientForInformer, _, err := clients.NewClient(agent_config)
@@ -91,15 +92,15 @@ func NewJobClusterAgent(config string, agentEventQueue *cache.FIFO) *JobClusterA
 
 	qa.jobInformer = arbinformers.NewFilteredSharedInformerFactory(queueJobClientForInformer, 0,
 		func(opt *metav1.ListOptions) {
-		opt.LabelSelector = "IsDispatched=true"
- 		},
-		).AppWrapper().AppWrappers()
+			opt.LabelSelector = "IsDispatched=true"
+		},
+	).AppWrapper().AppWrappers()
 	qa.jobInformer.Informer().AddEventHandler(
 		cache.FilteringResourceEventHandler{
 			FilterFunc: func(obj interface{}) bool {
 				switch t := obj.(type) {
 				case *arbv1.AppWrapper:
-					glog.V(4).Infof("Filter AppWrapper name(%s) namespace(%s)\n", t.Name, t.Namespace)
+					klog.V(4).Infof("Filter AppWrapper name(%s) namespace(%s)\n", t.Name, t.Namespace)
 					return true
 				default:
 					return false
@@ -121,61 +122,57 @@ func NewJobClusterAgent(config string, agentEventQueue *cache.FIFO) *JobClusterA
 	return qa
 }
 
-
 func (cc *JobClusterAgent) addQueueJob(obj interface{}) {
 	qj, ok := obj.(*arbv1.AppWrapper)
 	if !ok {
-		glog.Errorf("obj is not AppWrapper")
+		klog.Errorf("obj is not AppWrapper")
 		return
 	}
-	glog.V(10).Infof("[TTime]: %s Adding New Job: %s to EventQ\n", time.Now().String(), qj.Name)
+	klog.V(10).Infof("[TTime]: %s Adding New Job: %s to EventQ\n", time.Now().String(), qj.Name)
 	cc.agentEventQueue.Add(qj)
 }
 
 func (cc *JobClusterAgent) updateQueueJob(oldObj, newObj interface{}) {
 	newQJ, ok := newObj.(*arbv1.AppWrapper)
 	if !ok {
-		glog.Errorf("newObj is not AppWrapper")
+		klog.Errorf("newObj is not AppWrapper")
 		return
 	}
-	glog.V(10).Infof("[TTime]: %s Adding Update Job: %s to EventQ\n", time.Now().String(), newQJ.Name)
+	klog.V(10).Infof("[TTime]: %s Adding Update Job: %s to EventQ\n", time.Now().String(), newQJ.Name)
 	cc.agentEventQueue.Add(newQJ)
 }
 
 func (cc *JobClusterAgent) deleteQueueJob(obj interface{}) {
 	qj, ok := obj.(*arbv1.AppWrapper)
 	if !ok {
-		glog.Errorf("obj is not AppWrapper")
+		klog.Errorf("obj is not AppWrapper")
 		return
 	}
-	glog.V(10).Infof("[TTime]: %s Adding Delete Job: %s to EventQ\n", time.Now().String(), qj.Name)
+	klog.V(10).Infof("[TTime]: %s Adding Delete Job: %s to EventQ\n", time.Now().String(), qj.Name)
 	cc.agentEventQueue.Add(qj)
 }
 
-
-
 func (qa *JobClusterAgent) Run(stopCh chan struct{}) {
 	go qa.jobInformer.Informer().Run(stopCh)
 	cache.WaitForCacheSync(stopCh, qa.jobSynced)
-	// go wait.Until(qa.UpdateAgent, 2*time.Second, stopCh)
 }
 
 func (qa *JobClusterAgent) DeleteJob(cqj *arbv1.AppWrapper) {
-	qj_temp:=cqj.DeepCopy()
-	glog.V(2).Infof("[Dispatcher: Agent] Request deletion of XQJ %s to Agent %s\n", qj_temp.Name, qa.AgentId)
-	qa.queuejobclients.ArbV1().AppWrappers(qj_temp.Namespace).Delete(qj_temp.Name,  &metav1.DeleteOptions{})
+	qj_temp := cqj.DeepCopy()
+	klog.V(2).Infof("[Dispatcher: Agent] Request deletion of XQJ %s to Agent %s\n", qj_temp.Name, qa.AgentId)
+	qa.queuejobclients.ArbV1().AppWrappers(qj_temp.Namespace).Delete(qj_temp.Name, &metav1.DeleteOptions{})
 	return
 }
 
 func (qa *JobClusterAgent) CreateJob(cqj *arbv1.AppWrapper) {
-	qj_temp:=cqj.DeepCopy()
-	agent_qj:=&arbv1.AppWrapper{
-		TypeMeta: qj_temp.TypeMeta,
-		ObjectMeta: metav1.ObjectMeta{Name: qj_temp.Name, Namespace: qj_temp.Namespace,},
-		Spec: qj_temp.Spec,
+	qj_temp := cqj.DeepCopy()
+	agent_qj := &arbv1.AppWrapper{
+		TypeMeta:   qj_temp.TypeMeta,
+		ObjectMeta: metav1.ObjectMeta{Name: qj_temp.Name, Namespace: qj_temp.Namespace},
+		Spec:       qj_temp.Spec,
 	}
-	agent_qj.Status.CanRun=qj_temp.Status.CanRun
-	agent_qj.Status.IsDispatched=qj_temp.Status.IsDispatched
+	agent_qj.Status.CanRun = qj_temp.Status.CanRun
+	agent_qj.Status.IsDispatched = qj_temp.Status.IsDispatched
 
 	if agent_qj.Labels == nil {
 		agent_qj.Labels = map[string]string{}
@@ -185,18 +182,8 @@ func (qa *JobClusterAgent) CreateJob(cqj *arbv1.AppWrapper) {
 	}
 	agent_qj.Labels["IsDispatched"] = "true"
 
-	// glog.Infof("[Agent] XQJ resourceVersion cleaned--Name:%s, Kind:%s\n", agent_qj.Name, agent_qj.Kind)
-	glog.V(2).Infof("[Dispatcher: Agent] Create XQJ: %s (Status: %+v) in Agent %s\n", agent_qj.Name, agent_qj.Status, qa.AgentId)
+	klog.V(2).Infof("[Dispatcher: Agent] Create XQJ: %s (Status: %+v) in Agent %s\n", agent_qj.Name, agent_qj.Status, qa.AgentId)
 	qa.queuejobclients.ArbV1().AppWrappers(agent_qj.Namespace).Create(agent_qj)
-	// pods, err := qa.deploymentclients.CoreV1().Pods("").List(metav1.ListOptions{})
-	// if err != nil {
-	// 	glog.Infof("[Agent] Cannot Access Agent================\n")
-	// }
-	// glog.Infof("There are %d pods in the cluster\n", len(pods.Items))
-	// // for _, pod := range pods.Items {
-	// 	glog.Infof("[Agent] Pod Name=%s\n",pod.Name)
-	// }
-
 	return
 }
 
@@ -206,82 +193,82 @@ type ClusterMetricsList struct {
 	Metadata   struct {
 		SelfLink string `json:"selfLink"`
 	} `json:"metadata"`
-	Items [] struct {
-		MetricName	string    `json:"metricName"`
-		MetricLabels map[string]string  `json:"metriclabels"`
-		Timestamp  string `json:"timestamp"`
-		Value string `json:"value"`
+	Items []struct {
+		MetricName   string            `json:"metricName"`
+		MetricLabels map[string]string `json:"metriclabels"`
+		Timestamp    string            `json:"timestamp"`
+		Value        string            `json:"value"`
 	} `json:"items"`
 }
 
 func (qa *JobClusterAgent) UpdateAggrResources() error {
-    glog.V(6).Infof("[Dispatcher: Agent] Getting aggregated resources for Agent ID: %s with Agent QueueJob Name: %s\n", qa.AgentId, qa.DeploymentName)
+	klog.V(6).Infof("[Dispatcher: Agent] Getting aggregated resources for Agent ID: %s with Agent QueueJob Name: %s\n", qa.AgentId, qa.DeploymentName)
 
-    // Read the Agent XQJ Deployment object
-    if(qa.k8sClients ==nil) {
-    	return nil
+	// Read the Agent XQJ Deployment object
+	if qa.k8sClients == nil {
+		return nil
 
 	}
 
-	data, err := qa.k8sClients.RESTClient().Get().AbsPath("apis/external.metrics.k8s.io/v1beta1/namespaces/default/cluster-external-metric").DoRaw()
+	data, err := qa.k8sClients.RESTClient().Get().AbsPath("apis/external.metrics.k8s.io/v1beta1/namespaces/default/cluster-external-metric").DoRaw(context.Background())
 
-    if err != nil {
-		glog.V(2).Infof("Failed to get metrics from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n", qa.AgentId, qa.DeploymentName, err)
+	if err != nil {
+		klog.V(2).Infof("Failed to get metrics from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n", qa.AgentId, qa.DeploymentName, err)
 
 	} else {
 		res := &ClusterMetricsList{}
 		unmarshalerr := json.Unmarshal(data, res)
 		if unmarshalerr != nil {
-			glog.V(2).Infof("Failed to unmarshal metrics to struct: %v from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n",
+			klog.V(2).Infof("Failed to unmarshal metrics to struct: %v from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n",
 				res, qa.AgentId, qa.DeploymentName, unmarshalerr)
 		} else {
 			if len(res.Items) > 0 {
 				for i := 0; i < len(res.Items); i++ {
-					glog.V(9).Infof("Obtained the metric:%s, label:%v, value: %s, from the Agent: %s  with Agent QueueJob Name: %s.\n",
+					klog.V(9).Infof("Obtained the metric:%s, label:%v, value: %s, from the Agent: %s  with Agent QueueJob Name: %s.\n",
 						res.Items[i].MetricName, res.Items[i].MetricLabels, res.Items[i].Value, qa.AgentId, qa.DeploymentName)
 					clusterMetricType := res.Items[i].MetricLabels["cluster"]
-					if strings.Compare(clusterMetricType, "cpu") == 0  || strings.Compare(clusterMetricType, "memory") == 0 {
+					if strings.Compare(clusterMetricType, "cpu") == 0 || strings.Compare(clusterMetricType, "memory") == 0 {
 						num, err := strconv.ParseFloat(res.Items[i].Value, 64)
-						if err !=nil {
-							glog.Warningf("Possible issue converting %s string value of %s due to error: %v\n",
+						if err != nil {
+							klog.Warningf("Possible issue converting %s string value of %s due to error: %v\n",
 								clusterMetricType, res.Items[i].Value, err)
 						} else {
 							f_num := math.Float64bits(num)
 							f_zero := math.Float64bits(0.0)
-							if (f_num > f_zero) {
+							if f_num > f_zero {
 								if strings.Compare(clusterMetricType, "cpu") == 0 {
 									qa.AggrResources.MilliCPU = num
-									glog.V(10).Infof("Updated %s from %f to %f for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
+									klog.V(10).Infof("Updated %s from %f to %f for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
 										clusterMetricType, qa.AggrResources.MilliCPU, num, res, qa.AgentId, qa.DeploymentName)
 								} else {
 									qa.AggrResources.Memory = num
-									glog.V(10).Infof("Updated %s from %f to %f for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
+									klog.V(10).Infof("Updated %s from %f to %f for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
 										clusterMetricType, qa.AggrResources.Memory, num, res, qa.AgentId, qa.DeploymentName)
 								}
 							} else {
-								glog.Warningf("Possible issue converting %s string value of %s to float type.  Conversion result: %f\n",
+								klog.Warningf("Possible issue converting %s string value of %s to float type.  Conversion result: %f\n",
 									clusterMetricType, res.Items[i].Value, num)
-							}  // Float value resulted in zero value.
+							} // Float value resulted in zero value.
 						} // Converting string to float success
 					} else {
-						glog.V(9).Infof("Unknown label value: %s for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
+						klog.V(9).Infof("Unknown label value: %s for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s\n",
 							clusterMetricType, res, qa.AgentId, qa.DeploymentName)
 					} // Unknown label
 
 				}
 			} else {
-				glog.V(2).Infof("Failed to obtain values for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n", res, qa.AgentId, qa.DeploymentName, unmarshalerr)
+				klog.V(2).Infof("Failed to obtain values for metrics: %v from deployment Agent ID: %s with Agent QueueJob Name: %s, Error: %v\n", res, qa.AgentId, qa.DeploymentName, unmarshalerr)
 			}
 		}
 	}
 
-	glog.V(4).Infof("[Dispatcher: Agent] Updated Aggr Resources of %s: %v\n", qa.AgentId, qa.AggrResources)
-    return nil
+	klog.V(4).Infof("[Dispatcher: Agent] Updated Aggr Resources of %s: %v\n", qa.AgentId, qa.AggrResources)
+	return nil
 }
 
 func buildResource(cpu string, memory string) *clusterstateapi.Resource {
-    return clusterstateapi.NewResource(v1.ResourceList{
-        v1.ResourceCPU:    resource.MustParse(cpu),
-        v1.ResourceMemory: resource.MustParse(memory),
-    })
+	return clusterstateapi.NewResource(v1.ResourceList{
+		v1.ResourceCPU:    resource.MustParse(cpu),
+		v1.ResourceMemory: resource.MustParse(memory),
+	})
 }
@@ -14,17 +14,22 @@ limitations under the License.
 package configmap
 
 import (
+	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	//schedulerapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
 
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	// "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/klog"
+
+	"sync"
+	"time"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -36,8 +41,6 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"sync"
-	"time"
 )
 
 var queueJobKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
@@ -53,15 +56,15 @@ const (
 
 //QueueJobResService contains service info
 type QueueJobResConfigMap struct {
-	clients    			*kubernetes.Clientset
-	arbclients 			*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of services, populated by the serviceController
-	configmapStore    	corelisters.ConfigMapLister
-	configmapInformer 	corev1informer.ConfigMapInformer
-	rtScheme        	*runtime.Scheme
-	jsonSerializer  	*json.Serializer
+	configmapStore    corelisters.ConfigMapLister
+	configmapInformer corev1informer.ConfigMapInformer
+	rtScheme          *runtime.Scheme
+	jsonSerializer    *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 			queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -131,13 +134,11 @@ func (qjrConfigMap *QueueJobResConfigMap) deleteConfigMap(obj interface{}) {
 	return
 }
 
-
 func (qjrConfigMap *QueueJobResConfigMap) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        return total
+	total := clusterstateapi.EmptyResource()
+	return total
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrConfigMap *QueueJobResConfigMap) getConfigMapTemplate(qjobRes *arbv1.AppWrapperResource) (*v1.ConfigMap, error) {
 
@@ -162,15 +163,15 @@ func (qjrConfigMap *QueueJobResConfigMap) createConfigMapWithControllerRef(names
 		configmap.OwnerReferences = append(configmap.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrConfigMap.clients.Core().ConfigMaps(namespace).Create(configmap); err != nil {
+	if _, err := qjrConfigMap.clients.CoreV1().ConfigMaps(namespace).Create(context.Background(), configmap, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 func (qjrConfigMap *QueueJobResConfigMap) delConfigMap(namespace string, name string) error {
-	if err := qjrConfigMap.clients.Core().ConfigMaps(namespace).Delete(name, nil); err != nil {
+	if err := qjrConfigMap.clients.CoreV1().ConfigMaps(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 
@@ -186,8 +187,7 @@ func (qjrConfigMap *QueueJobResConfigMap) SyncQueueJob(queuejob *arbv1.AppWrappe
 	startTime := time.Now()
 
 	defer func() {
-		// glog.V(4).Infof("Finished syncing queue job resource %q (%v)", qjobRes.Template, time.Now().Sub(startTime))
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	_namespace, configMapInQjr, configMapsInEtcd, err := qjrConfigMap.getConfigMapForQueueJobRes(qjobRes, queuejob)
@@ -200,14 +200,14 @@ func (qjrConfigMap *QueueJobResConfigMap) SyncQueueJob(queuejob *arbv1.AppWrappe
 
 	diff := int(replicas) - int(configMapLen)
 
-	glog.V(4).Infof("QJob: %s had %d configMaps and %d desired configMaps", queuejob.Name, configMapLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d configMaps and %d desired configMaps", queuejob.Name, configMapLen, replicas)
 
 	if diff > 0 {
 		//TODO: need set reference after Service has been really added
 		tmpConfigMap := v1.ConfigMap{}
 		err = qjrConfigMap.refManager.AddReference(qjobRes, &tmpConfigMap)
 		if err != nil {
-			glog.Errorf("Cannot add reference to configmap resource %+v", err)
+			klog.Errorf("Cannot add reference to configmap resource %+v", err)
 			return err
 		}
 		if configMapInQjr.Labels == nil {
@@ -240,32 +240,31 @@ func (qjrConfigMap *QueueJobResConfigMap) SyncQueueJob(queuejob *arbv1.AppWrappe
 	return nil
 }
 
-
 func (qjrConfigMap *QueueJobResConfigMap) getConfigMapForQueueJobRes(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) (*string, *v1.ConfigMap, []*v1.ConfigMap, error) {
 
 	// Get "a" ConfigMap from AppWrapper Resource
 	configMapInQjr, err := qjrConfigMap.getConfigMapTemplate(qjobRes)
 	if err != nil {
-		glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+		klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 		return nil, nil, nil, err
 	}
 
 	// Get ConfigMap"s" in Etcd Server
 	var _namespace *string
-	if configMapInQjr.Namespace!=""{
+	if configMapInQjr.Namespace != "" {
 		_namespace = &configMapInQjr.Namespace
 	} else {
 		_namespace = &queuejob.Namespace
 	}
-	configMapList, err := qjrConfigMap.clients.CoreV1().ConfigMaps(*_namespace).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name),})
+	configMapList, err := qjrConfigMap.clients.CoreV1().ConfigMaps(*_namespace).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name)})
 	// configMapList, err := qjrConfigMap.clients.CoreV1().ConfigMaps(*_namespace).List(metav1.ListOptions{})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
 	configMapsInEtcd := []*v1.ConfigMap{}
 	for i, _ := range configMapList.Items {
-				configMapsInEtcd = append(configMapsInEtcd, &configMapList.Items[i])
+		configMapsInEtcd = append(configMapsInEtcd, &configMapList.Items[i])
 	}
 
 	myConfigMapsInEtcd := []*v1.ConfigMap{}
@@ -278,7 +277,6 @@ func (qjrConfigMap *QueueJobResConfigMap) getConfigMapForQueueJobRes(qjobRes *ar
 	return _namespace, configMapInQjr, myConfigMapsInEtcd, nil
 }
 
-
 func (qjrConfigMap *QueueJobResConfigMap) deleteQueueJobResConfigMaps(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) error {
 
 	job := *queuejob
@@ -297,7 +295,7 @@ func (qjrConfigMap *QueueJobResConfigMap) deleteQueueJobResConfigMaps(qjobRes *a
 			defer wait.Done()
 			if err := qjrConfigMap.delConfigMap(*_namespace, activeConfigMaps[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, application wrapper %q/%q deadline exceeded", activeConfigMaps[ix].Name, *_namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, application wrapper %q/%q deadline exceeded", activeConfigMaps[ix].Name, *_namespace, job.Name)
 			}
 		}(i)
 	}

@@ -14,14 +14,17 @@ limitations under the License.
 package deployment
 
 import (
+	"context"
 	"fmt"
+	"sync"
+	"time"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
-	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	"github.com/golang/glog"
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
+	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
 	apps "k8s.io/api/apps/v1"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -34,8 +37,7 @@ import (
 	extlister "k8s.io/client-go/listers/apps/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"sync"
-	"time"
+	"k8s.io/klog"
 )
 
 var queueJobKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
@@ -51,15 +53,15 @@ const (
 
 //QueueJobResDeployment contains the resources of this queuejob
 type QueueJobResDeployment struct {
-	clients    		*kubernetes.Clientset
-	arbclients 		*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of deployments, populated by the deploymentController
-	deploymentStore	extlister.DeploymentLister
-	deployInformer 	extinformer.DeploymentInformer
-	rtScheme       	*runtime.Scheme
-	jsonSerializer 	*json.Serializer
+	deploymentStore extlister.DeploymentLister
+	deployInformer  extinformer.DeploymentInformer
+	rtScheme        *runtime.Scheme
+	jsonSerializer  *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 		queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -104,56 +106,55 @@ func NewQueueJobResDeployment(config *rest.Config) queuejobresources.Interface {
 	return qjrDeployment
 }
 
-
 func (qjrDeployment *QueueJobResDeployment) GetPodTemplate(qjobRes *arbv1.AppWrapperResource) (*v1.PodTemplateSpec, int32, error) {
 	res, err := qjrDeployment.getDeploymentTemplate(qjobRes)
 	if err != nil {
-	        return nil, -1, err
+		return nil, -1, err
 	}
 	return &res.Spec.Template, *res.Spec.Replicas, nil
 }
 
 func (qjrDeployment *QueueJobResDeployment) GetAggregatedResources(job *arbv1.AppWrapper) *clusterstateapi.Resource {
 	total := clusterstateapi.EmptyResource()
 	if job.Spec.AggrResources.Items != nil {
-	    //calculate scaling
-	    for _, ar := range job.Spec.AggrResources.Items {
-	        if ar.Type == arbv1.ResourceTypeDeployment {
-	                template, replicas, err := qjrDeployment.GetPodTemplate(&ar)
-					if err != nil {
-						glog.Errorf("Pod Template not found in item: %+v error: %+v.  Aggregated resources set to 0.", ar, err)
-					} else {
-						myres := queuejobresources.GetPodResources(template)
-						myres.MilliCPU = float64(replicas) * myres.MilliCPU
-						myres.Memory = float64(replicas) * myres.Memory
-						myres.GPU = int64(replicas) * myres.GPU
-						total = total.Add(myres)
-					}
-	        }
-	    }
+		//calculate scaling
+		for _, ar := range job.Spec.AggrResources.Items {
+			if ar.Type == arbv1.ResourceTypeDeployment {
+				template, replicas, err := qjrDeployment.GetPodTemplate(&ar)
+				if err != nil {
+					klog.Errorf("Pod Template not found in item: %+v error: %+v.  Aggregated resources set to 0.", ar, err)
+				} else {
+					myres := queuejobresources.GetPodResources(template)
+					myres.MilliCPU = float64(replicas) * myres.MilliCPU
+					myres.Memory = float64(replicas) * myres.Memory
+					myres.GPU = int64(replicas) * myres.GPU
+					total = total.Add(myres)
+				}
+			}
+		}
 	}
 	return total
 }
 
 func (qjrDeployment *QueueJobResDeployment) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        if job.Spec.AggrResources.Items != nil {
-            //calculate scaling
-            for _, ar := range job.Spec.AggrResources.Items {
-                  if ar.Priority < priority {
-                        continue
-                  }
-                  if ar.Type == arbv1.ResourceTypeDeployment {
-                        template, replicas, _ := qjrDeployment.GetPodTemplate(&ar)
-                        myres := queuejobresources.GetPodResources(template)
-                        myres.MilliCPU = float64(replicas) * myres.MilliCPU
-                        myres.Memory = float64(replicas) * myres.Memory
-                        myres.GPU = int64(replicas) * myres.GPU
-                        total = total.Add(myres)
-                }
-            }
-        }
-        return total
+	total := clusterstateapi.EmptyResource()
+	if job.Spec.AggrResources.Items != nil {
+		//calculate scaling
+		for _, ar := range job.Spec.AggrResources.Items {
+			if ar.Priority < priority {
+				continue
+			}
+			if ar.Type == arbv1.ResourceTypeDeployment {
+				template, replicas, _ := qjrDeployment.GetPodTemplate(&ar)
+				myres := queuejobresources.GetPodResources(template)
+				myres.MilliCPU = float64(replicas) * myres.MilliCPU
+				myres.Memory = float64(replicas) * myres.Memory
+				myres.GPU = int64(replicas) * myres.GPU
+				total = total.Add(myres)
+			}
+		}
+	}
+	return total
 }
 
 //func (qjrDeployment *QueueJobResDeployment) GetAggregatedResourcesByPhase(phase v1.PodPhase, job *arbv1.AppWrapper) *clusterstateapi.Resource {
@@ -194,7 +195,6 @@ func (qjrDeployment *QueueJobResDeployment) deleteDeployment(obj interface{}) {
 	return
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrDeployment *QueueJobResDeployment) getDeploymentTemplate(qjobRes *arbv1.AppWrapperResource) (*apps.Deployment, error) {
 	deploymentGVK := schema.GroupVersion{Group: apps.GroupName, Version: "v1"}.WithKind("Deployment")
@@ -217,15 +217,15 @@ func (qjrDeployment *QueueJobResDeployment) createDeploymentWithControllerRef(na
 		deployment.OwnerReferences = append(deployment.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrDeployment.clients.AppsV1().Deployments(namespace).Create(deployment); err != nil {
+	if _, err := qjrDeployment.clients.AppsV1().Deployments(namespace).Create(context.Background(), deployment, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
 	return nil
 }
 
 func (qjrDeployment *QueueJobResDeployment) delDeployment(namespace string, name string) error {
-	if err := qjrDeployment.clients.AppsV1().Deployments(namespace).Delete(name, nil); err != nil {
+	if err := qjrDeployment.clients.AppsV1().Deployments(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 	return nil
@@ -240,8 +240,7 @@ func (qjrDeployment *QueueJobResDeployment) SyncQueueJob(queuejob *arbv1.AppWrap
 	startTime := time.Now()
 
 	defer func() {
-		// glog.V(4).Infof("Finished syncing queue job resource %q (%v)", qjobRes.Template, time.Now().Sub(startTime))
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	_namespace, deploymentInQjr, deploymentsInEtcd, err := qjrDeployment.getDeploymentForQueueJobRes(qjobRes, queuejob)
@@ -254,14 +253,14 @@ func (qjrDeployment *QueueJobResDeployment) SyncQueueJob(queuejob *arbv1.AppWrap
 
 	diff := int(replicas) - int(deploymentLen)
 
-	glog.V(4).Infof("QJob: %s had %d Deployments and %d desired Deployments", queuejob.Name, deploymentLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d Deployments and %d desired Deployments", queuejob.Name, deploymentLen, replicas)
 
 	if diff > 0 {
 		//TODO: need set reference after Service has been really added
 		tmpDeployment := apps.Deployment{}
 		err = qjrDeployment.refManager.AddReference(qjobRes, &tmpDeployment)
 		if err != nil {
-			glog.Errorf("Cannot add reference to configmap resource %+v", err)
+			klog.Errorf("Cannot add reference to configmap resource %+v", err)
 			return err
 		}
 		if deploymentInQjr.Labels == nil {
@@ -298,34 +297,32 @@ func (qjrDeployment *QueueJobResDeployment) SyncQueueJob(queuejob *arbv1.AppWrap
 	return nil
 }
 
-
 func (qjrDeployment *QueueJobResDeployment) getDeploymentForQueueJobRes(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) (*string, *apps.Deployment, []*apps.Deployment, error) {
 
 	// Get "a" Deployment from AppWrapper  Resource
 	deploymentInQjr, err := qjrDeployment.getDeploymentTemplate(qjobRes)
 	if err != nil {
-		glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+		klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 		return nil, nil, nil, err
 	}
 
 	// Get Deployment"s" in Etcd Server
 	var _namespace *string
-	if deploymentInQjr.Namespace!=""{
+	if deploymentInQjr.Namespace != "" {
 		_namespace = &deploymentInQjr.Namespace
 	} else {
 		_namespace = &queuejob.Namespace
 	}
 
-	deploymentList, err := qjrDeployment.clients.AppsV1().Deployments(*_namespace).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name),})
+	deploymentList, err := qjrDeployment.clients.AppsV1().Deployments(*_namespace).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name)})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	deploymentsInEtcd := []*apps.Deployment{}
 	for i, _ := range deploymentList.Items {
-				deploymentsInEtcd = append(deploymentsInEtcd, &deploymentList.Items[i])
+		deploymentsInEtcd = append(deploymentsInEtcd, &deploymentList.Items[i])
 	}
 
-
 	myDeploymentsInEtcd := []*apps.Deployment{}
 	for i, deployment := range deploymentsInEtcd {
 		if qjrDeployment.refManager.BelongTo(qjobRes, deployment) {
@@ -336,7 +333,6 @@ func (qjrDeployment *QueueJobResDeployment) getDeploymentForQueueJobRes(qjobRes
 	return _namespace, deploymentInQjr, myDeploymentsInEtcd, nil
 }
 
-
 func (qjrDeployment *QueueJobResDeployment) deleteQueueJobResDeployments(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) error {
 
 	job := *queuejob
@@ -355,7 +351,7 @@ func (qjrDeployment *QueueJobResDeployment) deleteQueueJobResDeployments(qjobRes
 			defer wait.Done()
 			if err := qjrDeployment.delDeployment(*_namespace, activeDeployments[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeDeployments[ix].Name, *_namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeDeployments[ix].Name, *_namespace, job.Name)
 			}
 		}(i)
 	}
@@ -368,4 +364,3 @@ func (qjrDeployment *QueueJobResDeployment) deleteQueueJobResDeployments(qjobRes
 func (qjrDeployment *QueueJobResDeployment) Cleanup(queuejob *arbv1.AppWrapper, qjobRes *arbv1.AppWrapperResource) error {
 	return qjrDeployment.deleteQueueJobResDeployments(qjobRes, queuejob)
 }
-
@@ -17,16 +17,17 @@ limitations under the License.
 package genericresource
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"time"
 
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
-	"github.com/golang/glog"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
 
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -44,16 +45,16 @@ var resourceName = "resourceName"
 var appWrapperKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
 
 type GenericResources struct {
-	clients    *kubernetes.Clientset
+	clients          *kubernetes.Clientset
 	kubeClientConfig *rest.Config
-	arbclients *clientset.Clientset
+	arbclients       *clientset.Clientset
 }
 
 func NewAppWrapperGenericResource(config *rest.Config) *GenericResources {
 	return &GenericResources{
-		clients:    kubernetes.NewForConfigOrDie(config),
+		clients:          kubernetes.NewForConfigOrDie(config),
 		kubeClientConfig: config,
-		arbclients: clientset.NewForConfigOrDie(config),
+		arbclients:       clientset.NewForConfigOrDie(config),
 	}
 }
 
@@ -69,31 +70,28 @@ func join(strs ...string) string {
 }
 
 func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWrapperGenericResource) (podList []*v1.Pod, err error) {
-
 	startTime := time.Now()
 	defer func() {
-		glog.V(4).Infof("Finished syncing AppWrapper job resource %s (%v)", aw.Name, time.Now().Sub(startTime))
-		// glog.V(4).Infof("Finished syncing AppWrapper job resource %q (%v)", awobRes.Template, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing AppWrapper job resource %s (%v)", aw.Name, time.Now().Sub(startTime))
 	}()
 
 	namespaced := true
 	//todo:DELETEME	dd := common.KubeClient.Discovery()
 	dd := gr.clients.Discovery()
 	apigroups, err := restmapper.GetAPIGroupResources(dd)
 	if err != nil {
-		glog.Fatal(err)
+		klog.Fatal(err)
 	}
 	ext := awr.GenericTemplate
 	restmapper := restmapper.NewDiscoveryRESTMapper(apigroups)
-	versions := &runtime.VersionedObjects{}
-	_, gvk, err := unstructured.UnstructuredJSONScheme.Decode(ext.Raw, nil, versions)
+	_, gvk, err := unstructured.UnstructuredJSONScheme.Decode(ext.Raw, nil, nil)
 	if err != nil {
-		glog.Errorf("Decoding error, please check your CR! Aborting handling the resource creation, err:  `%v`", err)
+		klog.Errorf("Decoding error, please check your CR! Aborting handling the resource creation, err:  `%v`", err)
 		return []*v1.Pod{}, err
 	}
 	mapping, err := restmapper.RESTMapping(gvk.GroupKind(), gvk.Version)
 	if err != nil {
-		glog.Errorf("mapping error from raw object: `%v`", err)
+		klog.Errorf("mapping error from raw object: `%v`", err)
 		return []*v1.Pod{}, err
 	}
 
@@ -105,12 +103,12 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 	}
 	dclient, err := dynamic.NewForConfig(restconfig)
 	if err != nil {
-		glog.Fatal(err)
+		klog.Fatal(err)
 	}
 
-	apiresourcelist, err := dd.ServerResources()
+	_, apiresourcelist, err := dd.ServerGroupsAndResources()
 	if err != nil {
-		glog.Fatal(err)
+		klog.Fatal(err)
 	}
 
 	rsrc := mapping.Resource
@@ -128,7 +126,7 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 	unstruct.Object = make(map[string]interface{})
 	var blob interface{}
 	if err = json.Unmarshal(ext.Raw, &blob); err != nil {
-		glog.Fatal(err)
+		klog.Fatal(err)
 	}
 	ownerRef := metav1.NewControllerRef(aw, appWrapperKind)
 	unstruct.Object = blob.(map[string]interface{}) //set object to the content of the blob after Unmarshalling
@@ -142,7 +140,6 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 			name = objectName.(string)
 		}
 		if objectns, ok := metadata["namespace"]; ok {
-			//glog.V(9).Infof("metadata[namespace] exists")
 			namespace = objectns.(string)
 		}
 	}
@@ -159,12 +156,12 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 	// Add labels to pod templete if one exists.
 	podTemplateFound := addLabelsToPodTemplateField(&unstruct, labels)
 	if !podTemplateFound {
-		glog.V(4).Infof("[SyncQueueJob] No pod template spec exists for resource: %s to add labels.", name)
+		klog.V(4).Infof("[SyncQueueJob] No pod template spec exists for resource: %s to add labels.", name)
 	}
 
 	// Get the resource  to see if it exists
 	labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobName, aw.Name, resourceName, unstruct.GetName())
-	inEtcd, err := dclient.Resource(rsrc).List(metav1.ListOptions{LabelSelector: labelSelector})
+	inEtcd, err := dclient.Resource(rsrc).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector})
 	if err != nil {
 		return []*v1.Pod{}, err
 	}
@@ -179,9 +176,9 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 		err = createObject(namespaced, namespace, newName, rsrc, unstruct, dclient)
 		if err != nil {
 			if errors.IsAlreadyExists(err) {
-				glog.V(4).Infof("%v\n", err.Error())
+				klog.V(4).Infof("%v\n", err.Error())
 			} else {
-				glog.Errorf("Error creating the object `%v`, the error is `%v`", newName, errors.ReasonForError(err))
+				klog.Errorf("Error creating the object `%v`, the error is `%v`", newName, errors.ReasonForError(err))
 				return []*v1.Pod{}, err
 			}
 		}
@@ -191,24 +188,23 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 	var thisObj *unstructured.Unstructured
 	var err1 error
 	if namespaced {
-		thisObj, err1 = dclient.Resource(rsrc).Namespace(namespace).Get(name, metav1.GetOptions{})
+		thisObj, err1 = dclient.Resource(rsrc).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{})
 	} else {
-		thisObj, err1 = dclient.Resource(rsrc).Get(name, metav1.GetOptions{})
+		thisObj, err1 = dclient.Resource(rsrc).Get(context.Background(), name, metav1.GetOptions{})
 	}
 	if err1 != nil {
-		glog.Errorf("Could not get created resource with error %v", err)
+		klog.Errorf("Could not get created resource with error %v", err)
 	}
 	thisOwnerRef := metav1.NewControllerRef(thisObj, thisObj.GroupVersionKind())
 
-
-	podL, _ := gr.clients.CoreV1().Pods("").List(metav1.ListOptions{})
+	podL, _ := gr.clients.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
 	pods := []*v1.Pod{}
 	for _, pod := range (*podL).Items {
 		parent := metav1.GetControllerOf(&pod)
 		if reflect.DeepEqual(thisOwnerRef, parent) {
 			pods = append(pods, &pod)
 		}
-		glog.V(10).Infof("[SyncQueueJob] pod %s created from a Generic Item\n", pod.Name)
+		klog.V(10).Infof("[SyncQueueJob] pod %s created from a Generic Item\n", pod.Name)
 	}
 	return pods, nil
 }
@@ -217,24 +213,24 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra
 func addLabelsToPodTemplateField(unstruct *unstructured.Unstructured, labels map[string]string) (hasFields bool) {
 	spec, isFound, _ := unstructured.NestedMap(unstruct.UnstructuredContent(), "spec")
 	if !isFound {
-		glog.V(10).Infof("[addLabelsToPodTemplateField] 'spec' field not found.")
+		klog.V(10).Infof("[addLabelsToPodTemplateField] 'spec' field not found.")
 		return false
 	}
 	template, isFound, _ := unstructured.NestedMap(spec, "template")
 	if !isFound {
-		glog.V(10).Infof("[addLabelsToPodTemplateField] 'spec.template' field not found.")
+		klog.V(10).Infof("[addLabelsToPodTemplateField] 'spec.template' field not found.")
 		return false
 	}
 
 	marshal, _ := json.Marshal(template)
 	unmarshal := v1.PodTemplateSpec{}
 	if err := json.Unmarshal(marshal, &unmarshal); err != nil {
-		glog.Warning(err)
+		klog.Warning(err)
 		return false
 	}
 	existingLabels, isFound, _ := unstructured.NestedStringMap(template, "metadata", "labels")
 	if !isFound {
-		glog.V(10).Infof("[addLabelsToPodTemplateField] 'spec.template.metadata.labels' field not found.")
+		klog.V(10).Infof("[addLabelsToPodTemplateField] 'spec.template.metadata.labels' field not found.")
 		return false
 	}
 	newLength := len(existingLabels) + len(labels)
@@ -248,26 +244,30 @@ func addLabelsToPodTemplateField(unstruct *unstructured.Unstructured, labels map
 	}
 
 	if err := unstructured.SetNestedStringMap(unstruct.Object, m, "spec", "template", "metadata", "labels"); err != nil {
-		glog.Warning(err)
+		klog.Warning(err)
 		return false
 	}
 
 	return isFound
 }
+
 //checks if object has replicas and containers field
 func hasFields(obj runtime.RawExtension) (hasFields bool, replica float64, containers []v1.Container) {
 	var unstruct unstructured.Unstructured
 	unstruct.Object = make(map[string]interface{})
 	var blob interface{}
 	if err := json.Unmarshal(obj.Raw, &blob); err != nil {
-		glog.Fatal(err)
+		klog.Fatal(err)
 	}
 	unstruct.Object = blob.(map[string]interface{})
 	spec, isFound, _ := unstructured.NestedMap(unstruct.UnstructuredContent(), "spec")
 	replicas, isFound, _ := unstructured.NestedFloat64(spec, "replicas")
+
+	// Set default to 1 if no replicas field is found.
 	if !isFound {
-		return false, 0, nil
+		replicas = 1
 	}
+
 	template, isFound, _ := unstructured.NestedMap(spec, "template")
 	subspec, isFound, _ := unstructured.NestedMap(template, "spec")
 	containerList, isFound, _ := unstructured.NestedSlice(subspec, "containers")
@@ -288,32 +288,32 @@ func createObject(namespaced bool, namespace string, name string, rsrc schema.Gr
 	var err error
 	if !namespaced {
 		res := dclient.Resource(rsrc)
-		_, err = res.Create(&unstruct, metav1.CreateOptions{})
+		_, err = res.Create(context.Background(), &unstruct, metav1.CreateOptions{})
 		if err != nil {
 			if errors.IsAlreadyExists(err) {
-				glog.Errorf("%v\n", err.Error())
+				klog.Errorf("%v\n", err.Error())
 				return nil
 			} else {
-				glog.Errorf("Error creating the object `%v`, the error is `%v`", name, errors.ReasonForError(err))
+				klog.Errorf("Error creating the object `%v`, the error is `%v`", name, errors.ReasonForError(err))
 				return err
 			}
 		} else {
-			glog.V(4).Infof("Resource `%v` created\n", name)
+			klog.V(4).Infof("Resource `%v` created\n", name)
 			return nil
 		}
 	} else {
 		res := dclient.Resource(rsrc).Namespace(namespace)
-		_, err = res.Create(&unstruct, metav1.CreateOptions{})
+		_, err = res.Create(context.Background(), &unstruct, metav1.CreateOptions{})
 		if err != nil {
 			if errors.IsAlreadyExists(err) {
-				glog.Errorf("%v\n", err.Error())
+				klog.Errorf("%v\n", err.Error())
 				return nil
 			} else {
-				glog.Errorf("Error creating the object `%v`, the error is `%v`", name, errors.ReasonForError(err))
+				klog.Errorf("Error creating the object `%v`, the error is `%v`", name, errors.ReasonForError(err))
 				return err
 			}
 		} else {
-			glog.V(4).Infof("Resource `%v` created\n", name)
+			klog.V(4).Infof("Resource `%v` created\n", name)
 			return nil
 
 		}
@@ -330,14 +330,14 @@ func GetResources(awr *arbv1.AppWrapperGenericResource) (resource *clusterstatea
 				res := getContainerResources(item, replicas)
 				totalresource = totalresource.Add(res)
 			}
-			glog.V(8).Infof("[GetResources] Requested total allocation resource from containers `%v`.\n", totalresource)
+			klog.V(8).Infof("[GetResources] Requested total allocation resource from containers `%v`.\n", totalresource)
 		} else {
 			podresources := awr.CustomPodResources
 			for _, item := range podresources {
 				res := getPodResources(item)
 				totalresource = totalresource.Add(res)
 			}
-			glog.V(8).Infof("[GetResources] Requested total allocation resource from pods `%v`.\n", totalresource)
+			klog.V(8).Infof("[GetResources] Requested total allocation resource from pods `%v`.\n", totalresource)
 		}
 	}
 	return totalresource, nil

@@ -14,16 +14,21 @@ limitations under the License.
 package namespace
 
 import (
+	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	//schedulerapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+
+	"sync"
+	"time"
+
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	// "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/klog"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -35,8 +40,6 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"sync"
-	"time"
 )
 
 var queueJobKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
@@ -52,15 +55,15 @@ const (
 
 //QueueJobResService contains service info
 type QueueJobResNamespace struct {
-	clients    			*kubernetes.Clientset
-	arbclients 			*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of services, populated by the serviceController
-	namespaceStore    	corelisters.NamespaceLister
-	namespaceInformer 	corev1informer.NamespaceInformer
-	rtScheme        	*runtime.Scheme
-	jsonSerializer  	*json.Serializer
+	namespaceStore    corelisters.NamespaceLister
+	namespaceInformer corev1informer.NamespaceInformer
+	rtScheme          *runtime.Scheme
+	jsonSerializer    *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 			queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -130,13 +133,11 @@ func (qjrNamespace *QueueJobResNamespace) deleteNamespace(obj interface{}) {
 	return
 }
 
-
 func (qjrNamespace *QueueJobResNamespace) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        return total
+	total := clusterstateapi.EmptyResource()
+	return total
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrNamespace *QueueJobResNamespace) getNamespaceTemplate(qjobRes *arbv1.AppWrapperResource) (*v1.Namespace, error) {
 
@@ -158,12 +159,12 @@ func (qjrNamespace *QueueJobResNamespace) getNamespaceTemplate(qjobRes *arbv1.Ap
 
 func (qjrNamespace *QueueJobResNamespace) createNamespaceWithControllerRef(namespace *v1.Namespace, controllerRef *metav1.OwnerReference) error {
 
-	// glog.V(4).Infof("==========create Namespace: %+v \n", namespace)
+	// klog.V(4).Infof("==========create Namespace: %+v \n", namespace)
 	if controllerRef != nil {
 		namespace.OwnerReferences = append(namespace.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrNamespace.clients.Core().Namespaces().Create(namespace); err != nil {
+	if _, err := qjrNamespace.clients.CoreV1().Namespaces().Create(context.Background(), namespace, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
@@ -172,8 +173,8 @@ func (qjrNamespace *QueueJobResNamespace) createNamespaceWithControllerRef(names
 
 func (qjrNamespace *QueueJobResNamespace) delNamespace(name string) error {
 
-	glog.V(4).Infof("==========delete namespace: %s \n", name)
-	if err := qjrNamespace.clients.Core().Namespaces().Delete(name, nil); err != nil {
+	klog.V(4).Infof("==========delete namespace: %s \n", name)
+	if err := qjrNamespace.clients.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 
@@ -189,8 +190,7 @@ func (qjrNamespace *QueueJobResNamespace) SyncQueueJob(queuejob *arbv1.AppWrappe
 
 	startTime := time.Now()
 	defer func() {
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
-		// glog.V(4).Infof("Finished syncing queue job resource %s (%v)", qjobRes.Template, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	namespaces, err := qjrNamespace.getNamespaceForQueueJobRes(qjobRes, queuejob)
@@ -203,19 +203,19 @@ func (qjrNamespace *QueueJobResNamespace) SyncQueueJob(queuejob *arbv1.AppWrappe
 
 	diff := int(replicas) - int(namespaceLen)
 
-	glog.V(4).Infof("QJob: %s had %d namespaces and %d desired namespaces", queuejob.Name, namespaceLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d namespaces and %d desired namespaces", queuejob.Name, namespaceLen, replicas)
 
 	if diff > 0 {
 		template, err := qjrNamespace.getNamespaceTemplate(qjobRes)
 		if err != nil {
-			glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+			klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 			return err
 		}
 		//TODO: need set reference after Service has been really added
 		tmpNamespace := v1.Namespace{}
 		err = qjrNamespace.refManager.AddReference(qjobRes, &tmpNamespace)
 		if err != nil {
-			glog.Errorf("Cannot add reference to namespace resource %+v", err)
+			klog.Errorf("Cannot add reference to namespace resource %+v", err)
 			return err
 		}
 
@@ -248,15 +248,14 @@ func (qjrNamespace *QueueJobResNamespace) SyncQueueJob(queuejob *arbv1.AppWrappe
 }
 
 func (qjrNamespace *QueueJobResNamespace) getNamespaceForQueueJob(j *arbv1.AppWrapper) ([]*v1.Namespace, error) {
-	// namespacelist, err := qjrNamespace.clients.CoreV1().Namespaces().List(metav1.ListOptions{})
-	namespacelist, err := qjrNamespace.clients.CoreV1().Namespaces().List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, j.Name),})
+	namespacelist, err := qjrNamespace.clients.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, j.Name)})
 	if err != nil {
 		return nil, err
 	}
 
 	namespaces := []*v1.Namespace{}
 	for i, _ := range namespacelist.Items {
-				namespaces = append(namespaces, &namespacelist.Items[i])
+		namespaces = append(namespaces, &namespacelist.Items[i])
 	}
 
 	return namespaces, nil
@@ -299,7 +298,7 @@ func (qjrNamespace *QueueJobResNamespace) deleteQueueJobResNamespaces(qjobRes *a
 			defer wait.Done()
 			if err := qjrNamespace.delNamespace(activeNamespaces[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeNamespaces[ix].Name, job.Namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeNamespaces[ix].Name, job.Namespace, job.Name)
 			}
 		}(i)
 	}

@@ -14,12 +14,13 @@ limitations under the License.
 package networkpolicy
 
 import (
+	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	//schedulerapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,12 +30,14 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/cache"
 	"sync"
 	"time"
 
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
 	networkingv1 "k8s.io/api/networking/v1"
 	networkingv1informer "k8s.io/client-go/informers/networking/v1"
 	networkingv1lister "k8s.io/client-go/listers/networking/v1"
@@ -53,15 +56,15 @@ const (
 
 //QueueJobResService contains service info
 type QueueJobResNetworkPolicy struct {
-	clients    				*kubernetes.Clientset
-	arbclients 				*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of services, populated by the serviceController
-	networkpolicyStore    	networkingv1lister.NetworkPolicyLister
-	networkpolicyInformer 	networkingv1informer.NetworkPolicyInformer
-	rtScheme        		*runtime.Scheme
-	jsonSerializer  		*json.Serializer
+	networkpolicyStore    networkingv1lister.NetworkPolicyLister
+	networkpolicyInformer networkingv1informer.NetworkPolicyInformer
+	rtScheme              *runtime.Scheme
+	jsonSerializer        *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 				queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -132,20 +135,17 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) deleteNetworkPolicy(obj interf
 	return
 }
 
-
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        return total
+	total := clusterstateapi.EmptyResource()
+	return total
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) getNetworkPolicyTemplate(qjobRes *arbv1.AppWrapperResource) (*networkingv1.NetworkPolicy, error) {
 
 	networkpolicyGVK := schema.GroupVersion{Group: networkingv1.GroupName, Version: "v1"}.WithKind("NetworkPolicy")
 	obj, _, err := qjrNetworkPolicy.jsonSerializer.Decode(qjobRes.Template.Raw, &networkpolicyGVK, nil)
 	if err != nil {
-		// glog.Infof("Decoding Error for NetworkPolicy=================================================")
 		return nil, err
 	}
 
@@ -160,12 +160,11 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) getNetworkPolicyTemplate(qjobR
 
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) createNetworkPolicyWithControllerRef(namespace string, networkpolicy *networkingv1.NetworkPolicy, controllerRef *metav1.OwnerReference) error {
 
-	// glog.V(4).Infof("==========create NetworkPolicy: %+v \n", networkpolicy)
 	if controllerRef != nil {
 		networkpolicy.OwnerReferences = append(networkpolicy.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrNetworkPolicy.clients.Networking().NetworkPolicies(namespace).Create(networkpolicy); err != nil {
+	if _, err := qjrNetworkPolicy.clients.NetworkingV1().NetworkPolicies(namespace).Create(context.Background(), networkpolicy, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
@@ -174,8 +173,8 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) createNetworkPolicyWithControl
 
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) delNetworkPolicy(namespace string, name string) error {
 
-	glog.V(4).Infof("==========delete networkpolicy: %s \n", name)
-	if err := qjrNetworkPolicy.clients.Networking().NetworkPolicies(namespace).Delete(name, nil); err != nil {
+	klog.V(4).Infof("==========delete networkpolicy: %s \n", name)
+	if err := qjrNetworkPolicy.clients.NetworkingV1().NetworkPolicies(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 
@@ -186,14 +185,12 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) UpdateQueueJobStatus(queuejob
 	return nil
 }
 
-
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) SyncQueueJob(queuejob *arbv1.AppWrapper, qjobRes *arbv1.AppWrapperResource) error {
 
 	startTime := time.Now()
 
 	defer func() {
-		// glog.V(4).Infof("Finished syncing queue job resource %q (%v)", qjobRes.Template, time.Now().Sub(startTime))
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	_namespace, networkPolicyInQjr, networkPoliciesInEtcd, err := qjrNetworkPolicy.getNetworkPolicyForQueueJobRes(qjobRes, queuejob)
@@ -206,14 +203,14 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) SyncQueueJob(queuejob *arbv1.A
 
 	diff := int(replicas) - int(networkPolicyLen)
 
-	glog.V(4).Infof("QJob: %s had %d NetworkPolicies and %d desired NetworkPolicies", queuejob.Name, networkPolicyLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d NetworkPolicies and %d desired NetworkPolicies", queuejob.Name, networkPolicyLen, replicas)
 
 	if diff > 0 {
 		//TODO: need set reference after Service has been really added
 		tmpNetworkPolicy := networkingv1.NetworkPolicy{}
 		err = qjrNetworkPolicy.refManager.AddReference(qjobRes, &tmpNetworkPolicy)
 		if err != nil {
-			glog.Errorf("Cannot add reference to configmap resource %+v", err)
+			klog.Errorf("Cannot add reference to configmap resource %+v", err)
 			return err
 		}
 
@@ -247,31 +244,29 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) SyncQueueJob(queuejob *arbv1.A
 	return nil
 }
 
-
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) getNetworkPolicyForQueueJobRes(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) (*string, *networkingv1.NetworkPolicy, []*networkingv1.NetworkPolicy, error) {
 
 	// Get "a" NetworkPolicy from AppWrapper Resource
 	networkPolicyInQjr, err := qjrNetworkPolicy.getNetworkPolicyTemplate(qjobRes)
 	if err != nil {
-		glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+		klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 		return nil, nil, nil, err
 	}
 
 	// Get NetworkPolicy"s" in Etcd Server
 	var _namespace *string
-	if networkPolicyInQjr.Namespace!=""{
+	if networkPolicyInQjr.Namespace != "" {
 		_namespace = &networkPolicyInQjr.Namespace
 	} else {
 		_namespace = &queuejob.Namespace
 	}
-	// networkPolicyList, err := qjrNetworkPolicy.clients.Networking().NetworkPolicies(*_namespace).List(metav1.ListOptions{})
-	networkPolicyList, err := qjrNetworkPolicy.clients.Networking().NetworkPolicies(*_namespace).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name),})
+	networkPolicyList, err := qjrNetworkPolicy.clients.NetworkingV1().NetworkPolicies(*_namespace).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name)})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	networkPoliciesInEtcd := []*networkingv1.NetworkPolicy{}
-	for i,  _ := range networkPolicyList.Items {
-				networkPoliciesInEtcd = append(networkPoliciesInEtcd, &networkPolicyList.Items[i])
+	for i, _ := range networkPolicyList.Items {
+		networkPoliciesInEtcd = append(networkPoliciesInEtcd, &networkPolicyList.Items[i])
 	}
 	myNetworkPoliciesInEtcd := []*networkingv1.NetworkPolicy{}
 	for i, networkPolicy := range networkPoliciesInEtcd {
@@ -283,7 +278,6 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) getNetworkPolicyForQueueJobRes
 	return _namespace, networkPolicyInQjr, myNetworkPoliciesInEtcd, nil
 }
 
-
 func (qjrNetworkPolicy *QueueJobResNetworkPolicy) deleteQueueJobResNetworkPolicies(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) error {
 
 	job := *queuejob
@@ -302,7 +296,7 @@ func (qjrNetworkPolicy *QueueJobResNetworkPolicy) deleteQueueJobResNetworkPolici
 			defer wait.Done()
 			if err := qjrNetworkPolicy.delNetworkPolicy(*_namespace, activeNetworkPolicies[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeNetworkPolicies[ix].Name, *_namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activeNetworkPolicies[ix].Name, *_namespace, job.Name)
 			}
 		}(i)
 	}

@@ -14,16 +14,21 @@ limitations under the License.
 package persistentvolume
 
 import (
+	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	//schedulerapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	// "k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/klog"
+
+	"sync"
+	"time"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -35,8 +40,6 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"sync"
-	"time"
 )
 
 var queueJobKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
@@ -52,15 +55,15 @@ const (
 
 //QueueJobResService contains service info
 type QueueJobResPersistentvolume struct {
-	clients    				*kubernetes.Clientset
-	arbclients 				*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of services, populated by the serviceController
 	persistentvolumeStore    corelisters.PersistentVolumeLister
 	persistentvolumeInformer corev1informer.PersistentVolumeInformer
-	rtScheme        		*runtime.Scheme
-	jsonSerializer  		*json.Serializer
+	rtScheme                 *runtime.Scheme
+	jsonSerializer           *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 				queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -130,13 +133,11 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) deletePersistentVolume(o
 	return
 }
 
-
 func (qjrPersistentvolume *QueueJobResPersistentvolume) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        return total
+	total := clusterstateapi.EmptyResource()
+	return total
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrPersistentvolume *QueueJobResPersistentvolume) getPersistentVolumeTemplate(qjobRes *arbv1.AppWrapperResource) (*v1.PersistentVolume, error) {
 
@@ -158,12 +159,11 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) getPersistentVolumeTempl
 
 func (qjrPersistentvolume *QueueJobResPersistentvolume) createPersistentVolumeWithControllerRef(persistentvolume *v1.PersistentVolume, controllerRef *metav1.OwnerReference) error {
 
-	// glog.V(4).Infof("==========create PersistentVolume: %+v \n", persistentvolume)
 	if controllerRef != nil {
 		persistentvolume.OwnerReferences = append(persistentvolume.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrPersistentvolume.clients.Core().PersistentVolumes().Create(persistentvolume); err != nil {
+	if _, err := qjrPersistentvolume.clients.CoreV1().PersistentVolumes().Create(context.Background(), persistentvolume, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
@@ -172,8 +172,8 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) createPersistentVolumeWi
 
 func (qjrPersistentvolume *QueueJobResPersistentvolume) delPersistentVolume(name string) error {
 
-	glog.V(4).Infof("==========delete persistentvolume: %s \n", name)
-	if err := qjrPersistentvolume.clients.Core().PersistentVolumes().Delete(name, nil); err != nil {
+	klog.V(4).Infof("==========delete persistentvolume: %s \n", name)
+	if err := qjrPersistentvolume.clients.CoreV1().PersistentVolumes().Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 
@@ -189,8 +189,7 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) SyncQueueJob(queuejob *a
 
 	startTime := time.Now()
 	defer func() {
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
-		// glog.V(4).Infof("Finished syncing queue job resource %q (%v)", qjobRes.Template, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	persistentvolumes, err := qjrPersistentvolume.getPersistentVolumeForQueueJobRes(qjobRes, queuejob)
@@ -203,19 +202,19 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) SyncQueueJob(queuejob *a
 
 	diff := int(replicas) - int(persistentvolumeLen)
 
-	glog.V(4).Infof("QJob: %s had %d persistentvolumes and %d desired persistentvolumes", queuejob.Name, persistentvolumeLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d persistentvolumes and %d desired persistentvolumes", queuejob.Name, persistentvolumeLen, replicas)
 
 	if diff > 0 {
 		template, err := qjrPersistentvolume.getPersistentVolumeTemplate(qjobRes)
 		if err != nil {
-			glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+			klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 			return err
 		}
 		//TODO: need set reference after Service has been really added
 		tmpPersistentVolume := v1.PersistentVolume{}
 		err = qjrPersistentvolume.refManager.AddReference(qjobRes, &tmpPersistentVolume)
 		if err != nil {
-			glog.Errorf("Cannot add reference to persistentvolume resource %+v", err)
+			klog.Errorf("Cannot add reference to persistentvolume resource %+v", err)
 			return err
 		}
 
@@ -248,14 +247,14 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) SyncQueueJob(queuejob *a
 }
 
 func (qjrPersistentvolume *QueueJobResPersistentvolume) getPersistentVolumeForQueueJob(j *arbv1.AppWrapper) ([]*v1.PersistentVolume, error) {
-	persistentvolumelist, err := qjrPersistentvolume.clients.CoreV1().PersistentVolumes().List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, j.Name),})
+	persistentvolumelist, err := qjrPersistentvolume.clients.CoreV1().PersistentVolumes().List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, j.Name)})
 	if err != nil {
 		return nil, err
 	}
 
 	persistentvolumes := []*v1.PersistentVolume{}
 	for i, _ := range persistentvolumelist.Items {
-				persistentvolumes = append(persistentvolumes, &persistentvolumelist.Items[i])
+		persistentvolumes = append(persistentvolumes, &persistentvolumelist.Items[i])
 	}
 
 	// for i, persistentvolume := range persistentvolumelist.Items {
@@ -311,7 +310,7 @@ func (qjrPersistentvolume *QueueJobResPersistentvolume) deleteQueueJobResPersist
 			defer wait.Done()
 			if err := qjrPersistentvolume.delPersistentVolume(activePersistentVolumes[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activePersistentVolumes[ix].Name, job.Namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activePersistentVolumes[ix].Name, job.Namespace, job.Name)
 			}
 		}(i)
 	}

@@ -14,14 +14,18 @@ limitations under the License.
 package persistentvolumeclaim
 
 import (
+	"context"
 	"fmt"
-	"github.com/golang/glog"
+
 	arbv1 "github.com/IBM/multi-cluster-app-dispatcher/pkg/apis/controller/v1alpha1"
 	clientset "github.com/IBM/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned"
 	"github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/queuejobresources"
-	//schedulerapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/scheduler/api"
+
+	"sync"
+	"time"
+
 	clusterstateapi "github.com/IBM/multi-cluster-app-dispatcher/pkg/controller/clusterstate/api"
-	"k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -34,8 +38,7 @@ import (
 	corelisters "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
-	"sync"
-	"time"
+	"k8s.io/klog"
 )
 
 var queueJobKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper")
@@ -51,15 +54,15 @@ const (
 
 //QueueJobResService contains service info
 type QueueJobResPersistentVolumeClaim struct {
-	clients    						*kubernetes.Clientset
-	arbclients 						*clientset.Clientset
+	clients    *kubernetes.Clientset
+	arbclients *clientset.Clientset
 	// A store of services, populated by the serviceController
-	persistentvolumeclaimStore    	corelisters.PersistentVolumeClaimLister
-	persistentvolumeclaimInformer 	corev1informer.PersistentVolumeClaimInformer
-	rtScheme        				*runtime.Scheme
-	jsonSerializer  				*json.Serializer
+	persistentvolumeclaimStore    corelisters.PersistentVolumeClaimLister
+	persistentvolumeclaimInformer corev1informer.PersistentVolumeClaimInformer
+	rtScheme                      *runtime.Scheme
+	jsonSerializer                *json.Serializer
 	// Reference manager to manage membership of queuejob resource and its members
-	refManager 						queuejobresources.RefManager
+	refManager queuejobresources.RefManager
 }
 
 //Register registers a queue job resource type
@@ -129,13 +132,11 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) deletePersiste
 	return
 }
 
-
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) GetAggregatedResourcesByPriority(priority float64, job *arbv1.AppWrapper) *clusterstateapi.Resource {
-        total := clusterstateapi.EmptyResource()
-        return total
+	total := clusterstateapi.EmptyResource()
+	return total
 }
 
-
 // Parse queue job api object to get Service template
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) getPersistentVolumeClaimTemplate(qjobRes *arbv1.AppWrapperResource) (*v1.PersistentVolumeClaim, error) {
 
@@ -157,12 +158,11 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) getPersistentV
 
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) createPersistentVolumeClaimWithControllerRef(namespace string, persistentvolumeclaim *v1.PersistentVolumeClaim, controllerRef *metav1.OwnerReference) error {
 
-	// glog.V(4).Infof("==========create PersistentVolumeClaim: %+v \n", persistentvolumeclaim)
 	if controllerRef != nil {
 		persistentvolumeclaim.OwnerReferences = append(persistentvolumeclaim.OwnerReferences, *controllerRef)
 	}
 
-	if _, err := qjrPersistentVolumeClaim.clients.Core().PersistentVolumeClaims(namespace).Create(persistentvolumeclaim); err != nil {
+	if _, err := qjrPersistentVolumeClaim.clients.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), persistentvolumeclaim, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 
@@ -171,8 +171,8 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) createPersiste
 
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) delPersistentVolumeClaim(namespace string, name string) error {
 
-	glog.V(4).Infof("==========delete persistentvolumeclaim: %s \n", name)
-	if err := qjrPersistentVolumeClaim.clients.Core().PersistentVolumeClaims(namespace).Delete(name, nil); err != nil {
+	klog.V(4).Infof("==========delete persistentvolumeclaim: %s \n", name)
+	if err := qjrPersistentVolumeClaim.clients.CoreV1().PersistentVolumeClaims(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil {
 		return err
 	}
 
@@ -188,8 +188,7 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) SyncQueueJob(q
 	startTime := time.Now()
 
 	defer func() {
-		// glog.V(4).Infof("Finished syncing queue job resource %q (%v)", qjobRes.Template, time.Now().Sub(startTime))
-		glog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
+		klog.V(4).Infof("Finished syncing queue job resource %s (%v)", queuejob.Name, time.Now().Sub(startTime))
 	}()
 
 	_namespace, persistentVolumeClaimInQjr, persistentVolumeClaimsInEtcd, err := qjrPersistentVolumeClaim.getPersistentVolumeClaimForQueueJobRes(qjobRes, queuejob)
@@ -202,14 +201,14 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) SyncQueueJob(q
 
 	diff := int(replicas) - int(persistentVolumeClaimLen)
 
-	glog.V(4).Infof("QJob: %s had %d PersistVolumeClaims and %d desired PersistVolumeClaims", queuejob.Name, persistentVolumeClaimLen, replicas)
+	klog.V(4).Infof("QJob: %s had %d PersistVolumeClaims and %d desired PersistVolumeClaims", queuejob.Name, persistentVolumeClaimLen, replicas)
 
 	if diff > 0 {
 		//TODO: need set reference after Service has been really added
 		tmpPersistentVolumeClaim := v1.PersistentVolumeClaim{}
 		err = qjrPersistentVolumeClaim.refManager.AddReference(qjobRes, &tmpPersistentVolumeClaim)
 		if err != nil {
-			glog.Errorf("Cannot add reference to configmap resource %+v", err)
+			klog.Errorf("Cannot add reference to configmap resource %+v", err)
 			return err
 		}
 
@@ -243,30 +242,29 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) SyncQueueJob(q
 	return nil
 }
 
-
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) getPersistentVolumeClaimForQueueJobRes(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) (*string, *v1.PersistentVolumeClaim, []*v1.PersistentVolumeClaim, error) {
 
 	// Get "a" PersistentVolumeClaim from AppWrapper Resource
 	persistentVolumeClaimInQjr, err := qjrPersistentVolumeClaim.getPersistentVolumeClaimTemplate(qjobRes)
 	if err != nil {
-		glog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
+		klog.Errorf("Cannot read template from resource %+v %+v", qjobRes, err)
 		return nil, nil, nil, err
 	}
 
 	// Get PersistentVolumeClaim"s" in Etcd Server
 	var _namespace *string
-	if persistentVolumeClaimInQjr.Namespace!=""{
+	if persistentVolumeClaimInQjr.Namespace != "" {
 		_namespace = &persistentVolumeClaimInQjr.Namespace
 	} else {
 		_namespace = &queuejob.Namespace
 	}
-	persistentVolumeClaimList, err := qjrPersistentVolumeClaim.clients.CoreV1().PersistentVolumeClaims(*_namespace).List(metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name),})
+	persistentVolumeClaimList, err := qjrPersistentVolumeClaim.clients.CoreV1().PersistentVolumeClaims(*_namespace).List(context.Background(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", queueJobName, queuejob.Name)})
 	if err != nil {
 		return nil, nil, nil, err
 	}
 	persistentVolumeClaimsInEtcd := []*v1.PersistentVolumeClaim{}
 	for i, _ := range persistentVolumeClaimList.Items {
-				persistentVolumeClaimsInEtcd = append(persistentVolumeClaimsInEtcd, &persistentVolumeClaimList.Items[i])
+		persistentVolumeClaimsInEtcd = append(persistentVolumeClaimsInEtcd, &persistentVolumeClaimList.Items[i])
 	}
 
 	myPersistentVolumeClaimsInEtcd := []*v1.PersistentVolumeClaim{}
@@ -279,7 +277,6 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) getPersistentV
 	return _namespace, persistentVolumeClaimInQjr, myPersistentVolumeClaimsInEtcd, nil
 }
 
-
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) deleteQueueJobResPersistentVolumeClaims(qjobRes *arbv1.AppWrapperResource, queuejob *arbv1.AppWrapper) error {
 
 	job := *queuejob
@@ -298,7 +295,7 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) deleteQueueJob
 			defer wait.Done()
 			if err := qjrPersistentVolumeClaim.delPersistentVolumeClaim(*_namespace, activePersistentVolumeClaims[ix].Name); err != nil {
 				defer utilruntime.HandleError(err)
-				glog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activePersistentVolumeClaims[ix].Name, *_namespace, job.Name)
+				klog.V(2).Infof("Failed to delete %v, queue job %q/%q deadline exceeded", activePersistentVolumeClaims[ix].Name, *_namespace, job.Name)
 			}
 		}(i)
 	}
@@ -311,4 +308,3 @@ func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) deleteQueueJob
 func (qjrPersistentVolumeClaim *QueueJobResPersistentVolumeClaim) Cleanup(queuejob *arbv1.AppWrapper, qjobRes *arbv1.AppWrapperResource) error {
 	return qjrPersistentVolumeClaim.deleteQueueJobResPersistentVolumeClaims(qjobRes, queuejob)
 }
-
-Original file line number
+Diff line change
@@ @@ -9,7 +9,7 @@ services: @@
       - docker
     go:
-      - "1.11"
+      - "1.16.3"
     go_import_path: github.com/IBM/multi-cluster-app-dispatcher