From e7e40af1a725828685e70e25883d6f75eb0bd8d3 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Fri, 22 Mar 2019 10:52:57 +0800 Subject: [PATCH 1/2] Add configmap for scheduler --- .../chart/volcano/config/kube-batch-conf.yaml | 9 + .../chart/volcano/templates/scheduler.yaml | 30 ++- test/e2e/job_scheduling.go | 190 ------------------ 3 files changed, 31 insertions(+), 198 deletions(-) create mode 100644 installer/chart/volcano/config/kube-batch-conf.yaml diff --git a/installer/chart/volcano/config/kube-batch-conf.yaml b/installer/chart/volcano/config/kube-batch-conf.yaml new file mode 100644 index 0000000000..25df16501b --- /dev/null +++ b/installer/chart/volcano/config/kube-batch-conf.yaml @@ -0,0 +1,9 @@ +actions: "reclaim, allocate, backfill, preempt" +tiers: +- plugins: + - name: priority + - name: gang +- plugins: + - name: drf + - name: predicates + - name: proportion diff --git a/installer/chart/volcano/templates/scheduler.yaml b/installer/chart/volcano/templates/scheduler.yaml index 1e51377de5..32c4c08139 100644 --- a/installer/chart/volcano/templates/scheduler.yaml +++ b/installer/chart/volcano/templates/scheduler.yaml @@ -1,4 +1,11 @@ apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-scheduler-configmap +data: + {{- (.Files.Glob "config/*").AsConfig | nindent 2 }} +--- +apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Release.Name }}-scheduler @@ -89,11 +96,18 @@ spec: spec: serviceAccount: {{ .Release.Name }}-scheduler containers: - - name: {{ .Release.Name }}-scheduler - image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} - args: - - --alsologtostderr - - -v=2 - - 2>&1 - imagePullPolicy: "IfNotPresent" - + - name: {{ .Release.Name }}-scheduler + image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} + args: + - --alsologtostderr + - --scheduler-conf=/volcano.scheduler/kube-batch-conf.yaml + - -v=2 + - 2>&1 + imagePullPolicy: "IfNotPresent" + volumeMounts: + -name: scheduler-config + mountPath: /volcano.scheduler + volumes: + - name: scheduler-config + configMap: + name: {{ .Release.Name }}-scheduler-configmap diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index 83a7796909..19b7144a0e 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -17,134 +17,11 @@ limitations under the License. package e2e import ( - "time" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Job E2E Test", func() { - It("Schedule Job", func() { - context := initTestContext() - defer cleanupTestContext(context) - rep := clusterSize(context, oneCPU) - - job := createJob(context, &jobSpec{ - name: "qj-1", - tasks: []taskSpec{ - { - img: defaultBusyBoxImage, - req: oneCPU, - min: 2, - rep: rep, - }, - }, - }) - - err := waitJobReady(context, job) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Schedule Multiple Jobs", func() { - context := initTestContext() - defer cleanupTestContext(context) - - rep := clusterSize(context, oneCPU) - - job := &jobSpec{ - tasks: []taskSpec{ - { - img: defaultBusyBoxImage, - req: oneCPU, - min: 2, - rep: rep, - }, - }, - } - - job.name = "mqj-1" - job1 := createJob(context, job) - job.name = "mqj-2" - job2 := createJob(context, job) - job.name = "mqj-3" - job3 := createJob(context, job) - - err := waitJobReady(context, job1) - Expect(err).NotTo(HaveOccurred()) - - err = waitJobReady(context, job2) - Expect(err).NotTo(HaveOccurred()) - - err = waitJobReady(context, job3) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Gang scheduling", func() { - context := initTestContext() - defer cleanupTestContext(context) - rep := clusterSize(context, oneCPU)/2 + 1 - - replicaset := createReplicaSet(context, "rs-1", rep, defaultNginxImage, oneCPU) - err := waitReplicaSetReady(context, replicaset.Name) - Expect(err).NotTo(HaveOccurred()) - - jobSpec := &jobSpec{ - name: "gang-qj", - namespace: "test", - tasks: []taskSpec{ - { - img: defaultBusyBoxImage, - req: oneCPU, - min: rep, - rep: rep, - }, - }, - } - - job := createJob(context, jobSpec) - err = waitJobPending(context, job) - Expect(err).NotTo(HaveOccurred()) - - err = waitJobUnschedulable(context, job) - Expect(err).NotTo(HaveOccurred()) - - err = deleteReplicaSet(context, replicaset.Name) - Expect(err).NotTo(HaveOccurred()) - - err = waitJobReady(context, job) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Gang scheduling: Full Occupied", func() { - context := initTestContext() - defer cleanupTestContext(context) - rep := clusterSize(context, oneCPU) - - job := &jobSpec{ - namespace: "test", - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: oneCPU, - min: rep, - rep: rep, - }, - }, - } - - job.name = "gang-fq-qj1" - job1 := createJob(context, job) - err := waitJobReady(context, job1) - Expect(err).NotTo(HaveOccurred()) - - job.name = "gang-fq-qj2" - job2 := createJob(context, job) - err = waitJobPending(context, job2) - Expect(err).NotTo(HaveOccurred()) - - err = waitJobReady(context, job1) - Expect(err).NotTo(HaveOccurred()) - }) It("Preemption", func() { context := initTestContext() @@ -218,71 +95,4 @@ var _ = Describe("Job E2E Test", func() { err = waitTasksReady(context, job3, int(rep)/3) Expect(err).NotTo(HaveOccurred()) }) - - It("Schedule BestEffort Job", func() { - context := initTestContext() - defer cleanupTestContext(context) - - slot := oneCPU - rep := clusterSize(context, slot) - - spec := &jobSpec{ - name: "test", - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: 2, - rep: rep, - }, - { - img: defaultNginxImage, - min: 2, - rep: rep / 2, - }, - }, - } - - job := createJob(context, spec) - - err := waitJobReady(context, job) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Statement", func() { - context := initTestContext() - defer cleanupTestContext(context) - - slot := oneCPU - rep := clusterSize(context, slot) - - spec := &jobSpec{ - namespace: "test", - tasks: []taskSpec{ - { - img: defaultNginxImage, - req: slot, - min: rep, - rep: rep, - }, - }, - } - - spec.name = "st-qj-1" - job1 := createJob(context, spec) - err := waitJobReady(context, job1) - Expect(err).NotTo(HaveOccurred()) - - now := time.Now() - - spec.name = "st-qj-2" - job2 := createJob(context, spec) - err = waitJobUnschedulable(context, job2) - Expect(err).NotTo(HaveOccurred()) - - // No preemption event - evicted, err := jobEvicted(context, job1, now)() - Expect(err).NotTo(HaveOccurred()) - Expect(evicted).NotTo(BeTrue()) - }) }) From e9f1ef12b09ad67d51dedec0a632ace6f8ad4ec2 Mon Sep 17 00:00:00 2001 From: TommyLike Date: Fri, 22 Mar 2019 11:47:29 +0800 Subject: [PATCH 2/2] Fix debug issue --- .../{kube-batch-conf.yaml => kube-batch.conf} | 0 .../chart/volcano/templates/scheduler.yaml | 8 +- test/e2e/job_scheduling.go | 196 +++++++++++++++++- 3 files changed, 197 insertions(+), 7 deletions(-) rename installer/chart/volcano/config/{kube-batch-conf.yaml => kube-batch.conf} (100%) diff --git a/installer/chart/volcano/config/kube-batch-conf.yaml b/installer/chart/volcano/config/kube-batch.conf similarity index 100% rename from installer/chart/volcano/config/kube-batch-conf.yaml rename to installer/chart/volcano/config/kube-batch.conf diff --git a/installer/chart/volcano/templates/scheduler.yaml b/installer/chart/volcano/templates/scheduler.yaml index 32c4c08139..8a4423316a 100644 --- a/installer/chart/volcano/templates/scheduler.yaml +++ b/installer/chart/volcano/templates/scheduler.yaml @@ -100,13 +100,13 @@ spec: image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} args: - --alsologtostderr - - --scheduler-conf=/volcano.scheduler/kube-batch-conf.yaml - - -v=2 + - --scheduler-conf=/volcano.scheduler/kube-batch.conf + - -v=3 - 2>&1 imagePullPolicy: "IfNotPresent" volumeMounts: - -name: scheduler-config - mountPath: /volcano.scheduler + - name: scheduler-config + mountPath: /volcano.scheduler volumes: - name: scheduler-config configMap: diff --git a/test/e2e/job_scheduling.go b/test/e2e/job_scheduling.go index 19b7144a0e..e7a6d4e161 100644 --- a/test/e2e/job_scheduling.go +++ b/test/e2e/job_scheduling.go @@ -17,11 +17,134 @@ limitations under the License. package e2e import ( + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("Job E2E Test", func() { + It("Schedule Job", func() { + context := initTestContext() + defer cleanupTestContext(context) + rep := clusterSize(context, oneCPU) + + job := createJob(context, &jobSpec{ + name: "qj-1", + tasks: []taskSpec{ + { + img: defaultBusyBoxImage, + req: oneCPU, + min: 2, + rep: rep, + }, + }, + }) + + err := waitJobReady(context, job) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Schedule Multiple Jobs", func() { + context := initTestContext() + defer cleanupTestContext(context) + + rep := clusterSize(context, oneCPU) + + job := &jobSpec{ + tasks: []taskSpec{ + { + img: defaultBusyBoxImage, + req: oneCPU, + min: 2, + rep: rep, + }, + }, + } + + job.name = "mqj-1" + job1 := createJob(context, job) + job.name = "mqj-2" + job2 := createJob(context, job) + job.name = "mqj-3" + job3 := createJob(context, job) + + err := waitJobReady(context, job1) + Expect(err).NotTo(HaveOccurred()) + + err = waitJobReady(context, job2) + Expect(err).NotTo(HaveOccurred()) + + err = waitJobReady(context, job3) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Gang scheduling", func() { + context := initTestContext() + defer cleanupTestContext(context) + rep := clusterSize(context, oneCPU)/2 + 1 + + replicaset := createReplicaSet(context, "rs-1", rep, defaultNginxImage, oneCPU) + err := waitReplicaSetReady(context, replicaset.Name) + Expect(err).NotTo(HaveOccurred()) + + jobSpec := &jobSpec{ + name: "gang-qj", + namespace: "test", + tasks: []taskSpec{ + { + img: defaultBusyBoxImage, + req: oneCPU, + min: rep, + rep: rep, + }, + }, + } + + job := createJob(context, jobSpec) + err = waitJobPending(context, job) + Expect(err).NotTo(HaveOccurred()) + + err = waitJobUnschedulable(context, job) + Expect(err).NotTo(HaveOccurred()) + + err = deleteReplicaSet(context, replicaset.Name) + Expect(err).NotTo(HaveOccurred()) + + err = waitJobReady(context, job) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Gang scheduling: Full Occupied", func() { + context := initTestContext() + defer cleanupTestContext(context) + rep := clusterSize(context, oneCPU) + + job := &jobSpec{ + namespace: "test", + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: oneCPU, + min: rep, + rep: rep, + }, + }, + } + + job.name = "gang-fq-qj1" + job1 := createJob(context, job) + err := waitJobReady(context, job1) + Expect(err).NotTo(HaveOccurred()) + + job.name = "gang-fq-qj2" + job2 := createJob(context, job) + err = waitJobPending(context, job2) + Expect(err).NotTo(HaveOccurred()) + + err = waitJobReady(context, job1) + Expect(err).NotTo(HaveOccurred()) + }) It("Preemption", func() { context := initTestContext() @@ -73,16 +196,16 @@ var _ = Describe("Job E2E Test", func() { }, } - job.name = "preemptee-qj" + job.name = "multipreemptee-qj" job1 := createJob(context, job) err := waitTasksReady(context, job1, int(rep)) Expect(err).NotTo(HaveOccurred()) - job.name = "preemptor-qj1" + job.name = "multipreemptor-qj1" job2 := createJob(context, job) Expect(err).NotTo(HaveOccurred()) - job.name = "preemptor-qj2" + job.name = "multipreemptor-qj2" job3 := createJob(context, job) Expect(err).NotTo(HaveOccurred()) @@ -95,4 +218,71 @@ var _ = Describe("Job E2E Test", func() { err = waitTasksReady(context, job3, int(rep)/3) Expect(err).NotTo(HaveOccurred()) }) + + It("Schedule BestEffort Job", func() { + context := initTestContext() + defer cleanupTestContext(context) + + slot := oneCPU + rep := clusterSize(context, slot) + + spec := &jobSpec{ + name: "test", + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: 2, + rep: rep, + }, + { + img: defaultNginxImage, + min: 2, + rep: rep / 2, + }, + }, + } + + job := createJob(context, spec) + + err := waitJobReady(context, job) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Statement", func() { + context := initTestContext() + defer cleanupTestContext(context) + + slot := oneCPU + rep := clusterSize(context, slot) + + spec := &jobSpec{ + namespace: "test", + tasks: []taskSpec{ + { + img: defaultNginxImage, + req: slot, + min: rep, + rep: rep, + }, + }, + } + + spec.name = "st-qj-1" + job1 := createJob(context, spec) + err := waitJobReady(context, job1) + Expect(err).NotTo(HaveOccurred()) + + now := time.Now() + + spec.name = "st-qj-2" + job2 := createJob(context, spec) + err = waitJobUnschedulable(context, job2) + Expect(err).NotTo(HaveOccurred()) + + // No preemption event + evicted, err := jobEvicted(context, job1, now)() + Expect(err).NotTo(HaveOccurred()) + Expect(evicted).NotTo(BeTrue()) + }) })