This repository has been archived by the owner on Mar 22, 2023. It is now read-only.
/
kubernetes_scheduler_integration_test.clj
127 lines (124 loc) · 8.27 KB
/
kubernetes_scheduler_integration_test.clj
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
(ns waiter.kubernetes-scheduler-integration-test
(:require [clojure.data.json :as json]
[clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as walk]
[clojure.test :refer :all]
[clojure.tools.logging :as log]
[waiter.util.client-tools :refer :all]))
(deftest ^:parallel ^:integration-fast test-kubernetes-watch-state-update
(testing-using-waiter-url
(when (using-k8s? waiter-url)
(let [{:keys [body] :as response} (make-request waiter-url "/state/scheduler" :method :get)
_ (assert-response-status response 200)
body-json (-> body str try-parse-json)
watch-state-json (get-in body-json ["state" "watch-state"])
initial-pods-snapshot-version (get-in watch-state-json ["pods-metadata" "version" "snapshot"])
initial-pods-watch-version (get-in watch-state-json ["pods-metadata" "version" "watch"])
initial-rs-snapshot-version (get-in watch-state-json ["rs-metadata" "version" "snapshot"])
initial-rs-watch-version (get-in watch-state-json ["rs-metadata" "version" "watch"])
{:keys [service-id request-headers]} (make-request-with-debug-info
{:x-waiter-name (rand-name)}
#(make-kitchen-request waiter-url % :path "/hello"))]
(with-service-cleanup
service-id
(let [{:keys [body] :as response} (make-request waiter-url "/state/scheduler" :method :get)
_ (assert-response-status response 200)
body-json (-> body str try-parse-json)
watch-state-json (get-in body-json ["state" "watch-state"])
pods-snapshot-version' (get-in watch-state-json ["pods-metadata" "version" "snapshot"])
pods-watch-version' (get-in watch-state-json ["pods-metadata" "version" "watch"])
rs-snapshot-version' (get-in watch-state-json ["rs-metadata" "version" "snapshot"])
rs-watch-version' (get-in watch-state-json ["rs-metadata" "version" "watch"])]
(is (or (nil? initial-pods-watch-version)
(< initial-pods-snapshot-version initial-pods-watch-version)))
(is (<= initial-pods-snapshot-version pods-snapshot-version'))
(is (< pods-snapshot-version' pods-watch-version'))
(is (or (nil? initial-rs-watch-version)
(< initial-rs-snapshot-version initial-rs-watch-version)))
(is (<= initial-rs-snapshot-version rs-snapshot-version'))
(is (< rs-snapshot-version' rs-watch-version'))))))))
(deftest ^:parallel ^:integration-slow ^:resource-heavy test-s3-logs
(testing-using-waiter-url
(when (using-k8s? waiter-url)
(let [headers {:x-waiter-name (rand-name)
:x-waiter-max-instances 2
:x-waiter-scale-up-factor 0.99
:x-waiter-scale-down-factor 0.99
:x-kitchen-delay-ms 5000}
_ (log/info "making canary request...")
{:keys [cookies instance-id service-id]} (make-request-with-debug-info headers #(make-kitchen-request waiter-url %))
request-fn (fn [] (->> #(make-kitchen-request waiter-url %)
(make-request-with-debug-info headers)
:instance-id))]
(with-service-cleanup
service-id
(assert-service-on-all-routers waiter-url service-id cookies)
;; Get a service with at least one active and one killed instance.
;; This portion of the test logic was copied from basic-test/test-killed-instances
(log/info "starting parallel requests")
(let [instance-ids-atom (atom #{})
instance-request-fn (fn []
(let [instance-id (request-fn)]
(swap! instance-ids-atom conj instance-id)))
instance-ids (->> (parallelize-requests 4 10 instance-request-fn
:canceled? (fn [] (> (count @instance-ids-atom) 2))
:verbose true
:service-id service-id)
(reduce set/union))]
(is (> (count instance-ids) 1) (str instance-ids)))
(log/info "waiting for at least one instance to get killed")
(is (wait-for #(->> (get-in (service-settings waiter-url service-id) [:instances :killed-instances])
(map :id)
set
seq)
:interval 2 :timeout 45)
(str "No killed instances found for " service-id))
;; Test that the active instances' logs are available.
;; This portion of the test logic was copied from basic-test/test-basic-logs
(let [active-instances (get-in (service-settings waiter-url service-id :cookies cookies)
[:instances :active-instances])
log-url (:log-url (first active-instances))
_ (log/debug "Log Url Active:" log-url)
make-request-fn (fn [url] (make-request url "" :verbose true))
{:keys [body] :as logs-response} (make-request-fn log-url)
_ (assert-response-status logs-response 200)
_ (log/debug "Response body:" body)
log-files-list (walk/keywordize-keys (json/read-str body))
stdout-file-link (:url (first (filter #(= (:name %) "stdout") log-files-list)))
stderr-file-link (:url (first (filter #(= (:name %) "stderr") log-files-list)))]
(is (every? #(string/includes? body %) ["stderr" "stdout"])
(str "Directory listing is missing entries: stderr and stdout, got response: " logs-response))
(doseq [file-link [stderr-file-link stdout-file-link]]
(if (string/starts-with? (str file-link) "http")
(assert-response-status (make-request-fn file-link) 200)
(log/warn "test-basic-logs did not verify file link:" stdout-file-link))))
(delete-service waiter-url service-id)
;; Test that the killed instances' logs were persisted to S3.
;; This portion of the test logic was modified from the active-instances tests above.
(log/info "waiting s3 logs to appear")
(let [log-bucket-url (k8s-log-bucket-url waiter-url)]
(is (wait-for
#(let [killed-instances (get-in (service-settings waiter-url service-id :cookies cookies)
[:instances :killed-instances])
log-url (:log-url (first killed-instances))]
(string/starts-with? log-url log-bucket-url))
:interval 1 :timeout 15)
(str "Log URL never pointed to S3 bucket " log-bucket-url)))
(let [killed-instances (get-in (service-settings waiter-url service-id :cookies cookies)
[:instances :killed-instances])
log-url (:log-url (first killed-instances))
_ (log/debug "Log Url Killed:" log-url)
make-request-fn (fn [url] (make-request url "" :verbose true))
{:keys [body] :as logs-response} (make-request-fn log-url)
_ (assert-response-status logs-response 200)
_ (log/debug "Response body:" body)
log-files-list (walk/keywordize-keys (json/read-str body))
stdout-file-link (:url (first (filter #(= (:name %) "stdout") log-files-list)))
stderr-file-link (:url (first (filter #(= (:name %) "stderr") log-files-list)))]
(is (every? #(string/includes? body %) ["stderr" "stdout"])
(str "Directory listing is missing entries: stderr and stdout, got response: " logs-response))
(doseq [file-link [stderr-file-link stdout-file-link]]
(if (string/starts-with? (str file-link) "http")
(assert-response-status (make-request-fn file-link) 200)
(log/warn "test-basic-logs did not verify file link:" stdout-file-link)))))))))