basic_test.clj

;;
;; Copyright (c) Two Sigma Open Source, LLC
;;
;; Licensed under the Apache License, Version 2.0 (the "License");
;; you may not use this file except in compliance with the License.
;; You may obtain a copy of the License at
;;
;;  http://www.apache.org/licenses/LICENSE-2.0
;;
;; Unless required by applicable law or agreed to in writing, software
;; distributed under the License is distributed on an "AS IS" BASIS,
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
;;
(ns token-syncer.basic-test
  (:require [clojure.string :as str]
            [clojure.test :refer :all]
            [clojure.tools.logging :as log]
            [plumbing.core :as pc]
            [token-syncer.commands.ping :as ping]
            [token-syncer.commands.syncer :as syncer]
            [token-syncer.main :as main])
  (:import (java.util UUID)))

(defn- kitchen-cmd []
  (str (System/getenv "WAITER_TEST_KITCHEN_CMD") " -p $PORT0"))

(def basic-description {"cmd" (kitchen-cmd)
                        "cpus" 1
                        "mem" 2048
                        "metric-group" "syncer-test"})

(defn- waiter-urls
  "Retrieves urls to the waiter clusters provided in the WAITER_URIS environment variable."
  []
  (let [waiter-uris (System/getenv "WAITER_URIS")]
    (is waiter-uris)
    (-> waiter-uris
        (str/split #",")
        sort)))

(defn- waiter-url->cluster
  "Retrieves the cluster name corresponding to the provided cluster,"
  [waiter-url]
  (str "cluster-" (hash waiter-url)))

(defn- waiter-api
  "Initializes and returns the Waiter API functions."
  []
  (main/init-waiter-api {:connection-timeout-ms 5000, :idle-timeout-ms 5000}))

(deftest ^:integration test-environment
  (log/info "Running: test-environment")
  (testing "verifies presence of environment variables for running integration tests"
    (testing "waiter cluster uris"
      (log/info "env.WAITER_URIS" (System/getenv "WAITER_URIS"))
      (is (System/getenv "WAITER_URIS"))
      (is (> (count (waiter-urls)) 1)))

    (testing "kitchen command"
      (log/info "env.WAITER_TEST_KITCHEN_CMD" (System/getenv "WAITER_TEST_KITCHEN_CMD"))
      (is (not (str/blank? (System/getenv "WAITER_TEST_KITCHEN_CMD")))))))

(defn- basic-token-metadata
  "Returns the common metadata used in the tests."
  [current-time-ms]
  {"cluster" "cl.1"
   "last-update-time" current-time-ms
   "last-update-user" "auth-user"
   "owner" "test-user"
   "previous" {"last-update-time" (- current-time-ms 30000)
               "last-update-user" "another-auth-user"}
   "root" "src1"})

(defn- token->etag
  "Retrieves the etag for a token on a waiter router."
  [{:keys [load-token]} waiter-url token-name]
  (-> (load-token waiter-url token-name)
      (get :token-etag)))

(defn- cleanup-token
  "'Hard' deletes the token on all provided clusters."
  [{:keys [hard-delete-token] :as waiter-api} waiter-urls token-name]
  (log/info "Cleaning up token:" token-name)
  (doseq [waiter-url waiter-urls]
    (try
      (let [token-etag (token->etag waiter-api waiter-url token-name)]
        (hard-delete-token waiter-url token-name token-etag))
      (catch Exception _))))

(deftest ^:integration test-token-hard-delete
  (testing "token sync hard-delete"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-hard-delete-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [last-update-time-ms (- (System/currentTimeMillis) 10000)
              token-metadata (-> (basic-token-metadata last-update-time-ms)
                                 (assoc "deleted" true))
              token-description (merge basic-description token-metadata)]

          (doseq [waiter-url waiter-urls]
            (let [token-etag (token->etag waiter-api waiter-url token-name)]
              (store-token waiter-url token-name token-etag token-description)))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [waiter-sync-result (constantly
                                         {:code :success/hard-delete
                                          :details {:etag (str token-etag)
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description token-description
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result waiter-urls)}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (let [response (load-token waiter-url token-name)]
                    (is (= 404 (:status response)) (str waiter-url " responded with " response))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-soft-delete
  (testing "token sync soft-delete"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-soft-delete-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              token-metadata (basic-token-metadata current-time-ms)
              token-description (merge basic-description token-metadata)]

          (store-token (first waiter-urls) token-name nil (assoc token-description "deleted" true))
          (doseq [waiter-url (rest waiter-urls)]
            (let [last-update-time-ms (- current-time-ms 10000)]
              (store-token waiter-url token-name nil
                           (assoc token-description "last-update-time" last-update-time-ms))))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [waiter-sync-result (constantly
                                         {:code :success/soft-delete
                                          :details {:etag token-etag
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description (assoc token-description "deleted" true)
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result (rest waiter-urls))}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description (assoc token-description "deleted" true)
                          :headers {"content-type" "application/json"}
                          :status 200}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-hard-delete-previously-soft-deleted
  (testing "token sync hard-delete previously soft-deleted"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-update-previously-soft-deleted-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              token-metadata (basic-token-metadata current-time-ms)
              token-description (merge basic-description token-metadata)]

          (store-token (first waiter-urls) token-name nil (assoc token-description "deleted" true))
          (doseq [waiter-url (rest waiter-urls)]
            (let [last-update-time-ms (- current-time-ms 10000)]
              (store-token waiter-url token-name nil
                           (assoc token-description
                             "deleted" true
                             "last-update-time" last-update-time-ms))))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [waiter-sync-result (constantly
                                         {:code :success/hard-delete
                                          :details {:etag ""
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description (assoc token-description "deleted" true)
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result waiter-urls)}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description {}
                          :headers {"content-type" "application/json"
                                    "etag" ""}
                          :status 404
                          :token-etag ""}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-token-on-single-cluster
  (testing "token exists on single cluster"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-token-on-single-cluster-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              token-metadata (basic-token-metadata current-time-ms)
              token-description (merge basic-description token-metadata)]

          (store-token (first waiter-urls) token-name nil token-description)

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [waiter-sync-result (constantly
                                         {:code :success/sync-update
                                          :details {:etag token-etag
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description token-description
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result (rest waiter-urls))}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description token-description
                          :headers {"content-type" "application/json"
                                    "etag" token-etag}
                          :status 200
                          :token-etag token-etag}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-already-synced
  (testing "token already synced"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-already-synced-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              token-metadata (basic-token-metadata current-time-ms)
              token-description (merge basic-description token-metadata)]

          (doseq [waiter-url waiter-urls]
            (store-token waiter-url token-name nil token-description))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [expected-result {:details {}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{}}
                                               :tokens {:pending {:count 0 :value #{}}
                                                        :previously-synced {:count 1 :value #{token-name}}
                                                        :processed {:count 0 :value #{}}
                                                        :selected {:count 0 :value #{}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description token-description
                          :headers {"content-type" "application/json"
                                    "etag" token-etag}
                          :status 200
                          :token-etag token-etag}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-update
  (testing "token sync update"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-update-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              token-metadata (basic-token-metadata current-time-ms)
              token-description (merge basic-description token-metadata)]

          (let [last-update-time-ms (- current-time-ms 10000)]
            (store-token (first waiter-urls) token-name nil token-description)
            (doseq [waiter-url (rest waiter-urls)]
              (store-token waiter-url token-name nil
                           (assoc token-description
                             "cpus" 2
                             "mem" 2048
                             "last-update-time" last-update-time-ms))))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [waiter-sync-result (constantly
                                         {:code :success/sync-update
                                          :details {:etag token-etag
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description token-description
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result (rest waiter-urls))}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description token-description
                          :headers {"content-type" "application/json"
                                    "etag" token-etag}
                          :status 200
                          :token-etag token-etag}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-owners-but-same-root
  (testing "token sync update with different owners but same root"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-different-owners-but-same-root-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              last-update-time-ms (- current-time-ms 10000)]

          (doall
            (map-indexed
              (fn [index waiter-url]
                (store-token waiter-url token-name nil
                             (assoc basic-description
                               "cluster" (waiter-url->cluster waiter-url)
                               "cpus" (inc index)
                               "last-update-time" (- last-update-time-ms index)
                               "last-update-user" (str "auth-user-" index)
                               "owner" (str "test-user-" index)
                               "previous" {"last-update-time" (- current-time-ms 30000)
                                           "last-update-user" "foo-user"}
                               "root" "common-root")))
              waiter-urls))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [latest-description (assoc basic-description
                                         "cluster" (waiter-url->cluster (first waiter-urls))
                                         "cpus" 1
                                         "last-update-time" last-update-time-ms
                                         "last-update-user" "auth-user-0"
                                         "owner" "test-user-0"
                                         "previous" {"last-update-time" (- current-time-ms 30000)
                                                     "last-update-user" "foo-user"}
                                         "root" "common-root")
                    waiter-sync-result (constantly
                                         {:code :success/sync-update
                                          :details {:etag token-etag
                                                    :status 200}})
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description latest-description
                                                                    :token-etag token-etag}
                                                           :sync-result (pc/map-from-keys waiter-sync-result (rest waiter-urls))}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doseq [waiter-url waiter-urls]
                  (is (= {:description latest-description
                          :headers {"content-type" "application/json"
                                    "etag" token-etag}
                          :status 200
                          :token-etag token-etag}
                         (load-token waiter-url token-name))))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-roots
  (testing "token sync update with different roots"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-different-roots-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              last-update-time-ms (- current-time-ms 10000)]

          (doall
            (map-indexed
              (fn [index waiter-url]
                (store-token waiter-url token-name nil
                             (assoc basic-description
                               "cluster" (waiter-url->cluster waiter-url)
                               "cpus" (inc index)
                               "last-update-time" (- last-update-time-ms index)
                               "last-update-user" "auth-user"
                               "owner" "test-user"
                               "previous" {"last-update-time" (- current-time-ms 30000)
                                           "last-update-user" "foo-user"}
                               "root" waiter-url)))
              waiter-urls))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [latest-description (assoc basic-description
                                         "cluster" (waiter-url->cluster (first waiter-urls))
                                         "cpus" 1
                                         "last-update-time" last-update-time-ms
                                         "last-update-user" "auth-user"
                                         "owner" "test-user"
                                         "previous" {"last-update-time" (- current-time-ms 30000)
                                                     "last-update-user" "foo-user"}
                                         "root" (first waiter-urls))
                    sync-result (->> (rest waiter-urls)
                                     (map-indexed
                                       (fn [index waiter-url]
                                         [waiter-url
                                          {:code :error/root-mismatch
                                           :details {:cluster (assoc basic-description
                                                                "cluster" (waiter-url->cluster waiter-url)
                                                                "cpus" (+ index 2)
                                                                "last-update-time" (- last-update-time-ms index 1)
                                                                "last-update-user" "auth-user"
                                                                "owner" "test-user"
                                                                "previous" {"last-update-time" (- current-time-ms 30000)
                                                                            "last-update-user" "foo-user"}
                                                                "root" waiter-url)
                                                     :latest latest-description}}]))
                                     (into {}))
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description latest-description
                                                                    :token-etag token-etag}
                                                           :sync-result sync-result}}
                                     :summary {:sync {:failed #{token-name}
                                                      :unmodified #{}
                                                      :updated #{}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doall
                  (map-indexed
                    (fn [index waiter-url]
                      (let [token-last-modified-time (- last-update-time-ms index)
                            token-etag (token->etag waiter-api waiter-url token-name)]
                        (is (= {:description (assoc basic-description
                                               "cluster" (waiter-url->cluster waiter-url)
                                               "cpus" (inc index)
                                               "last-update-time" token-last-modified-time
                                               "last-update-user" "auth-user"
                                               "owner" "test-user"
                                               "previous" {"last-update-time" (- current-time-ms 30000)
                                                           "last-update-user" "foo-user"}
                                               "root" waiter-url)
                                :headers {"content-type" "application/json"
                                          "etag" token-etag}
                                :status 200
                                :token-etag token-etag}
                               (load-token waiter-url token-name)))))
                    waiter-urls))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-roots-and-deleted
  (testing "token sync hard-delete deleted tokens with different different roots"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-different-roots-and-deleted-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              last-update-time-ms (- current-time-ms 10000)]

          (doall
            (map-indexed
              (fn [index waiter-url]
                (store-token waiter-url token-name nil
                             (assoc basic-description
                               "cluster" (waiter-url->cluster waiter-url)
                               "cpus" (inc index)
                               "deleted" true
                               "last-update-time" (- last-update-time-ms index)
                               "last-update-user" "auth-user"
                               "owner" "test-user"
                               "previous" {"last-update-time" (- current-time-ms 30000)
                                           "last-update-user" "foo-user"}
                               "root" waiter-url)))
              waiter-urls))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [latest-description (assoc basic-description
                                         "cluster" (waiter-url->cluster (first waiter-urls))
                                         "cpus" 1
                                         "deleted" true
                                         "last-update-time" last-update-time-ms
                                         "last-update-user" "auth-user"
                                         "owner" "test-user"
                                         "previous" {"last-update-time" (- current-time-ms 30000)
                                                     "last-update-user" "foo-user"}
                                         "root" (first waiter-urls))
                    sync-result (pc/map-from-keys
                                  (constantly
                                    {:code :success/hard-delete
                                     :details {:etag ""
                                               :status 200}})
                                  waiter-urls)
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description latest-description
                                                                    :token-etag token-etag}
                                                           :sync-result sync-result}}
                                     :summary {:sync {:failed #{}
                                                      :unmodified #{}
                                                      :updated #{token-name}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doall
                  (map
                    (fn [waiter-url]
                      (is (= {:description {}
                              :headers {"content-type" "application/json"
                                        "etag" ""}
                              :status 404
                              :token-etag ""}
                             (load-token waiter-url token-name))))
                    waiter-urls))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-same-user-params-but-different-root
  (testing "token sync update with different difference only in roots and system metadata"
    (let [waiter-urls (waiter-urls)
          {:keys [load-token store-token] :as waiter-api} (waiter-api)
          limit 10
          token-name (str "test-token-same-user-params-but-different-root-" (UUID/randomUUID))]
      (try
        ;; ARRANGE
        (let [current-time-ms (System/currentTimeMillis)
              last-update-time-ms (- current-time-ms 10000)]

          (doall
            (map-indexed
              (fn [index waiter-url]
                (store-token waiter-url token-name nil
                             (assoc basic-description
                               "cluster" (waiter-url->cluster waiter-url)
                               "last-update-time" (- last-update-time-ms index)
                               "last-update-user" (str "auth-user-" index)
                               "owner" "test-user"
                               "previous" {"last-update-time" (- current-time-ms 30000)
                                           "last-update-user" "foo-user"}
                               "root" waiter-url)))
              waiter-urls))

          (let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

            ;; ACT
            (let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

              ;; ASSERT
              (let [latest-description (assoc basic-description
                                         "cluster" (waiter-url->cluster (first waiter-urls))
                                         "last-update-time" last-update-time-ms
                                         "last-update-user" "auth-user-0"
                                         "owner" "test-user"
                                         "previous" {"last-update-time" (- current-time-ms 30000)
                                                     "last-update-user" "foo-user"}
                                         "root" (first waiter-urls))
                    sync-result (pc/map-from-keys
                                  (constantly {:code :skip/token-sync})
                                  (rest waiter-urls))
                    expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
                                                                    :description latest-description
                                                                    :token-etag token-etag}
                                                           :sync-result sync-result}}
                                     :summary {:sync {:failed #{token-name}
                                                      :unmodified #{}
                                                      :updated #{}}
                                               :tokens {:pending {:count 1 :value #{token-name}}
                                                        :previously-synced {:count 0 :value #{}}
                                                        :processed {:count 1 :value #{token-name}}
                                                        :selected {:count 1 :value #{token-name}}
                                                        :total {:count 1 :value #{token-name}}}}}]
                (is (= expected-result actual-result))
                (doall
                  (map-indexed
                    (fn [index waiter-url]
                      (let [token-etag (token->etag waiter-api waiter-url token-name)]
                        (is (= {:description (assoc basic-description
                                               "cluster" (waiter-url->cluster waiter-url)
                                               "last-update-time" (- last-update-time-ms index)
                                               "last-update-user" (str "auth-user-" index)
                                               "owner" "test-user"
                                               "previous" {"last-update-time" (- current-time-ms 30000)
                                                           "last-update-user" "foo-user"}
                                               "root" waiter-url)
                                :headers {"content-type" "application/json"
                                          "etag" token-etag}
                                :status 200
                                :token-etag token-etag}
                               (load-token waiter-url token-name)))))
                    waiter-urls))))))
        (finally
          (cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-ping-tokens
         (testing "token ping on clusters"
           (let [waiter-urls (waiter-urls)
                 queue-timeout-ms 120000
                 {:keys [store-token] :as waiter-api} (waiter-api)]

             (testing "successful health check"
               (let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
                 (try
                   ;; ARRANGE
                   (doall
                     (map (fn [waiter-url]
                            (->> (assoc basic-description
                                   "health-check-url" "/status"
                                   "idle-timeout-mins" 2
                                   "run-as-user" "*"
                                   "version" "lorem-ipsum")
                                 (store-token waiter-url token-name nil)))
                          waiter-urls))

                   ;; ACT
                   (let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

                     ;; ASSERT
                     (let [expected-result {:details
                                            (pc/map-from-keys
                                              (fn [cluster-url]
                                                {:exit-code 0
                                                 :message (str "successfully pinged token " token-name " on " cluster-url
                                                               ", reason: health check returned status code 200")})
                                              waiter-urls)
                                            :exit-code 0
                                            :message (str "pinging token " token-name " on "
                                                          (-> waiter-urls vec println with-out-str str/trim)
                                                          " was successful")
                                            :token token-name}]
                       (is (= expected-result actual-result))))
                   (finally
                     (cleanup-token waiter-api waiter-urls token-name)))))

             (testing "unsuccessful health check"
               (let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
                 (try
                   ;; ARRANGE
                   (doall
                     (map (fn [waiter-url]
                            (->> (assoc basic-description
                                   "health-check-url" "/bad-status"
                                   "idle-timeout-mins" 2
                                   "run-as-user" "*"
                                   "version" "lorem-ipsum")
                                 (store-token waiter-url token-name nil)))
                          waiter-urls))

                   ;; ACT
                   (let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

                     ;; ASSERT
                     (let [expected-result {:details
                                            (pc/map-from-keys
                                              (fn [cluster-url]
                                                {:exit-code 1
                                                 :message (str "unable to ping token " token-name " on " cluster-url
                                                               ", reason: health check returned status code 503")})
                                              waiter-urls)
                                            :exit-code (count waiter-urls)
                                            :message (str "pinging token " token-name " on "
                                                          (-> waiter-urls vec println with-out-str str/trim)
                                                          " failed")
                                            :token token-name}]
                       (is (= expected-result actual-result))))
                   (finally
                     (cleanup-token waiter-api waiter-urls token-name))))))))