Skip to content
This repository has been archived by the owner on Mar 22, 2023. It is now read-only.

Commit

Permalink
enables synchronization when roots mismatch but updated by same… (#989)
Browse files Browse the repository at this point in the history
  • Loading branch information
@shamsimam @dposada
shamsimam authored and dposada committed Nov 6, 2019
1 parent bdb579a commit 8ae970b
Show file tree
Hide file tree
Showing 5 changed files with 270 additions and 106 deletions.
9 changes: 6 additions & 3 deletions token-syncer/bin/test.sh
View file
Expand Up @@ -38,11 +38,14 @@ export WAITER_URIS=${WAITER_URIS:-http://127.0.0.1:9091,http://127.0.0.1:9092}
# Wait for waiter to be listening
for WAITER_URI in ${WAITER_URIS//,/ }
do
timeout 180s bash -c "wait_for_server ${WAITER_URI}"
if [ $? -ne 0 ]; then
echo "$(date +%H:%M:%S) timed out waiting for waiter to start listening on ${WAITER_URI}, displaying waiter log"
timeout 240s bash -c "wait_for_server ${WAITER_URI}"
if [[ $? -ne 0 ]]; then
echo "$(date +%H:%M:%S) ERROR timed out waiting for waiter to start listening on ${WAITER_URI}"
echo "$(date +%H:%M:%S) displaying waiter log:"
cat ${WAITER_DIR}/log/*waiter.log
exit 1
else
echo "$(date +%H:%M:%S) waiter server listening on ${WAITER_URI} started"
fi
done

Expand Down
232 changes: 154 additions & 78 deletions token-syncer/integration/token_syncer/basic_test.clj
View file
Expand Up @@ -447,12 +447,12 @@
(finally
(cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-roots
(deftest ^:integration test-token-different-roots-and-last-update-user
(testing "token sync update with different roots"
(let [waiter-urls (waiter-urls)
{:keys [load-token store-token] :as waiter-api} (waiter-api)
limit 10
token-name (str "test-token-different-roots-" (UUID/randomUUID))]
token-name (str "test-token-diff-root-diff-user" (UUID/randomUUID))]
(try
;; ARRANGE
(let [current-time-ms (System/currentTimeMillis)
Expand All @@ -466,7 +466,7 @@
"cluster" (waiter-url->cluster waiter-url)
"cpus" (inc index)
"last-update-time" (- last-update-time-ms index)
"last-update-user" "auth-user"
"last-update-user" (str "auth-user-" index)
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
Expand All @@ -483,7 +483,7 @@
"cluster" (waiter-url->cluster (first waiter-urls))
"cpus" 1
"last-update-time" last-update-time-ms
"last-update-user" "auth-user"
"last-update-user" "auth-user-0"
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
Expand All @@ -497,7 +497,7 @@
"cluster" (waiter-url->cluster waiter-url)
"cpus" (+ index 2)
"last-update-time" (- last-update-time-ms index 1)
"last-update-user" "auth-user"
"last-update-user" (str "auth-user-" (inc index))
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
Expand Down Expand Up @@ -526,7 +526,7 @@
"cluster" (waiter-url->cluster waiter-url)
"cpus" (inc index)
"last-update-time" token-last-modified-time
"last-update-user" "auth-user"
"last-update-user" (str "auth-user-" index)
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
Expand All @@ -540,6 +540,82 @@
(finally
(cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-roots-but-same-last-update-user
(testing "token sync update with different roots"
(let [waiter-urls (waiter-urls)
{:keys [load-token store-token] :as waiter-api} (waiter-api)
limit 10
token-name (str "diff-root-same-user" (UUID/randomUUID))]
(try
;; ARRANGE
(let [current-time-ms (System/currentTimeMillis)
last-update-time-ms (- current-time-ms 10000)]

(doall
(map-indexed
(fn [index waiter-url]
(store-token waiter-url token-name nil
(assoc basic-description
"cluster" (waiter-url->cluster waiter-url)
"cpus" (inc index)
"last-update-time" (- last-update-time-ms index)
"last-update-user" "auth-user"
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
"root" waiter-url)))
waiter-urls))

(let [token-etag (token->etag waiter-api (first waiter-urls) token-name)]

;; ACT
(let [actual-result (syncer/sync-tokens waiter-api waiter-urls limit)]

;; ASSERT
(let [latest-description (assoc basic-description
"cluster" (waiter-url->cluster (first waiter-urls))
"cpus" 1
"last-update-time" last-update-time-ms
"last-update-user" "auth-user"
"owner" "test-user"
"previous" {"last-update-time" (- current-time-ms 30000)
"last-update-user" "foo-user"}
"root" (first waiter-urls))
sync-result (->> (rest waiter-urls)
(map
(fn [waiter-url]
[waiter-url
{:code :success/sync-update
:details {:etag token-etag
:status 200}}]))
(into {}))
expected-result {:details {token-name {:latest {:cluster-url (first waiter-urls)
:description latest-description
:token-etag token-etag}
:sync-result sync-result}}
:summary {:sync {:failed #{}
:unmodified #{}
:updated #{token-name}}
:tokens {:pending {:count 1 :value #{token-name}}
:previously-synced {:count 0 :value #{}}
:processed {:count 1 :value #{token-name}}
:selected {:count 1 :value #{token-name}}
:total {:count 1 :value #{token-name}}}}}]
(is (= expected-result actual-result))
(doall
(map
(fn [waiter-url]
(let [token-etag (token->etag waiter-api waiter-url token-name)]
(is (= {:description latest-description
:headers {"content-type" "application/json"
"etag" token-etag}
:status 200
:token-etag token-etag}
(load-token waiter-url token-name)))))
waiter-urls))))))
(finally
(cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-token-different-roots-and-deleted
(testing "token sync hard-delete deleted tokens with different different roots"
(let [waiter-urls (waiter-urls)
Expand Down Expand Up @@ -692,75 +768,75 @@
(cleanup-token waiter-api waiter-urls token-name))))))

(deftest ^:integration test-ping-tokens
(testing "token ping on clusters"
(let [waiter-urls (waiter-urls)
queue-timeout-ms 120000
{:keys [store-token] :as waiter-api} (waiter-api)]

(testing "successful health check"
(let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
(try
;; ARRANGE
(doall
(map (fn [waiter-url]
(->> (assoc basic-description
"health-check-url" "/status"
"idle-timeout-mins" 2
"run-as-user" "*"
"version" "lorem-ipsum")
(store-token waiter-url token-name nil)))
waiter-urls))

;; ACT
(let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

;; ASSERT
(let [expected-result {:details
(pc/map-from-keys
(fn [cluster-url]
{:exit-code 0
:message (str "successfully pinged token " token-name " on " cluster-url
", reason: health check returned status code 200")})
waiter-urls)
:exit-code 0
:message (str "pinging token " token-name " on "
(-> waiter-urls vec println with-out-str str/trim)
" was successful")
:token token-name}]
(is (= expected-result actual-result))))
(finally
(cleanup-token waiter-api waiter-urls token-name)))))

(testing "unsuccessful health check"
(let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
(try
;; ARRANGE
(doall
(map (fn [waiter-url]
(->> (assoc basic-description
"health-check-url" "/bad-status"
"idle-timeout-mins" 2
"run-as-user" "*"
"version" "lorem-ipsum")
(store-token waiter-url token-name nil)))
waiter-urls))

;; ACT
(let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

;; ASSERT
(let [expected-result {:details
(pc/map-from-keys
(fn [cluster-url]
{:exit-code 1
:message (str "unable to ping token " token-name " on " cluster-url
", reason: health check returned status code 503")})
waiter-urls)
:exit-code (count waiter-urls)
:message (str "pinging token " token-name " on "
(-> waiter-urls vec println with-out-str str/trim)
" failed")
:token token-name}]
(is (= expected-result actual-result))))
(finally
(cleanup-token waiter-api waiter-urls token-name))))))))
(testing "token ping on clusters"
(let [waiter-urls (waiter-urls)
queue-timeout-ms 120000
{:keys [store-token] :as waiter-api} (waiter-api)]

(testing "successful health check"
(let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
(try
;; ARRANGE
(doall
(map (fn [waiter-url]
(->> (assoc basic-description
"health-check-url" "/status"
"idle-timeout-mins" 2
"run-as-user" "*"
"version" "lorem-ipsum")
(store-token waiter-url token-name nil)))
waiter-urls))

;; ACT
(let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

;; ASSERT
(let [expected-result {:details
(pc/map-from-keys
(fn [cluster-url]
{:exit-code 0
:message (str "successfully pinged token " token-name " on " cluster-url
", reason: health check returned status code 200")})
waiter-urls)
:exit-code 0
:message (str "pinging token " token-name " on "
(-> waiter-urls vec println with-out-str str/trim)
" was successful")
:token token-name}]
(is (= expected-result actual-result))))
(finally
(cleanup-token waiter-api waiter-urls token-name)))))

(testing "unsuccessful health check"
(let [token-name (str "test-ping-tokens-" (UUID/randomUUID))]
(try
;; ARRANGE
(doall
(map (fn [waiter-url]
(->> (assoc basic-description
"health-check-url" "/bad-status"
"idle-timeout-mins" 2
"run-as-user" "*"
"version" "lorem-ipsum")
(store-token waiter-url token-name nil)))
waiter-urls))

;; ACT
(let [actual-result (ping/ping-token waiter-api waiter-urls token-name queue-timeout-ms)]

;; ASSERT
(let [expected-result {:details
(pc/map-from-keys
(fn [cluster-url]
{:exit-code 1
:message (str "unable to ping token " token-name " on " cluster-url
", reason: health check returned status code 503")})
waiter-urls)
:exit-code (count waiter-urls)
:message (str "pinging token " token-name " on "
(-> waiter-urls vec println with-out-str str/trim)
" failed")
:token token-name}]
(is (= expected-result actual-result))))
(finally
(cleanup-token waiter-api waiter-urls token-name))))))))
7 changes: 4 additions & 3 deletions token-syncer/src/token_syncer/commands/syncer.clj
View file
Expand Up @@ -83,8 +83,8 @@
cluster-result
(try
(let [{:keys [description error status] :as token-data} (get cluster-url->token-data cluster-url)
latest-root (get latest-token-description "root")
cluster-root (get description "root")]
{latest-root "root" latest-update-user "last-update-user"} latest-token-description
{cluster-root "root" cluster-update-user "last-update-user"} description]
(cond
error
{:code :error/token-read
Expand Down Expand Up @@ -117,9 +117,10 @@
(apply dissoc description system-metadata-keys)))
{:code :skip/token-sync}

;; token user-specified content different, and roots also different
;; token user-specified content, last update user, and root different
(and (seq description)
(not= latest-root cluster-root)
(not= latest-update-user cluster-update-user)
(not= (apply dissoc description ignored-root-mismatch-equality-comparison-keys)
(apply dissoc latest-token-description ignored-root-mismatch-equality-comparison-keys)))
{:code :error/root-mismatch
Expand Down
8 changes: 5 additions & 3 deletions token-syncer/test/token_syncer/commands/backup_test.clj
View file
Expand Up @@ -129,11 +129,13 @@
(is (= {:exit-code 1
:message "test-command: expected 2 arguments FILE URL, provided 3: [\"some-file.txt\" \"http://cluster-1.com\" \"http://cluster-2.com\"]"}
(cli/process-command test-command-config context args))))
(let [args ["some-file.txt" "http://cluster-1.com"]]
(let [args ["some-file.txt" "http://cluster-1.com"]
invocation-promise (promise)]
(with-redefs [backup-tokens (fn [in-waiter-api in-file-operations-api cluster-url file accrete]
(is (= waiter-api in-waiter-api))
(is (= file-operations-api in-file-operations-api))
(println "backup-tokens:" cluster-url file accrete))]
(deliver invocation-promise ::invoked))]
(is (= {:exit-code 0
:message "test-command: exiting with code 0"}
(cli/process-command test-command-config context args))))))))
(cli/process-command test-command-config context args)))
(is (= ::invoked (deref invocation-promise 0 ::un-initialized))))))))

0 comments on commit 8ae970b

Please sign in to comment.