Skip to content
Permalink
Browse files

verifies support for temporarily unhealthy instances (#782)

  • Loading branch information...
shamsimam authored and DaoWen committed Jun 12, 2019
1 parent 81db4fc commit d3367d2d778a10acfb1a9bbeba34ad1b59c4845d
Showing with 76 additions and 3 deletions.
  1. +38 −2 containers/test-apps/kitchen/bin/kitchen
  2. +38 −1 waiter/integration/waiter/health_check_test.clj
@@ -56,6 +56,9 @@ _auth_handler = None
_async_state_lock = threading.Lock()
_async_state = {}

_allow_response_status_change = False
_default_response_status = 200

_counter_lock = threading.Lock()
_pending_http_requests = 0
_total_http_requests = 0
@@ -97,6 +100,16 @@ def terminate(source):
os._exit(1)


def reset_default_response_status(handler):
"""Clears the status default value to 200 for all kitchen responses."""
global _default_response_status
if _allow_response_status_change:
handler.logger().info('Resetting default response status from {} to 200'.format(_default_response_status))
_default_response_status = 200
else:
handler.logger().info('Ignroing request to reset default response status')


_gzip_window_bits = 16 + zlib.MAX_WBITS


@@ -502,7 +515,7 @@ class Kitchen(HTTPWebSocketsHandler):
self.__response_body_callback = None
self.__response_bytes = None
self.__response_length = max_response_size
self.__status = 200
self.__status = None
self.__response_trailers = {}
self.__trailer_delay_secs = 0
self.__truncated_length = max_response_size + 1
@@ -599,6 +612,7 @@ class Kitchen(HTTPWebSocketsHandler):
def __process_headers(self):
"""Handle logic for all supported Kitchen HTTP header values."""
assert self.__path is not None, 'Processes headers AFTER processing the request path.'
global _default_response_status

# Respond with chunked encoding when request is chunked
if self.headers.get('transfer-encoding') == 'chunked':
@@ -741,6 +755,25 @@ class Kitchen(HTTPWebSocketsHandler):
if str(name).startswith('x-kitchen-trailer-'):
self.__response_trailers[name[len('x-kitchen-trailer-'):]] = value

# Process default response status
default_status_value = self.headers.get('x-kitchen-default-status-value')
if default_status_value is not None:
if _allow_response_status_change:
self.logger().info('Changing default response status to {}'.format(default_status_value))
_default_response_status = int(default_status_value)
default_timeout = self.headers.get('x-kitchen-default-status-timeout')
if default_timeout is not None:
default_timeout_ms = int(default_timeout)
if default_timeout_ms > 0:
self.logger().info('Will reset default response status after {} ms'.format(default_timeout_ms))
run_after_ms(default_timeout_ms, reset_default_response_status, self)
else:
self.logger().info('Ignoring request to change the default response status')

# Set the response status if it is not set
if self.__status is None:
self.__status = _default_response_status

def __process_path(self):
"""Handle logic for all supported Kitchen endpoint paths."""
path = self.__path
@@ -972,8 +1005,10 @@ class BasicAuthHandler():


def main():
global kitchen_logger, _auth_handler, binary_max_size, text_max_size
global kitchen_logger, _allow_response_status_change, _auth_handler, binary_max_size, text_max_size
parser = argparse.ArgumentParser(description='A toy HTTP Service for testing the Waiter platform')
parser.add_argument('--enable-status-change', action='store_true', default=False,
help='Enables option to configure default response status using the x-kitchen-default-status-value header')
parser.add_argument('--hostname', metavar='HOSTNAME', default='', help='Server host name')
parser.add_argument('--log-output', metavar='LOG_OUTPUT', choices=['stdout', 'stderr', 'file'], default='stdout', help='Log output destination')
parser.add_argument('-p', '--port', metavar='PORT_NUMBER', type=int, default=8080, help='Server port number')
@@ -1016,6 +1051,7 @@ def main():
kitchen_logger.info('Sleeping for {}ms...'.format(args.start_up_sleep_ms))
time.sleep(args.start_up_sleep_ms / 1000.0)

_allow_response_status_change = args.enable_status_change
binary_max_size = args.ws_max_binary_message_size
text_max_size = args.ws_max_text_message_size

@@ -185,4 +185,41 @@
(is (= current-user permitted-user))
(is (= current-user run-as-user)))))
(finally
(delete-token-and-assert waiter-url token))))))
(delete-token-and-assert waiter-url token))))))

(deftest ^:parallel ^:integration-fast test-temporarily-unhealthy-instance
(testing-using-waiter-url
(let [{:keys [cookies instance-id request-headers service-id] :as canary-response}
(make-request-with-debug-info
{:x-waiter-cmd (kitchen-cmd "--enable-status-change -p $PORT0")
:x-waiter-concurrency-level 128
:x-waiter-health-check-interval-secs 5
:x-waiter-health-check-max-consecutive-failures 10
:x-waiter-name (rand-name)}
#(make-kitchen-request waiter-url % :path "/hello"))
check-filtered-instances (fn [target-url healthy-filter-fn]
(let [instance-ids (->> (active-instances target-url service-id :cookies cookies)
(healthy-filter-fn :healthy?)
(map :id))]
(and (= 1 (count instance-ids))
(= instance-id (first instance-ids)))))]
(assert-response-status canary-response 200)
(is service-id)
(with-service-cleanup
service-id
(doseq [[_ router-url] (routers waiter-url)]
(is (wait-for #(check-filtered-instances router-url filter)))
(is (= 1 (count (active-instances router-url service-id :cookies cookies)))))
(let [request-headers (assoc request-headers
:x-kitchen-default-status-timeout 20000
:x-kitchen-default-status-value 400)
response (make-kitchen-request waiter-url request-headers :path "/hello")]
(assert-response-status response 400))
(doseq [[_ router-url] (routers waiter-url)]
(is (wait-for #(check-filtered-instances router-url remove)))
(is (= 1 (count (active-instances router-url service-id :cookies cookies)))))
(let [response (make-kitchen-request waiter-url request-headers :path "/hello")]
(assert-response-status response 200))
(doseq [[_ router-url] (routers waiter-url)]
(is (wait-for #(check-filtered-instances router-url filter)))
(is (= 1 (count (active-instances router-url service-id :cookies cookies)))))))))

0 comments on commit d3367d2

Please sign in to comment.
You can’t perform that action at this time.