verifies support for temporarily unhealthy instances (#782)

containers/test-apps/kitchen/bin/kitchen

@@ -56,6 +56,9 @@ _auth_handler = None
 _async_state_lock = threading.Lock()
 _async_state = {}
 
+_allow_response_status_change = False
+_default_response_status = 200
+
 _counter_lock = threading.Lock()
 _pending_http_requests = 0
 _total_http_requests = 0
@@ -97,6 +100,16 @@ def terminate(source):
     os._exit(1)
 
 
+def reset_default_response_status(handler):
+    """Clears the status default value to 200 for all kitchen responses."""
+    global _default_response_status
+    if _allow_response_status_change:
+        handler.logger().info('Resetting default response status from {} to 200'.format(_default_response_status))
+        _default_response_status = 200
+    else:
+        handler.logger().info('Ignroing request to reset default response status')
+
+
 _gzip_window_bits = 16 + zlib.MAX_WBITS
 
 
@@ -502,7 +515,7 @@ class Kitchen(HTTPWebSocketsHandler):
             self.__response_body_callback = None
             self.__response_bytes = None
             self.__response_length = max_response_size
-            self.__status = 200
+            self.__status = None
             self.__response_trailers = {}
             self.__trailer_delay_secs = 0
             self.__truncated_length = max_response_size + 1
@@ -599,6 +612,7 @@ class Kitchen(HTTPWebSocketsHandler):
     def __process_headers(self):
         """Handle logic for all supported Kitchen HTTP header values."""
         assert self.__path is not None, 'Processes headers AFTER processing the request path.'
+        global _default_response_status
 
         # Respond with chunked encoding when request is chunked
         if self.headers.get('transfer-encoding') == 'chunked':
@@ -741,6 +755,25 @@ class Kitchen(HTTPWebSocketsHandler):
             if str(name).startswith('x-kitchen-trailer-'):
                 self.__response_trailers[name[len('x-kitchen-trailer-'):]] = value
 
+        # Process default response status
+        default_status_value = self.headers.get('x-kitchen-default-status-value')
+        if default_status_value is not None:
+            if _allow_response_status_change:
+                self.logger().info('Changing default response status to {}'.format(default_status_value))
+                _default_response_status = int(default_status_value)
+                default_timeout = self.headers.get('x-kitchen-default-status-timeout')
+                if default_timeout is not None:
+                    default_timeout_ms = int(default_timeout)
+                    if default_timeout_ms > 0:
+                        self.logger().info('Will reset default response status after {} ms'.format(default_timeout_ms))
+                        run_after_ms(default_timeout_ms, reset_default_response_status, self)
+            else:
+                self.logger().info('Ignoring request to change the default response status')
+
+        # Set the response status if it is not set
+        if self.__status is None:
+            self.__status = _default_response_status
+
     def __process_path(self):
         """Handle logic for all supported Kitchen endpoint paths."""
         path = self.__path
@@ -972,8 +1005,10 @@ class BasicAuthHandler():
 
 
 def main():
-    global kitchen_logger, _auth_handler, binary_max_size, text_max_size
+    global kitchen_logger, _allow_response_status_change, _auth_handler, binary_max_size, text_max_size
     parser = argparse.ArgumentParser(description='A toy HTTP Service for testing the Waiter platform')
+    parser.add_argument('--enable-status-change', action='store_true', default=False,
+            help='Enables option to configure default response status using the x-kitchen-default-status-value header')
     parser.add_argument('--hostname', metavar='HOSTNAME', default='', help='Server host name')
     parser.add_argument('--log-output', metavar='LOG_OUTPUT', choices=['stdout', 'stderr', 'file'], default='stdout', help='Log output destination')
     parser.add_argument('-p', '--port', metavar='PORT_NUMBER', type=int, default=8080, help='Server port number')
@@ -1016,6 +1051,7 @@ def main():
         kitchen_logger.info('Sleeping for {}ms...'.format(args.start_up_sleep_ms))
         time.sleep(args.start_up_sleep_ms / 1000.0)
 
+    _allow_response_status_change = args.enable_status_change
     binary_max_size = args.ws_max_binary_message_size
     text_max_size = args.ws_max_text_message_size
 

waiter/integration/waiter/health_check_test.clj

@@ -185,4 +185,41 @@
               (is (= current-user permitted-user))
               (is (= current-user run-as-user)))))
         (finally
-          (delete-token-and-assert waiter-url token))))))
+          (delete-token-and-assert waiter-url token))))))
+
+(deftest ^:parallel ^:integration-fast test-temporarily-unhealthy-instance
+  (testing-using-waiter-url
+    (let [{:keys [cookies instance-id request-headers service-id] :as canary-response}
+          (make-request-with-debug-info
+            {:x-waiter-cmd (kitchen-cmd "--enable-status-change -p $PORT0")
+             :x-waiter-concurrency-level 128
+             :x-waiter-health-check-interval-secs 5
+             :x-waiter-health-check-max-consecutive-failures 10
+             :x-waiter-name (rand-name)}
+            #(make-kitchen-request waiter-url % :path "/hello"))
+          check-filtered-instances (fn [target-url healthy-filter-fn]
+                                     (let [instance-ids (->> (active-instances target-url service-id :cookies cookies)
+                                                          (healthy-filter-fn :healthy?)
+                                                          (map :id))]
+                                       (and (= 1 (count instance-ids))
+                                            (= instance-id (first instance-ids)))))]
+      (assert-response-status canary-response 200)
+      (is service-id)
+      (with-service-cleanup
+        service-id
+        (doseq [[_ router-url] (routers waiter-url)]
+          (is (wait-for #(check-filtered-instances router-url filter)))
+          (is (= 1 (count (active-instances router-url service-id :cookies cookies)))))
+        (let [request-headers (assoc request-headers
+                                :x-kitchen-default-status-timeout 20000
+                                :x-kitchen-default-status-value 400)
+              response (make-kitchen-request waiter-url request-headers :path "/hello")]
+          (assert-response-status response 400))
+        (doseq [[_ router-url] (routers waiter-url)]
+          (is (wait-for #(check-filtered-instances router-url remove)))
+          (is (= 1 (count (active-instances router-url service-id :cookies cookies)))))
+        (let [response (make-kitchen-request waiter-url request-headers :path "/hello")]
+          (assert-response-status response 200))
+        (doseq [[_ router-url] (routers waiter-url)]
+          (is (wait-for #(check-filtered-instances router-url filter)))
+          (is (= 1 (count (active-instances router-url service-id :cookies cookies)))))))))