[TS-3549]: Configurable option to avoid thundering herd problem

for multiple concurrent requests. The initial POC patch for this
solution came from Justin Laue. This patch will further be improved
with a few more changes to possibly support SWR feature
in the core.

mgmt/RecordsConfig.cc

@@ -649,6 +649,13 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http.cache.max_open_write_retries", RECD_INT, "1", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
   ,
+  //       #  open_write_fail_action has 3 options:
+  //       #
+  //       #  0 - default. disable cache and goto origin
+  //       #  1 - return error if cache miss
+  //       #  2 - serve stale until proxy.config.http.cache.max_stale_age, then goto origin, if refresh_miss
+  {RECT_CONFIG, "proxy.config.http.cache.open_write_fail_action", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
+  ,
   //       #  when_to_revalidate has 4 options:
   //       #
   //       #  0 - default. use use cache directives or heuristic

proxy/hdrs/HdrToken.cc

@@ -78,7 +78,7 @@ static const char *_hdrtoken_strs[] = {
   "Summary", // NNTP
   "Transfer-Encoding", "Upgrade", "User-Agent", "Vary", "Via", "Warning", "Www-Authenticate",
   "Xref",      // NNTP
-  "@DataInfo", // Internal Hack
+  "@Ats-Internal", // Internal Hack
 
   // Accept-Encoding
   "compress", "deflate", "gzip", "identity",
@@ -227,7 +227,6 @@ static HdrTokenFieldInfo _hdrtoken_strs_field_initializers[] = {
   {"Warning", MIME_SLOTID_NONE, MIME_PRESENCE_WARNING, (HTIF_COMMAS | HTIF_MULTVALS)},
   {"Www-Authenticate", MIME_SLOTID_WWW_AUTHENTICATE, MIME_PRESENCE_WWW_AUTHENTICATE, HTIF_NONE},
   {"Xref", MIME_SLOTID_NONE, MIME_PRESENCE_XREF, HTIF_NONE},
-  {"@DataInfo", MIME_SLOTID_NONE, MIME_PRESENCE_INT_DATA_INFO, HTIF_NONE},
   {"X-ID", MIME_SLOTID_NONE, MIME_PRESENCE_NONE, (HTIF_COMMAS | HTIF_MULTVALS | HTIF_HOPBYHOP)},
   {"X-Forwarded-For", MIME_SLOTID_NONE, MIME_PRESENCE_NONE, (HTIF_COMMAS | HTIF_MULTVALS)},
   {"Sec-WebSocket-Key", MIME_SLOTID_NONE, MIME_PRESENCE_NONE, HTIF_NONE},
@@ -321,7 +320,7 @@ static const char *_hdrtoken_commonly_tokenized_strs[] = {
   "Summary", // NNTP
   "Transfer-Encoding", "Upgrade", "User-Agent", "Vary", "Via", "Warning", "Www-Authenticate",
   "Xref",      // NNTP
-  "@DataInfo", // Internal Hack
+  "@Ats-Internal", // Internal Hack
 
   // Accept-Encoding
   "compress", "deflate", "gzip", "identity",

proxy/hdrs/HdrToken.h

@@ -372,7 +372,6 @@ hdrtoken_wks_to_flags(const char *wks)
 #define MIME_PRESENCE_UNUSED_5 (TOK_64_CONST(1) << 60)
 
 #define MIME_PRESENCE_XREF (TOK_64_CONST(1) << 61)
-#define MIME_PRESENCE_INT_DATA_INFO (TOK_64_CONST(1) << 62)
 
 #define MIME_PRESENCE_NONE TOK_64_CONST(0)
 #define MIME_PRESENCE_ALL ~(TOK_64_CONST(0))

proxy/hdrs/MIME.cc

@@ -145,7 +145,7 @@ const char *MIME_FIELD_VIA;
 const char *MIME_FIELD_WARNING;
 const char *MIME_FIELD_WWW_AUTHENTICATE;
 const char *MIME_FIELD_XREF;
-const char *MIME_FIELD_INT_DATA_INFO;
+const char *MIME_FIELD_ATS_INTERNAL;
 const char *MIME_FIELD_X_ID;
 const char *MIME_FIELD_X_FORWARDED_FOR;
 const char *MIME_FIELD_SEC_WEBSOCKET_KEY;
@@ -260,7 +260,7 @@ int MIME_LEN_VIA;
 int MIME_LEN_WARNING;
 int MIME_LEN_WWW_AUTHENTICATE;
 int MIME_LEN_XREF;
-int MIME_LEN_INT_DATA_INFO;
+int MIME_LEN_ATS_INTERNAL;
 int MIME_LEN_X_ID;
 int MIME_LEN_X_FORWARDED_FOR;
 int MIME_LEN_SEC_WEBSOCKET_KEY;
@@ -338,7 +338,7 @@ int MIME_WKSIDX_VIA;
 int MIME_WKSIDX_WARNING;
 int MIME_WKSIDX_WWW_AUTHENTICATE;
 int MIME_WKSIDX_XREF;
-int MIME_WKSIDX_INT_DATA_INFO;
+int MIME_WKSIDX_ATS_INTERNAL;
 int MIME_WKSIDX_X_ID;
 int MIME_WKSIDX_X_FORWARDED_FOR;
 int MIME_WKSIDX_SEC_WEBSOCKET_KEY;
@@ -712,7 +712,7 @@ mime_init()
     MIME_FIELD_WARNING = hdrtoken_string_to_wks("Warning");
     MIME_FIELD_WWW_AUTHENTICATE = hdrtoken_string_to_wks("Www-Authenticate");
     MIME_FIELD_XREF = hdrtoken_string_to_wks("Xref");
-    MIME_FIELD_INT_DATA_INFO = hdrtoken_string_to_wks("@DataInfo");
+    MIME_FIELD_ATS_INTERNAL = hdrtoken_string_to_wks("@Ats-Internal");
     MIME_FIELD_X_ID = hdrtoken_string_to_wks("X-ID");
     MIME_FIELD_X_FORWARDED_FOR = hdrtoken_string_to_wks("X-Forwarded-For");
 
@@ -793,7 +793,7 @@ mime_init()
     MIME_LEN_WARNING = hdrtoken_wks_to_length(MIME_FIELD_WARNING);
     MIME_LEN_WWW_AUTHENTICATE = hdrtoken_wks_to_length(MIME_FIELD_WWW_AUTHENTICATE);
     MIME_LEN_XREF = hdrtoken_wks_to_length(MIME_FIELD_XREF);
-    MIME_LEN_INT_DATA_INFO = hdrtoken_wks_to_length(MIME_FIELD_INT_DATA_INFO);
+    MIME_LEN_ATS_INTERNAL = hdrtoken_wks_to_length(MIME_FIELD_ATS_INTERNAL);
     MIME_LEN_X_ID = hdrtoken_wks_to_length(MIME_FIELD_X_ID);
     MIME_LEN_X_FORWARDED_FOR = hdrtoken_wks_to_length(MIME_FIELD_X_FORWARDED_FOR);
 

proxy/hdrs/MIME.h

@@ -382,7 +382,7 @@ extern const char *MIME_FIELD_VIA;
 extern const char *MIME_FIELD_WARNING;
 extern const char *MIME_FIELD_WWW_AUTHENTICATE;
 extern const char *MIME_FIELD_XREF;
-extern const char *MIME_FIELD_INT_DATA_INFO;
+extern const char *MIME_FIELD_ATS_INTERNAL;
 extern const char *MIME_FIELD_X_ID;
 extern const char *MIME_FIELD_X_FORWARDED_FOR;
 extern const char *MIME_FIELD_SEC_WEBSOCKET_KEY;
@@ -485,7 +485,7 @@ extern int MIME_LEN_VIA;
 extern int MIME_LEN_WARNING;
 extern int MIME_LEN_WWW_AUTHENTICATE;
 extern int MIME_LEN_XREF;
-extern int MIME_LEN_INT_DATA_INFO;
+extern int MIME_LEN_ATS_INTERNAL;
 extern int MIME_LEN_X_ID;
 extern int MIME_LEN_X_FORWARDED_FOR;
 
@@ -588,7 +588,7 @@ extern int MIME_WKSIDX_VIA;
 extern int MIME_WKSIDX_WARNING;
 extern int MIME_WKSIDX_WWW_AUTHENTICATE;
 extern int MIME_WKSIDX_XREF;
-extern int MIME_WKSIDX_INT_DATA_INFO;
+extern int MIME_WKSIDX_ATS_INTERNAL;
 extern int MIME_WKSIDX_X_ID;
 extern int MIME_WKSIDX_SEC_WEBSOCKET_KEY;
 extern int MIME_WKSIDX_SEC_WEBSOCKET_VERSION;

proxy/http/HttpConfig.cc

@@ -1098,6 +1098,7 @@ HttpConfig::startup()
   HttpEstablishStaticConfigByte(c.send_408_post_timeout_response, "proxy.config.http.send_408_post_timeout_response");
   HttpEstablishStaticConfigByte(c.disallow_post_100_continue, "proxy.config.http.disallow_post_100_continue");
   HttpEstablishStaticConfigByte(c.parser_allow_non_http, "proxy.config.http.parse.allow_non_http");
+  HttpEstablishStaticConfigLongLong(c.cache_open_write_fail_action, "proxy.config.http.cache.open_write_fail_action");
 
   HttpEstablishStaticConfigByte(c.oride.cache_when_to_revalidate, "proxy.config.http.cache.when_to_revalidate");
   HttpEstablishStaticConfigByte(c.oride.cache_required_headers, "proxy.config.http.cache.required_headers");
@@ -1357,6 +1358,7 @@ HttpConfig::reconfigure()
   params->send_408_post_timeout_response = INT_TO_BOOL(m_master.send_408_post_timeout_response);
   params->disallow_post_100_continue = INT_TO_BOOL(m_master.disallow_post_100_continue);
   params->parser_allow_non_http = INT_TO_BOOL(m_master.parser_allow_non_http);
+  params->cache_open_write_fail_action = m_master.cache_open_write_fail_action;
 
   params->oride.cache_when_to_revalidate = m_master.oride.cache_when_to_revalidate;
 

proxy/http/HttpConfig.h

@@ -743,6 +743,7 @@ struct HttpConfigParams : public ConfigInfo {
   MgmtByte send_408_post_timeout_response;
   MgmtByte disallow_post_100_continue;
   MgmtByte parser_allow_non_http;
+  MgmtInt cache_open_write_fail_action;
 
   OverridableHttpConfigParams oride;
 
@@ -852,7 +853,8 @@ inline HttpConfigParams::HttpConfigParams()
     cluster_time_delta(0), redirection_enabled(0), redirection_host_no_port(0), number_of_redirections(1), post_copy_size(2048),
     ignore_accept_mismatch(0), ignore_accept_language_mismatch(0), ignore_accept_encoding_mismatch(0),
     ignore_accept_charset_mismatch(0), send_100_continue_response(0), send_408_post_timeout_response(0),
-    disallow_post_100_continue(0), parser_allow_non_http(1), autoconf_port(0), autoconf_localhost_only(0)
+    disallow_post_100_continue(0), parser_allow_non_http(1), cache_open_write_fail_action(0), autoconf_port(0),
+    autoconf_localhost_only(0)
 {
 }
 

proxy/http/HttpSM.cc

@@ -2364,9 +2364,20 @@ HttpSM::state_cache_open_write(int event, void *data)
   case CACHE_EVENT_OPEN_WRITE_FAILED:
     // Failed on the write lock and retrying the vector
     //  for reading
-    t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_FAIL;
-    break;
-
+    if (t_state.http_config_param->cache_open_write_fail_action ==
+        HttpTransact::CACHE_OPEN_WRITE_FAIL_DEFAULT) {
+      t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_FAIL;
+      break;
+    } else {
+      t_state.cache_open_write_fail_action = t_state.http_config_param->cache_open_write_fail_action;
+      if (!t_state.cache_info.object_read) {
+        // cache miss, set wl_state to fail
+        t_state.cache_info.write_lock_state = HttpTransact::CACHE_WL_FAIL;
+        break;
+      }
+    }
+    // INTENTIONAL FALL THROUGH
+    // Allow for stale object to be served
   case CACHE_EVENT_OPEN_READ:
     // The write vector was locked and the cache_sm retried
     // and got the read vector again.

proxy/http/HttpTransact.cc

@@ -2910,8 +2910,31 @@ HttpTransact::handle_cache_write_lock(State *s)
     // No write lock, ignore the cache and proxy only;
     // FIX: Should just serve from cache if this is a revalidate
     s->cache_info.action = CACHE_DO_NO_ACTION;
-    s->cache_info.write_status = CACHE_WRITE_LOCK_MISS;
-    remove_ims = true;
+    if (s->cache_open_write_fail_action & CACHE_OPEN_WRITE_FAIL_ERROR_ON_MISS) {
+      DebugTxn("http_error", "cache_open_write_fail_action, cache miss, return error");
+      s->cache_info.write_status = CACHE_WRITE_ERROR;
+      build_error_response(s, HTTP_STATUS_BAD_GATEWAY, "Connection Failed", "connect#failed_connect",
+                           NULL);
+      MIMEField *ats_field;
+      HTTPHdr* header = &(s->hdr_info.client_response);
+
+      if ((ats_field = header->field_find(MIME_FIELD_ATS_INTERNAL, MIME_LEN_ATS_INTERNAL)) == NULL) {
+        if (likely((ats_field = header->field_create(MIME_FIELD_ATS_INTERNAL, MIME_LEN_ATS_INTERNAL)) != NULL))
+          header->field_attach(ats_field);
+      }
+      if (likely(ats_field)) {
+        Debug("http_error", "Adding Ats-Internal-Messages: %d", CACHE_WL_FAIL);
+        header->field_value_set_int(ats_field, CACHE_WL_FAIL);
+      } else {
+        Debug("http_error", "failed to add Ats-Internal-Messages: %d", CACHE_WL_FAIL);
+      }
+
+      TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, NULL);
+      return;
+    } else {
+      s->cache_info.write_status = CACHE_WRITE_LOCK_MISS;
+      remove_ims = true;
+    }
     break;
   case CACHE_WL_READ_RETRY:
     //  Write failed but retried and got a vector to read
@@ -7232,6 +7255,13 @@ HttpTransact::what_is_document_freshness(State *s, HTTPHdr *client_request, HTTP
   uint32_t cc_mask, cooked_cc_mask;
   uint32_t os_specifies_revalidate;
 
+  if (s->cache_open_write_fail_action & CACHE_OPEN_WRITE_FAIL_STALE_OR_REVALIDATE) {
+    if (is_stale_cache_response_returnable(s)) {
+      DebugTxn("http_match", "[what_is_document_freshness] cache_serve_stale_on_write_lock_fail, return FRESH");
+      return (FRESHNESS_FRESH);
+    }
+  }
+
   //////////////////////////////////////////////////////
   // If config file has a ttl-in-cache field set,     //
   // it has priority over any other http headers and  //

proxy/http/HttpTransact.h

@@ -301,6 +301,13 @@ class HttpTransact
     TOTAL_CACHE_ACTION_TYPES
   };
 
+  enum CacheOpenWriteFailAction_t {
+    CACHE_OPEN_WRITE_FAIL_DEFAULT = 0,
+    CACHE_OPEN_WRITE_FAIL_ERROR_ON_MISS = 1,
+    CACHE_OPEN_WRITE_FAIL_STALE_OR_REVALIDATE = 2,
+    TOTAL_OPEN_WRITE_FAIL_ACTION_TYPES
+  };
+
   enum CacheWriteLock_t {
     CACHE_WL_INIT,
     CACHE_WL_SUCCESS,
@@ -796,6 +803,7 @@ class HttpTransact
     DNSLookupInfo dns_info;
     RedirectInfo redirect_info;
     unsigned int updated_server_version;
+    unsigned int cache_open_write_fail_action;
     bool is_revalidation_necessary; // Added to check if revalidation is necessary - YTS Team, yamsat
     bool request_will_not_selfloop; // To determine if process done - YTS Team, yamsat
     ConnectionAttributes client_info;
@@ -960,8 +968,8 @@ class HttpTransact
     // Constructor
     State()
       : m_magic(HTTP_TRANSACT_MAGIC_ALIVE), state_machine(NULL), http_config_param(NULL), force_dns(false),
-        updated_server_version(HostDBApplicationInfo::HTTP_VERSION_UNDEFINED), is_revalidation_necessary(false),
-        request_will_not_selfloop(false), // YTS Team, yamsat
+        updated_server_version(HostDBApplicationInfo::HTTP_VERSION_UNDEFINED), cache_open_write_fail_action(0),
+        is_revalidation_necessary(false), request_will_not_selfloop(false), // YTS Team, yamsat
         source(SOURCE_NONE), pre_transform_source(SOURCE_NONE), req_flavor(REQ_FLAVOR_FWDPROXY), pending_work(NULL),
         cdn_saved_next_action(SM_ACTION_UNDEFINED), cdn_saved_transact_return_point(NULL), cdn_remap_complete(false),
         first_dns_lookup(true), parent_params(NULL), cache_lookup_result(CACHE_LOOKUP_NONE), backdoor_request(false),