Skip to content
This repository was archived by the owner on Apr 10, 2025. It is now read-only.

Commit e59644f

Browse files
jeffkaufmancrowell
authored andcommitted
ApacheFetch waits for asynchronous processing inside PSOL. Currently that wait has no timeout, but as described in #1048 PSOL sometimes fails to return. In Nginx and other event-loop based servers this causes a problem for that single request but doesn't take down the server, but in Apache we currently tie up a thread indefinitely waiting for the fetch to finish. This change makes Apache act more like the other servers: after a 2min timeout it gives up waiting on PSOL and frees up the thread.
This doesn't solve the underlying PSOL issue(s), but it makes it much less damaging to Apache servers.
1 parent 9a37778 commit e59644f

File tree

5 files changed

+227
-74
lines changed

5 files changed

+227
-74
lines changed

net/instaweb/apache/instaweb_handler.cc

Lines changed: 167 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
#include "net/instaweb/util/public/statistics.h"
5858
#include "net/instaweb/util/public/string_writer.h"
5959
#include "net/instaweb/util/public/timer.h"
60+
#include "pagespeed/kernel/base/basictypes.h"
6061
#include "pagespeed/kernel/base/ref_counted_ptr.h"
6162
#include "pagespeed/kernel/http/http_options.h"
6263
#include "pagespeed/kernel/http/response_headers.h"
@@ -93,24 +94,30 @@ const char kResourceUrlYes[] = "<YES>";
9394
// instance).
9495
const size_t kMaxPostSizeBytes = 131072;
9596

97+
// How long to try waiting before giving up.
98+
const int kMaxWaitMs = 2 * Timer::kMinuteMs;
99+
96100
} // namespace
97101

98102
ApacheFetch::ApacheFetch(const GoogleString& mapped_url,
103+
StringPiece debug_info,
99104
ServerContext* server_context,
100105
request_rec* request,
101106
const RequestContextPtr& request_context,
102107
const RewriteOptions* options)
103-
: AsyncFetchUsingWriter(request_context, &apache_writer_),
108+
: AsyncFetch(request_context),
104109
mapped_url_(mapped_url),
105110
apache_writer_(request),
111+
options_(options),
106112
server_context_(server_context),
107113
mutex_(server_context->thread_system()->NewMutex()),
108114
condvar_(mutex_->NewCondvar()),
115+
abandoned_(false),
109116
done_(false),
117+
headers_sent_(false),
110118
handle_error_(true),
111119
status_ok_(false),
112120
is_proxy_(false),
113-
options_(options),
114121
blocking_fetch_timeout_ms_(options_->blocking_fetch_timeout_ms()) {
115122
// We are proxying content, and the caching in the http configuration
116123
// should not apply; we want to use the caching from the proxy.
@@ -119,6 +126,7 @@ ApacheFetch::ApacheFetch(const GoogleString& mapped_url,
119126
ApacheRequestToRequestHeaders(*request, request_headers());
120127
request_headers()->RemoveAll(HttpAttributes::kCookie);
121128
request_headers()->RemoveAll(HttpAttributes::kCookie2);
129+
debug_info.CopyToString(&debug_info_);
122130
}
123131

124132
ApacheFetch::~ApacheFetch() {
@@ -158,9 +166,6 @@ void ApacheFetch::HandleHeadersComplete() {
158166
}
159167

160168
// TODO(sligocki): Add X-Mod-Pagespeed header.
161-
if (content_length_known() && !inject_error_message) {
162-
apache_writer_.set_content_length(content_length());
163-
}
164169

165170
// Default cache-control to nocache.
166171
if (!response_headers()->Has(HttpAttributes::kCacheControl)) {
@@ -169,49 +174,139 @@ void ApacheFetch::HandleHeadersComplete() {
169174
}
170175
response_headers()->ComputeCaching();
171176

172-
apache_writer_.OutputHeaders(response_headers());
177+
{
178+
ScopedMutex lock(mutex_.get());
179+
if (!abandoned_) {
180+
if (content_length_known() && !inject_error_message) {
181+
apache_writer_.set_content_length(content_length());
182+
}
183+
184+
apache_writer_.OutputHeaders(response_headers());
185+
headers_sent_ = true;
173186

174-
if (inject_error_message) {
175-
apache_writer_.Write("Missing Content-Type required for proxied "
176-
"resource", server_context_->message_handler());
177-
apache_writer_.set_squelch_output(true);
187+
if (inject_error_message) {
188+
apache_writer_.Write("Missing Content-Type required for proxied "
189+
"resource", server_context_->message_handler());
190+
apache_writer_.set_squelch_output(true);
191+
}
192+
return;
193+
}
178194
}
195+
server_context_->message_handler()->Message(
196+
kWarning,
197+
"HeadersComplete for url %s received after "
198+
"being abandoned for timing out.",
199+
mapped_url_.c_str());
200+
return; // Don't do anything.
179201
}
180202
}
181203

182204
void ApacheFetch::HandleDone(bool success) {
183-
ScopedMutex lock(mutex_.get());
184-
done_ = true;
185-
if (status_ok_ && !success) {
205+
{
206+
ScopedMutex lock(mutex_.get());
207+
done_ = true;
208+
209+
if (!abandoned_) {
210+
if (status_ok_ && !success) {
211+
server_context_->message_handler()->Message(
212+
kWarning,
213+
"Response for url %s issued with status %d %s but "
214+
"failed to complete.",
215+
mapped_url_.c_str(), response_headers()->status_code(),
216+
response_headers()->reason_phrase());
217+
}
218+
// We've not been abandoned; let our owner know we're done and they will
219+
// delete us.
220+
condvar_->Signal();
221+
return;
222+
}
186223
server_context_->message_handler()->Message(
187224
kWarning,
188-
"Response for url %s issued with status %d %s but "
189-
"failed to complete.",
225+
"Response for url %s completed with status %d %s after "
226+
"being abandoned for timing out.",
190227
mapped_url_.c_str(), response_headers()->status_code(),
191228
response_headers()->reason_phrase());
192229
}
193-
condvar_->Signal();
230+
// We've been abandoned, so we have ownership of ourself.
231+
delete this;
232+
}
233+
234+
bool ApacheFetch::HandleWrite(const StringPiece& sp,
235+
MessageHandler* handler) {
236+
{
237+
ScopedMutex lock(mutex_.get());
238+
if (!abandoned_) {
239+
return apache_writer_.Write(sp, handler);
240+
}
241+
}
242+
handler->Message(kWarning,
243+
"Write of %ld bytes for url %s received after "
244+
"being abandoned for timing out.",
245+
sp.size(), mapped_url_.c_str());
246+
return false; // Drop the write.
194247
}
195248

196-
void ApacheFetch::Wait() {
249+
bool ApacheFetch::HandleFlush(MessageHandler* handler) {
250+
{
251+
ScopedMutex lock(mutex_.get());
252+
if (!abandoned_) {
253+
return apache_writer_.Flush(handler);
254+
}
255+
}
256+
handler->Message(kWarning,
257+
"Flush for url %s received after "
258+
"being abandoned for timing out.",
259+
mapped_url_.c_str());
260+
return false; // Drop the flush.
261+
}
262+
263+
ApacheFetch::WaitResult ApacheFetch::Wait(const RewriteDriver& rewrite_driver) {
197264
MessageHandler* handler = server_context_->message_handler();
198265
Timer* timer = server_context_->timer();
199266
int64 start_ms = timer->NowMs();
200-
{
201-
ScopedMutex lock(mutex_.get());
202-
while (!done_) {
203-
condvar_->TimedWait(blocking_fetch_timeout_ms_);
204-
if (!done_) {
205-
int64 elapsed_ms = timer->NowMs() - start_ms;
267+
WaitResult result = kWaitSuccess;
268+
mutex_->Lock();
269+
while (!done_ && (result == kWaitSuccess)) {
270+
condvar_->TimedWait(blocking_fetch_timeout_ms_);
271+
if (!done_) {
272+
int64 elapsed_ms = timer->NowMs() - start_ms;
273+
if (elapsed_ms > kMaxWaitMs) {
274+
abandoned_ = true;
275+
// Now that we're abandoned the instaweb context needs to drop its
276+
// pointer to us and not free us. We tell it to do this by returning
277+
// kAbandonedAndHandled / kAbandonedAndDeclined.
278+
//
279+
// If we never sent headers out then it's safe to DECLINE this request
280+
// and pass it to a different content handler. Otherwise some data was
281+
// sent out and we have to accept this as it is.
282+
result = headers_sent_ ? kAbandonedAndHandled : kAbandonedAndDeclined;
283+
}
284+
285+
// Unlock briefly so we can write messages without holding the lock.
286+
mutex_->Unlock();
287+
if (result != kWaitSuccess) {
288+
handler->Message(
289+
kWarning, "Abandoned URL %s after %g sec (debug=%s, driver=%s) ",
290+
mapped_url_.c_str(), elapsed_ms / 1000.0, debug_info_.c_str(),
291+
rewrite_driver.ToString(true /* show_detached_contexts */).c_str());
292+
} else {
206293
handler->Message(
207294
kWarning, "Waiting for completion of URL %s for %g sec",
208295
mapped_url_.c_str(), elapsed_ms / 1000.0);
209296
}
297+
mutex_->Lock();
210298
}
211299
}
300+
mutex_->Unlock();
301+
return result;
212302
}
213303

214304
bool ApacheFetch::IsCachedResultValid(const ResponseHeaders& headers) {
305+
ScopedMutex lock(mutex_.get());
306+
// options_ isn't valid after we're abandoned, so make sure we haven't been.
307+
if (abandoned_) {
308+
return false;
309+
}
215310
return OptionsAwareHTTPCacheCallback::IsCacheValid(
216311
mapped_url_, *options_, request_context(), headers);
217312
}
@@ -221,7 +316,8 @@ InstawebHandler::InstawebHandler(request_rec* request)
221316
server_context_(InstawebContext::ServerContextFromServerRec(
222317
request->server)),
223318
rewrite_driver_(NULL),
224-
num_response_attributes_(0) {
319+
num_response_attributes_(0),
320+
fetch_(NULL) {
225321
apache_request_context_ = server_context_->NewApacheRequestContext(request);
226322
request_context_.reset(apache_request_context_);
227323

@@ -244,13 +340,31 @@ InstawebHandler::InstawebHandler(request_rec* request)
244340
}
245341

246342
InstawebHandler::~InstawebHandler() {
247-
WaitForFetch();
343+
// If fetch_ is null we either never tried to fetch anything or it took
344+
// ownership of itself after timing out.
345+
if (fetch_ != NULL) {
346+
if (WaitForFetch() == ApacheFetch::kWaitSuccess) {
347+
delete fetch_; // Fetch completed normally.
348+
} else {
349+
// Fetch took ownership of itself and will continue in the background.
350+
CHECK(fetch_ == NULL);
351+
}
352+
}
248353
}
249354

250-
void InstawebHandler::WaitForFetch() {
251-
if (fetch_.get() != NULL) {
252-
fetch_->Wait();
355+
ApacheFetch::WaitResult InstawebHandler::WaitForFetch() {
356+
if (fetch_ == NULL) {
357+
return ApacheFetch::kWaitSuccess; // Nothing to wait for.
358+
}
359+
360+
ApacheFetch::WaitResult wait_result = fetch_->Wait(*rewrite_driver_);
361+
if (wait_result != ApacheFetch::kWaitSuccess) {
362+
// Give up on waiting for the fetch so we stop tying up a thread. Fetch has
363+
// taken ownership of itself, will discard any messages it receives if it's
364+
// abandoned, and will delete itself when Done() is called, if ever.
365+
fetch_ = NULL;
253366
}
367+
return wait_result;
254368
}
255369

256370
void InstawebHandler::SetupSpdyConnectionIfNeeded() {
@@ -279,11 +393,12 @@ RewriteDriver* InstawebHandler::MakeDriver() {
279393
return rewrite_driver_;
280394
}
281395

282-
ApacheFetch* InstawebHandler::MakeFetch(const GoogleString& url) {
283-
DCHECK(fetch_.get() == NULL);
284-
fetch_.reset(new ApacheFetch(url, server_context_, request_, request_context_,
285-
options_));
286-
return fetch_.get();
396+
ApacheFetch* InstawebHandler::MakeFetch(const GoogleString& url,
397+
StringPiece debug_info) {
398+
DCHECK(fetch_ == NULL);
399+
fetch_ = new ApacheFetch(url, debug_info, server_context_, request_,
400+
request_context_, options_);
401+
return fetch_;
287402
}
288403

289404
/* static */
@@ -535,12 +650,14 @@ bool InstawebHandler::HandleAsInPlace() {
535650
!= NULL) || (request_->user != NULL));
536651

537652
RewriteDriver* driver = MakeDriver();
538-
MakeFetch(original_url_);
653+
MakeFetch(original_url_, "ipro");
539654
fetch_->set_handle_error(false);
540-
driver->FetchInPlaceResource(stripped_gurl_, false /* proxy_mode */,
541-
fetch_.get());
542-
WaitForFetch();
543-
if (fetch_->status_ok()) {
655+
driver->FetchInPlaceResource(stripped_gurl_, false /* proxy_mode */, fetch_);
656+
ApacheFetch::WaitResult wait_result = WaitForFetch();
657+
if (wait_result != ApacheFetch::kWaitSuccess) {
658+
// Note: fetch_ has been released; no longer safe to look at;
659+
handled = (wait_result == ApacheFetch::kAbandonedAndHandled);
660+
} else if (fetch_->status_ok()) {
544661
server_context_->rewrite_stats()->ipro_served()->Add(1);
545662
handled = true;
546663
} else if ((fetch_->response_headers()->status_code() ==
@@ -596,16 +713,15 @@ bool InstawebHandler::HandleAsProxy() {
596713
&host_header, &is_proxy) &&
597714
is_proxy) {
598715
RewriteDriver* driver = MakeDriver();
599-
MakeFetch(mapped_url);
716+
MakeFetch(mapped_url, "proxy");
600717
fetch_->set_is_proxy(true);
601718
driver->SetRequestHeaders(*fetch_->request_headers());
602719
server_context_->proxy_fetch_factory()->StartNewProxyFetch(
603-
mapped_url, fetch_.get(), driver, NULL, NULL);
604-
WaitForFetch();
605-
handled = true;
720+
mapped_url, fetch_, driver, NULL, NULL);
721+
handled = WaitForFetch() != ApacheFetch::kAbandonedAndDeclined;
606722
}
607723

608-
return handled;
724+
return handled; // false == declined
609725
}
610726

611727
// Determines whether the url can be handled as a mod_pagespeed or in-place
@@ -960,7 +1076,7 @@ apr_status_t InstawebHandler::instaweb_handler(request_rec* request) {
9601076
server_context->StatisticsPage(is_global_statistics,
9611077
instaweb_handler.query_params(),
9621078
instaweb_handler.options(),
963-
instaweb_handler.MakeFetch());
1079+
instaweb_handler.MakeFetch("statistics"));
9641080
return OK;
9651081
} else if ((request_handler_str == kAdminHandler) ||
9661082
(request_handler_str == kGlobalAdminHandler)) {
@@ -969,7 +1085,7 @@ apr_status_t InstawebHandler::instaweb_handler(request_rec* request) {
9691085
instaweb_handler.stripped_gurl(),
9701086
instaweb_handler.query_params(),
9711087
instaweb_handler.options(),
972-
instaweb_handler.MakeFetch());
1088+
instaweb_handler.MakeFetch("admin"));
9731089
ret = OK;
9741090
} else if (global_config->enable_cache_purge() &&
9751091
!global_config->purge_method().empty() &&
@@ -978,20 +1094,21 @@ apr_status_t InstawebHandler::instaweb_handler(request_rec* request) {
9781094
AdminSite* admin_site = server_context->admin_site();
9791095
admin_site->PurgeHandler(instaweb_handler.original_url_,
9801096
server_context->cache_path(),
981-
instaweb_handler.MakeFetch());
1097+
instaweb_handler.MakeFetch("purge"));
9821098
ret = OK;
9831099
} else if (request_handler_str == kConsoleHandler) {
9841100
InstawebHandler instaweb_handler(request);
9851101
server_context->ConsoleHandler(*instaweb_handler.options(),
9861102
AdminSite::kOther,
9871103
instaweb_handler.query_params(),
988-
instaweb_handler.MakeFetch());
1104+
instaweb_handler.MakeFetch("console"));
9891105
ret = OK;
9901106
} else if (request_handler_str == kMessageHandler) {
9911107
InstawebHandler instaweb_handler(request);
992-
server_context->MessageHistoryHandler(*instaweb_handler.options(),
993-
AdminSite::kOther,
994-
instaweb_handler.MakeFetch());
1108+
server_context->MessageHistoryHandler(
1109+
*instaweb_handler.options(),
1110+
AdminSite::kOther,
1111+
instaweb_handler.MakeFetch("messages"));
9951112
ret = OK;
9961113
} else if (request_handler_str == kLogRequestHeadersHandler) {
9971114
// For testing CustomFetchHeader.

0 commit comments

Comments
 (0)