diff --git a/r2/r2/config/middleware.py b/r2/r2/config/middleware.py index 4dec5e9a50..f0c0d8d118 100644 --- a/r2/r2/config/middleware.py +++ b/r2/r2/config/middleware.py @@ -42,6 +42,17 @@ import sys, tempfile, urllib, re, os, sha, subprocess from httplib import HTTPConnection +# hack in Paste support for HTTP 429 "Too Many Requests" +from paste import httpexceptions, wsgiwrappers + +class HTTPTooManyRequests(httpexceptions.HTTPClientError): + code = 429 + title = 'Too Many Requests' + explanation = ('The server has received too many requests from the client.') + +httpexceptions._exceptions[429] = HTTPTooManyRequests +wsgiwrappers.STATUS_CODE_TEXT[429] = HTTPTooManyRequests.title + #from pylons.middleware import error_mapper def error_mapper(code, message, environ, global_conf=None, **kw): from pylons import c @@ -52,7 +63,7 @@ def error_mapper(code, message, environ, global_conf=None, **kw): if global_conf is None: global_conf = {} - codes = [304, 401, 403, 404, 503] + codes = [304, 401, 403, 404, 429, 503] if not asbool(global_conf.get('debug')): codes.append(500) if code in codes: diff --git a/r2/r2/controllers/error.py b/r2/r2/controllers/error.py index 75c131f5f1..3a66416724 100644 --- a/r2/r2/controllers/error.py +++ b/r2/r2/controllers/error.py @@ -72,13 +72,32 @@ ''' toofast = \ -''' - service temporarily unavailable +''' + + + Too Many Requests + + - the service you request is temporarily unavailable. please try again later. +

whoa there, pardner!

+

reddit's awesome and all, but you may have a bit of a + problem. we've seen far too many requests come from your ip address + recently.

+

if you think that we've incorrectly blocked you or you would like + to discuss easier ways to get the data you want, please contact us + any of the following ways.

+ +

as a reminder, we recommend that clients make no more than one + request every two seconds to avoid being blocked like this.

-''' +''' class ErrorController(RedditController): """Generates error documents as and when they are required. @@ -146,6 +165,10 @@ def send404(self): else: return pages.Reddit404().render() + def send429(self): + c.response.status_code = 429 + return toofast + def send503(self): c.response.status_code = 503 if 'retry_after' in request.environ: @@ -186,6 +209,8 @@ def GET_document(self): return pages.TakedownPage(link).render() elif code == 403: return self.send403() + elif code == 429: + return self.send429() elif code == 500: randmin = {'admin': rand.choice(self.admins)} failien_name = 'youbrokeit%d.png' % rand.randint(1, NUM_FAILIENS) diff --git a/r2/r2/controllers/reddit_base.py b/r2/r2/controllers/reddit_base.py index 7ab2e52931..e939e6d59b 100644 --- a/r2/r2/controllers/reddit_base.py +++ b/r2/r2/controllers/reddit_base.py @@ -419,7 +419,7 @@ def ratelimit_agents(): if s and user_agent and s in user_agent.lower(): key = 'rate_agent_' + s if g.cache.get(s): - abort(503, 'service temporarily unavailable') + abort(429) else: g.cache.set(s, 't', time = 1) @@ -430,7 +430,7 @@ def ratelimit_throttled(): ip, slash16 = ip_and_slash16(request) if throttled(ip) or throttled(slash16): - abort(503, 'service temporarily unavailable') + abort(429) def paginated_listing(default_page_size=25, max_page_size=100, backend='sql'): @@ -616,7 +616,7 @@ def post(self): and request.method.upper() == 'GET' and (not c.user_is_loggedin or c.allow_loggedin_cache) and not c.used_cache - and response.status_code != 503 + and response.status_code not in (429, 503) and response.content and response.content[0]): try: g.rendercache.set(self.request_key(),