Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

adding error handling to send method #15

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 15 additions & 7 deletions grequests.py
Expand Up @@ -46,15 +46,23 @@ def wrapped(*args, **kwargs):
return wrapped


def send(r, pool=None, prefetch=False):
def send(r, pool=None, prefetch=False, exception_handler=None):
"""Sends the request object using the specified pool. If a pool isn't
specified this method blocks. Pools are useful because you can specify size
and can hence limit concurrency."""

if pool != None:
return pool.spawn(r.send, prefetch=prefetch)
glet = pool.spawn(r.send, prefetch=prefetch)

return gevent.spawn(r.send, prefetch=prefetch)
glet = gevent.spawn(r.send, prefetch=prefetch)

if exception_handler:
def eh_wrapper(g):
return exception_handler(r,g.exception)

glet.link_exception(eh_wrapper)

return glet


# Patched requests.api functions.
Expand All @@ -68,7 +76,7 @@ def send(r, pool=None, prefetch=False):
request = patched(api.request)


def map(requests, prefetch=True, size=None):
def map(requests, prefetch=True, size=None, exception_handler=None):
"""Concurrently converts a list of Requests to Responses.

:param requests: a collection of Request objects.
Expand All @@ -79,13 +87,13 @@ def map(requests, prefetch=True, size=None):
requests = list(requests)

pool = Pool(size) if size else None
jobs = [send(r, pool, prefetch=prefetch) for r in requests]
jobs = [send(r, pool, prefetch=prefetch, exception_handler=exception_handler) for r in requests]
gevent.joinall(jobs)

return [r.response for r in requests]


def imap(requests, prefetch=True, size=2):
def imap(requests, prefetch=True, size=2, exception_handler=None):
"""Concurrently converts a generator object of Requests to
a generator of Responses.

Expand All @@ -97,7 +105,7 @@ def imap(requests, prefetch=True, size=2):
pool = Pool(size)

def send(r):
r.send(prefetch)
r.send(prefetch,exception_handler=exception_handler)
return r.response

for r in pool.imap_unordered(send, requests):
Expand Down