Permalink
Cannot retrieve contributors at this time
"""Download flags of countries (with error handling). | |
ThreadPool version | |
Sample run:: | |
$ python3 flags2_threadpool.py -s ERROR -e | |
ERROR site: http://localhost:8003/flags | |
Searching for 676 flags: from AA to ZZ | |
30 concurrent connections will be used. | |
-------------------- | |
150 flags downloaded. | |
361 not found. | |
165 errors. | |
Elapsed time: 7.46s | |
""" | |
# BEGIN FLAGS2_THREADPOOL | |
import collections | |
from concurrent import futures | |
import requests | |
import tqdm # <1> | |
from flags2_common import main, HTTPStatus # <2> | |
from flags2_sequential import download_one # <3> | |
DEFAULT_CONCUR_REQ = 30 # <4> | |
MAX_CONCUR_REQ = 1000 # <5> | |
def download_many(cc_list, base_url, verbose, concur_req): | |
counter = collections.Counter() | |
with futures.ThreadPoolExecutor(max_workers=concur_req) as executor: # <6> | |
to_do_map = {} # <7> | |
for cc in sorted(cc_list): # <8> | |
future = executor.submit(download_one, | |
cc, base_url, verbose) # <9> | |
to_do_map[future] = cc # <10> | |
done_iter = futures.as_completed(to_do_map) # <11> | |
if not verbose: | |
done_iter = tqdm.tqdm(done_iter, total=len(cc_list)) # <12> | |
for future in done_iter: # <13> | |
try: | |
res = future.result() # <14> | |
except requests.exceptions.HTTPError as exc: # <15> | |
error_msg = 'HTTP {res.status_code} - {res.reason}' | |
error_msg = error_msg.format(res=exc.response) | |
except requests.exceptions.ConnectionError as exc: | |
error_msg = 'Connection error' | |
else: | |
error_msg = '' | |
status = res.status | |
if error_msg: | |
status = HTTPStatus.error | |
counter[status] += 1 | |
if verbose and error_msg: | |
cc = to_do_map[future] # <16> | |
print('*** Error for {}: {}'.format(cc, error_msg)) | |
return counter | |
if __name__ == '__main__': | |
main(download_many, DEFAULT_CONCUR_REQ, MAX_CONCUR_REQ) | |
# END FLAGS2_THREADPOOL |