-
Notifications
You must be signed in to change notification settings - Fork 0
/
multithreaded_io_task.py
61 lines (48 loc) · 1.68 KB
/
multithreaded_io_task.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
# imports
# if not installed use pip install requests or conda install requests
import requests as r
# needed for multithreading
from queue import Queue
from threading import Thread
# define number of threads
NUM_THREADS = 5
q = Queue()
# define the function responsible for downloading the files
def download_pics():
global q
while True:
# get current url
url = q.get()
# make the reqeuest
# By default, when you make a request, the body of the response is downloaded immediately.
# You can override this behaviour and defer downloading the response body until you access the Response.content attribute
# For this we set stream=True so that requests doesn't download the whole image into memory first.
res = r.get(url, stream = True)
# extract the filename of the url
filename = f"{url.split('/')[-1]}"
# check if we made a successful http request
if res.status_code == 200:
# write the pic with name filename to the current directory
with open(filename,"wb") as f:
# download the image bytewise and write each block (bytesize = 128 by default) to current directory
# if different bytesize is desireable, use res.iter_content(bytesize) instead of res
for block in res:
f.write(block)
else:
# if not successful we break
# print('For file %s the download was not successfull' %filename)
break
q.task_done()
# excute if main
if __name__ == "__main__":
urls = [
"https://logos-world.net/wp-content/uploads/2021/10/Python-Symbol.png"
]
# put the 100 urls one by one into a queue object
for url in urls*100:
q.put(url)
for t in range(NUM_THREADS):
worker = Thread(target=download_pics)
worker.daemon = True
worker.start()
q.join()