/
threading-s3-uploads.py
75 lines (61 loc) · 2.03 KB
/
threading-s3-uploads.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
from concurrent.futures import ThreadPoolExecutor
from functools import partial
from time import perf_counter
from typing import List
import boto3
from botocore.client import BaseClient
LOCAL_PATH: str = (
"/Users/eliasbenaddouidrissi/Desktop/dev/repos/python-s3-threading-examples/"
)
BUCKET_NAME: str = "s3-threading-examples"
def upload_object(s3_client: BaseClient, file_name: str) -> str:
"""
Uploads a file to an S3 bucket.
Args:
s3_client (BaseClient): An instance of the boto3 S3 client.
file_name (str): The name of the file to upload.
Returns:
str: A message indicating that the file has been uploaded.
"""
print(f"Uploading {file_name}")
upload_filename = f"{LOCAL_PATH}/{file_name}"
s3_client.upload_file(
Filename=upload_filename, Bucket=BUCKET_NAME, Key=f"{file_name}"
)
return f"{file_name} uploaded"
def upload_multithreading(s3_client: BaseClient, keys_lst: List[str]) -> None:
"""
Uploads multiple files to an S3 bucket using multithreading.
Args:
s3_client (BaseClient): An instance of the boto3 S3 client.
keys_lst (List[str]): List of file names to upload.
"""
with ThreadPoolExecutor(max_workers=8) as executor:
func = partial(upload_object, s3_client)
results = executor.map(func, keys_lst)
for result in results:
print(result)
if __name__ == "__main__":
session = boto3.Session(profile_name="default")
s3_client = session.client("s3")
t1_start = perf_counter()
keys_lst = [
"example1.txt",
"example2.txt",
"example3.txt",
"example4.txt",
"example5.txt",
"example6.txt",
"example7.txt",
"example8.txt",
"example9.txt",
"example10.txt",
"example11.txt",
"example12.txt",
"example14.txt",
"example14.txt",
"example15.txt",
]
upload_multithreading(s3_client, keys_lst)
t1_stop = perf_counter()
print("Elapsed time in seconds:", t1_stop - t1_start)