Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 439 lines (384 sloc) 16.84 kb
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
1 #!/usr/bin/env python
2 # Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/
3 #
4 # Permission is hereby granted, free of charge, to any person obtaining a
5 # copy of this software and associated documentation files (the
6 # "Software"), to deal in the Software without restriction, including
7 # without limitation the rights to use, copy, modify, merge, publish, dis-
8 # tribute, sublicense, and/or sell copies of the Software, and to permit
9 # persons to whom the Software is furnished to do so, subject to the fol-
10 # lowing conditions:
11 #
12 # The above copyright notice and this permission notice shall be included
13 # in all copies or substantial portions of the Software.
14 #
15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 # IN THE SOFTWARE.
22 #
c06d56f @glance- pep8 cleanup
glance- authored
23 import getopt
24 import sys
25 import os
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
26 import boto
c06d56f @glance- pep8 cleanup
glance- authored
27
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
28 from boto.compat import six
29
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
30 try:
31 # multipart portions copyright Fabian Topfstedt
32 # https://gist.github.com/924094
33
34 import math
35 import mimetypes
36 from multiprocessing import Pool
37 from boto.s3.connection import S3Connection
38 from filechunkio import FileChunkIO
39 multipart_capable = True
40 usage_flag_multipart_capable = """ [--multipart]"""
41 usage_string_multipart_capable = """
f35e914 @danielgtaylor Make s3put non-multipart uploads not require ListBucket access, and docu...
danielgtaylor authored
42 multipart - Upload files as multiple parts. This needs filechunkio.
43 Requires ListBucket, ListMultipartUploadParts,
44 ListBucketMultipartUploads and PutObject permissions."""
9a50a7e @glance- s3put: message when multipart upload not available
glance- authored
45 except ImportError as err:
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
46 multipart_capable = False
47 usage_flag_multipart_capable = ""
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
48 if six.PY2:
49 attribute = 'message'
50 else:
51 attribute = 'msg'
9a50a7e @glance- s3put: message when multipart upload not available
glance- authored
52 usage_string_multipart_capable = '\n\n "' + \
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
53 getattr(err, attribute)[len('No module named '):] + \
9a50a7e @glance- s3put: message when multipart upload not available
glance- authored
54 '" is missing for multipart support '
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
55
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
56
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
57 DEFAULT_REGION = 'us-east-1'
58
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
59 usage_string = """
60 SYNOPSIS
7b54fd4 Fixes issue 280. Incorporating patch from punteney to add a --no-overwr...
Mitch.Garnaat authored
61 s3put [-a/--access_key <access_key>] [-s/--secret_key <secret_key>]
62 -b/--bucket <bucket_name> [-c/--callback <num_cb>]
63 [-d/--debug <debug_level>] [-i/--ignore <ignore_dirs>]
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
64 [-n/--no_op] [-p/--prefix <prefix>] [-k/--key_prefix <key_prefix>]
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
65 [-q/--quiet] [-g/--grant grant] [-w/--no_overwrite] [-r/--reduced]
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
66 [--header] [--region <name>] [--host <s3_host>]""" + \
67 usage_flag_multipart_capable + """ path [path...]
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
68
69 Where
70 access_key - Your AWS Access Key ID. If not supplied, boto will
71 use the value of the environment variable
72 AWS_ACCESS_KEY_ID
73 secret_key - Your AWS Secret Access Key. If not supplied, boto
74 will use the value of the environment variable
75 AWS_SECRET_ACCESS_KEY
76 bucket_name - The name of the S3 bucket the file(s) should be
77 copied to.
78 path - A path to a directory or file that represents the items
79 to be uploaded. If the path points to an individual file,
80 that file will be uploaded to the specified bucket. If the
72b4d1a @pasc Fixed up some typos in s3put's help message
pasc authored
81 path points to a directory, it will recursively traverse
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
82 the directory and upload all files to the specified bucket.
83 debug_level - 0 means no debug output (default), 1 means normal
84 debug output from boto, and 2 means boto debug output
85 plus request/response output from httplib
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
86 ignore_dirs - a comma-separated list of directory names that will
87 be ignored and not uploaded to S3.
88 num_cb - The number of progress callbacks to display. The default
89 is zero which means no callbacks. If you supplied a value
90 of "-c 10" for example, the progress callback would be
91 called 10 times for each file transferred.
92 prefix - A file path prefix that will be stripped from the full
93 path of the file when determining the key name in S3.
94 For example, if the full path of a file is:
95 /home/foo/bar/fie.baz
96 and the prefix is specified as "-p /home/foo/" the
97 resulting key name in S3 will be:
98 /bar/fie.baz
99 The prefix must end in a trailing separator and if it
100 does not then one will be added.
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
101 key_prefix - A prefix to be added to the S3 key name, after any
102 stripping of the file path is done based on the
103 "-p/--prefix" option.
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
104 reduced - Use Reduced Redundancy storage
3b9f77c Added a -g (grant) option that allows you to pass in a canned ACL policy...
Mitch.Garnaat authored
105 grant - A canned ACL policy that will be granted on each file
106 transferred to S3. The value of provided must be one
107 of the "canned" ACL policies supported by S3:
108 private|public-read|public-read-write|authenticated-read
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
109 no_overwrite - No files will be overwritten on S3, if the file/key
110 exists on s3 it will be kept. This is useful for
111 resuming interrupted transfers. Note this is not a
112 sync, even if the file has been updated locally if
113 the key exists on s3 the file on s3 will not be
7b54fd4 Fixes issue 280. Incorporating patch from punteney to add a --no-overwr...
Mitch.Garnaat authored
114 updated.
72b4d1a @pasc Fixed up some typos in s3put's help message
pasc authored
115 header - key=value pairs of extra header(s) to pass along in the
de1894e @kopertop Added --host option to s3put
kopertop authored
116 request
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
117 region - Manually set a region for buckets that are not in the US
118 classic region. Normally the region is autodetected, but
119 setting this yourself is more efficient.
de1894e @kopertop Added --host option to s3put
kopertop authored
120 host - Hostname override, for using an endpoint other then AWS S3
121 """ + usage_string_multipart_capable + """
7645c5e @methane Add -r/--reduced option to s3put.
methane authored
122
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
123
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
124 If the -n option is provided, no files will be transferred to S3 but
125 informational messages will be printed about what would happen.
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
126 """
c06d56f @glance- pep8 cleanup
glance- authored
127
128
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
129 def usage(status=1):
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
130 print(usage_string)
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
131 sys.exit(status)
08e6393 s3put: expand prefix path; remove prefix from file path only if matching...
Louis Pery authored
132
c06d56f @glance- pep8 cleanup
glance- authored
133
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
134 def submit_cb(bytes_so_far, total_bytes):
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
135 print('%d bytes transferred / %d bytes total' % (bytes_so_far, total_bytes))
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
136
c06d56f @glance- pep8 cleanup
glance- authored
137
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
138 def get_key_name(fullpath, prefix, key_prefix):
08e6393 s3put: expand prefix path; remove prefix from file path only if matching...
Louis Pery authored
139 if fullpath.startswith(prefix):
140 key_name = fullpath[len(prefix):]
141 else:
142 key_name = fullpath
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
143 l = key_name.split(os.sep)
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
144 return key_prefix + '/'.join(l)
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
145
c06d56f @glance- pep8 cleanup
glance- authored
146
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
147 def _upload_part(bucketname, aws_key, aws_secret, multipart_id, part_num,
148 source_path, offset, bytes, debug, cb, num_cb,
149 amount_of_retries=10):
150 """
151 Uploads a part with retries.
152 """
153 if debug == 1:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
154 print("_upload_part(%s, %s, %s)" % (source_path, offset, bytes))
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
155
156 def _upload(retries_left=amount_of_retries):
157 try:
158 if debug == 1:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
159 print('Start uploading part #%d ...' % part_num)
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
160 conn = S3Connection(aws_key, aws_secret)
161 conn.debug = debug
162 bucket = conn.get_bucket(bucketname)
163 for mp in bucket.get_all_multipart_uploads():
164 if mp.id == multipart_id:
165 with FileChunkIO(source_path, 'r', offset=offset,
166 bytes=bytes) as fp:
167 mp.upload_part_from_file(fp=fp, part_num=part_num,
168 cb=cb, num_cb=num_cb)
169 break
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
170 except Exception as exc:
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
171 if retries_left:
172 _upload(retries_left=retries_left - 1)
173 else:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
174 print('Failed uploading part #%d' % part_num)
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
175 raise exc
176 else:
177 if debug == 1:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
178 print('... Uploaded part #%d' % part_num)
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
179
180 _upload()
181
d2ac8da @ralfharing check for invalid regions
ralfharing authored
182 def check_valid_region(conn, region):
183 if conn is None:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
184 print('Invalid region (%s)' % region)
d2ac8da @ralfharing check for invalid regions
ralfharing authored
185 sys.exit(1)
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
186
187 def multipart_upload(bucketname, aws_key, aws_secret, source_path, keyname,
188 reduced, debug, cb, num_cb, acl='private', headers={},
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
189 guess_mimetype=True, parallel_processes=4,
190 region=DEFAULT_REGION):
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
191 """
192 Parallel multipart upload.
193 """
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
194 conn = boto.s3.connect_to_region(region, aws_access_key_id=aws_key,
195 aws_secret_access_key=aws_secret)
d2ac8da @ralfharing check for invalid regions
ralfharing authored
196 check_valid_region(conn, region)
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
197 conn.debug = debug
198 bucket = conn.get_bucket(bucketname)
199
200 if guess_mimetype:
201 mtype = mimetypes.guess_type(keyname)[0] or 'application/octet-stream'
202 headers.update({'Content-Type': mtype})
203
204 mp = bucket.initiate_multipart_upload(keyname, headers=headers,
205 reduced_redundancy=reduced)
206
207 source_size = os.stat(source_path).st_size
208 bytes_per_chunk = max(int(math.sqrt(5242880) * math.sqrt(source_size)),
209 5242880)
210 chunk_amount = int(math.ceil(source_size / float(bytes_per_chunk)))
211
212 pool = Pool(processes=parallel_processes)
213 for i in range(chunk_amount):
214 offset = i * bytes_per_chunk
215 remaining_bytes = source_size - offset
216 bytes = min([bytes_per_chunk, remaining_bytes])
217 part_num = i + 1
218 pool.apply_async(_upload_part, [bucketname, aws_key, aws_secret, mp.id,
219 part_num, source_path, offset, bytes,
220 debug, cb, num_cb])
221 pool.close()
222 pool.join()
223
224 if len(mp.get_all_parts()) == chunk_amount:
225 mp.complete_upload()
226 key = bucket.get_key(keyname)
227 key.set_acl(acl)
228 else:
229 mp.cancel_upload()
230
231
232 def singlepart_upload(bucket, key_name, fullpath, *kargs, **kwargs):
233 """
234 Single upload.
235 """
236 k = bucket.new_key(key_name)
237 k.set_contents_from_filename(fullpath, *kargs, **kwargs)
238
239
08e6393 s3put: expand prefix path; remove prefix from file path only if matching...
Louis Pery authored
240 def expand_path(path):
241 path = os.path.expanduser(path)
242 path = os.path.expandvars(path)
243 return os.path.abspath(path)
244
c06d56f @glance- pep8 cleanup
glance- authored
245
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
246 def main():
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
247
248 # default values
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
249 aws_access_key_id = None
250 aws_secret_access_key = None
251 bucket_name = ''
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
252 ignore_dirs = []
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
253 debug = 0
254 cb = None
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
255 num_cb = 0
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
256 quiet = False
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
257 no_op = False
258 prefix = '/'
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
259 key_prefix = ''
3b9f77c Added a -g (grant) option that allows you to pass in a canned ACL policy...
Mitch.Garnaat authored
260 grant = None
7b54fd4 Fixes issue 280. Incorporating patch from punteney to add a --no-overwr...
Mitch.Garnaat authored
261 no_overwrite = False
7645c5e @methane Add -r/--reduced option to s3put.
methane authored
262 reduced = False
10a4a36 @kopertop Added headers option to s3put, and ignore all dot files in directory
kopertop authored
263 headers = {}
de1894e @kopertop Added --host option to s3put
kopertop authored
264 host = None
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
265 multipart_requested = False
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
266 region = None
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
267
268 try:
269 opts, args = getopt.getopt(
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
270 sys.argv[1:], 'a:b:c::d:g:hi:k:np:qs:wr',
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
271 ['access_key=', 'bucket=', 'callback=', 'debug=', 'help', 'grant=',
c06d56f @glance- pep8 cleanup
glance- authored
272 'ignore=', 'key_prefix=', 'no_op', 'prefix=', 'quiet',
4a48371 @garnaat Resolving merge conflict.
garnaat authored
273 'secret_key=', 'no_overwrite', 'reduced', 'header=', 'multipart',
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
274 'host=', 'region='])
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
275 except:
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
276 usage(1)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
277
278 # parse opts
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
279 for o, a in opts:
280 if o in ('-h', '--help'):
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
281 usage(0)
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
282 if o in ('-a', '--access_key'):
283 aws_access_key_id = a
284 if o in ('-b', '--bucket'):
285 bucket_name = a
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
286 if o in ('-c', '--callback'):
287 num_cb = int(a)
288 cb = submit_cb
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
289 if o in ('-d', '--debug'):
290 debug = int(a)
3b9f77c Added a -g (grant) option that allows you to pass in a canned ACL policy...
Mitch.Garnaat authored
291 if o in ('-g', '--grant'):
292 grant = a
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
293 if o in ('-i', '--ignore'):
294 ignore_dirs = a.split(',')
295 if o in ('-n', '--no_op'):
296 no_op = True
7645c5e @methane Add -r/--reduced option to s3put.
methane authored
297 if o in ('-w', '--no_overwrite'):
7b54fd4 Fixes issue 280. Incorporating patch from punteney to add a --no-overwr...
Mitch.Garnaat authored
298 no_overwrite = True
c9f46e7 Added ability to specify directories to ignore. Also added the ability ...
Mitch.Garnaat authored
299 if o in ('-p', '--prefix'):
300 prefix = a
301 if prefix[-1] != os.sep:
302 prefix = prefix + os.sep
08e6393 s3put: expand prefix path; remove prefix from file path only if matching...
Louis Pery authored
303 prefix = expand_path(prefix)
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
304 if o in ('-k', '--key_prefix'):
305 key_prefix = a
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
306 if o in ('-q', '--quiet'):
307 quiet = True
308 if o in ('-s', '--secret_key'):
309 aws_secret_access_key = a
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
310 if o in ('-r', '--reduced'):
311 reduced = True
b58ee24 @danielgtaylor Properly handle getopt long options in s3put
danielgtaylor authored
312 if o == '--header':
eee7d14 @krallin s3put: accept headers values containing "="
krallin authored
313 (k, v) = a.split("=", 1)
10a4a36 @kopertop Added headers option to s3put, and ignore all dot files in directory
kopertop authored
314 headers[k] = v
b58ee24 @danielgtaylor Properly handle getopt long options in s3put
danielgtaylor authored
315 if o == '--host':
de1894e @kopertop Added --host option to s3put
kopertop authored
316 host = a
b58ee24 @danielgtaylor Properly handle getopt long options in s3put
danielgtaylor authored
317 if o == '--multipart':
5c0c353 @glance- s3put s3multiput: Merge multiput functionality into s3put
glance- authored
318 if multipart_capable:
319 multipart_requested = True
320 else:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
321 print("multipart upload requested but not capable")
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
322 sys.exit(4)
b58ee24 @danielgtaylor Properly handle getopt long options in s3put
danielgtaylor authored
323 if o == '--region':
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
324 regions = boto.s3.regions()
325 for region_info in regions:
326 if region_info.name == a:
327 region = a
328 break
329 else:
eba1d45 @danielgtaylor Fix s3put to be compatible with Python 2.5.x
danielgtaylor authored
330 raise ValueError('Invalid region %s specified' % a)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
331
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
332 if len(args) < 1:
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
333 usage(2)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
334
335 if not bucket_name:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
336 print("bucket name is required!")
3fe70ca @jerm added non-zero exit statuses for unsuccessful runs
jerm authored
337 usage(3)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
338
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
339 connect_args = {
340 'aws_access_key_id': aws_access_key_id,
341 'aws_secret_access_key': aws_secret_access_key
342 }
343
de1894e @kopertop Added --host option to s3put
kopertop authored
344 if host:
397f2f8 @danielgtaylor Add a --region argument to s3put that allows you to manually specify a r...
danielgtaylor authored
345 connect_args['host'] = host
346
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
347 c = boto.s3.connect_to_region(region or DEFAULT_REGION, **connect_args)
d2ac8da @ralfharing check for invalid regions
ralfharing authored
348 check_valid_region(c, region or DEFAULT_REGION)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
349 c.debug = debug
f35e914 @danielgtaylor Make s3put non-multipart uploads not require ListBucket access, and docu...
danielgtaylor authored
350 b = c.get_bucket(bucket_name, validate=False)
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
351
352 # Attempt to determine location and warn if no --host or --region
353 # arguments were passed. Then try to automagically figure out
354 # what should have been passed and fix it.
355 if host is None and region is None:
356 try:
357 location = b.get_location()
358
359 # Classic region will be '', any other will have a name
360 if location:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
361 print('Bucket exists in %s but no host or region given!' % location)
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
362
363 # Override for EU, which is really Ireland according to the docs
364 if location == 'EU':
365 location = 'eu-west-1'
366
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
367 print('Automatically setting region to %s' % location)
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
368
369 # Here we create a new connection, and then take the existing
370 # bucket and set it to use the new connection
371 c = boto.s3.connect_to_region(location, **connect_args)
372 c.debug = debug
373 b.connection = c
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
374 except Exception as e:
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
375 if debug > 0:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
376 print(e)
377 print('Could not get bucket region info, skipping...')
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
378
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
379 existing_keys_to_check_against = []
380 files_to_check_for_upload = []
381
382 for path in args:
383 path = expand_path(path)
384 # upload a directory of files recursively
385 if os.path.isdir(path):
386 if no_overwrite:
387 if not quiet:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
388 print('Getting list of existing keys to check against')
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
389 for key in b.list(get_key_name(path, prefix, key_prefix)):
390 existing_keys_to_check_against.append(key.name)
391 for root, dirs, files in os.walk(path):
392 for ignore in ignore_dirs:
393 if ignore in dirs:
394 dirs.remove(ignore)
395 for path in files:
396 if path.startswith("."):
397 continue
398 files_to_check_for_upload.append(os.path.join(root, path))
399
400 # upload a single file
401 elif os.path.isfile(path):
402 fullpath = os.path.abspath(path)
403 key_name = get_key_name(fullpath, prefix, key_prefix)
404 files_to_check_for_upload.append(fullpath)
405 existing_keys_to_check_against.append(key_name)
406
407 # we are trying to upload something unknown
408 else:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
409 print("I don't know what %s is, so i can't upload it" % path)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
410
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
411 for fullpath in files_to_check_for_upload:
933128c @glance- s3put: add --key_prefix to prefix keys in s3
glance- authored
412 key_name = get_key_name(fullpath, prefix, key_prefix)
6eadfe1 @glance- s3put s3multiput: Cleanup and resync of the code.
glance- authored
413
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
414 if no_overwrite and key_name in existing_keys_to_check_against:
0865004 @jerm -w flag never actually checked s3 for a file's existence. Always refused...
jerm authored
415 if b.get_key(key_name):
416 if not quiet:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
417 print('Skipping %s as it exists in s3' % fullpath)
0865004 @jerm -w flag never actually checked s3 for a file's existence. Always refused...
jerm authored
418 continue
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
419
420 if not quiet:
bc126af @danielgtaylor Fix bin scripts fo Python 3.
danielgtaylor authored
421 print('Copying %s to %s/%s' % (fullpath, bucket_name, key_name))
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
422
423 if not no_op:
424 # 0-byte files don't work and also don't need multipart upload
425 if os.stat(fullpath).st_size != 0 and multipart_capable and \
426 multipart_requested:
427 multipart_upload(bucket_name, aws_access_key_id,
428 aws_secret_access_key, fullpath, key_name,
429 reduced, debug, cb, num_cb,
75d5c7b @danielgtaylor Attempt to auto-detect bucket region information and if no manual --host...
danielgtaylor authored
430 grant or 'private', headers,
431 region=region or DEFAULT_REGION)
d28364e @glance- s3put: refactor main and add muliple path support
glance- authored
432 else:
433 singlepart_upload(b, key_name, fullpath, cb=cb, num_cb=num_cb,
434 policy=grant, reduced_redundancy=reduced,
435 headers=headers)
f846d7e A simple command line utility to put files or entire directory structure...
Mitch.Garnaat authored
436
437 if __name__ == "__main__":
438 main()
Something went wrong with that request. Please try again.