Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Headers setting support when files match a regex #19

Closed
wants to merge 6 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,17 @@ Usage
[-t THREADS] [-u USERNAME] [-p PASSWORD]
[-i {rackspace,keystone}] [-a AUTH_URL] [-v]
{delete,upload,download} ...

Gevent-based, multithreaded tool for interacting with OpenStack Swift and
Rackspace Cloud Files

positional arguments:
{delete,upload,download}
delete Delete files from specified container
upload Upload files to specified container
download Download files to specified directory from the
specified container

optional arguments:
-h, --help show this help message and exit
--version show program's version number and exit
Expand Down Expand Up @@ -72,6 +72,9 @@ Usage
-v, --verbose Enable verbosity. Supply multiple times for additional
verbosity. 1) Show Thread Start/Finish, 2) Show Object
Name.
-H, --headers <file_name_regex>,<header_name>:<header_value>
Set headers returned by RackSpace when serving files matching
a specified regular expression.

Examples
--------
Expand All @@ -88,3 +91,6 @@ Examples

posthaste -c example -r DFW -u $OS_USERNAME -p $OS_PASSWORD -t 100 delete

Grand access to webfonts across different domains:
::
posthaste -c example -r DFW -u $OS_USERNAME -p $OS_PASSWORD -t 100 upload /path/to/some/dir/ -H ".*\.(eot|otf|woff|ttf)$,Access-Control-Allow-Origin:*"
54 changes: 44 additions & 10 deletions posthaste.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,41 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import gevent
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
from gevent.queue import Queue

import sys
import argparse
from collections import defaultdict
import functools
import json
import os
import argparse
import re
import requests
import functools
import time
import sys
import threading
import time


__version__ = '0.2.2'

HEADER_PAIR_FORMAT = '"<file_name_regex>,<header_name>:<header_value>"'


def regex_header_pair(string):
try:
regex, header_string = string.split(',')
# Just split header name and value and make it a dictionary
header_name, header_value = header_string.split(':')

return (regex, header_name, header_value)
except:
raise argparse.ArgumentTypeError(
'Headers must be specified in the format: %s' % HEADER_PAIR_FORMAT
)


def handle_args():
desc = ('Gevent-based, multithreaded tool for interacting with OpenStack '
Expand Down Expand Up @@ -91,6 +108,10 @@ def handle_args():
help='Upload files to specified container')
upload.set_defaults(action='upload')
upload.add_argument('directory', help='The directory to upload')
upload.add_argument('-H', '--headers', required=False,
type=regex_header_pair, action='append',
help='Add headers to files matching a specified regex. '
'Format: %s' % HEADER_PAIR_FORMAT)

download = subparsers.add_parser('download',
help='Download files to specified '
Expand All @@ -101,6 +122,7 @@ def handle_args():
help='The directory to download files to')

args = parser.parse_args()

return args


Expand Down Expand Up @@ -319,7 +341,7 @@ def _delete(i, files, errors):
pool.join()
return errors

def handle_upload(self, directory, container, threads, verbose):
def handle_upload(self, directory, container, threads, verbose, headers):
@self.requires_auth
def _upload(thread, queue, errors):
if verbose:
Expand All @@ -340,11 +362,17 @@ def _upload(thread, queue, errors):
print 'Thread %3s: uploading %s' % (thread,
file['name'])
try:
file_headers = {
'X-Auth-Token': self.token,
}

for regex, header_dict in headers.items():
if re.match(regex, file['name']):
file_headers.update(header_dict)

r = s.put('%s/%s/%s' %
(self.endpoint, container, file['name']),
data=body, headers={
'X-Auth-Token': self.token
})
data=body, headers=file_headers)
except:
e = sys.exc_info()[1]
errors.append({
Expand Down Expand Up @@ -450,9 +478,15 @@ def shell():
args = handle_args()
posthaste = Posthaste(args)
if args.action == 'upload':
# Zip the regex-headers pairs in a single dict and pass it to
# the upload handler
headers = defaultdict(dict)
for regex, header_name, header_value in args.headers:
headers[regex][header_name] = header_value

posthaste.get_files(args.directory, args.verbose)
errors = posthaste.handle_upload(args.directory, args.container,
args.threads, args.verbose)
args.threads, args.verbose, headers)
elif args.action == 'download':
posthaste.get_objects(args.container, args.verbose)
errors = posthaste.handle_download(args.directory, args.container,
Expand Down