Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Add back support for --endpoint-url option to s3 #469

Closed
wants to merge 3 commits into from

2 participants

@mguillaume

The --endpoint-url option was re-enabled with commit d6c46df but s3 uses different code to get the endpoint.
This adds back endpoint-url support for the s3 service.

@mguillaume

Looks like I have to adapt the unit tests too, I'll update ASAP.

@jamesls
Owner

Could you also add a unit tests for your change? That way we can ensure we don't regress on this again in the future.

Thanks.

@mguillaume

I think this one does it.
Note that for S3, without specifying a region, endpoint-url will not work as botocore has to do a lot of "magic" associated with region endpoints and DNS compatible endpoints. I have a (pretty simple) pull request in botocore to allow us to use --region=mock for this: boto/botocore#171

@jamesls
Owner

Thanks for adding the test, looks good. Looks like there were other changes to s3 recently so there are a few merge conflicts I'll have to work through. Unless of course you wouldn't mind rebasing against develop :)

@mguillaume

This is "rebased" (manually + push -f), but one test does not pass anymore.
I'm not 100% sure but I think the Mock service for s3 doesn't provide everything that is required by the new ListCommand.

@jamesls jamesls closed this pull request from a commit
@jamesls jamesls Merge branch 'mguillaume-develop' into develop
Closes #469

* mguillaume-develop:
  Update ls to support endpoint-url
  Add unit test to ensure endpoint-url is passed to botocore for s3
  Update S3 unit tests for endpoint_url param
  Add back support for --endpoint-url option to s3
3a97a37
@jamesls jamesls closed this in 3a97a37
@jamesls
Owner

Sorry for the delay, this has been merged, thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
26 awscli/customizations/s3/s3.py
@@ -269,6 +269,7 @@ def _do_command(self, parsed_args, parsed_globals):
params = self._build_call_parameters(parsed_args, {})
cmd_params = CommandParameters(self._session, self._name, params)
cmd_params.check_region(parsed_globals)
+ cmd_params.check_endpoint_url(parsed_globals)
cmd_params.add_paths(parsed_args.paths)
cmd_params.check_force(parsed_globals)
cmd = CommandArchitecture(self._session, self._name,
@@ -434,7 +435,8 @@ def __init__(self, session, cmd, parameters):
self.parameters = parameters
self.instructions = []
self._service = self.session.get_service('s3')
- self._endpoint = self._service.get_endpoint(self.parameters['region'])
+ self._endpoint = self._service.get_endpoint(region_name=self.parameters['region'],
+ endpoint_url=self.parameters['endpoint_url'])
def create_instructions(self):
"""
@@ -731,15 +733,31 @@ def check_region(self, parsed_globals):
parsed_region = None
if 'region' in parsed_globals:
parsed_region = getattr(parsed_globals, 'region')
- if not region and not parsed_region:
+ if 'endpoint_url' in parsed_globals:
+ parsed_endpoint_url = getattr(parsed_globals, 'endpoint_url')
+ else:
+ parsed_endpoint_url = None
+ if not region and not parsed_region and parsed_endpoint_url is None:
raise Exception("A region must be specified --region or "
"specifying the region\nin a configuration "
- "file or as an environment variable\n")
+ "file or as an environment variable.\n"
+ "Alternately, an endpoint can be specified "
+ "with --endpoint-url")
if parsed_region:
self.parameters['region'] = parsed_region
- else:
+ elif region:
self.parameters['region'] = region
+ else:
+ self.parameters['region'] = None
+ def check_endpoint_url(self, parsed_globals):
+ """
+ Adds endpoint_url to the parameters.
+ """
+ if 'endpoint_url' in parsed_globals:
+ self.parameters['endpoint_url'] = getattr(parsed_globals, 'endpoint_url')
+ else:
+ self.parameters['endpoint_url'] = None
# This is a dictionary useful for automatically adding the different commands,
# the amount of arguments it takes, and the optional parameters that can appear
View
4 tests/unit/customizations/s3/fake_session.py
@@ -77,9 +77,9 @@ class FakeService(object):
def __init__(self, session):
self.session = session
- def get_endpoint(self, region):
+ def get_endpoint(self, region_name, endpoint_url=None):
endpoint = Mock()
- endpoint.region_name = region
+ endpoint.region_name = region_name
return endpoint
def get_operation(self, name):
View
28 tests/unit/customizations/s3/test_s3.py
@@ -169,9 +169,9 @@ def test_create_instructions(self):
'mb': ['s3_handler'],
'rb': ['s3_handler']}
- params = {'filters': True, 'region': 'us-east-1'}
+ params = {'filters': True, 'region': 'us-east-1', 'endpoint_url': None}
for cmd in cmds:
- cmd_arc = CommandArchitecture(self.session, cmd, {'region': 'us-east-1'})
+ cmd_arc = CommandArchitecture(self.session, cmd, {'region': 'us-east-1', 'endpoint_url': None})
cmd_arc.create_instructions()
self.assertEqual(cmd_arc.instructions, instructions[cmd])
@@ -191,7 +191,8 @@ def test_run_cp_put(self):
filters = [['--include', '*']]
params = {'dir_op': False, 'dryrun': True, 'quiet': False,
'src': local_file, 'dest': s3_file, 'filters': filters,
- 'paths_type': 'locals3', 'region': 'us-east-1'}
+ 'paths_type': 'locals3', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'cp', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -208,7 +209,8 @@ def test_run_cp_get(self):
filters = [['--include', '*']]
params = {'dir_op': False, 'dryrun': True, 'quiet': False,
'src': s3_file, 'dest': local_file, 'filters': filters,
- 'paths_type': 's3local', 'region': 'us-east-1'}
+ 'paths_type': 's3local', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'cp', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -223,7 +225,8 @@ def test_run_cp_copy(self):
filters = [['--include', '*']]
params = {'dir_op': False, 'dryrun': True, 'quiet': False,
'src': s3_file, 'dest': s3_file, 'filters': filters,
- 'paths_type': 's3s3', 'region': 'us-east-1'}
+ 'paths_type': 's3s3', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'cp', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -238,7 +241,8 @@ def test_run_mv(self):
filters = [['--include', '*']]
params = {'dir_op': False, 'dryrun': True, 'quiet': False,
'src': s3_file, 'dest': s3_file, 'filters': filters,
- 'paths_type': 's3s3', 'region': 'us-east-1'}
+ 'paths_type': 's3s3', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'mv', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -253,7 +257,8 @@ def test_run_remove(self):
filters = [['--include', '*']]
params = {'dir_op': False, 'dryrun': True, 'quiet': False,
'src': s3_file, 'dest': s3_file, 'filters': filters,
- 'paths_type': 's3', 'region': 'us-east-1'}
+ 'paths_type': 's3', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'rm', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -272,7 +277,8 @@ def test_run_sync(self):
filters = [['--include', '*']]
params = {'dir_op': True, 'dryrun': True, 'quiet': False,
'src': local_dir, 'dest': s3_prefix, 'filters': filters,
- 'paths_type': 'locals3', 'region': 'us-east-1'}
+ 'paths_type': 'locals3', 'region': 'us-east-1',
+ 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'sync', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -286,7 +292,7 @@ def test_run_mb(self):
s3_prefix = 's3://' + self.bucket + '/'
params = {'dir_op': True, 'dryrun': True, 'quiet': False,
'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3',
- 'region': 'us-east-1'}
+ 'region': 'us-east-1', 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'mb', params)
cmd_arc.create_instructions()
cmd_arc.run()
@@ -300,7 +306,7 @@ def test_run_rb(self):
s3_prefix = 's3://' + self.bucket + '/'
params = {'dir_op': True, 'dryrun': True, 'quiet': False,
'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3',
- 'region': 'us-east-1'}
+ 'region': 'us-east-1', 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'rb', params)
cmd_arc.create_instructions()
rc = cmd_arc.run()
@@ -315,7 +321,7 @@ def test_run_rb_nonzero_rc(self):
s3_prefix = 's3://' + self.bucket + '/'
params = {'dir_op': True, 'dryrun': False, 'quiet': False,
'src': s3_prefix, 'dest': s3_prefix, 'paths_type': 's3',
- 'region': 'us-east-1'}
+ 'region': 'us-east-1', 'endpoint_url': None}
cmd_arc = CommandArchitecture(self.session, 'rb', params)
cmd_arc.create_instructions()
rc = cmd_arc.run()
View
11 tests/unit/test_clidriver.py
@@ -330,6 +330,17 @@ def test_aws_with_region(self):
endpoint.assert_called_with(region_name='us-east-1',
endpoint_url=None)
+ def test_s3_with_region_and_endpoint_url(self):
+ with mock.patch('botocore.service.Service.get_endpoint') as endpoint:
+ http_response = models.Response()
+ http_response.status_code = 200
+ endpoint.return_value.make_request.return_value = (
+ http_response, {})
+ self.assert_params_for_cmd(
+ 's3 ls s3://test --region us-east-1 --endpoint-url https://foobar.com/',
+ expected_rc=0)
+ endpoint.assert_called_with(region_name='us-east-1',
+ endpoint_url='https://foobar.com/')
def inject_new_param(self, argument_table, **kwargs):
argument = CustomArgument('unknown-arg', {})
Something went wrong with that request. Please try again.