Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

EMRJobRunner._aws_region is '' instead of None

  • Loading branch information...
commit c41e451fcf9e12b1af6c0dd9b4bcfba15a640146 1 parent 62197d8
David Marin authored

Showing 3 changed files with 24 additions and 5 deletions. Show diff stats Hide diff stats

  1. +4 4 mrjob/emr.py
  2. +19 0 tests/emr_test.py
  3. +1 1  tests/mockboto.py
8 mrjob/emr.py
@@ -81,7 +81,7 @@
81 81 'EU': 'eu-west-1.elasticmapreduce.amazonaws.com',
82 82 'us-east-1': 'us-east-1.elasticmapreduce.amazonaws.com',
83 83 'us-west-1': 'us-west-1.elasticmapreduce.amazonaws.com',
84   - None: 'elasticmapreduce.amazonaws.com', # when no region specified
  84 + '': 'elasticmapreduce.amazonaws.com', # when no region specified
85 85 }
86 86
87 87 # map from AWS region to S3 endpoint
@@ -91,7 +91,7 @@
91 91 'us-east-1': 's3.amazonaws.com', # no region-specific endpoint
92 92 'us-west-1': 's3-us-west-1.amazonaws.com',
93 93 'ap-southeast-1': 's3-ap-southeast-1.amazonaws.com', # no EMR endpoint yet
94   - None: 's3.amazonaws.com',
  94 + '': 's3.amazonaws.com',
95 95 }
96 96
97 97 def parse_s3_uri(uri):
@@ -270,7 +270,7 @@ def __init__(self, **kwargs):
270 270
271 271 # make aws_region an instance variable; we might want to set it
272 272 # based on the scratch bucket
273   - self._aws_region = self._opts['aws_region']
  273 + self._aws_region = self._opts['aws_region'] or ''
274 274
275 275 # if we're going to create a bucket to use as temp space, we don't
276 276 # want to actually create it until we run the job (Issue #50).
@@ -397,7 +397,7 @@ def _fix_s3_scratch_and_log_uri_opts(self):
397 397 # if we're not using an ancient version of boto, set region
398 398 # based on the bucket's region
399 399 if (hasattr(scratch_bucket, 'get_location')):
400   - self._aws_region = scratch_bucket.get_location()
  400 + self._aws_region = scratch_bucket.get_location() or ''
401 401 if self._aws_region:
402 402 log.info("using scratch bucket's region (%s) to connect to AWS" %
403 403 self._aws_region)
19 tests/emr_test.py
@@ -490,6 +490,25 @@ def test_no_region(self):
490 490 'elasticmapreduce.amazonaws.com')
491 491 assert_equal(runner.make_s3_conn().endpoint,
492 492 's3.amazonaws.com')
  493 + assert_equal(runner._aws_region, '')
  494 +
  495 + def test_none_region(self):
  496 + # blank region should be treated the same as no region
  497 + runner = EMRJobRunner(conf_path=False, aws_region=None)
  498 + assert_equal(runner.make_emr_conn().endpoint,
  499 + 'elasticmapreduce.amazonaws.com')
  500 + assert_equal(runner.make_s3_conn().endpoint,
  501 + 's3.amazonaws.com')
  502 + assert_equal(runner._aws_region, '')
  503 +
  504 + def test_blank_region(self):
  505 + # blank region should be treated the same as no region
  506 + runner = EMRJobRunner(conf_path=False, aws_region='')
  507 + assert_equal(runner.make_emr_conn().endpoint,
  508 + 'elasticmapreduce.amazonaws.com')
  509 + assert_equal(runner.make_s3_conn().endpoint,
  510 + 's3.amazonaws.com')
  511 + assert_equal(runner._aws_region, '')
493 512
494 513 def test_eu(self):
495 514 runner = EMRJobRunner(conf_path=False, aws_region='EU')
2  tests/mockboto.py
@@ -64,7 +64,7 @@ def __init__(self, aws_access_key_id=None, aws_secret_access_key=None,
64 64 from bucket name to key name to bytes.
65 65 """
66 66 self.mock_s3_fs = mock_s3_fs or {}
67   - self.endpoint = host
  67 + self.endpoint = host or 's3.amazonaws.com'
68 68
69 69 def get_bucket(self, bucket_name):
70 70 if bucket_name in self.mock_s3_fs:

0 comments on commit c41e451

Please sign in to comment.
Something went wrong with that request. Please try again.