Merge pull request #2020 from davidmarin/gcs-upgrade

upgrade google-cloud-services to 1.11.0, remove shim (fixes #2019)

mrjob/dataproc.py

@@ -406,7 +406,6 @@ def fs(self):
 
             self._fs.add_fs('gcs', GCSFilesystem(
                 credentials=self._credentials,
-                local_tmp_dir=self._get_local_tmp_dir(),
                 project_id=self._project_id,
             ))
 

mrjob/fs/gcs.py

@@ -242,12 +242,7 @@ def create_bucket(self, name,
         and time-to-live."""
         bucket = self.client.bucket(name)
 
-        # as of google-cloud 0.32.0, there isn't a direct way to set location
-        if location:
-            bucket._changes.add('location')
-            bucket._properties['location'] = location
-
-        bucket.create()
+        bucket.create(location=location)
 
         bucket.lifecycle_rules = [
             dict(

setup.py

@@ -31,7 +31,7 @@
             'PyYAML>=3.08',
             'google-cloud-dataproc>=0.2.0',
             'google-cloud-logging>=1.5.0',
-            'google-cloud-storage>=1.9.0',
+            'google-cloud-storage>=1.11.0',
         ],
         'provides': ['mrjob'],
         'test_suite': 'tests',

tests/mock_google/case.py

@@ -58,8 +58,8 @@ def setUp(self):
         # mock project ID, returned by mock google.auth.default()
         self.mock_project_id = 'mock-project-12345'
 
-        # Maps bucket name to a dictionary with the key
-        # *blobs*. *blobs* maps object name to
+        # Maps bucket name to a dictionary with the keys
+        # *blobs* and *location*. *blobs* maps object name to
         # a dictionary with the key *data*, which is
         # a bytestring.
         self.mock_gcs_fs = {}

tests/mock_google/storage.py

@@ -61,17 +61,17 @@ def blob(self, blob_name, chunk_size=None):
         # always returns something whether it exists or not
         return MockGoogleStorageBlob(blob_name, self, chunk_size=chunk_size)
 
-    def create(self):
+    def create(self, location=None):
         if self.exists():
             raise Conflict(
                 'POST https://www.googleapis.com/storage/v1'
                 '/b?project=%s: Sorry, that name is not available.'
                 ' Please try a different one.')
 
-        if 'location' in self._changes and 'location' in self._properties:
-            location = self._properties['location'].upper()
+        if location:
+            location = location.upper()
         else:
-            location = 'US'
+            location = 'US'  # default bucket location
 
         self.client.mock_gcs_fs[self.name] = dict(
             blobs={}, lifecycle_rules=[], location=location)
@@ -92,10 +92,11 @@ def lifecycle_rules(self):
         fs = self.client.mock_gcs_fs
 
         if self.name in fs:
-            return deepcopy(fs[self.name]['lifecycle_rules'])
-        else:
-            # google-cloud-sdk silently ignores missing buckets
-            return []
+            # as of google-cloud-storage 1.11.0, this is a generator
+            for rule in fs[self.name]['lifecycle_rules']:
+                yield deepcopy(rule)
+
+        # if bucket doesn't exist, yield nothing
 
     @lifecycle_rules.setter
     def lifecycle_rules(self, rules):
@@ -126,7 +127,7 @@ def location(self):
             return fs[self.name]['location']
         else:
             # google-cloud-sdk silently ignores missing buckets
-            return []
+            return None
 
 
 class MockGoogleStorageBlob(object):

tests/test_dataproc.py

@@ -664,7 +664,7 @@ def assert_new_tmp_bucket(self, location, **runner_kwargs):
         self.assertEqual(current_bucket.location, location.upper())
 
         # Verify that we setup bucket lifecycle rules of 28-day retention
-        first_lifecycle_rule = current_bucket.lifecycle_rules[0]
+        first_lifecycle_rule = list(current_bucket.lifecycle_rules)[0]
         self.assertEqual(first_lifecycle_rule['action'], dict(type='Delete'))
         self.assertEqual(first_lifecycle_rule['condition'],
                          dict(age=_DEFAULT_CLOUD_TMP_DIR_OBJECT_TTL_DAYS))