Merge pull request #1760 from davidmarin/google-bootstrap-dir

run Dataproc bootstrap scripts in /tmp/mrjob (fixes #1601)

mrjob/cloud.py

@@ -285,6 +285,11 @@ def _master_bootstrap_script_content(self, bootstrap):
         out.extend(self._start_of_sh_script())
         out.append('')
 
+        # for example, create a tmp dir and cd to it
+        if self._bootstrap_pre_commands():
+            out.extend(self._bootstrap_pre_commands())
+            out.append('')
+
         # store $PWD
         out.append('# store $PWD')
         out.append('__mrjob_PWD=$PWD')
@@ -370,6 +375,11 @@ def _master_bootstrap_script_content(self, bootstrap):
 
         return out
 
+    def _bootstrap_pre_commands(self):
+        """A list of hard-coded commands to run at the beginning of the
+        bootstrap script. Currently used by dataproc to cd into a tmp dir."""
+        return []
+
     def _start_of_sh_script(self):
         """Return a list of lines (without trailing newlines) containing the
         shell script shebang and pre-commands."""

mrjob/dataproc.py

@@ -894,6 +894,13 @@ def _store_cluster_info(self):
         self._hadoop_version = map_version(
             self._image_version, _DATAPROC_IMAGE_TO_HADOOP_VERSION)
 
+    def _bootstrap_pre_commands(self):
+        # don't run the bootstrap script in / (see #1601)
+        return [
+            'mkdir /tmp/mrjob',
+            'cd /tmp/mrjob',
+        ]
+
     ### Bootstrapping ###
 
     def _bootstrap_python(self):

tests/test_dataproc.py

@@ -804,6 +804,11 @@ def assertScriptDownloads(path, name=None):
         # check scripts get run
 
         # bootstrap
+
+        # see #1601
+        self.assertIn('mkdir /tmp/mrjob', lines)
+        self.assertIn('cd /tmp/mrjob', lines)
+
         self.assertIn('  ' + PYTHON_BIN + ' $__mrjob_PWD/bar.py', lines)
         self.assertIn('  $__mrjob_PWD/ohnoes.sh', lines)