Remove bootstrap's dependency on gsutil

That will assist supporting distros that doesn't have gsutil installed.

daisy_workflows/linux_common/bootstrap.py

@@ -21,17 +21,59 @@
   script: The main script to be run
   prefix: a string prefix for outputing status
 """
+import ast
+import json
 import logging
 import os
 import subprocess
 import urllib2
 
 
 DIR = '/files'
+TOKEN = None
+
+
+def GetAccessToken():
+  url = '%(metadata)s/v1/instance/service-accounts/default/token' % {
+      'metadata': 'http://metadata.google.internal/computeMetadata',
+  }
+  request = urllib2.Request(url)
+  request.add_unredirected_header('Metadata-Flavor', 'Google')
+  # converts the stringified dictionary of the response to a dictionary
+  response = ast.literal_eval(urllib2.urlopen(request).read())
+  return '%s %s' % (response[u'token_type'], response[u'access_token'])
+
+
+def GetBucketContent(bucket):
+  url = '%(storage)s/v1/b/%(bucket_name)s/o' % {
+      'storage': 'https://www.googleapis.com/storage',
+      'bucket_name': bucket,
+  }
+  print ("Bucket listing: %s" % url)
+  request = urllib2.Request(url)
+  request.add_unredirected_header('Metadata-Flavor', 'Google')
+  request.add_unredirected_header('Authorization', TOKEN)
+  content = json.load(urllib2.urlopen(request))
+  return [i['name'] for i in content['items']]
+
+
+def SaveBucketFile(bucket, bucket_file, dest_filepath):
+  url = 'https://storage.googleapis.com/%s/%s' % (bucket, bucket_file)
+  print ("Bucket save: %s => %s" % (url, dest_filepath))
+  request = urllib2.Request(url)
+  request.add_unredirected_header('Metadata-Flavor', 'Google')
+  request.add_unredirected_header('Authorization', TOKEN)
+  content = urllib2.urlopen(request).read()
+  f = open(dest_filepath, 'w')
+  f.write(content)
+  f.close()
 
 
 def GetMetadataAttribute(attribute):
-  url = 'http://metadata.google.internal/computeMetadata/v1/instance/attributes/%s' % attribute
+  url = '%(metadata)s/v1/instance/attributes/%(attribute_name)s' % {
+      'metadata': 'http://metadata.google.internal/computeMetadata',
+      'attribute_name': attribute,
+  }
   request = urllib2.Request(url)
   request.add_unredirected_header('Metadata-Flavor', 'Google')
   return urllib2.urlopen(request).read()
@@ -52,6 +94,7 @@ def DebianInstallGoogleApiPythonClient(prefix):
 def Bootstrap():
   """Get files, run."""
   try:
+    global TOKEN
     prefix = GetMetadataAttribute('prefix')
     status = prefix + 'Status'
     logging.info('%s: Starting bootstrap.py.', status)
@@ -63,12 +106,27 @@ def Bootstrap():
     except urllib2.HTTPError:
       pass
 
+    TOKEN = GetAccessToken()
     gcs_dir = GetMetadataAttribute('files_gcs_dir')
     script = GetMetadataAttribute('script')
     full_script = os.path.join(DIR, script)
     subprocess.check_call(['mkdir', DIR])
-    subprocess.check_call(
-        ['gsutil', '-m', 'cp', '-r', os.path.join(gcs_dir, '*'), DIR])
+
+    # Copies all files from bucket's gcs_dir to DIR
+    path_stripped = gcs_dir[len('gs://'):]
+    token = path_stripped.find('/')
+
+    # skip leading slash on bucket_dir
+    bucket, bucket_dir = path_stripped[:token], path_stripped[token + 1:]
+
+    def BelongsToBucketDir(filename):
+      return filename.startswith(bucket_dir)
+
+    bucket_files = filter(BelongsToBucketDir, GetBucketContent(bucket))
+    for f in bucket_files:
+      dest_filepath = f.replace(bucket_dir, DIR)
+      SaveBucketFile(bucket, f, dest_filepath)
+
     logging.info('%s: Making script %s executable.', status, full_script)
     subprocess.check_call(['chmod', '+x', script], cwd=DIR)
     logging.info('%s: Running %s.', status, full_script)