Portability

pywren/invokers.py

@@ -5,7 +5,7 @@
 
 import botocore
 import botocore.session
-from pywren import local
+#from pywren import local
 
 SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
 

pywren/queues.py

@@ -5,7 +5,7 @@
 import time
 
 import boto3
-from pywren import local
+#from pywren import local
 
 SOURCE_DIR = os.path.dirname(os.path.abspath(__file__))
 

pywren/scripts/pywrencli.py

@@ -210,11 +210,14 @@ def deploy_lambda(ctx, update_if_exists=True):
     module_dir = os.path.join(SOURCE_DIR, "../")
 
     for f in ['wrenutil.py', 'wrenconfig.py', 'wrenhandler.py',
-              'version.py', 'jobrunner.py', 'wren.py']:
+              'version.py', 'jobrunner.py', 'wren.py', 'storage/storage.py',
+              'storage/s3_backend.py', 'storage/storage_utils.py', 'storage/exceptions.py']:
         f = os.path.abspath(os.path.join(module_dir, f))
         a = os.path.relpath(f, SOURCE_DIR + "/..")
 
         zipfile_obj.write(f, arcname=a)
+    a = os.path.relpath("storage/__init__.py", SOURCE_DIR + "/..")
+    zipfile_obj.writestr(a, "")
     zipfile_obj.close()
     #open("/tmp/deploy.zip", 'w').write(file_like_object.getvalue())
 
@@ -575,3 +578,6 @@ def cleanup_all(ctx, force):
 
 def main():
     return cli() # pylint: disable=no-value-for-parameter
+
+if __name__ == '__main__':
+    main()

pywren/storage/s3_backend.py

@@ -14,6 +14,21 @@ def __init__(self, s3config):
         self.s3client = self.session.create_client(
             's3', config=botocore.client.Config(max_pool_connections=200))
 
+    def head_object(self, key):
+        """
+        Get object metadata from S3 with a key. Throws StorageNoSuchKeyError if the given key does not exist.
+        :param key: key of the object
+        :return: Data of the object
+        :rtype: str/bytes
+        """
+        try:
+            return self.s3client.head_object(Bucket = self.s3_bucket, Key = key)
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "NoSuchKey":
+                raise StorageNoSuchKeyError(key)
+            else:
+                raise e
+
     def put_object(self, key, data):
         """
         Put an object in S3. Override the object if the key already exists.
@@ -24,15 +39,19 @@ def put_object(self, key, data):
         """
         self.s3client.put_object(Bucket=self.s3_bucket, Key=key, Body=data)
 
-    def get_object(self, key):
+    def get_object(self, key, data_byte_range = None):
         """
         Get object from S3 with a key. Throws StorageNoSuchKeyError if the given key does not exist.
         :param key: key of the object
         :return: Data of the object
         :rtype: str/bytes
         """
         try:
-            r = self.s3client.get_object(Bucket=self.s3_bucket, Key=key)
+            if data_byte_range != None:
+                range_str = 'bytes={}-{}'.format(*data_byte_range)
+                r = self.s3client.get_object(Bucket=self.s3_bucket, Key=key, Range = range_str)
+            else:
+                r = self.s3client.get_object(Bucket=self.s3_bucket, Key=key)
             data = r['Body'].read()
             return data
         except botocore.exceptions.ClientError as e:

pywren/storage/storage.py

@@ -33,6 +33,20 @@ def get_storage_config(self):
         """
         return self.storage_config
 
+    def head_object(self, key):
+        """
+        Retrieves metadata for given key.
+        The metadata dict must have Contentlength and Etag as keys
+        :return: dict
+        """
+        return self.backend_handler.head_object(key)
+
+    def get_object(self, key, data_byte_range = None):
+        """
+        Retrieves object for given key.
+        """
+        return self.backend_handler.get_object(key, data_byte_range)
+
     def put_data(self, key, data):
         """
         Put input data into storage.