Merge 4cac8b4 into b3ac694

README.rst

@@ -18,6 +18,10 @@ Services
   - Forward (place names ⇢ longitude, latitude)
   - Reverse (longitude, latitude ⇢ place names)
 
+- `Upload <https://www.mapbox.com/developers/api/uploads/>`__
+
+  - Upload data to be processed and hosted by Mapbox.
+
 -  Other services coming soon
 
 Installation
@@ -30,6 +34,9 @@ Installation
 API Usage
 =========
 
+Geocoder
+--------
+
 To begin geocoding, import the mapbox module and create a new
 ``Geocoder`` object with your `Mapbox access token 
 <https://www.mapbox.com/developers/api/#access-tokens>`__.
@@ -54,6 +61,24 @@ To begin geocoding, import the mapbox module and create a new
 
 See ``import mapbox; help(mapbox.Geocoder)`` for more detailed usage.
 
+Upload
+------
+To upload data, you must created a token with ``uploads:*`` scopes at https://www.mapbox.com/account/apps/.
+Then upload any supported file to your account using the ``Uploader`` 
+
+.. code:: python
+    
+    from mapbox import Uploader
+    conxn = Uploader('username', access_token='MY_TOKEN')
+    resp = conxn.upload('RGB.byte.tif', 'RGB-byte-tif')
+    upload_id = resp.json()['id']
+
+    resp = conxn.status(upload_id).json()
+    resp['complete']  # True
+    resp['tileset']   # "username.RGB-byte-tif"
+
+See ``import mapbox; help(mapbox.Uploader)`` for more detailed usage.
+
 Command Line Interface
 ======================
 

mapbox/__init__.py

@@ -3,3 +3,4 @@
 
 from .services.base import Service
 from .services.geocoding import Geocoder, InvalidPlaceTypeError
+from .services.upload import Uploader

mapbox/services/upload.py

@@ -0,0 +1,132 @@
+# mapbox
+from uritemplate import URITemplate
+from .base import Service
+from boto3.session import Session
+
+
+class Uploader(Service):
+    """Mapbox Upload API
+
+    Example usage:
+
+        from mapbox import Uploader
+
+        u = Uploader('username')
+        url = u.stage('test.tif')
+        job = u.extract(url, 'test1').json()
+
+        assert job in u.list().json()
+
+        # ... wait unti finished ...
+        finished = u.status(job).json()['complete']
+
+        u.delete(job)
+        assert job not in u.list().json()
+    """
+
+    def __init__(self, username, access_token=None):
+        self.username = username
+        self.baseuri = 'https://api.mapbox.com/uploads/v1'
+        self.session = self.get_session(access_token)
+
+    def _get_credentials(self):
+        """Gets temporary S3 credentials to stage user-uploaded files
+        """
+        uri = URITemplate('%s/{username}/credentials' % self.baseuri).expand(
+            username=self.username)
+        return self.session.get(uri)
+
+    def stage(self, filepath, creds=None):
+        """Stages the user's file on S3
+        If creds are not provided, temporary credientials will be generated
+        Returns the URL to the staged resource.
+        """
+        if not creds:
+            res = self._get_credentials()
+            creds = res.json()
+
+        session = Session(aws_access_key_id=creds['accessKeyId'],
+                          aws_secret_access_key=creds['secretAccessKey'],
+                          aws_session_token=creds['sessionToken'],
+                          region_name="us-east-1")
+
+        s3 = session.resource('s3')
+        with open(filepath, 'rb') as data:
+            res = s3.Object(creds['bucket'], creds['key']).put(Body=data)
+
+        return creds['url']
+
+    def extract(self, stage_url, tileset, name=None):
+        """Initiates the extraction process from the
+        staging S3 bucket into the user's tileset.
+
+        Note: this step is refered to as "upload" in the API docs;
+        This classes upload() method is a high-level function
+        which acts like the web-based upload form
+
+        Parameters
+        stage_url: URL to resource on S3, does not work on arbitrary URLs (TODO)
+        tileset: the map/tileset name to create. Username will be prefixed if not
+                 done already (e.g. 'test1' becomes 'username.test1')
+
+        Returns a response object where the json() contents are
+        an upload dict
+        """
+        if not tileset.startswith(self.username + "."):
+            tileset = "{0}.{1}".format(self.username, tileset)
+
+        msg = {'tileset': tileset,
+               'url': stage_url}
+
+        if name is not None:
+            msg['name'] = name
+
+        uri = URITemplate('%s/{username}' % self.baseuri).expand(
+            username=self.username)
+
+        return self.session.post(uri, json=msg)
+
+    def list(self):
+        """List of all uploads
+
+        Returns a response object where the json() contents are
+        a list of uploads
+        """
+        uri = URITemplate('%s/{username}' % self.baseuri).expand(
+            username=self.username)
+        return self.session.get(uri)
+
+    def delete(self, upload):
+        """Delete the specified upload
+        """
+        if isinstance(upload, dict):
+            upload_id = upload['id']
+        else:
+            upload_id = upload
+
+        uri = URITemplate('%s/{username}/{upload_id}' % self.baseuri).expand(
+            username=self.username, upload_id=upload_id)
+        return self.session.delete(uri)
+
+    def status(self, upload):
+        """Check status of upload
+
+        Returns a response object where the json() contents are
+        another (updated) upload dict
+        """
+        if isinstance(upload, dict):
+            upload_id = upload['id']
+        else:
+            upload_id = upload
+
+        uri = URITemplate('%s/{username}/{upload_id}' % self.baseuri).expand(
+            username=self.username, upload_id=upload_id)
+        return self.session.get(uri)
+
+    def upload(self, filepath, tileset):
+        """High level function to upload a local file to mapbox tileset
+        Effectively replicates the upload functionality using the HTML form
+        Returns a response object where the json() is a dict with upload metadata
+        """
+        url = self.stage(filepath)
+        return self.extract(url, tileset)

setup.py

@@ -29,7 +29,7 @@
       include_package_data=True,
       zip_safe=False,
       install_requires=[
-          'click', 'click-plugins', 'cligj', 'requests', 'uritemplate.py'
+          'click', 'click-plugins', 'cligj', 'requests', 'uritemplate.py', 'boto3'
       ],
       extras_require={
           'test': ['coveralls', 'pytest', 'pytest-cov', 'responses'],

tests/test_upload.py

@@ -0,0 +1,130 @@
+import responses
+import json
+
+import mapbox
+
+username = 'testuser'
+upload_response_body = """
+    {{"progress": 0,
+    "modified": "date.test",
+    "error": null,
+    "tileset": "{username}.test1",
+    "complete": false,
+    "owner": "{username}",
+    "created": "date.test",
+    "id": "id.test",
+    "name": null}}""".format(username=username)
+
+@responses.activate
+def test_get_credentials():
+    query_body = """
+       {{"key": "_pending/{username}/key.test",
+         "accessKeyId": "ak.test",
+         "bucket": "tilestream-tilesets-production",
+         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{username}/key.test",
+         "secretAccessKey": "sak.test",
+         "sessionToken": "st.test"}}""".format(username=username)
+
+    responses.add(
+        responses.GET,
+        'https://api.mapbox.com/uploads/v1/{0}/credentials?access_token=pk.test'.format(username),
+        match_querystring=True,
+        body=query_body, status=200,
+        content_type='application/json')
+
+    res = mapbox.Uploader(username, access_token='pk.test')._get_credentials()
+    assert res.status_code == 200
+    creds = res.json()
+    assert username in creds['url']
+    for k in ['key', 'bucket', 'url', 'accessKeyId',
+              'secretAccessKey', 'sessionToken']:
+        assert k in creds.keys()
+
+
+# This is not working, moto is not properly patching boto3?
+#
+# import boto3
+# from moto import mock_s3
+# @mock_s3
+# def test_stage():
+#     creds = {
+#         "key": "_pending/{0}/key.test".format(username),
+#         "accessKeyId": "ak.test",
+#         "bucket": "tilestream-tilesets-production",
+#         "url": "https://tilestream-tilesets-production.s3.amazonaws.com/_pending/{0}/key.test".format(username),
+#         "secretAccessKey": "sak.test",
+#         "sessionToken": "st.test"}
+#     s3 = boto3.resource('s3', region_name='us-east-1')
+#     s3.create_bucket(Bucket=creds['bucket'])
+#     url = mapbox.Uploader(username, access_token='pk.test').stage('tests/test.csv', creds)
+#     assert url == creds['url']
+#     assert s3.Object(creds['bucket'], creds['key']).get()['Body'].read().decode("utf-8") == \
+#         'testing123'
+
+
+@responses.activate
+def test_extract():
+    responses.add(
+        responses.POST,
+        'https://api.mapbox.com/uploads/v1/{0}?access_token=pk.test'.format(username),
+        match_querystring=True,
+        body=upload_response_body, status=201,
+        content_type='application/json')
+
+    res = mapbox.Uploader(username, access_token='pk.test').extract(
+        'http://example.com/test.json', 'test1')  # without username prefix
+    assert res.status_code == 201
+    job = res.json()
+    assert job['tileset'] == "{0}.test1".format(username)
+
+    res2 = mapbox.Uploader(username, access_token='pk.test').extract(
+        'http://example.com/test.json', 'testuser.test1')  # also takes full tileset
+    assert res2.status_code == 201
+    job = res2.json()
+    assert job['tileset'] == "{0}.test1".format(username)
+
+
+@responses.activate
+def test_list():
+    responses.add(
+        responses.GET,
+        'https://api.mapbox.com/uploads/v1/{0}?access_token=pk.test'.format(username),
+        match_querystring=True,
+        body="[{0}]".format(upload_response_body), status=200,
+        content_type='application/json')
+
+    res = mapbox.Uploader(username, access_token='pk.test').list()
+    assert res.status_code == 200
+    uploads = res.json()
+    assert len(uploads) == 1
+    assert json.loads(upload_response_body) in uploads
+
+
+@responses.activate
+def test_status():
+    job = json.loads(upload_response_body)
+    responses.add(
+        responses.GET,
+        'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token=pk.test'.format(username, job['id']),
+        match_querystring=True,
+        body=upload_response_body, status=200,
+        content_type='application/json')
+
+    res = mapbox.Uploader(username, access_token='pk.test').status(job)
+    assert res.status_code == 200
+    status = res.json()
+    assert job == status
+
+
+@responses.activate
+def test_delete():
+    job = json.loads(upload_response_body)
+    responses.add(
+        responses.DELETE,
+        'https://api.mapbox.com/uploads/v1/{0}/{1}?access_token=pk.test'.format(username, job['id']),
+        match_querystring=True,
+        body=None, status=204,
+        content_type='application/json')
+
+    res = mapbox.Uploader(username, access_token='pk.test').delete(job)
+    assert res.status_code == 204