Merge debc29d into b3ac694

mapbox/__init__.py

@@ -1,5 +1,8 @@
 # mapbox
-__version__ = "0.2.0"
 
 from .services.base import Service
+from .services.datasets import Datasets, Dataset
 from .services.geocoding import Geocoder, InvalidPlaceTypeError
+
+
+__version__ = "0.3.0"

mapbox/compat.py

@@ -5,3 +5,4 @@
 
 
 map = itertools.imap if sys.version_info < (3,) else map
+zip = itertools.izip if sys.version_info < (3,) else zip

mapbox/scripts/datasets.py

@@ -0,0 +1,90 @@
+import logging
+import json
+
+import click
+
+import mapbox
+from mapbox.services.datasets import batch, iter_features
+
+
+class MapboxCLIException(click.ClickException):
+    pass
+
+
+@click.command(short_help="Create an empty dataset")
+@click.argument('owner')
+@click.option('--name')
+@click.option('--description')
+@click.pass_context
+def create_dataset(ctx, owner, name, description):
+    """Create a new empty dataset."""
+    verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 2
+    logger = logging.getLogger('mapbox')
+
+    access_token = (ctx.obj and ctx.obj.get('access_token')) or None
+
+    datasets = mapbox.Datasets(owner, access_token=access_token)
+    properties = {}
+    if name:
+        properties['name'] = name
+    if description:
+        properties['description'] = description
+    resp = datasets.create(**properties)
+    click.echo(resp.text)
+
+
+@click.command(short_help="List datasets")
+@click.argument('owner')
+@click.pass_context
+def ls_datasets(ctx, owner):
+    """List datasets."""
+    verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 2
+    logger = logging.getLogger('mapbox')
+
+    access_token = (ctx.obj and ctx.obj.get('access_token')) or None
+
+    datasets = mapbox.Datasets(owner, access_token=access_token)
+    resp = datasets.list()
+    click.echo(resp.text)
+
+
+@click.command(short_help="Get a dataset's features")
+@click.argument('owner')
+@click.argument('id')
+@click.option('--output', '-o', default='-', help="Save output to a file.")
+@click.pass_context
+def retrieve_features(ctx, owner, id, output):
+    """Return features as GeoJSON."""
+    verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 2
+    logger = logging.getLogger('mapbox')
+
+    access_token = (ctx.obj and ctx.obj.get('access_token')) or None
+    stdout = click.open_file(output, 'w')
+
+    dataset = mapbox.Dataset(owner, id, access_token=access_token)
+    resp = dataset.retrieve_features()
+    click.echo(resp.text, file=stdout)
+
+
+@click.command(short_help="Update a dataset's features")
+@click.argument('owner')
+@click.argument('id')
+@click.option(
+    '--sequence / --no-sequence', default=False,
+    help="Specify whether the input stream is a LF-delimited sequence of GeoJSON "
+         "features (the default) or a single GeoJSON feature collection.")
+@click.pass_context
+def update_features(ctx, owner, id, sequence):
+    """Update a dataset's features from provided GeoJSON."""
+    verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 2
+    logger = logging.getLogger('mapbox')
+
+    access_token = (ctx.obj and ctx.obj.get('access_token')) or None
+
+    dataset = mapbox.Dataset(owner, id, access_token=access_token)
+
+    stdin = click.get_text_stream('stdin')
+    for update_batch in batch(iter_features(stdin, is_sequence=sequence)):
+        payload = {'put': list(update_batch)}
+        logger.debug("Payload: %r", payload)
+        resp = dataset.update_features(**payload)

mapbox/services/datasets.py

@@ -0,0 +1,114 @@
+# mapbox.datasets
+from itertools import chain, count, groupby
+import json
+
+import requests
+from uritemplate import URITemplate
+
+from mapbox.services.base import Service
+
+
+# Constants
+BASE_URI = 'https://api.mapbox.com/datasets/v1'
+MAX_BATCH_SIZE = 100
+
+
+def iter_features(src, is_sequence=False):
+    """Yield features from a src that may be either a GeoJSON feature
+    text sequence or GeoJSON feature collection."""
+    first_line = next(src)
+    # If input is RS-delimited JSON sequence.
+    if first_line.startswith(u'\x1e'):
+        buffer = first_line.strip(u'\x1e')
+        for line in src:
+            if line.startswith(u'\x1e'):
+                if buffer:
+                    feat = json.loads(buffer)
+                    yield feat
+                buffer = line.strip(u'\x1e')
+            else:
+                buffer += line
+        else:
+            feat = json.loads(buffer)
+            yield feat
+    elif is_sequence:
+        yield json.loads(first_line)
+        for line in src:
+            feat = json.loads(line)
+            yield feat
+    else:
+        text = "".join(chain([first_line], src))
+        for feat in json.loads(text)['features']:
+            yield feat
+
+
+def batch(iterable, size=MAX_BATCH_SIZE):
+    """Yield batches of features."""
+    c = count()
+    for k, g in groupby(iterable, lambda x:next(c)//size):
+         yield g
+
+
+class Datasets(Service):
+    """A Datasets API proxy"""
+
+    def __init__(self, name, access_token=None):
+        self.name = name
+        self.baseuri = 'https://api.mapbox.com/datasets/v1'
+        self.session = self.get_session(access_token)
+
+    def list(self):
+        """Returns a Requests response object that contains a listing of
+        the owner's datasets.
+
+        `response.json()` returns the geocoding result as GeoJSON.
+        `response.status_code` returns the HTTP API status code.
+
+        See: https://www.mapbox.com/developers/api/datasets/."""
+        uri = URITemplate(self.baseuri + '/{owner}').expand(owner=self.name)
+        return self.session.get(uri)
+
+    def create(self, **kwargs):
+        """Create a new dataset and return a Requests response object
+        that contains information about the new dataset.
+
+        `response.json()` returns the geocoding result as GeoJSON.
+        `response.status_code` returns the HTTP API status code.
+
+        See: https://www.mapbox.com/developers/api/datasets/."""
+        uri = URITemplate(self.baseuri + '/{owner}').expand(owner=self.name)
+        return self.session.post(uri, json=kwargs)
+
+
+class Dataset(Service):
+    """A Datasets API proxy"""
+
+    def __init__(self, owner, id, access_token=None):
+        self.owner = owner
+        self.id = id
+        self.baseuri = URITemplate(BASE_URI + '/{owner}/{id}').expand(
+            owner=self.owner, id=self.id)
+        self.session = self.get_session(access_token)
+
+    def retrieve_features(self):
+        """Return a Requests response object that contains the features
+        of the dataset.
+
+        `response.json()` returns the geocoding result as GeoJSON.
+        `response.status_code` returns the HTTP API status code.
+
+        See: https://www.mapbox.com/developers/api/datasets/."""
+        uri = URITemplate(self.baseuri + '/features').expand()
+        return self.session.get(uri)
+
+
+    def update_features(self, **updates):
+        """Return a Requests response object that contains the features
+        of the dataset.
+
+        `response.json()` returns the geocoding result as GeoJSON.
+        `response.status_code` returns the HTTP API status code.
+
+        See: https://www.mapbox.com/developers/api/datasets/."""
+        uri = URITemplate(self.baseuri + '/features').expand()
+        return self.session.post(uri, json=updates)

setup.py

@@ -40,5 +40,9 @@
 
       [mapbox.mapbox_commands]
       geocode=mapbox.scripts.geocoder:geocode
+      create_dataset=mapbox.scripts.datasets:create_dataset
+      ls_datasets=mapbox.scripts.datasets:ls_datasets
+      retrieve_features=mapbox.scripts.datasets:retrieve_features
+      update_features=mapbox.scripts.datasets:update_features
       """
       )

tests/test_cli.py

@@ -184,3 +184,114 @@ def test_cli_geocode_rev_headers():
         input='{0},{1}'.format(lon, lat))
     assert result.exit_code == 0
     assert result.output.startswith('Content-Type')
+
+
+@responses.activate
+def test_cli_datasets_list():
+    """Listing datasets works"""
+
+    body = '''
+[
+  {
+    "owner": "juser",
+    "id": "ds1",
+    "created": "2015-09-19",
+    "modified": "2015-09-19"
+  },
+  {
+    "owner": "juser",
+    "id": "ds2",
+    "created": "2015-09-19",
+    "modified": "2015-09-19"
+  }
+]
+'''
+
+    responses.add(
+        responses.GET,
+        'https://api.mapbox.com/datasets/v1/juser?access_token=pk.test',
+        match_querystring=True,
+        body=body, status=200,
+        content_type='application/json')
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        ['--access-token', 'pk.test', 'ls_datasets', 'juser'])
+    assert result.exit_code == 0
+    body = json.loads(result.output)
+    assert [item['id'] for item in body] == ['ds1', 'ds2']
+
+
+@responses.activate
+def test_cli_dataset_creation():
+
+    def request_callback(request):
+        payload = json.loads(request.body)
+        resp_body = {
+            'owner': 'juser',
+            'id': 'new',
+            'name': payload['name'],
+            'description': payload['description'],
+            'created': '2015-09-19',
+            'modified': '2015-09-19'}
+        headers = {}
+        return (200, headers, json.dumps(resp_body))
+
+    responses.add_callback(
+        responses.POST,
+        'https://api.mapbox.com/datasets/v1/juser?access_token=pk.test',
+        match_querystring=True,
+        callback=request_callback)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        ['--access-token', 'pk.test', 'create_dataset', 'juser', '--name', 'things', '--description', 'all the things'])
+    assert result.exit_code == 0
+    body = json.loads(result.output)
+    assert body['name'] == 'things'
+    assert body['description'] == 'all the things'
+
+
+@responses.activate
+def test_cli_dataset_retrieve_features():
+    """Features retrieval work"""
+
+    responses.add(
+        responses.GET,
+        'https://api.mapbox.com/datasets/v1/juser/test/features?access_token=pk.test',
+        match_querystring=True,
+        body=json.dumps({'type': 'FeatureCollection'}),
+        status=200,
+        content_type='application/json')
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        ['--access-token', 'pk.test', 'retrieve_features', 'juser', 'test'])
+    assert result.exit_code == 0
+    assert json.loads(result.output)['type'] == 'FeatureCollection'
+
+
+@responses.activate
+def test_cli_dataset_update_features():
+    """Features update works"""
+
+    def request_callback(request):
+        payload = json.loads(request.body)
+        assert payload['put'] == [{'type': 'Feature'}]
+        return (200, {}, "")
+
+    responses.add_callback(
+        responses.POST,
+        'https://api.mapbox.com/datasets/v1/juser/test/features?access_token=pk.test',
+        match_querystring=True,
+        callback=request_callback)
+
+    runner = CliRunner()
+    result = runner.invoke(
+        main_group,
+        ['--access-token', 'pk.test', 'update_features', 'juser', 'test', '--sequence'],
+        input=json.dumps({'type': 'Feature'}))
+    assert result.exit_code == 0