From a27db09be9d18d23a81b11239f300be01303bf93 Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 09:33:57 -0400 Subject: [PATCH 1/7] initial take on features_in_arg --- cligj/__init__.py | 19 +++++++++- cligj/features.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 cligj/features.py diff --git a/cligj/__init__.py b/cligj/__init__.py index 7b2873e..6e5337b 100755 --- a/cligj/__init__.py +++ b/cligj/__init__.py @@ -4,6 +4,8 @@ import click +from .features import normalize_feature_inputs + # Arguments. # Multiple input files. @@ -22,8 +24,23 @@ required=True, metavar="INPUTS... OUTPUT") -# Options. +# Features input +# Accepts multiple representations of GeoJSON features +# - Path to file(s), each containing single FeatureCollection or Feature +# - Coordinate pair(s) of the form "[0, 0]" or "0, 0" or "0 0" +# - STDIN stream containing +# - line-delimited features +# - ASCII Record Separator (0x1e) delimited features +# - FeatureCollection or Feature +# Returns the input data as an iterable of GeoJSON Feature-like dictionaries +features_in_arg = click.argument( + 'features', + nargs=-1, + callback=normalize_feature_inputs, + metavar="FEATURES...") + +# Options. verbose_opt = click.option( '--verbose', '-v', count=True, diff --git a/cligj/features.py b/cligj/features.py new file mode 100644 index 0000000..89adfaa --- /dev/null +++ b/cligj/features.py @@ -0,0 +1,95 @@ +from itertools import chain +import json +import re + +import click + + +def normalize_feature_inputs(ctx, param, features_like): + if len(features_like) == 0: + features_like = ('-',) + + for flike in features_like: + try: + # It's a file/stream with GeoJSON + src = iter(click.open_file(flike, mode='r')) + for feature in iter_features(src): + yield feature + except IOError: + # It's a coordinate string + coords = list(coords_from_query(flike)) + feature = { + 'type': 'Feature', + 'properties': {}, + 'geometry': { + 'type': 'Point', + 'coordinates': coords}} + yield feature + + +def iter_features(src): + """Yield features from a src that may be either a GeoJSON feature + text sequence or GeoJSON feature collection.""" + first_line = next(src) + # If input is RS-delimited JSON sequence. + if first_line.startswith(u'\x1e'): + buffer = first_line.strip(u'\x1e') + for line in src: + if line.startswith(u'\x1e'): + if buffer: + feat = json.loads(buffer) + yield feat + buffer = line.strip(u'\x1e') + else: + buffer += line + else: + feat = json.loads(buffer) + yield feat + else: + try: + feat = json.loads(first_line) + assert feat['type'] == 'Feature' + yield feat + for line in src: + feat = json.loads(line) + yield feat + except (TypeError, KeyError, AssertionError, ValueError): + text = "".join(chain([first_line], src)) + feats = json.loads(text) + if feats['type'] == 'Feature': + yield feats + elif feats['type'] == 'FeatureCollection': + for feat in feats['features']: + yield feat + +def iter_query(query): + """Accept a filename, stream, or string. + Returns an iterator over lines of the query.""" + try: + itr = click.open_file(query).readlines() + except IOError: + itr = [query] + return itr + + +def coords_from_query(query): + """Transform a query line into a (lng, lat) pair of coordinates.""" + try: + coords = json.loads(query) + except ValueError: + vals = re.split(r"\,*\s*", query.strip()) + coords = [float(v) for v in vals] + return tuple(coords[:2]) + + +def normalize_feature_objects(feature_objs): + for obj in feature_objs: + if hasattr(obj, "__geo_interface__") and \ + obj.__geo_interface__['type'] == 'Feature': + yield obj.__geo_interface__ + elif isinstance(obj, dict) and 'type' in obj and \ + obj['type'] == 'Feature': + yield obj + else: + raise ValueError("Did not recognize object {0}" + "as GeoJSON Feature".format(obj)) From 490d8f1fd31197203efb2788636bdccf1576bc6c Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 16:21:17 -0400 Subject: [PATCH 2/7] tests --- cligj/features.py | 9 +++ tests/onepoint.geojson | 4 ++ tests/test_features.py | 126 ++++++++++++++++++++++++++++++++++++++ tests/twopoints.geojson | 1 + tests/twopoints_seq.txt | 2 + tests/twopoints_seqrs.txt | 7 +++ 6 files changed, 149 insertions(+) create mode 100644 tests/onepoint.geojson create mode 100644 tests/test_features.py create mode 100644 tests/twopoints.geojson create mode 100644 tests/twopoints_seq.txt create mode 100644 tests/twopoints_seqrs.txt diff --git a/cligj/features.py b/cligj/features.py index 89adfaa..5e9a0d3 100644 --- a/cligj/features.py +++ b/cligj/features.py @@ -6,6 +6,11 @@ def normalize_feature_inputs(ctx, param, features_like): + """ Click callback which accepts the following values for features_like + [] == stdin + ["path", "path2.geojson"] == file paths + ["...", "..."] == or coords or geojson + """ if len(features_like) == 0: features_like = ('-',) @@ -83,8 +88,12 @@ def coords_from_query(query): def normalize_feature_objects(feature_objs): + """Takes an iterable of GeoJSON-like Feature mappings or + an iterable of objects with a geo interface and + normalizes it to the former.""" for obj in feature_objs: if hasattr(obj, "__geo_interface__") and \ + 'type' in obj.__geo_interface__.keys() and \ obj.__geo_interface__['type'] == 'Feature': yield obj.__geo_interface__ elif isinstance(obj, dict) and 'type' in obj and \ diff --git a/tests/onepoint.geojson b/tests/onepoint.geojson new file mode 100644 index 0000000..2323266 --- /dev/null +++ b/tests/onepoint.geojson @@ -0,0 +1,4 @@ +{"geometry": {"coordinates": [-122.7282, 45.5801], "type": "Point"}, +"id": "0", +"properties": {}, +"type": "Feature"} diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000..d33bebc --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,126 @@ +import json +import sys + +import pytest + +from cligj.features import \ + coords_from_query, iter_query, \ + normalize_feature_inputs, normalize_feature_objects + + +def test_iter_query_string(): + assert iter_query("lolwut") == ["lolwut"] + + +def test_iter_query_file(tmpdir): + filename = str(tmpdir.join('test.txt')) + with open(filename, 'w') as f: + f.write("lolwut") + assert iter_query(filename) == ["lolwut"] + + +def test_coords_from_query_json(): + assert coords_from_query("[-100, 40]") == (-100, 40) + + +def test_coords_from_query_csv(): + assert coords_from_query("-100, 40") == (-100, 40) + + +def test_coords_from_query_ws(): + assert coords_from_query("-100 40") == (-100, 40) + + +@pytest.fixture +def expected_features(): + with open("tests/twopoints.geojson") as src: + fc = json.loads(src.read()) + return fc['features'] + + +def _geoms(features): + geoms = [] + for feature in features: + geoms.append(feature['geometry']) + return geoms + + +def test_featurecollection_file(expected_features): + features = normalize_feature_inputs(None, 'features', ["tests/twopoints.geojson"]) + assert _geoms(features) == _geoms(expected_features) + + +def test_featurecollection_stdin(expected_features): + sys.stdin = open("tests/twopoints.geojson", 'r') + features = normalize_feature_inputs(None, 'features', []) + assert _geoms(features) == _geoms(expected_features) + + +def test_featuresequence(expected_features): + features = normalize_feature_inputs(None, 'features', ["tests/twopoints_seq.txt"]) + assert _geoms(features) == _geoms(expected_features) + +# TODO test path to sequence files fail + +def test_featuresequence_stdin(expected_features): + sys.stdin = open("tests/twopoints_seq.txt", 'r') + features = normalize_feature_inputs(None, 'features', []) + assert _geoms(features) == _geoms(expected_features) + + +def test_singlefeature(expected_features): + features = normalize_feature_inputs(None, 'features', ["tests/onepoint.geojson"]) + assert _geoms(features) == _geoms([expected_features[0]]) + + +def test_singlefeature_stdin(expected_features): + sys.stdin = open("tests/onepoint.geojson", 'r') + features = normalize_feature_inputs(None, 'features', []) + assert _geoms(features) == _geoms([expected_features[0]]) + + +def test_featuresequencers(expected_features): + features = normalize_feature_inputs(None, 'features', ["tests/twopoints_seqrs.txt"]) + assert _geoms(features) == _geoms(expected_features) + + +def test_featuresequencers_stdin(expected_features): + sys.stdin = open("tests/twopoints_seqrs.txt", 'r') + features = normalize_feature_inputs(None, 'features', []) + assert _geoms(features) == _geoms(expected_features) + + +def test_coordarrays(expected_features): + inputs = ["[-122.7282, 45.5801]", "[-121.3153, 44.0582]"] + features = normalize_feature_inputs(None, 'features', inputs) + assert _geoms(features) == _geoms(expected_features) + + +def test_coordpairs_comma(expected_features): + inputs = ["-122.7282, 45.5801", "-121.3153, 44.0582"] + features = normalize_feature_inputs(None, 'features', inputs) + assert _geoms(features) == _geoms(expected_features) + + +def test_coordpairs_space(expected_features): + inputs = ["-122.7282 45.5801", "-121.3153 44.0582"] + features = normalize_feature_inputs(None, 'features', inputs) + assert _geoms(features) == _geoms(expected_features) + + +class MockGeo(object): + def __init__(self, feature): + self.__geo_interface__ = feature + + +def test_normalize_feature_objects(expected_features): + objs = [MockGeo(f) for f in expected_features] + assert expected_features == list(normalize_feature_objects(objs)) + assert expected_features == list(normalize_feature_objects(expected_features)) + +def test_normalize_feature_objects_bad(expected_features): + objs = [MockGeo(f) for f in expected_features] + objs.append(MockGeo(dict())) + with pytest.raises(ValueError): + list(normalize_feature_objects(objs)) + diff --git a/tests/twopoints.geojson b/tests/twopoints.geojson new file mode 100644 index 0000000..70fb618 --- /dev/null +++ b/tests/twopoints.geojson @@ -0,0 +1 @@ +{"features": [{"bbox": [-122.9292140099711, 45.37948199034149, -122.44106199104115, 45.858097009742835], "center": [-122.7282, 45.5801], "context": [{"id": "postcode.2503633822", "text": "97203"}, {"id": "region.3470299826", "text": "Oregon"}, {"id": "country.4150104525", "short_code": "us", "text": "United States"}], "geometry": {"coordinates": [-122.7282, 45.5801], "type": "Point"}, "id": "place.42767", "place_name": "Portland, Oregon, United States", "properties": {}, "relevance": 0.999, "text": "Portland", "type": "Feature"}, {"bbox": [-121.9779540096568, 43.74737999114854, -120.74788099000016, 44.32812500969035], "center": [-121.3153, 44.0582], "context": [{"id": "postcode.3332732485", "text": "97701"}, {"id": "region.3470299826", "text": "Oregon"}, {"id": "country.4150104525", "short_code": "us", "text": "United States"}], "geometry": {"coordinates": [-121.3153, 44.0582], "type": "Point"}, "id": "place.3965", "place_name": "Bend, Oregon, United States", "properties": {}, "relevance": 0.999, "text": "Bend", "type": "Feature"}], "type": "FeatureCollection"} diff --git a/tests/twopoints_seq.txt b/tests/twopoints_seq.txt new file mode 100644 index 0000000..9fa84aa --- /dev/null +++ b/tests/twopoints_seq.txt @@ -0,0 +1,2 @@ +{"geometry": {"coordinates": [-122.7282, 45.5801], "type": "Point"}, "id": "0", "properties": {}, "type": "Feature"} +{"geometry": {"coordinates": [-121.3153, 44.0582], "type": "Point"}, "id": "1", "properties": {}, "type": "Feature"} diff --git a/tests/twopoints_seqrs.txt b/tests/twopoints_seqrs.txt new file mode 100644 index 0000000..2670e39 --- /dev/null +++ b/tests/twopoints_seqrs.txt @@ -0,0 +1,7 @@ +{"geometry": {"coordinates": [-122.7282, 45.5801], "type": "Point"}, "id": "0", "properties": {}, "type": "Feature"} +{"geometry": { "coordinates": + [-121.3153, 44.0582], + "type": "Point"}, + "id": "1", + "properties": {}, + "type": "Feature"} From 015575bc4e2dd915e2484cf09e3a919a098a563e Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 16:23:59 -0400 Subject: [PATCH 3/7] docs --- README.rst | 64 +++++++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 42 deletions(-) diff --git a/README.rst b/README.rst index 82475b8..652ee54 100755 --- a/README.rst +++ b/README.rst @@ -9,6 +9,16 @@ cligj Common arguments and options for GeoJSON processing commands, using Click. +TODO what it does and who it is for + +Arguments +--------- +TODO + +Options +-------- +TODO + Example ------- @@ -30,11 +40,10 @@ a delimiter, use the ``--rs`` option import json @click.command() + @cligj.features_in_arg @cligj.sequence_opt @cligj.use_rs_opt - def features(sequence, use_rs): - features = [ - {'type': 'Feature', 'id': '1'}, {'type': 'Feature', 'id': '2'}] + def pass_features(features, sequence, use_rs): if sequence: for feature in features: if use_rs: @@ -42,20 +51,26 @@ a delimiter, use the ``--rs`` option click.echo(json.dumps(feature)) else: click.echo(json.dumps( - {'type': 'FeatureCollection', 'features': features})) + {'type': 'FeatureCollection', 'features': list(features)})) On the command line it works like this. .. code-block:: console - $ features + $ cat data.geojson {'type': 'FeatureCollection', 'features': [{'type': 'Feature', 'id': '1'}, {'type': 'Feature', 'id': '2'}]} - $ features --sequence + $ pass_features data.geojson + {'type': 'FeatureCollection', 'features': [{'type': 'Feature', 'id': '1'}, {'type': 'Feature', 'id': '2'}]} + + $ cat data.geojson | pass_features + {'type': 'FeatureCollection', 'features': [{'type': 'Feature', 'id': '1'}, {'type': 'Feature', 'id': '2'}]} + + $ cat data.geojson | pass_features --sequence {'type': 'Feature', 'id': '1'} {'type': 'Feature', 'id': '2'} - $ features --sequence --rs + $ cat data.geojson | pass_features --sequence --rs ^^{'type': 'Feature', 'id': '1'} ^^{'type': 'Feature', 'id': '2'} @@ -69,38 +84,3 @@ Plugins The cligj.plugins module is deprecated and will be removed at version 1.0. Use `click-plugins `_ instead. - -``cligj`` can also facilitate loading external `click-based -`_ plugins via `setuptools entry points -`_. -The ``cligj.plugins`` module contains a special ``group()`` decorator that -behaves exactly like ``click.group()`` except that it offers the opportunity -load plugins and attach them to the group as it is istantiated. - -.. code-block:: python - - from pkg_resources import iter_entry_points - - import cligj.plugins - import click - - @cligj.plugins.group(plugins=iter_entry_points('module.entry_points')) - def cli(): - - """A CLI application.""" - - pass - - @cli.command() - @click.argument('arg') - def printer(arg): - - """Print arg.""" - - click.echo(arg) - - @cli.group(plugins=iter_entry_points('other_module.more_plugins')) - def plugins(): - - """A sub-group that contains plugins from a different module.""" - pass From 71c69abbd5a7bbfcce164886684cb7929fb01a28 Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 16:32:12 -0400 Subject: [PATCH 4/7] encourage processing in an external function --- README.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 652ee54..75a77c0 100755 --- a/README.rst +++ b/README.rst @@ -39,19 +39,25 @@ a delimiter, use the ``--rs`` option import cligj import json + def process_features(features) + for feature in features + # TODO process feature here + yield feature + @click.command() @cligj.features_in_arg @cligj.sequence_opt @cligj.use_rs_opt def pass_features(features, sequence, use_rs): if sequence: - for feature in features: + for feature in process_features(features): if use_rs: click.echo(b'\x1e', nl=False) click.echo(json.dumps(feature)) else: click.echo(json.dumps( - {'type': 'FeatureCollection', 'features': list(features)})) + {'type': 'FeatureCollection', + 'features': list(process_features(features))})) On the command line it works like this. From 9e70160e77d9c140e3f381b6c18e8b29b3ee618b Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 18:45:37 -0400 Subject: [PATCH 5/7] better docs --- README.rst | 39 ++++++++++++++++++++++++++++++++++++--- cligj/__init__.py | 6 ------ cligj/features.py | 12 ++++++++---- tests/test_features.py | 2 +- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 75a77c0..f194623 100755 --- a/README.rst +++ b/README.rst @@ -13,11 +13,44 @@ TODO what it does and who it is for Arguments --------- -TODO + + +``files_in_arg`` +Multiple files + +``files_inout_arg`` +Multiple files, last of which is an output file. + +``features_in_arg`` +GeoJSON Features input which accepts multiple representations of GeoJSON features +and returns the input data as an iterable of GeoJSON Feature-like dictionaries Options -------- -TODO +``verbose_opt`` +``quiet_opt`` +``format_opt`` + +JSON formatting options. +``indent_opt`` +``compact_opt`` + +Coordinate precision option. +``precision_opt`` + +Geographic (default), projected, or Mercator switch. +``projection_geographic_opt`` +``projection_projected_opt`` +``projection_mercator_opt`` + +Feature collection or feature sequence switch. +``sequence_opt`` +``use_rs_opt`` + +GeoJSON output mode option. +``geojson_type_collection_opt`` +``geojson_type_feature_opt`` +``def geojson_type_bbox_opt`` Example ------- @@ -39,7 +72,7 @@ a delimiter, use the ``--rs`` option import cligj import json - def process_features(features) + def process_features(features): for feature in features # TODO process feature here yield feature diff --git a/cligj/__init__.py b/cligj/__init__.py index 6e5337b..aa4ef97 100755 --- a/cligj/__init__.py +++ b/cligj/__init__.py @@ -26,12 +26,6 @@ # Features input # Accepts multiple representations of GeoJSON features -# - Path to file(s), each containing single FeatureCollection or Feature -# - Coordinate pair(s) of the form "[0, 0]" or "0, 0" or "0 0" -# - STDIN stream containing -# - line-delimited features -# - ASCII Record Separator (0x1e) delimited features -# - FeatureCollection or Feature # Returns the input data as an iterable of GeoJSON Feature-like dictionaries features_in_arg = click.argument( 'features', diff --git a/cligj/features.py b/cligj/features.py index 5e9a0d3..97bb553 100644 --- a/cligj/features.py +++ b/cligj/features.py @@ -6,10 +6,14 @@ def normalize_feature_inputs(ctx, param, features_like): - """ Click callback which accepts the following values for features_like - [] == stdin - ["path", "path2.geojson"] == file paths - ["...", "..."] == or coords or geojson + """ Click callback which accepts the following values: + * Path to file(s), each containing single FeatureCollection or Feature + * Coordinate pair(s) of the form "[0, 0]" or "0, 0" or "0 0" + * if not specified or '-', process STDIN stream containing + - line-delimited features + - ASCII Record Separator (0x1e) delimited features + - FeatureCollection or Feature object + and yields GeoJSON Features. """ if len(features_like) == 0: features_like = ('-',) diff --git a/tests/test_features.py b/tests/test_features.py index d33bebc..3ccc76f 100644 --- a/tests/test_features.py +++ b/tests/test_features.py @@ -118,9 +118,9 @@ def test_normalize_feature_objects(expected_features): assert expected_features == list(normalize_feature_objects(objs)) assert expected_features == list(normalize_feature_objects(expected_features)) + def test_normalize_feature_objects_bad(expected_features): objs = [MockGeo(f) for f in expected_features] objs.append(MockGeo(dict())) with pytest.raises(ValueError): list(normalize_feature_objects(objs)) - From 3221b1ec60c62d29d701590b3c3998b47b2fa0cd Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 18:55:58 -0400 Subject: [PATCH 6/7] how do I rst? --- README.rst | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f194623..6a72770 100755 --- a/README.rst +++ b/README.rst @@ -28,28 +28,42 @@ and returns the input data as an iterable of GeoJSON Feature-like dictionaries Options -------- ``verbose_opt`` + ``quiet_opt`` + ``format_opt`` -JSON formatting options. +JSON formatting options +~~~~~~~~~~~~~~~~~~~~~~~ + ``indent_opt`` + ``compact_opt`` -Coordinate precision option. +Coordinate precision option +~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``precision_opt`` -Geographic (default), projected, or Mercator switch. +Geographic (default), projected, or Mercator switch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``projection_geographic_opt`` + ``projection_projected_opt`` + ``projection_mercator_opt`` -Feature collection or feature sequence switch. +Feature collection or feature sequence switch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``sequence_opt`` + ``use_rs_opt`` -GeoJSON output mode option. +GeoJSON output mode option +~~~~~~~~~~~~~~~~~~~~~~~~~~ ``geojson_type_collection_opt`` + ``geojson_type_feature_opt`` + ``def geojson_type_bbox_opt`` Example From 24cc9ef2557e9d3def8ceb1a7d613ebf1092440f Mon Sep 17 00:00:00 2001 From: Matthew Perry Date: Sat, 12 Dec 2015 19:21:55 -0400 Subject: [PATCH 7/7] add usage and intro --- README.rst | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 6a72770..2e22d44 100755 --- a/README.rst +++ b/README.rst @@ -9,12 +9,13 @@ cligj Common arguments and options for GeoJSON processing commands, using Click. -TODO what it does and who it is for +`cligj` is for Python developers who create command line interfaces for geospatial data. +`cligj` allows you to quickly build consistent, well-tested and interoperable CLIs for handling GeoJSON. + Arguments --------- - ``files_in_arg`` Multiple files @@ -27,6 +28,7 @@ and returns the input data as an iterable of GeoJSON Feature-like dictionaries Options -------- + ``verbose_opt`` ``quiet_opt`` @@ -87,7 +89,7 @@ a delimiter, use the ``--rs`` option import json def process_features(features): - for feature in features + for feature in features: # TODO process feature here yield feature @@ -106,7 +108,25 @@ a delimiter, use the ``--rs`` option {'type': 'FeatureCollection', 'features': list(process_features(features))})) -On the command line it works like this. +On the command line, the generated help text explains the usage + +.. code-block:: console + + Usage: pass_features [OPTIONS] FEATURES... + + Options: + --sequence / --no-sequence Write a LF-delimited sequence of texts + containing individual objects or write a single + JSON text containing a feature collection object + (the default). + --rs / --no-rs Use RS (0x1E) as a prefix for individual texts + in a sequence as per http://tools.ietf.org/html + /draft-ietf-json-text-sequence-13 (default is + False). + --help Show this message and exit. + + +And can be used like this .. code-block:: console