GL JS 0.44.1 and JSON df_to_geojson performance and data type improve…

…ments (#44) * upgrade to GL JS 0.44.1 * added geojson lib requirement * refactored df_to_geojson to use vectorized logic and pandas to_dict parsing to handle date conversion * fixed bugs in new geojson export from DF * updated examples * bug fix - df.columns.values * updated tests to use pandas dataframes instead of mock
mapbox · Feb 19, 2018 · a83965d · a83965d
1 parent 31042a3
commit a83965d
Show file tree

Hide file tree

Showing 9 changed files with 6,804 additions and 6,511 deletions.
diff --git a/examples/point-viz-types-example.ipynb b/examples/point-viz-types-example.ipynb
diff --git a/examples/points.csv b/examples/points.csv
diff --git a/examples/points1.geojson b/examples/points1.geojson
diff --git a/examples/viz4.html b/examples/viz4.html
@@ -0,0 +1,288 @@
+<!DOCTYPE html>
+<html>
+<head>
+<title>mapboxgl-jupyter viz</title>
+<meta charset='UTF-8' />
+<meta name='viewport'
+      content='initial-scale=1,maximum-scale=1,user-scalable=no' />
+<script type='text/javascript'
+        src='https://api.tiles.mapbox.com/mapbox-gl-js/v0.44.1/mapbox-gl.js'></script>
+<link type='text/css'
+      href='https://api.tiles.mapbox.com/mapbox-gl-js/v0.44.1/mapbox-gl.css' rel='stylesheet' />
+<style type='text/css'>
+    body { margin:0; padding:0; }
+    .map { position:absolute; top:0; bottom:0; width:100%; }
+    .legend {
+        background-color: white;
+        color: black;
+        border-radius: 3px;
+        bottom: 50px;
+        width: 100px;
+        box-shadow: 0 1px 2px rgba(0, 0, 0, 0.10);
+        font: 12px/20px 'Helvetica Neue', Arial, Helvetica, sans-serif;
+        padding: 12px;
+        position: absolute;
+        right: 10px;
+        z-index: 1;
+    }
+    .legend h4 { margin: 0 0 10px; }
+    .legend-title {
+        margin: 6px;
+        padding: 6px:
+        font-weight: bold;
+        font-size: 14px;
+        font: 12px/20px 'Helvetica Neue', Arial, Helvetica, sans-serif;
+    }
+    .legend div span {
+        border-radius: 50%;
+        display: inline-block;
+        height: 10px;
+        margin-right: 5px;
+        width: 10px;
+    }
+</style>
+</head>
+<body>
+
+<div id='map' class='map'></div>
+<div id='legend' class='legend'></div>
+
+<script type='text/javascript'>
+function calcCircleColorLegend(myColorStops, title) {
+    //Calculate a legend element on a Mapbox GL Style Spec property function stops array
+    var mytitle = document.createElement('div');
+    mytitle.textContent = title;
+    mytitle.className = 'legend-title'
+    var legend = document.getElementById('legend');
+    legend.appendChild(mytitle);
+
+    for (p = 0; p < myColorStops.length; p++) {
+        if (!!document.getElementById('legend-points-value-' + p)) {
+            //update the legend if it already exists
+            document.getElementById('legend-points-value-' + p).textContent = myColorStops[p][0];
+            document.getElementById('legend-points-id-' + p).style.backgroundColor = myColorStops[p][1];
+        } else {
+            //create the legend if it doesn't yet exist
+            var item = document.createElement('div');
+            var key = document.createElement('span');
+            key.className = 'legend-key';
+            var value = document.createElement('span');
+
+            key.id = 'legend-points-id-' + p;
+            key.style.backgroundColor = myColorStops[p][1];
+            value.id = 'legend-points-value-' + p;
+
+            item.appendChild(key);
+            item.appendChild(value);
+            legend.appendChild(item);
+
+            data = document.getElementById('legend-points-value-' + p)
+            data.textContent = myColorStops[p][0];
+        }
+    }
+}
+
+function generateInterpolateExpression(propertyValue, stops) {
+    var expression;
+    if (propertyValue == 'zoom') {
+        expression = ['interpolate', ['linear'], ['zoom']]
+    }
+    else if (propertyValue == 'heatmap-density') {
+        expression = ['interpolate', ['linear'], ['heatmap-density']]
+    }
+    else {
+        expression = ['interpolate', ['linear'], ['get', propertyValue]]
+    }
+
+    for (var i=0; i<stops.length; i++) {
+        expression.push(stops[i][0], stops[i][1])
+    }
+    return expression
+}
+
+
+function generateMatchExpression(propertyValue, stops, defaultValue) {
+    var expression;
+    expression = ['match', ['get', propertyValue]]
+    for (var i=0; i<stops.length; i++) {
+        expression.push(stops[i][0], stops[i][1])
+    }
+    expression.push(defaultValue)
+
+    return expression
+}
+
+
+function generatePropertyExpression(expressionType, propertyValue, stops, defaultValue) {
+    var expression;
+    if (expressionType == 'match') {
+        expression = generateMatchExpression(propertyValue, stops, defaultValue)
+    }
+    else {
+        expression = generateInterpolateExpression(propertyValue, stops)
+    }
+
+    return expression
+}
+
+
+</script>
+
+<!-- main map creation code, extended by mapboxgl/templates/clustered_circle.html -->
+<script type="text/javascript">
+
+mapboxgl.accessToken = 'pk.eyJ1IjoicnNiYXVtYW5uIiwiYSI6IjdiOWEzZGIyMGNkOGY3NWQ4ZTBhN2Y5ZGU2Mzg2NDY2In0.jycgv7qwF8MMIWt4cT0RaQ';
+
+var map = new mapboxgl.Map({
+    container: 'map',
+    style: 'mapbox://styles/mapbox/light-v9?optimize=true', 
+    center: [-95, 40],
+    zoom: 3,
+    transformRequest: (url, resourceType)=> {
+        if ( url.slice(0,22) == 'https://api.mapbox.com' ) {
+            //Add Python Plugin identifier for Mapbox API traffic
+            return {
+               url: [url.slice(0, url.indexOf("?")+1), "pluginName=PythonMapboxgl&", url.slice(url.indexOf("?")+1)].join('')
+             }
+         }
+         else {
+             //Do not transform URL for non Mapbox GET requests
+             return {url: url}
+         }
+    }
+});
+
+map.addControl(new mapboxgl.NavigationControl());
+
+calcCircleColorLegend([[1, 'rgb(166,97,26)'], [10, 'rgb(223,194,125)'], [50, 'rgb(128,205,193)'], [100, 'rgb(1,133,113)']], "Point Density");
+
+map.on('style.load', function() {
+
+    map.addSource("data", {
+        "type": "geojson",
+        "data": "points1.geojson", //data from dataframe output to geojson
+        "buffer": 0,
+        "maxzoom": 10 + 1,
+        "cluster": true,
+        "clusterMaxZoom": 10,
+        "clusterRadius": 30
+    });
+
+    map.addLayer({
+        "id": "label",
+        "source": "data",
+        "type": "symbol",
+        "maxzoom": 24,
+        "minzoom": 0,
+        "layout": {
+            "text-field": "{point_count_abbreviated}",
+            "text-size" : generateInterpolateExpression('zoom', [[0,8],[22,16]] )
+        },
+        "paint": {
+            "text-halo-color": "white",
+            "text-halo-width": 1
+        }
+    }, "" )
+
+    map.addLayer({
+        "id": "circle-cluster",
+        "source": "data",
+        "type": "circle",
+        "maxzoom": 24,
+        "minzoom": 0,
+        "filter": ["has", "point_count"],
+        "paint": {
+            "circle-color": generateInterpolateExpression( "point_count", [[1, 'rgb(166,97,26)'], [10, 'rgb(223,194,125)'], [50, 'rgb(128,205,193)'], [100, 'rgb(1,133,113)']] ),
+            "circle-radius" : generateInterpolateExpression( "point_count", [[1, 5], [10, 10], [50, 15], [100, 20]] ),
+            "circle-stroke-color": "grey",
+            "circle-stroke-width": generateInterpolateExpression('zoom', [[0,0.01], [18,1]]),
+            "circle-opacity" : 0.9
+        }
+    }, "label");
+
+    map.addLayer({
+        "id": "circle-unclustered",
+        "source": "data",
+        "type": "circle",
+        "maxzoom": 24,
+        "minzoom": 0,
+        "filter": ["!has", "point_count"],
+        "paint": {
+            "circle-color": "rgb(166,97,26)",
+            "circle-radius" : generateInterpolateExpression( 'zoom', [[0,1], [22,10]]),
+            "circle-stroke-color": "grey",
+            "circle-stroke-width": generateInterpolateExpression('zoom', [[0,0.01], [18,1]]),
+            "circle-opacity" : 0.9
+        }
+    }, "circle-cluster");
+
+    // Create a popup
+    var popup = new mapboxgl.Popup({
+        closeButton: false,
+        closeOnClick: false
+    });
+
+    // Show the popup on mouseover
+    map.on('mousemove', 'circle-unclustered', function(e) {
+        map.getCanvas().style.cursor = 'pointer';
+        let f = e.features[0];
+        let popup_html = '<div><li><b>Location</b>: ' + f.geometry.coordinates[0].toPrecision(6) + 
+            ', ' + f.geometry.coordinates[1].toPrecision(6) + '</li>';
+
+        for (key in f.properties) {
+            popup_html += '<li><b> ' + key + '</b>: ' + f.properties[key] + ' </li>'
+        }
+
+        popup_html += '</div>'
+
+        popup.setLngLat(e.features[0].geometry.coordinates)
+            .setHTML(popup_html)
+            .addTo(map);
+    });
+
+    map.on('mousemove', 'circle-cluster', function(e) {
+        map.getCanvas().style.cursor = 'pointer';
+        let f = e.features[0];
+        let popup_html = '<div><li><b>Location</b>: ' + f.geometry.coordinates[0].toPrecision(6) + 
+            ', ' + f.geometry.coordinates[1].toPrecision(6) + '</li>';
+
+        popup_html += '<li><b>Point Count:</b>: ' + f.properties.point_count + ' </li>'
+
+        popup_html += '</div>'
+
+        popup.setLngLat(e.features[0].geometry.coordinates)
+            .setHTML(popup_html)
+            .addTo(map);
+    });
+
+    map.on('mouseleave', 'circle-unclustered', function() {
+        map.getCanvas().style.cursor = '';
+        popup.remove();
+    });
+
+    map.on('mouseleave', 'circle-cluster', function() {
+        map.getCanvas().style.cursor = '';
+        popup.remove();
+    });
+
+
+    map.on('click', 'circle-unclustered', function(e) {
+        map.easeTo({
+            center: e.features[0].geometry.coordinates,
+            zoom: map.getZoom() + 1
+        });
+    });
+
+    map.on('click', 'circle-cluster', function(e) {
+        map.easeTo({
+            center: e.features[0].geometry.coordinates,
+            zoom: map.getZoom() + 1
+        });
+    });
+});
+
+
+</script>
+
+</body>
+</html>
diff --git a/mapboxgl/utils.py b/mapboxgl/utils.py
@@ -1,66 +1,66 @@
 from .colors import color_ramps
+import geojson
 import json
 
 
-def df_to_geojson(df, properties=None, lat='lat', lon='lon', precision=None, filename=None):
-    """Serialize a Pandas dataframe to a geojson format Python dictionary
+def row_to_geojson(row, lon, lat):
+    """Convert a pandas dataframe row to a geojson format object.  Converts all datetimes to epoch seconds.
     """
-    geojson = {'type': 'FeatureCollection', 'features': []}
 
-    if filename:
-        with open(filename, 'w+') as f:
-            # Overwrite file if it already exists
-            pass
+    # Let pandas handle json serialization
+    row_json = json.loads(row.to_json(date_format='epoch', date_unit='s'))
+    return geojson.Feature(geometry=geojson.Point((row_json[lon], row_json[lat])),
+                           properties={key: row_json[key] for key in row_json.keys() if key not in [lon, lat]})
 
+
+def df_to_geojson(df, properties=None, lat='lat', lon='lon', precision=None, filename=None):
+    """Serialize a Pandas dataframe to a geojson format Python dictionary
+    """
     if precision:
-        df[lat] = df[lat].round(precision)
         df[lon] = df[lon].round(precision)
+        df[lat] = df[lat].round(precision)
 
     if not properties:
-        properties = [c for c in df.columns if c not in [lat, lon]]
+        # if no properties are selected, use all properties in dataframe
+        properties = [c for c in df.columns if c not in [lon, lat]]
+
+    for prop in properties:
+        # Check if list of properties exists in dataframe columns
+        if prop not in list(df.columns):
+            raise ValueError(
+                'properties must be a valid list of column names from dataframe')
+        if prop in [lon, lat]:
+            raise ValueError(
+                'properties cannot be the geometry longitude or latitude column')
 
     if filename:
+        with open(filename, 'w+') as f:
+            # Overwrite file if it already exists
+            pass
+
         with open(filename, 'a+') as f:
+            features = []
+            df[[lon, lat] + properties].apply(lambda x: features.append(
+                row_to_geojson(x, lon, lat)), axis=1)
+
             f.write('{"type": "FeatureCollection", "features": [\n')
-            rowcount = 0
-            for idx, row in df.iterrows():
-                feature = {
-                    'type': 'Feature',
-                    'properties': {},
-                    'geometry': {
-                        'type': 'Point',
-                        'coordinates': [row[lon], row[lat]]}
-                }
-                for prop in properties:
-                    feature['properties'][prop] = row[prop]
-                if rowcount == 0:
-                    f.write(json.dumps(feature, ensure_ascii=False, sort_keys=True) + '\n')
-                    rowcount+=1
+            for idx, feat in enumerate(features):
+                if idx == 0:
+                    f.write(geojson.dumps(feat) + '\n')
                 else:
-                    f.write(',' + json.dumps(feature,
-                                             ensure_ascii=False, sort_keys=True) + '\n')
-                    rowcount+=1
+                    f.write(',' + geojson.dumps(feat) + '\n')
             f.write(']}')
+
             return {
                 "type": "file",
                 "filename": filename,
-                "feature_count": rowcount
+                "feature_count": len(features)
             }
     else:
-        for idx, row in df.iterrows():
-            feature = {
-                'type': 'Feature',
-                'properties': {},
-                'geometry': {
-                    'type': 'Point',
-                    'coordinates': [row[lon], row[lat]]}
-            }
-            for prop in properties:
-                feature['properties'][prop] = row[prop]
-
-            geojson['features'].append(feature)
-
-        return geojson
+        features = []
+        df[[lon, lat] + properties].apply(lambda x: features.append(
+            row_to_geojson(x, lon, lat)), axis=1)
+        return geojson.FeatureCollection(features)
 
 
 def scale_between(minval, maxval, numStops):