Merge pull request #181 from jpolchlo/png

Add PNG rendering functionality

Makefile

@@ -21,6 +21,8 @@ ${BUILD-ASSEMBLY}: $(call rwildcard, geopyspark-backend/, *.scala)
 ${WHEEL}: ${DIST-ASSEMBLY} $(call rwildcard, geopyspark, *.py) setup.py
 	${PYTHON} setup.py bdist_wheel
 
+wheel: ${WHEEL}
+
 pyspark: ${DIST-ASSEMBLY}
 	pyspark --jars ${DIST-ASSEMBLY}
 

docker/Makefile

@@ -5,7 +5,7 @@ WHL := archives/geopyspark-0.1.0-py3-none-any.whl
 JAR := archives/geotrellis-backend-assembly-0.1.0.jar
 GDAL-BLOB := archives/gdal-and-friends.tar.gz
 PYTHON-BLOB := archives/geopyspark-and-friends.tar.gz
-VERSION := 3
+VERSION := 4
 STAGE0 := jamesmcclain/jupyter-geopyspark:stage0
 STAGE1 := quay.io/geodocker/jupyter-geopyspark:$(VERSION)
 GEONOTEBOOK := dcfcd81ac3e928fc35fe83a5a937561ccee3aa19

docker/config/patch.diff

@@ -102,15 +102,16 @@ index 0000000..3e3a150
 +__all__ = ('GeoTrellis')
 diff --git a/geonotebook/vis/geotrellis/geotrellis.py b/geonotebook/vis/geotrellis/geotrellis.py
 new file mode 100644
-index 0000000..2c7fceb
+index 0000000..029493e
 --- /dev/null
 +++ b/geonotebook/vis/geotrellis/geotrellis.py
-@@ -0,0 +1,89 @@
+@@ -0,0 +1,84 @@
 +import requests
 +import threading
 +import time
 +
 +from notebook.base.handlers import IPythonHandler
++from random import randint
 +from .server import moop
 +
 +
@@ -145,7 +146,7 @@ index 0000000..2c7fceb
 +        self.base_url = url
 +        self.pyramids = {}
 +        self.server_active = False
-+        self.port = 8033 # XXX
++        self.port = randint(49152, 65535) # XXX
 +
 +    def start_kernel(self, kernel):
 +        pass
@@ -169,46 +170,39 @@ index 0000000..2c7fceb
 +
 +    def ingest(self, data, name, **kwargs):
 +        from geopyspark.geotrellis.rdd import RasterRDD, TiledRasterRDD
-+        from geopyspark.geotrellis.constants import ZOOM
++        from geopyspark.geotrellis.render import PngRDD
 +
 +        rdd = data.rdd
 +        if isinstance(rdd, RasterRDD):
 +            metadata = rdd.collect_metadata()
 +            laid_out = rdd.tile_to_layout(metadata)
-+            reprojected = laid_out.reproject("EPSG:3857", scheme=ZOOM)
++            png = PngRDD.makePyramid(laid_out, data.rampname)
 +        elif isinstance(rdd, TiledRasterRDD):
 +            laid_out = rdd
-+            reprojected = laid_out.reproject("EPSG:3857", scheme=ZOOM)
++            png = PngRDD.makePyramid(laid_out, data.rampname)
++        elif isinstance(rdd, PngRDD):
++            png = rdd
 +        else:
-+            raise Exception
++            raise TypeError("Expected a RasterRDD, TiledRasterRDD, or PngRDD")
 +
-+        rdds = {}
-+        for layer_rdd in reprojected.pyramid(reprojected.zoom_level, 0):
-+            rdds[layer_rdd.zoom_level] = layer_rdd
-+        # self.pyramids.update({name: rdds})
-+        self.pyramids[name] = rdds
-+
-+        if self.server_active == False:
-+            t = threading.Thread(target=moop, args=(self.pyramids,))
-+            t.start()
-+            self.server_active = True
++        t = threading.Thread(target=moop, args=(png, self.port))
++        t.start()
 +
 +        self.base_url = "http://localhost:8000/user/hadoop/geotrellis" # XXX
 +        return self.base_url + "/" + str(self.port) + "/" + name
 diff --git a/geonotebook/vis/geotrellis/server.py b/geonotebook/vis/geotrellis/server.py
 new file mode 100644
-index 0000000..e121d55
+index 0000000..c7f7451
 --- /dev/null
 +++ b/geonotebook/vis/geotrellis/server.py
-@@ -0,0 +1,97 @@
+@@ -0,0 +1,69 @@
 +import io
 +import numpy as np
 +import rasterio
 +import threading
 +import time
 +
-+from PIL import Image
-+
++from PIL import Image, ImageDraw, ImageFont
 +
 +def make_image(arr):
 +    return Image.fromarray(arr.astype('uint8')).convert('L')
@@ -232,71 +226,44 @@ index 0000000..e121d55
 +alpha = np.vectorize(alpha)
 +lock = threading.Lock()
 +
-+def moop(pyramids):
++def moop(png, port):
 +    from flask import Flask, make_response, abort
 +
 +    app = Flask(__name__)
 +    app.reader = None
 +
-+    @app.route('/time')
++    @app.route("/time")
 +    def ping():
-+        return time.strftime("%H:%M:%S")
-+
-+    @app.route("/exists/<layer_name>")
-+    def exists(layer_name):
-+        return str(layer_name in pyramids)
-+
-+    @app.route("/remove/<layer_name>")
-+    def remove(layer_name):
-+        if layer_name in pyramids:
-+            del pyramids[layer_name]
-+            return 'yes'
-+        else:
-+            return 'no'
++        return time.strftime("%H:%M:%S") + "\n"
 +
 +    @app.route("/tile/<layer_name>/<int:x>/<int:y>/<int:zoom>.png")
 +    def tile(layer_name, x, y, zoom):
 +
 +        # fetch data
 +        try:
-+            lock.acquire()
-+            pyramid = pyramids[layer_name]
-+            rdd = pyramid[zoom]
-+            tile = rdd.lookup(col=x, row=y)
-+            arr = tile[0]['data']
++            img = png.lookup(x, y, zoom)
 +        except:
-+            arr = None
-+        finally:
-+            lock.release()
-+
-+        if arr == None:
-+            abort(404)
++            img = None
++
++        if img == None or len(img) == 0:
++            if png.debug:
++                image = Image.new('RGBA', (256,256))
++                draw = ImageDraw.Draw(image)
++                draw.rectangle([0, 0, 255, 255], outline=(255,0,0,255))
++                draw.line([(0,0),(255,255)], fill=(255,0,0,255))
++                draw.line([(0,255),(255,0)], fill=(255,0,0,255))
++                draw.text((136,122), str(x) + ', ' + str(y) + ', ' + str(zoom), fill=(255,0,0,255))
++                del draw
++                bio = io.BytesIO()
++                image.save(bio, 'PNG')
++                img = [bio.getvalue()]
++            else:
++                abort(404)
 +
-+        bands = arr.shape[0]
-+        if bands >= 3:
-+            bands = 3
-+        else:
-+            bands = 1
-+        arrs = [np.array(arr[i, :, :]).reshape(256, 256) for i in range(bands)]
-+
-+        # create tile
-+        if bands == 3:
-+            images = [make_image(clamp(arr)) for arr in arrs]
-+            images.append(make_image(alpha(arrs[0])))
-+            image = Image.merge('RGBA', images)
-+        else:
-+            gray = make_image(clamp(arrs[0]))
-+            alfa = make_image(alpha(arrs[0]))
-+            image = Image.merge('RGBA', list(gray, gray, gray, alfa))
-+
-+        # return tile
-+        bio = io.BytesIO()
-+        image.save(bio, 'PNG')
-+        response = make_response(bio.getvalue())
++        response = make_response(img[0])
 +        response.headers['Content-Type'] = 'image/png'
 +        return response
 +
-+    port = 8033
 +    app.run(host='0.0.0.0', port=port)
 diff --git a/geonotebook/wrappers/__init__.py b/geonotebook/wrappers/__init__.py
 index f95e8af..6a0e86b 100644
@@ -311,30 +278,28 @@ index f95e8af..6a0e86b 100644
 -__all__ = ('RasterData', 'RasterDataCollection', 'VectorData')
 +__all__ = ('RddRasterData', 'RasterData', 'RasterDataCollection', 'VectorData')
 diff --git a/geonotebook/wrappers/raster.py b/geonotebook/wrappers/raster.py
-index b2a83d3..c7232b4 100644
+index b2a83d3..115dd83 100644
 --- a/geonotebook/wrappers/raster.py
 +++ b/geonotebook/wrappers/raster.py
-@@ -9,6 +9,26 @@ import pkg_resources as pr
+@@ -9,6 +9,24 @@ import pkg_resources as pr
  from shapely.geometry import Polygon
 
 
 +class RddRasterData(object):
 +
-+    def __init__(self, rdd, name=None, remapper=None):
++    def __init__(self, rdd, name=None, rampname="Viridis"):
 +        from geopyspark.geotrellis.rdd import RasterRDD, TiledRasterRDD
++        from geopyspark.geotrellis.render import PngRDD
 +
-+        if not (isinstance(rdd, RasterRDD) or isinstance(rdd, TiledRasterRDD)):
-+            raise Exception
++        if not (isinstance(rdd, RasterRDD) or isinstance(rdd, TiledRasterRDD) or isinstance(rdd, PngRDD)):
++            raise TypeError("Expected a RasterRDD, TiledRasterRDD, or PngRDD")
 +
 +        self.rdd = rdd
 +
-+        if not remapper:
-+            self.remapper = lambda x: x
-+        else:
-+            self.remapper = remapper
++        self.rampname = rampname
 +
 +        if not name:
-+            self.name = str(abs(hash(rdd) + hash(remapper)))
++            self.name = str(abs(hash(rdd) + hash(rampname)))
 +        else:
 +            self.name = name
 +

geopyspark-backend/geotrellis/src/main/scala/geopyspark/geotrellis/PngRDD.scala

@@ -0,0 +1,90 @@
+package geopyspark.geotrellis
+
+import org.apache.spark.rdd.RDD
+
+import geotrellis.raster._
+import geotrellis.raster.histogram._
+import geotrellis.raster.render._
+import geotrellis.spark._
+import geotrellis.spark.render._
+
+import scala.reflect._
+
+object Coloring {
+  def getNamedRamp(name: String): ColorRamp = {
+    name match {
+      case "hot" => ColorRamps.HeatmapDarkRedToYellowWhite
+      case "coolwarm" => ColorRamps.BlueToRed
+      case "magma" => ColorRamps.Magma
+      case "inferno" => ColorRamps.Inferno
+      case "plasma" => ColorRamps.Plasma
+      case "viridis" => ColorRamps.Viridis
+
+      case "BlueToOrange" => ColorRamps.BlueToOrange
+      case "LightYellowToOrange" => ColorRamps.LightYellowToOrange
+      case "BlueToRed" => ColorRamps.BlueToRed
+      case "GreenToRedOrange" => ColorRamps.GreenToRedOrange
+      case "LightToDarkSunset" => ColorRamps.LightToDarkSunset
+      case "LightToDarkGreen" => ColorRamps.LightToDarkGreen
+      case "HeatmapYellowToRed" => ColorRamps.HeatmapYellowToRed
+      case "HeatmapBlueToYellowToRedSpectrum" => ColorRamps.HeatmapBlueToYellowToRedSpectrum
+      case "HeatmapDarkRedToYellowWhite" => ColorRamps.HeatmapDarkRedToYellowWhite
+      case "HeatmapLightPurpleToDarkPurpleToWhite" => ColorRamps.HeatmapLightPurpleToDarkPurpleToWhite
+      case "ClassificationBoldLandUse" => ColorRamps.ClassificationBoldLandUse
+      case "ClassificationMutedTerrain" => ColorRamps.ClassificationMutedTerrain
+      case "Magma" => ColorRamps.Magma
+      case "Inferno" => ColorRamps.Inferno
+      case "Plasma" => ColorRamps.Plasma
+      case "Viridis" => ColorRamps.Viridis
+    }
+  }
+
+  def makeColorMap(breaks: Array[Int], name: String): ColorMap = ColorMap(breaks, getNamedRamp(name))
+  def makeColorMap(breaks: Array[Double], name: String): ColorMap = ColorMap(breaks, getNamedRamp(name))
+
+  def makeColorMap(hist: Histogram[Int], name: String): ColorMap = ColorMap.fromQuantileBreaks(hist, getNamedRamp(name))
+  def makeColorMap(hist: Histogram[Double], name: String)(implicit dummy: DummyImplicit): ColorMap = ColorMap.fromQuantileBreaks(hist, getNamedRamp(name))
+
+}
+
+abstract class PngRDD[K: SpatialComponent :ClassTag] {
+  def rdd: RDD[(K, Png)]
+}
+
+object PngRDD {
+  def asSingleband(tiled: SpatialTiledRasterRDD, rampName: String): SpatialPngRDD = {
+    val rdd = tiled.rdd
+    val histogram = rdd.histogram().head
+    val mapped = rdd.map({ case (key, mbtile) =>
+      val tile = mbtile
+        .band(0)
+        .renderPng(Coloring.makeColorMap(histogram, rampName))
+      key -> tile
+    })
+    new SpatialPngRDD(mapped.asInstanceOf[RDD[(tiled.keyType, Png)]])
+  }
+
+  def asSingleband(tiled: TemporalTiledRasterRDD, rampName: String): TemporalPngRDD = {
+    val rdd = tiled.rdd
+    val histogram = rdd.histogram().head
+    val mapped = rdd.map({ case (key, mbtile) =>
+      val tile = mbtile
+        .band(0)
+        .renderPng(Coloring.makeColorMap(histogram, rampName))
+      key -> tile
+    })
+    new TemporalPngRDD(mapped.asInstanceOf[RDD[(tiled.keyType, Png)]])
+  }
+
+}
+
+class SpatialPngRDD(val rdd: RDD[(SpatialKey, Png)]) extends PngRDD[SpatialKey] {
+  def lookup(col: Int, row: Int): Array[Array[Byte]] =
+    rdd.lookup(SpatialKey(col, row)).map(_.bytes).toArray
+}
+
+class TemporalPngRDD(val rdd: RDD[(SpaceTimeKey, Png)]) extends PngRDD[SpaceTimeKey] {
+  def lookup(col: Int, row: Int, instant: Long): Array[Array[Byte]] =
+    rdd.lookup(SpaceTimeKey(col, row, instant)).map(_.bytes).toArray
+}
+

geopyspark-backend/geotrellis/src/main/scala/geopyspark/geotrellis/TiledRasterRDD.scala

@@ -36,6 +36,8 @@ import scala.collection.JavaConverters._
 abstract class TiledRasterRDD[K: SpatialComponent: AvroRecordCodec: JsonFormat: ClassTag] extends TileRDD[K] {
   import Constants._
 
+  type keyType = K
+
   def rdd: RDD[(K, MultibandTile)] with Metadata[TileLayerMetadata[K]]
   def zoomLevel: Option[Int]
 
@@ -584,4 +586,5 @@ object TemporalTiledRasterRDD {
 
     TemporalTiledRasterRDD(None, MultibandTileLayerRDD(rdd.tileToLayout(metadata), metadata))
   }
+
 }