Skip to content
Permalink
Browse files

Add overzoomingReader capability via self-type restricting Overzoomin…

…gValueReader trait

Signed-off-by: jpolchlo <jpolchlopek@azavea.com>
  • Loading branch information
jpolchlo authored and echeipesh committed Sep 26, 2017
1 parent 0119e1f commit adec9e01bff133438751d8b38993703f195207c2
@@ -58,8 +58,8 @@ class AccumuloLayerProvider extends AttributeStoreProvider
AccumuloLayerWriter(instance, store, table)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
val instance = AccumuloInstance(uri)
new AccumuloValueReader(instance, store)
new AccumuloValueReader(instance, store) with OverzoomingValueReader
}
}
@@ -16,7 +16,9 @@

package geotrellis.spark.io.accumulo

import geotrellis.spark.LayerId
import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.spark.{LayerId, SpatialComponent}
import geotrellis.spark.io._
import geotrellis.spark.io.avro.{AvroEncoder, AvroRecordCodec}
import geotrellis.spark.io.avro.codecs.KeyValueRecordCodec
@@ -76,10 +78,18 @@ object AccumuloValueReader {
attributeStore: AttributeStore,
layerId: LayerId
): Reader[K, V] =
new AccumuloValueReader(instance, attributeStore).reader[K, V](layerId)
(new AccumuloValueReader(instance, attributeStore) with OverzoomingValueReader).reader[K, V](layerId)

def apply(instance: AccumuloInstance): AccumuloValueReader =
def apply[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag, V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]](
instance: AccumuloInstance,
attributeStore: AttributeStore,
layerId: LayerId,
resampleMethod: ResampleMethod
): Reader[K, V] =
(new AccumuloValueReader(instance, attributeStore) with OverzoomingValueReader).overzoomingReader[K, V](layerId, resampleMethod)

def apply(instance: AccumuloInstance): AccumuloValueReader with OverzoomingValueReader =
new AccumuloValueReader(
instance = instance,
attributeStore = AccumuloAttributeStore(instance.connector))
attributeStore = AccumuloAttributeStore(instance.connector)) with OverzoomingValueReader
}
@@ -59,8 +59,8 @@ class CassandraLayerProvider extends AttributeStoreProvider
new CassandraLayerWriter(store, instance, keyspace, table)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
val instance = CassandraInstance(uri)
new CassandraValueReader(instance, store)
new CassandraValueReader(instance, store) with OverzoomingValueReader
}
}
@@ -16,7 +16,9 @@

package geotrellis.spark.io.cassandra

import geotrellis.spark.LayerId
import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.spark.{LayerId, SpatialComponent}
import geotrellis.spark.io._
import geotrellis.spark.io.avro.codecs.KeyValueRecordCodec
import geotrellis.spark.io.avro.{AvroEncoder, AvroRecordCodec}
@@ -74,15 +76,23 @@ object CassandraValueReader {
attributeStore: AttributeStore,
layerId: LayerId
): Reader[K, V] =
new CassandraValueReader(instance, attributeStore).reader[K, V](layerId)
(new CassandraValueReader(instance, attributeStore) with OverzoomingValueReader).reader[K, V](layerId)

def apply(instance: CassandraInstance): CassandraValueReader =
def apply[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag, V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]](
instance: CassandraInstance,
attributeStore: AttributeStore,
layerId: LayerId,
resampleMethod: ResampleMethod
): Reader[K, V] =
(new CassandraValueReader(instance, attributeStore) with OverzoomingValueReader).overzoomingReader[K, V](layerId, resampleMethod)

def apply(instance: CassandraInstance): CassandraValueReader with OverzoomingValueReader =
new CassandraValueReader(
instance = instance,
attributeStore = CassandraAttributeStore(instance))
attributeStore = CassandraAttributeStore(instance)) with OverzoomingValueReader

def apply(attributeStore: CassandraAttributeStore): CassandraValueReader =
def apply(attributeStore: CassandraAttributeStore): CassandraValueReader with OverzoomingValueReader =
new CassandraValueReader(
instance = attributeStore.instance,
attributeStore = attributeStore)
attributeStore = attributeStore) with OverzoomingValueReader
}
@@ -55,8 +55,8 @@ class HBaseLayerProvider extends AttributeStoreProvider
new HBaseLayerWriter(store, instance, table)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
val instance = HBaseInstance(uri)
new HBaseValueReader(instance, store)
new HBaseValueReader(instance, store) with OverzoomingValueReader
}
}
@@ -16,7 +16,9 @@

package geotrellis.spark.io.hbase

import geotrellis.spark.LayerId
import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.spark.{LayerId, SpatialComponent}
import geotrellis.spark.io._
import geotrellis.spark.io.avro.codecs.KeyValueRecordCodec
import geotrellis.spark.io.avro.{AvroEncoder, AvroRecordCodec}
@@ -63,15 +65,23 @@ object HBaseValueReader {
attributeStore: AttributeStore,
layerId: LayerId
): Reader[K, V] =
new HBaseValueReader(instance, attributeStore).reader[K, V](layerId)
(new HBaseValueReader(instance, attributeStore) with OverzoomingValueReader).reader[K, V](layerId)

def apply(instance: HBaseInstance): HBaseValueReader =
def apply[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag, V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]](
instance: HBaseInstance,
attributeStore: AttributeStore,
layerId: LayerId,
resampleMethod: ResampleMethod
): Reader[K, V] =
(new HBaseValueReader(instance, attributeStore) with OverzoomingValueReader).overzoomingReader[K, V](layerId, resampleMethod)

def apply(instance: HBaseInstance): HBaseValueReader with OverzoomingValueReader =
new HBaseValueReader(
instance = instance,
attributeStore = HBaseAttributeStore(instance))
attributeStore = HBaseAttributeStore(instance)) with OverzoomingValueReader

def apply(attributeStore: HBaseAttributeStore): HBaseValueReader =
def apply(attributeStore: HBaseAttributeStore): HBaseValueReader with OverzoomingValueReader =
new HBaseValueReader(
instance = attributeStore.instance,
attributeStore = attributeStore)
attributeStore = attributeStore) with OverzoomingValueReader
}
@@ -51,7 +51,7 @@ class S3LayerProvider extends AttributeStoreProvider
new S3LayerWriter(store, bucket = s3Uri.getBucket, keyPrefix = s3Uri.getKey)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
new S3ValueReader(store)
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
new S3ValueReader(store) with OverzoomingValueReader
}
}
@@ -16,6 +16,8 @@

package geotrellis.spark.io.s3

import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.avro._
@@ -70,11 +72,18 @@ object S3ValueReader {
attributeStore: AttributeStore,
layerId: LayerId
): Reader[K, V] =
new S3ValueReader(attributeStore).reader[K, V](layerId)
(new S3ValueReader(attributeStore) with OverzoomingValueReader).reader[K, V](layerId)

def apply(bucket: String, root: String): S3ValueReader =
new S3ValueReader(new S3AttributeStore(bucket, root))
def apply[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag, V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]](
attributeStore: AttributeStore,
layerId: LayerId,
resampleMethod: ResampleMethod
): Reader[K, V] =
(new S3ValueReader(attributeStore) with OverzoomingValueReader).overzoomingReader[K, V](layerId, resampleMethod)

def apply(bucket: String, root: String): S3ValueReader with OverzoomingValueReader =
new S3ValueReader(new S3AttributeStore(bucket, root)) with OverzoomingValueReader

def apply(bucket: String): S3ValueReader =
def apply(bucket: String): S3ValueReader with OverzoomingValueReader =
apply(bucket, "")
}
@@ -11,16 +11,22 @@ import spray.json._
import scala.reflect._
import java.net.URI

case class OverzoomingValueReader(valueReader: ValueReader[LayerId], resampleMethod: ResampleMethod) {
val attributeStore = valueReader.attributeStore
trait OverzoomingValueReader {
self: ValueReader[LayerId] =>

def reader[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag,
def overzoomingReader[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag,
V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]
](layerId: LayerId): Reader[K, V] = new Reader[K, V] {
](layerId: LayerId, resampleMethod: ResampleMethod = ResampleMethod.DEFAULT): Reader[K, V] = new Reader[K, V] {
val LayerId(layerName, requestedZoom) = layerId
val maxAvailableZoom = valueReader.attributeStore.layerIds.filter { case LayerId(name, _) => name == layerName }.map(_.zoom).max
lazy val baseReader = valueReader.reader[K, V](layerId)
lazy val maxReader = valueReader.reader[K, V](LayerId(layerName, maxAvailableZoom))
val maxAvailableZoom = attributeStore.layerIds.filter { case LayerId(name, _) => name == layerName }.map(_.zoom).max
val metadata = attributeStore.readMetadata[TileLayerMetadata[K]](LayerId(layerName, maxAvailableZoom))

val layoutScheme = ZoomedLayoutScheme(metadata.crs, metadata.tileRows)
val requestedMaptrans = layoutScheme.levelForZoom(requestedZoom).layout.mapTransform
val maxMaptrans = metadata.mapTransform

lazy val baseReader = reader[K, V](layerId)
lazy val maxReader = reader[K, V](LayerId(layerName, maxAvailableZoom))

def read(key: K): V = {
if (requestedZoom <= maxAvailableZoom) {
@@ -32,12 +38,6 @@ case class OverzoomingValueReader(valueReader: ValueReader[LayerId], resampleMet
key.setComponent[SpatialKey](SpatialKey(srcSK._1 / denom, srcSK._2 / denom))
}

val metadata = valueReader.attributeStore.readMetadata[TileLayerMetadata[K]](LayerId(layerName, maxAvailableZoom))
val layoutScheme = ZoomedLayoutScheme(metadata.crs, metadata.tileRows)

val requestedMaptrans = layoutScheme.levelForZoom(requestedZoom).layout.mapTransform
val maxMaptrans = metadata.mapTransform

val toResample = maxReader.read(maxKey)

return toResample.resample(maxMaptrans(maxKey), RasterExtent(requestedMaptrans(key), toResample.cols, toResample.rows), resampleMethod)
@@ -46,46 +46,3 @@ case class OverzoomingValueReader(valueReader: ValueReader[LayerId], resampleMet
}
}

object OverzoomingValueReader {

def apply(attributeStore: AttributeStore, valueReaderUri: URI, resampleMethod: ResampleMethod): OverzoomingValueReader =
OverzoomingValueReader(ValueReader(attributeStore, valueReaderUri), resampleMethod)

def apply(attributeStoreUri: URI, valueReaderUri: URI, resampleMethod: ResampleMethod): OverzoomingValueReader =
apply(AttributeStore(attributeStoreUri), valueReaderUri, resampleMethod)

def apply(uri: URI, resampleMethod: ResampleMethod): OverzoomingValueReader =
apply(attributeStoreUri = uri, valueReaderUri = uri, resampleMethod)

def apply(attributeStore: AttributeStore, valueReaderUri: String, resampleMethod: ResampleMethod): OverzoomingValueReader =
apply(attributeStore, new URI(valueReaderUri), resampleMethod)

def apply(attributeStoreUri: String, valueReaderUri: String, resampleMethod: ResampleMethod): OverzoomingValueReader =
apply(AttributeStore(new URI(attributeStoreUri)), new URI(valueReaderUri), resampleMethod)

def apply(uri: String, resampleMethod: ResampleMethod): OverzoomingValueReader = {
val _uri = new URI(uri)
apply(attributeStoreUri = _uri, valueReaderUri = _uri, resampleMethod)
}

def apply(attributeStore: AttributeStore, valueReaderUri: URI): OverzoomingValueReader =
OverzoomingValueReader(ValueReader(attributeStore, valueReaderUri), ResampleMethod.DEFAULT)

def apply(attributeStoreUri: URI, valueReaderUri: URI): OverzoomingValueReader =
apply(AttributeStore(attributeStoreUri), valueReaderUri, ResampleMethod.DEFAULT)

def apply(uri: URI): OverzoomingValueReader =
apply(attributeStoreUri = uri, valueReaderUri = uri, ResampleMethod.DEFAULT)

def apply(attributeStore: AttributeStore, valueReaderUri: String): OverzoomingValueReader =
apply(attributeStore, new URI(valueReaderUri), ResampleMethod.DEFAULT)

def apply(attributeStoreUri: String, valueReaderUri: String): OverzoomingValueReader =
apply(AttributeStore(new URI(attributeStoreUri)), new URI(valueReaderUri), ResampleMethod.DEFAULT)

def apply(uri: String): OverzoomingValueReader = {
val _uri = new URI(uri)
apply(attributeStoreUri = _uri, valueReaderUri = _uri, ResampleMethod.DEFAULT)
}

}
@@ -36,28 +36,28 @@ trait ValueReader[ID] {

object ValueReader {

def apply(attributeStore: AttributeStore, valueReaderUri: URI): ValueReader[LayerId] = {
def apply(attributeStore: AttributeStore, valueReaderUri: URI): ValueReader[LayerId] with OverzoomingValueReader = {
import scala.collection.JavaConversions._
ServiceLoader.load(classOf[ValueReaderProvider]).iterator()
.find(_.canProcess(valueReaderUri))
.getOrElse(throw new RuntimeException(s"Unable to find ValueReaderProvider for $valueReaderUri"))
.valueReader(valueReaderUri, attributeStore)
}

def apply(attributeStoreUri: URI, valueReaderUri: URI): ValueReader[LayerId] =
def apply(attributeStoreUri: URI, valueReaderUri: URI): ValueReader[LayerId] with OverzoomingValueReader =
apply(AttributeStore(attributeStoreUri), valueReaderUri)

def apply(uri: URI): ValueReader[LayerId] =
apply(attributeStoreUri = uri, valueReaderUri = uri)

def apply(attributeStore: AttributeStore, valueReaderUri: String): ValueReader[LayerId] =
def apply(attributeStore: AttributeStore, valueReaderUri: String): ValueReader[LayerId] with OverzoomingValueReader =
apply(attributeStore, new URI(valueReaderUri))


def apply(attributeStoreUri: String, valueReaderUri: String): ValueReader[LayerId] =
def apply(attributeStoreUri: String, valueReaderUri: String): ValueReader[LayerId] with OverzoomingValueReader =
apply(AttributeStore(new URI(attributeStoreUri)), new URI(valueReaderUri))

def apply(uri: String): ValueReader[LayerId] = {
def apply(uri: String): ValueReader[LayerId] with OverzoomingValueReader = {
val _uri = new URI(uri)
apply(attributeStoreUri = _uri, valueReaderUri = _uri)
}
@@ -23,5 +23,5 @@ import java.net.URI
trait ValueReaderProvider {
def canProcess(uri: URI): Boolean

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId]
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader
}
@@ -47,8 +47,8 @@ class FileLayerProvider extends AttributeStoreProvider
new FileLayerWriter(store, file.getCanonicalPath)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
val catalogPath = new File(uri).getCanonicalPath
new FileValueReader(store, catalogPath)
new FileValueReader(store, catalogPath) with OverzoomingValueReader
}
}
@@ -16,6 +16,8 @@

package geotrellis.spark.io.file

import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.spark._
import geotrellis.spark.io._
import geotrellis.spark.io.avro.codecs.KeyValueRecordCodec
@@ -67,11 +69,19 @@ object FileValueReader {
catalogPath: String,
layerId: LayerId
): Reader[K, V] =
new FileValueReader(attributeStore, catalogPath).reader(layerId)
(new FileValueReader(attributeStore, catalogPath) with OverzoomingValueReader).reader(layerId)

def apply(catalogPath: String): FileValueReader =
new FileValueReader(new FileAttributeStore(catalogPath), catalogPath)
def apply[K: AvroRecordCodec: JsonFormat: SpatialComponent: ClassTag, V <: CellGrid: AvroRecordCodec: ? => TileResampleMethods[V]](
attributeStore: AttributeStore,
catalogPath: String,
layerId: LayerId,
resampleMethod: ResampleMethod
): Reader[K, V] =
(new FileValueReader(attributeStore, catalogPath) with OverzoomingValueReader).overzoomingReader(layerId, resampleMethod)

def apply(catalogPath: String): FileValueReader with OverzoomingValueReader =
new FileValueReader(new FileAttributeStore(catalogPath), catalogPath) with OverzoomingValueReader

def apply(attributeStore: FileAttributeStore): FileValueReader =
new FileValueReader(attributeStore, attributeStore.catalogPath)
def apply(attributeStore: FileAttributeStore): FileValueReader with OverzoomingValueReader =
new FileValueReader(attributeStore, attributeStore.catalogPath) with OverzoomingValueReader
}
@@ -62,12 +62,12 @@ class HadoopLayerProvider extends AttributeStoreProvider
new HadoopLayerWriter(path, store, interval)
}

def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] = {
def valueReader(uri: URI, store: AttributeStore): ValueReader[LayerId] with OverzoomingValueReader = {
val _uri = trim(uri)
val path = new Path(_uri)
val params = UriUtils.getParams(_uri)
val conf = new Configuration()
val maxOpenFiles = params.getOrElse("maxOpenFiles", "16").toInt
new HadoopValueReader(store, conf, maxOpenFiles)
new HadoopValueReader(store, conf, maxOpenFiles) with OverzoomingValueReader
}
}

0 comments on commit adec9e0

Please sign in to comment.
You can’t perform that action at this time.