Permalink
Browse files

Merge pull request #107 from non/topic/server-cleanup

Clean up some details of catalogs and raster layers.
  • Loading branch information...
2 parents c76998d + d5be7ef commit d9c4854308855496ca5e886232ad696f0e87f71e @joshmarcus joshmarcus committed May 25, 2012
View
105 src/main/scala/geotrellis/process/catalog.scala
@@ -1,8 +1,7 @@
package geotrellis.process
-import scala.collection.mutable.{Map => MMap}
+import scala.collection.mutable
import scala.io.Source
-import scala.util.matching.Regex
import java.io.File
import net.liftweb.json.{parse, DefaultFormats}
@@ -11,15 +10,12 @@ import geotrellis._
import geotrellis.process._
import geotrellis.util._
-// this is a work in progress. the definitions for Layer, DataStore and Catalog
-// are not complete, and we will likely need more objects.
-//
-// example json is available in the geotrellis.process.catalog tests. please keep
-// it up-to-date with changes you make here.
+// example json is available in the geotrellis.process.catalog tests. please
+// keep it up-to-date with changes you make here.
/**
* Represents a layer of raster data, or a feature layer (e.g. a postgis table
- * of polygons.
+ * of polygons).
*/
trait Layer {
def name:String
@@ -31,40 +27,51 @@ trait Layer {
*/
case class DataStore(name:String, params:Map[String, String]) {
- val layers = MMap.empty[String, (String, RasterLayer)]
- val paths = MMap.empty[String, RasterLayer]
+ private val layers = mutable.Map.empty[String, RasterLayer]
+ private val paths = mutable.Map.empty[String, RasterLayer]
- findRasterLayers.foreach {
- layer =>
- val path = layer.jsonPath
- layers(layer.name) = (path, layer)
- paths(path) = layer
- }
-
- def cacheAll = params.contains("cacheAll")
+ initRasterLayers()
- def getLayer(name:String) = layers(name)
+ /**
+ * Initialize raster layers from the directory specified by the 'path' param.
+ */
+ private def initRasterLayers() {
+ val path = params("path")
+ val f = new java.io.File(path)
+ if (!f.isDirectory) sys.error("store %s is not a dirctory" format path)
+ find(f, ".json", initRasterLayer _)
+ }
/**
- * Recursively find RasterLayers defined in the DataStore path.
+ * Initialize a raster layer from its JSON metadata file.
*/
- def findRasterLayers: Array[RasterLayer] = {
- val path = new java.io.File(params("path"))
- if (!path.isDirectory) return Array.empty[RasterLayer]
-
- val layerJson = recursiveListFiles(path, """^.*\.json$""".r)
- layerJson.map(_.getAbsolutePath).map {
- p => RasterLayer.fromPath(p)
- //p => (p, RasterLayer.fromPath(p))
- }
+ private def initRasterLayer(f:File) {
+ val layer = RasterLayer.fromFile(f)
+ layers(layer.name) = layer
+ paths(layer.rasterPath) = layer
}
- def recursiveListFiles(f: File, r: Regex): Array[File] = {
- val these = f.listFiles
- val good = these.filter( _.getPath.endsWith(".json") )
- good ++ these.filter(_.isDirectory).flatMap(recursiveListFiles(_,r))
+ /**
+ * Find files based on an extension. Directories will be searched recursively
+ * and matching files will run the provided callback 'action'.
+ */
+ private def find(f:File, ext:String, action:File => Unit) {
+ val fs = f.listFiles
+ fs.filter(_.getPath.endsWith(ext)).foreach(f => action(f))
+ fs.filter(_.isDirectory).foreach(f => find(f, ext, action))
}
+ def getNames = layers.keys
+ def getPaths = paths.keys
+ def getLayers = layers.values
+
+ def hasCacheAll = params.contains("cacheAll")
+
+ def getRasterLayer(path:String): Option[RasterLayer] = paths.get(path)
+
+ def getRasterLayerByName(name:String):Option[RasterLayer] = {
+ layers.get(name)
+ }
}
@@ -74,29 +81,12 @@ case class DataStore(name:String, params:Map[String, String]) {
*/
case class Catalog(name:String, stores:Map[String, DataStore]) {
- def findRasterLayer(path:String): Option[RasterLayer] = {
- stores.values.flatMap(_.paths.get(path)).headOption
+ def getRasterLayer(path:String): Option[RasterLayer] = {
+ stores.values.flatMap(_.getRasterLayer(path)).headOption
}
- private def exists(path:String) = if (new java.io.File(path).exists) Some(path) else None
-
- def findPath(base:String) = exists(base + ".arg32") orElse exists(base + ".arg")
-
- def getRasterLayerByName(layerName:String):Option[(String, RasterLayer)] = {
- stores.values.foreach {
- store => {
- if (store.layers.contains(layerName)) {
- val (jsonPath, layer) = store.layers(layerName)
-
- // fixme
- val base = jsonPath.substring(0, jsonPath.length - 5)
- return findPath(base).map(s => (s, layer))
- //val path = jsonPath.substring(0, jsonPath.length - 5) + ".arg32"
- //return Some((path, layer))
- }
- }
- }
- None
+ def getRasterLayerByName(name:String):Option[RasterLayer] = {
+ stores.values.flatMap(_.getRasterLayerByName(name)).headOption
}
}
@@ -154,6 +144,8 @@ object RasterLayer {
fromJSON(data, base)
}
+ def fromFile(f:File) = RasterLayer.fromPath(f.getAbsolutePath)
+
/**
* Build a RasterLayer instance given a JSON string.
*/
@@ -162,8 +154,9 @@ object RasterLayer {
}
}
-case class RasterLayer(name:String, typ:String, datatyp:String, basePath:String,
- rasterExtent:RasterExtent, epsg:Int, xskew:Double, yskew:Double) extends Layer {
+case class RasterLayer(name:String, typ:String, datatyp:String,
+ basePath:String, rasterExtent:RasterExtent,
+ epsg:Int, xskew:Double, yskew:Double) extends Layer {
def jsonPath = basePath + ".json"
def rasterPath = basePath + "." + typ
}
View
114 src/main/scala/geotrellis/process/server.scala
@@ -88,9 +88,9 @@ akka {
def startUp:Unit = ()
def initStaticCache():Unit = {
- catalog.stores.values.filter(_.cacheAll).flatMap(_.paths.values).foreach {
- layer => loadInStaticCache(layer)
- }
+ val cacheStores = catalog.stores.values.filter(_.hasCacheAll)
+ cacheStores.foreach(_.getLayers.foreach(l => loadInStaticCache(l)))
+
val n = staticCache.size
val (amt, units) = Units.bytes(staticCache.foldLeft(0)(_ + _._2.length))
println("loaded %d layers (%.2f %s) into static cache" format (n, amt, units))
@@ -132,25 +132,30 @@ akka {
}
}
- def metadataPath(path:String) = path.substring(0, path.lastIndexOf(".")) + ".json"
+ def metadataPath(path:String) = {
+ path.substring(0, path.lastIndexOf(".")) + ".json"
+ }
+
/**
* Return the appropriate reader object for the given path.
*/
- def getReader(path:String, layerOpt:Option[RasterLayer]): FileReader = path match {
- case ArgPattern() => {
- //REVIEW: replace with unified ARG reader
- layerOpt.getOrElse(RasterLayer.fromPath(metadataPath(path))) match {
- case RasterLayer(_, "arg", "int32", _, _, _, _, _) => Arg32Reader
- case RasterLayer(_, "arg", "int8", _, _, _, _, _) => Arg8Reader
- case RasterLayer(_, typ, datatyp, _, _, _, _, _) =>
+ def getReader(path:String, layerOpt:Option[RasterLayer]): FileReader = {
+ path match {
+ case ArgPattern() => {
+ //REVIEW: replace with unified ARG reader
+ layerOpt.getOrElse(RasterLayer.fromPath(metadataPath(path))) match {
+ case RasterLayer(_, "arg", "int32", _, _, _, _, _) => Arg32Reader
+ case RasterLayer(_, "arg", "int8", _, _, _, _, _) => Arg8Reader
+ case RasterLayer(_, typ, datatyp, _, _, _, _, _) =>
throw new Exception("Unsupported raster layer: with type %s, datatype %s".format(typ,datatyp))
- }
+ }
+ }
+ case GeoTiffPattern() => GeoTiffReader
+ case AsciiPattern() => AsciiReader
+ case _ => sys.error("unknown path type %s".format(path))
}
- case GeoTiffPattern() => GeoTiffReader
- case AsciiPattern() => AsciiReader
- case _ => sys.error("unknown path type %s".format(path))
}
-
+
def getRaster(path:String, layerOpt:Option[RasterLayer], reOpt:Option[RasterExtent]):IntRaster = {
getReader(path, layerOpt).readPath(path, layerOpt, reOpt)
}
@@ -160,15 +165,16 @@ akka {
def getRasterExtentByName(name:String):RasterExtent = {
catalog.getRasterLayerByName(name) match {
- case Some((path, layer)) => layer.rasterExtent
+ case Some(layer) => layer.rasterExtent
case None => sys.error("couldn't find %s" format name)
}
}
def getRasterByName(name:String, reOpt:Option[RasterExtent]):IntRaster = {
catalog.getRasterLayerByName(name) match {
- case Some((path, layer)) => {
- val reader = getReader(path,Some(layer))
+ case Some(layer) => {
+ val path = layer.rasterPath
+ val reader = getReader(path, Some(layer))
staticCache.get(layer.name) match {
case Some(bytes) => reader.readCache(bytes, layer, reOpt)
case None => reader.readPath(path, Some(layer), reOpt)
@@ -177,78 +183,8 @@ akka {
case None => sys.error("couldn't find '%s'" format name)
}
}
-
}
-//trait FileCaching {
-// val mb = 1<<20
-//
-// // TODO: set this in config file
-// val maxBytesInCache = 1000 * mb // 2G
-// //var caching = true //xyz
-// var caching = false //xyz
-// var cacheSize:Int = 0
-//
-// val rasterCache:HashBackedCache[String,IntRaster] = new LRUCache(maxBytesInCache, _.length * 4)
-//
-// val catalog:Catalog
-//
-// var id = "default"
-//
-// //// TODO: remove this and add cache configuration to server's JSON config
-// //// and/or constructor arguments
-// //def enableCaching() { caching = true }
-// //def disableCaching() { caching = false }
-//
-// def getRasterExtentByName(name:String):RasterExtent = {
-// catalog.getRasterLayerByName(name) match {
-// case Some((path, layer)) => layer.rasterExtent
-// case None => sys.error("couldn't find %s" format name)
-// }
-// }
-//
-// def getRasterByName(name:String, reOpt:Option[RasterExtent]):IntRaster = {
-// catalog.getRasterLayerByName(name) match {
-// case Some((path, layer)) => getRaster(path, Option(layer), reOpt)
-// case None => sys.error("couldn't find %s" format name)
-// }
-// }
-//
-// def loadRaster(path:String, g:RasterExtent):IntRaster = getRaster(path, None, Option(g))
-//
-// /**
-// * THIS is the new thing that we are wanting to use.
-// */
-// def getRaster(path:String, layerOpt:Option[RasterLayer], reOpt:Option[RasterExtent]):IntRaster = {
-//
-// def xyz(path:String, layerOpt:Option[RasterLayer]) = {
-// getReader(path).read(path, layerOpt, None)
-// }
-//
-// if (this.caching) {
-// val t0 = System.currentTimeMillis
-// val raster = rasterCache.getOrInsert(path, xyz(path, layerOpt))
-// val t = System.currentTimeMillis - t0
-// if (t > 10) println("rasterCache.getOrInsert(%s) took %d ms" format (path, t))
-// IntRasterReader.read(raster, reOpt)
-// } else {
-// getReader(path).read(path, layerOpt, reOpt)
-// }
-// }
-//
-// def recursiveListFiles(f: File, r: Regex): Array[File] = {
-// val these = f.listFiles
-// val good = these.filter(f => r.findFirstIn(f.getName).isDefined)
-// good ++ these.filter(_.isDirectory).flatMap(recursiveListFiles(_,r))
-// }
-//
-// def cacheRasters() = catalog.stores.foreach {
-// case (name, store) => store.layers.values.foreach {
-// case (path, layer) => getRaster(path, Some(layer), None)
-// }
-// }
-//}
-
object Server {
val config = ConfigFactory.load()
val catalogPath = config.getString("geotrellis.catalog")
View
4 src/test/scala/geotrellis/process/catalog.scala
@@ -72,8 +72,8 @@ val json0 = """
it("should find Arg32s in a source directory") {
val catalog = Catalog.fromJSON(json1)
val store = catalog.stores("stroud:fs")
- val layers = store.findRasterLayers
- layers.length must be === 3
+ val layers = store.getLayers
+ layers.toList.length must be === 3
}
}

0 comments on commit d9c4854

Please sign in to comment.