Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Merge pull request #107 from non/topic/server-cleanup

Clean up some details of catalogs and raster layers.
  • Loading branch information...
commit d9c4854308855496ca5e886232ad696f0e87f71e 2 parents c76998d + d5be7ef
Josh Marcus authored May 25, 2012
105  src/main/scala/geotrellis/process/catalog.scala
... ...
@@ -1,8 +1,7 @@
1 1
 package geotrellis.process
2 2
 
3  
-import scala.collection.mutable.{Map => MMap}
  3
+import scala.collection.mutable
4 4
 import scala.io.Source
5  
-import scala.util.matching.Regex
6 5
 import java.io.File
7 6
 
8 7
 import net.liftweb.json.{parse, DefaultFormats}
@@ -11,15 +10,12 @@ import geotrellis._
11 10
 import geotrellis.process._
12 11
 import geotrellis.util._
13 12
 
14  
-// this is a work in progress. the definitions for Layer, DataStore and Catalog
15  
-// are not complete, and we will likely need more objects.
16  
-//
17  
-// example json is available in the geotrellis.process.catalog tests. please keep
18  
-// it up-to-date with changes you make here.
  13
+// example json is available in the geotrellis.process.catalog tests. please
  14
+// keep it up-to-date with changes you make here.
19 15
 
20 16
 /**
21 17
  * Represents a layer of raster data, or a feature layer (e.g. a postgis table
22  
- * of polygons.
  18
+ * of polygons).
23 19
  */
24 20
 trait Layer { 
25 21
   def name:String
@@ -31,40 +27,51 @@ trait Layer {
31 27
  */
32 28
 case class DataStore(name:String, params:Map[String, String]) {
33 29
 
34  
-  val layers = MMap.empty[String, (String, RasterLayer)]
35  
-  val paths = MMap.empty[String, RasterLayer]
  30
+  private val layers = mutable.Map.empty[String, RasterLayer]
  31
+  private val paths = mutable.Map.empty[String, RasterLayer]
36 32
 
37  
-  findRasterLayers.foreach {
38  
-    layer =>
39  
-    val path = layer.jsonPath
40  
-    layers(layer.name) = (path, layer)
41  
-    paths(path) = layer
42  
-  }
43  
-
44  
-  def cacheAll = params.contains("cacheAll")
  33
+  initRasterLayers()
45 34
 
46  
-  def getLayer(name:String) = layers(name)
  35
+  /**
  36
+   * Initialize raster layers from the directory specified by the 'path' param.
  37
+   */
  38
+  private def initRasterLayers() {
  39
+    val path = params("path")
  40
+    val f = new java.io.File(path)
  41
+    if (!f.isDirectory) sys.error("store %s is not a dirctory" format path)
  42
+    find(f, ".json", initRasterLayer _)
  43
+  }
47 44
 
48 45
   /**
49  
-   * Recursively find RasterLayers defined in the DataStore path.
  46
+   * Initialize a raster layer from its JSON metadata file.
50 47
    */
51  
-  def findRasterLayers: Array[RasterLayer] = {
52  
-    val path = new java.io.File(params("path"))
53  
-    if (!path.isDirectory) return Array.empty[RasterLayer]
54  
-
55  
-    val layerJson = recursiveListFiles(path, """^.*\.json$""".r)
56  
-    layerJson.map(_.getAbsolutePath).map {
57  
-      p => RasterLayer.fromPath(p)
58  
-      //p => (p, RasterLayer.fromPath(p))
59  
-    }
  48
+  private def initRasterLayer(f:File) {
  49
+    val layer = RasterLayer.fromFile(f)
  50
+    layers(layer.name) = layer
  51
+    paths(layer.rasterPath) = layer
60 52
   }
61 53
 
62  
-  def recursiveListFiles(f: File, r: Regex): Array[File] = {
63  
-    val these = f.listFiles
64  
-    val good = these.filter(  _.getPath.endsWith(".json") )
65  
-    good ++ these.filter(_.isDirectory).flatMap(recursiveListFiles(_,r))
  54
+  /**
  55
+   * Find files based on an extension. Directories will be searched recursively
  56
+   * and matching files will run the provided callback 'action'.
  57
+   */
  58
+  private def find(f:File, ext:String, action:File => Unit) {
  59
+    val fs = f.listFiles
  60
+    fs.filter(_.getPath.endsWith(ext)).foreach(f => action(f))
  61
+    fs.filter(_.isDirectory).foreach(f => find(f, ext, action))
66 62
   }
67 63
 
  64
+  def getNames = layers.keys
  65
+  def getPaths = paths.keys
  66
+  def getLayers = layers.values
  67
+
  68
+  def hasCacheAll = params.contains("cacheAll")
  69
+
  70
+  def getRasterLayer(path:String): Option[RasterLayer] = paths.get(path)
  71
+
  72
+  def getRasterLayerByName(name:String):Option[RasterLayer] = {
  73
+    layers.get(name)
  74
+  }
68 75
 }
69 76
 
70 77
 
@@ -74,29 +81,12 @@ case class DataStore(name:String, params:Map[String, String]) {
74 81
  */
75 82
 case class Catalog(name:String, stores:Map[String, DataStore]) {
76 83
 
77  
-  def findRasterLayer(path:String): Option[RasterLayer] = {
78  
-    stores.values.flatMap(_.paths.get(path)).headOption
  84
+  def getRasterLayer(path:String): Option[RasterLayer] = {
  85
+    stores.values.flatMap(_.getRasterLayer(path)).headOption
79 86
   }
80 87
 
81  
-  private def exists(path:String) = if (new java.io.File(path).exists) Some(path) else None
82  
-
83  
-  def findPath(base:String) = exists(base + ".arg32") orElse exists(base + ".arg")
84  
-
85  
-  def getRasterLayerByName(layerName:String):Option[(String, RasterLayer)] = {
86  
-    stores.values.foreach {
87  
-      store => {
88  
-        if (store.layers.contains(layerName)) {
89  
-          val (jsonPath, layer) = store.layers(layerName)
90  
-  
91  
-          // fixme
92  
-          val base = jsonPath.substring(0, jsonPath.length - 5)
93  
-          return findPath(base).map(s => (s, layer))
94  
-          //val path = jsonPath.substring(0, jsonPath.length - 5) + ".arg32"
95  
-          //return Some((path, layer))
96  
-        }
97  
-      }
98  
-    }
99  
-    None
  88
+  def getRasterLayerByName(name:String):Option[RasterLayer] = {
  89
+    stores.values.flatMap(_.getRasterLayerByName(name)).headOption
100 90
   }
101 91
 }
102 92
 
@@ -154,6 +144,8 @@ object RasterLayer {
154 144
     fromJSON(data, base)
155 145
   }
156 146
 
  147
+  def fromFile(f:File) = RasterLayer.fromPath(f.getAbsolutePath)
  148
+
157 149
   /**
158 150
    * Build a RasterLayer instance given a JSON string.
159 151
    */
@@ -162,8 +154,9 @@ object RasterLayer {
162 154
   }
163 155
 }
164 156
 
165  
-case class RasterLayer(name:String, typ:String, datatyp:String, basePath:String,
166  
-                       rasterExtent:RasterExtent, epsg:Int, xskew:Double, yskew:Double) extends Layer {
  157
+case class RasterLayer(name:String, typ:String, datatyp:String,
  158
+                       basePath:String, rasterExtent:RasterExtent,
  159
+                       epsg:Int, xskew:Double, yskew:Double) extends Layer {
167 160
   def jsonPath = basePath + ".json"
168 161
   def rasterPath = basePath + "." + typ
169 162
 }
114  src/main/scala/geotrellis/process/server.scala
@@ -88,9 +88,9 @@ akka {
88 88
   def startUp:Unit = ()
89 89
 
90 90
   def initStaticCache():Unit = {
91  
-    catalog.stores.values.filter(_.cacheAll).flatMap(_.paths.values).foreach {
92  
-      layer => loadInStaticCache(layer)
93  
-    }
  91
+    val cacheStores = catalog.stores.values.filter(_.hasCacheAll)
  92
+    cacheStores.foreach(_.getLayers.foreach(l => loadInStaticCache(l)))
  93
+
94 94
     val n = staticCache.size
95 95
     val (amt, units) = Units.bytes(staticCache.foldLeft(0)(_ + _._2.length))
96 96
     println("loaded %d layers (%.2f %s) into static cache" format (n, amt, units))
@@ -132,25 +132,30 @@ akka {
132 132
     }
133 133
   }
134 134
 
135  
-  def metadataPath(path:String) = path.substring(0, path.lastIndexOf(".")) + ".json"
  135
+  def metadataPath(path:String) = {
  136
+    path.substring(0, path.lastIndexOf(".")) + ".json"
  137
+  }
  138
+
136 139
   /**
137 140
    * Return the appropriate reader object for the given path.
138 141
    */
139  
-  def getReader(path:String, layerOpt:Option[RasterLayer]): FileReader = path match {
140  
-    case ArgPattern() => {
141  
-      //REVIEW: replace with unified ARG reader
142  
-      layerOpt.getOrElse(RasterLayer.fromPath(metadataPath(path))) match {
143  
-        case RasterLayer(_, "arg", "int32", _, _, _, _, _) => Arg32Reader
144  
-        case RasterLayer(_, "arg", "int8",  _, _, _, _, _) => Arg8Reader
145  
-        case RasterLayer(_, typ, datatyp,   _, _, _, _, _) => 
  142
+  def getReader(path:String, layerOpt:Option[RasterLayer]): FileReader = {
  143
+    path match {
  144
+      case ArgPattern() => {
  145
+        //REVIEW: replace with unified ARG reader
  146
+        layerOpt.getOrElse(RasterLayer.fromPath(metadataPath(path))) match {
  147
+          case RasterLayer(_, "arg", "int32", _, _, _, _, _) => Arg32Reader
  148
+          case RasterLayer(_, "arg", "int8",  _, _, _, _, _) => Arg8Reader
  149
+          case RasterLayer(_, typ, datatyp,   _, _, _, _, _) => 
146 150
           throw new Exception("Unsupported raster layer: with type %s, datatype %s".format(typ,datatyp))
147  
-      } 
  151
+        } 
  152
+      }
  153
+      case GeoTiffPattern() => GeoTiffReader
  154
+      case AsciiPattern() => AsciiReader
  155
+      case _ => sys.error("unknown path type %s".format(path))
148 156
     }
149  
-    case GeoTiffPattern() => GeoTiffReader
150  
-    case AsciiPattern() => AsciiReader
151  
-    case _ => sys.error("unknown path type %s".format(path))
152 157
   }
153  
-
  158
+    
154 159
   def getRaster(path:String, layerOpt:Option[RasterLayer], reOpt:Option[RasterExtent]):IntRaster = {
155 160
     getReader(path, layerOpt).readPath(path, layerOpt, reOpt)  
156 161
   }
@@ -160,15 +165,16 @@ akka {
160 165
 
161 166
   def getRasterExtentByName(name:String):RasterExtent = {
162 167
     catalog.getRasterLayerByName(name) match {
163  
-      case Some((path, layer)) => layer.rasterExtent
  168
+      case Some(layer) => layer.rasterExtent
164 169
       case None => sys.error("couldn't find %s" format name)
165 170
     }
166 171
   }
167 172
 
168 173
   def getRasterByName(name:String, reOpt:Option[RasterExtent]):IntRaster = {
169 174
     catalog.getRasterLayerByName(name) match {
170  
-      case Some((path, layer)) => {
171  
-        val reader = getReader(path,Some(layer))
  175
+      case Some(layer) => {
  176
+        val path = layer.rasterPath
  177
+        val reader = getReader(path, Some(layer))
172 178
         staticCache.get(layer.name) match {
173 179
           case Some(bytes) => reader.readCache(bytes, layer, reOpt)
174 180
           case None => reader.readPath(path, Some(layer), reOpt)
@@ -177,78 +183,8 @@ akka {
177 183
       case None => sys.error("couldn't find '%s'" format name)
178 184
     }
179 185
   }
180  
-
181 186
 }
182 187
 
183  
-//trait FileCaching {
184  
-//  val mb = 1<<20
185  
-//
186  
-//  // TODO: set this in config file
187  
-//  val maxBytesInCache = 1000 * mb // 2G
188  
-//  //var caching = true //xyz
189  
-//  var caching = false //xyz
190  
-//  var cacheSize:Int = 0
191  
-//
192  
-//  val rasterCache:HashBackedCache[String,IntRaster] = new LRUCache(maxBytesInCache, _.length * 4)
193  
-//
194  
-//  val catalog:Catalog
195  
-//
196  
-//  var id = "default"
197  
-//
198  
-//  //// TODO: remove this and add cache configuration to server's JSON config
199  
-//  ////       and/or constructor arguments
200  
-//  //def enableCaching() { caching = true }
201  
-//  //def disableCaching() { caching = false }
202  
-//
203  
-//  def getRasterExtentByName(name:String):RasterExtent = {
204  
-//    catalog.getRasterLayerByName(name) match {
205  
-//      case Some((path, layer)) => layer.rasterExtent
206  
-//      case None => sys.error("couldn't find %s" format name)
207  
-//    }
208  
-//  }
209  
-//
210  
-//  def getRasterByName(name:String, reOpt:Option[RasterExtent]):IntRaster = {
211  
-//    catalog.getRasterLayerByName(name) match {
212  
-//      case Some((path, layer)) => getRaster(path, Option(layer), reOpt)
213  
-//      case None => sys.error("couldn't find %s" format name)
214  
-//    }
215  
-//  }
216  
-//
217  
-//  def loadRaster(path:String, g:RasterExtent):IntRaster = getRaster(path, None, Option(g))
218  
-//
219  
-//  /**
220  
-//   * THIS is the new thing that we are wanting to use.
221  
-//   */
222  
-//  def getRaster(path:String, layerOpt:Option[RasterLayer], reOpt:Option[RasterExtent]):IntRaster = {
223  
-//
224  
-//    def xyz(path:String, layerOpt:Option[RasterLayer]) = {
225  
-//      getReader(path).read(path, layerOpt, None)
226  
-//    }
227  
-//
228  
-//    if (this.caching) {
229  
-//      val t0 = System.currentTimeMillis
230  
-//      val raster = rasterCache.getOrInsert(path, xyz(path, layerOpt))
231  
-//      val t = System.currentTimeMillis - t0
232  
-//      if (t > 10) println("rasterCache.getOrInsert(%s) took %d ms" format (path, t))
233  
-//      IntRasterReader.read(raster, reOpt)
234  
-//    } else {
235  
-//      getReader(path).read(path, layerOpt, reOpt)
236  
-//    }
237  
-//  }
238  
-//
239  
-//  def recursiveListFiles(f: File, r: Regex): Array[File] = {
240  
-//    val these = f.listFiles
241  
-//    val good = these.filter(f => r.findFirstIn(f.getName).isDefined)
242  
-//    good ++ these.filter(_.isDirectory).flatMap(recursiveListFiles(_,r))
243  
-//  }
244  
-//
245  
-//  def cacheRasters() = catalog.stores.foreach {
246  
-//    case (name, store) => store.layers.values.foreach {
247  
-//      case (path, layer) => getRaster(path, Some(layer), None)
248  
-//    }
249  
-//  }
250  
-//}
251  
-
252 188
 object Server {
253 189
   val config = ConfigFactory.load()
254 190
   val catalogPath = config.getString("geotrellis.catalog")
4  src/test/scala/geotrellis/process/catalog.scala
@@ -72,8 +72,8 @@ val json0 = """
72 72
     it("should find Arg32s in a source directory") {
73 73
       val catalog = Catalog.fromJSON(json1)
74 74
       val store = catalog.stores("stroud:fs")
75  
-      val layers = store.findRasterLayers
76  
-      layers.length must be === 3
  75
+      val layers = store.getLayers
  76
+      layers.toList.length must be === 3
77 77
     } 
78 78
   }
79 79
 

0 notes on commit d9c4854

Please sign in to comment.
Something went wrong with that request. Please try again.