Merge pull request #1905 from echeipesh/chunky-streaming

Streaming/Windowed GeoTiff Reading

project/Boilerplate.scala

@@ -325,7 +325,11 @@ object GenMacroSegmentCombiner extends Template {
       val diffs        = (1 until arity) map { i => s"val diff$i = b$i - b0 "} mkString "; "
       val diffsArgs    = ((((1 until arity) map { i => s"i + diff$i, segment" } mkString ", ") split ", ") init) mkString ", "
       val startVals    = (0 until arity) map { i => s"val start$i = bandSegmentCount * b$i" } mkString "; "
-      val segmentVals  = (0 until arity) map { i => s"val segment$i = getSegment(start$i + segmentIndex)" } mkString "; "
+      val getSegmentsZip = (0 until arity) map { i =>
+        if(i == 0) s"getSegments(start$i until start$i + bandSegmentCount)" else s"zip(getSegments(start$i until start$i + bandSegmentCount))"
+      } mkString "."
+      val bracesForZipArgs = (0 until arity - 1) map { _ => "(" } mkString ""
+      val segmentsZipArgs = bracesForZipArgs concat ((0 until arity) map { i => if(i == 0) s"(segmentIndex$i, segment$i)" else s"(_, segment$i))" } mkString ", ")
       val segmentArgs  = (0 until arity) map { i => s"i, segment$i" } mkString ", "
       val combinerArgs = (0 until arity) map { i => s"combiner.b$i" } mkString ", "
       val siArgs       = (1 to arity) map { i => s"s$i: GeoTiffSegment, i$i: Int" } mkString ", "
@@ -341,6 +345,7 @@ object GenMacroSegmentCombiner extends Template {
          |trait MacroGeotiffMultibandCombiners {
          |  def cellType: CellType
          |  def getSegment(i: Int): GeoTiffSegment
+         |  def getSegments(ids: Traversable[Int]): Iterator[(Int, GeoTiffSegment)]
          |
          |  val segmentLayout: GeoTiffSegmentLayout
          |  val segmentCount: Int
@@ -366,8 +371,7 @@ object GenMacroSegmentCombiner extends Template {
         -        ${diffs}
         -        val compressor = compression.createCompressor(segmentCount)
         -        val arr = Array.ofDim[Array[Byte]](segmentCount)
-        -        cfor(0)(_ < segmentCount, _ + 1) { segmentIndex =>
-        -          val segment = getSegment(segmentIndex)
+        -        getSegments(0 until segmentCount).foreach { case (segmentIndex, segment) =>
         -          val segmentSize = segment.size
         -          val segmentCombiner = createSegmentCombiner(segmentSize / bandCount)
         -          var j = 0
@@ -383,8 +387,8 @@ object GenMacroSegmentCombiner extends Template {
         -      val compressor = compression.createCompressor(bandSegmentCount)
         -      val arr = Array.ofDim[Array[Byte]](bandSegmentCount)
         -      ${startVals}
-        -      cfor(0)(_ < bandSegmentCount, _ + 1) { segmentIndex =>
-        -        ${segmentVals}
+        -      ${getSegmentsZip}.foreach { case (${segmentsZipArgs}) =>
+        -        val segmentIndex = segmentIndex0 - start0
         -        val segmentSize = segment0.size
         -        val segmentCombiner = createSegmentCombiner(segmentSize)
         -        cfor(0)(_ < segmentSize, _ + 1) { i =>

raster-test/src/test/scala/geotrellis/raster/io/geotiff/reader/MultibandGeoTiffReaderSpec.scala

@@ -149,8 +149,12 @@ class MultibandGeoTiffReaderSpec extends FunSpec
       geoTiffPath(s"3bands/bit/3bands-${s}-${i}.tif")
 
     it("should read pixel interleave, striped") {
-      val tile =
-        MultibandGeoTiff(p("striped", "pixel")).tile
+      val tile: GeoTiffMultibandTile =
+        MultibandGeoTiff(
+          path = p("striped", "pixel"),
+          decompress = false,
+          streaming = false
+        ).tile.asInstanceOf[GeoTiffMultibandTile]
 
       // println("         BIT BANDS")
       // println(tile.band(0).asciiDraw)

raster/src/main/scala/geotrellis/raster/crop/MultibandTileCropMethods.scala

@@ -18,6 +18,7 @@ package geotrellis.raster.crop
 
 import geotrellis.vector._
 import geotrellis.raster._
+import geotrellis.raster.io.geotiff.GeoTiffMultibandTile
 
 
 /**
@@ -31,13 +32,22 @@ trait MultibandTileCropMethods extends TileCropMethods[MultibandTile] {
     * [[MultibandTile]] and return a new MultibandTile.
     */
   def crop(gb: GridBounds, options: Options): MultibandTile = {
-    val croppedBands = Array.ofDim[Tile](self.bandCount)
-
-    for(b <- 0 until self.bandCount) {
-      croppedBands(b) = self.band(b).crop(gb, options)
+    self match {
+      case geotiffTile: GeoTiffMultibandTile =>
+        val cropBounds =
+          if(options.clamp)
+            gb.intersection(self)
+              .getOrElse(throw new GeoAttrsError(s"Grid bounds do not intersect: $self crop $gb"))
+          else
+            gb
+        geotiffTile.crop(cropBounds)
+      case _ =>
+        val croppedBands = Array.ofDim[Tile](self.bandCount)
+        for(b <- 0 until self.bandCount) {
+          croppedBands(b) = self.band(b).crop(gb, options)
+        }
+        ArrayMultibandTile(croppedBands)
     }
-
-    ArrayMultibandTile(croppedBands)
   }
 
   /**

raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala

@@ -17,10 +17,6 @@
 package geotrellis.raster.io.geotiff
 
 import geotrellis.util.ByteReader
-import geotrellis.vector.Extent
-import geotrellis.raster._
-import geotrellis.raster.io.geotiff._
-import geotrellis.raster.io.geotiff.reader._
 import geotrellis.raster.io.geotiff.tags._
 import geotrellis.raster.io.geotiff.util._
 
@@ -35,7 +31,7 @@ import spire.syntax.cfor._
   */
 class ArraySegmentBytes(compressedBytes: Array[Array[Byte]]) extends SegmentBytes {
 
-  override val size = compressedBytes.size
+  def length = compressedBytes.length
 
   /**
     * Returns an Array[Byte] that represents a [[GeoTiffSegment]] via
@@ -45,6 +41,10 @@ class ArraySegmentBytes(compressedBytes: Array[Array[Byte]]) extends SegmentByte
     * @return    An Array[Byte] that contains the bytes of the segment
     */
   def getSegment(i: Int) = compressedBytes(i)
+
+  def getSegments(indices: Traversable[Int]): Iterator[(Int, Array[Byte])] =
+    indices.toIterator
+      .map { i => i -> compressedBytes(i) }
 }
 
 object ArraySegmentBytes {
@@ -57,43 +57,12 @@ object ArraySegmentBytes {
     *  @return             A new instance of ArraySegmentBytes
     */
   def apply(byteReader: ByteReader, tiffTags: TiffTags): ArraySegmentBytes = {
-
-      val compressedBytes: Array[Array[Byte]] = {
-        def readSections(offsets: Array[Long],
-          byteCounts: Array[Long]): Array[Array[Byte]] = {
-            val result = Array.ofDim[Array[Byte]](offsets.size)
-
-            cfor(0)(_ < offsets.size, _ + 1) { i =>
-              result(i) = byteReader.getSignedByteArray(offsets(i), byteCounts(i))
-            }
-
-            result
-          }
-
-          if (tiffTags.hasStripStorage) {
-
-            val stripOffsets = (tiffTags &|->
-              TiffTags._basicTags ^|->
-              BasicTags._stripOffsets get)
-
-            val stripByteCounts = (tiffTags &|->
-              TiffTags._basicTags ^|->
-              BasicTags._stripByteCounts get)
-
-            readSections(stripOffsets.get, stripByteCounts.get)
-
-          } else {
-            val tileOffsets = (tiffTags &|->
-              TiffTags._tileTags ^|->
-              TileTags._tileOffsets get)
-
-            val tileByteCounts = (tiffTags &|->
-              TiffTags._tileTags ^|->
-              TileTags._tileByteCounts get)
-
-            readSections(tileOffsets.get, tileByteCounts.get)
-          }
-      }
-      new ArraySegmentBytes(compressedBytes)
+    // use streaming read here to improve performance via chunking
+    val streaming = LazySegmentBytes(byteReader, tiffTags)
+    val compressedBytes = Array.ofDim[Array[Byte]](streaming.length)
+    streaming.getSegments(compressedBytes.indices).foreach {
+      case (i, bytes) => compressedBytes(i) = bytes
     }
+    new ArraySegmentBytes(compressedBytes)
+  }
 }

raster/src/main/scala/geotrellis/raster/io/geotiff/BitGeoTiffSegmentCollection.scala

@@ -16,8 +16,6 @@
 
 package geotrellis.raster.io.geotiff
 
-import geotrellis.raster._
-
 trait BitGeoTiffSegmentCollection extends GeoTiffSegmentCollection {
   type T = BitGeoTiffSegment
 
@@ -28,9 +26,9 @@ trait BitGeoTiffSegmentCollection extends GeoTiffSegmentCollection {
   val bandCount: Int
   val hasPixelInterleave: Boolean
 
-  val createSegment: Int => BitGeoTiffSegment = { i =>
-    val (segmentCols, segmentRows) = segmentLayout.getSegmentDimensions(i)
-    // val size = segmentCols * segmentRows
+  lazy val decompressGeoTiffSegment = { (i: Int, bytes: Array[Byte]) =>
+    val (_, segmentRows) = segmentLayout.getSegmentDimensions(i)
+
     val cols = {
       val c = segmentLayout.tileLayout.tileCols
       if(hasPixelInterleave) c * bandCount
@@ -39,7 +37,6 @@ trait BitGeoTiffSegmentCollection extends GeoTiffSegmentCollection {
 
     val rows = if(segmentLayout.isStriped) { segmentRows } else { segmentLayout.tileLayout.tileRows }
 
-    new BitGeoTiffSegment(getDecompressedBytes(i), cols, rows)
+    new BitGeoTiffSegment(decompressor.decompress(bytes, i), cols, rows)
   }
-
 }

raster/src/main/scala/geotrellis/raster/io/geotiff/BitGeoTiffTile.scala

@@ -32,60 +32,6 @@ class BitGeoTiffTile(
   // We need multiband information because BitGeoTiffSegments are unique
   val hasPixelInterleave = false
 
-  // TODO: Optimize this.
-  def mutable: MutableArrayTile = {
-    val result = BitArrayTile.empty(cols, rows)
-
-    val layoutCols = segmentLayout.tileLayout.layoutCols
-    val tileCols = segmentLayout.tileLayout.tileCols
-
-    val layoutRows = segmentLayout.tileLayout.layoutRows
-    val tileRows = segmentLayout.tileLayout.tileRows
-
-    cfor(0)(_ < layoutCols, _ + 1) { layoutCol =>
-      val colStart = layoutCol * tileCols
-      val colEnd = (colStart + tileCols).min(cols)
-      cfor(0)(_ < layoutRows, _ + 1) { layoutRow =>
-        val rowStart = layoutRow * tileRows
-        val rowEnd = (rowStart + tileRows).min(rows)
-        cfor(colStart)(_ < colEnd, _ + 1) { col =>
-          cfor(rowStart)(_ < rowEnd, _ + 1) { row =>
-            result.set(col, row, get(col, row))
-          }
-        }
-      }
-    }
-   result
-  }
-
-  def crop(gridBounds: GridBounds): MutableArrayTile = {
-    val result = BitArrayTile.empty(gridBounds.width, gridBounds.height)
-
-    val colMin = gridBounds.colMin
-    val rowMin = gridBounds.rowMin
-    val tileCols = segmentLayout.tileLayout.tileCols
-    val tileRows = segmentLayout.tileLayout.tileRows
-
-    cfor(0)(_ < segmentCount, _ + 1) { i =>
-      val segmentTransform = segmentLayout.getSegmentTransform(i)
-      val colStart = segmentTransform.bitIndexToCol(0)
-      val rowStart = segmentTransform.bitIndexToRow(0)
-      val colEnd = (colStart + tileCols).min(cols)
-      val rowEnd = (rowStart + tileRows).min(rows)
-
-      if (gridBounds.intersects(GridBounds(colStart, rowStart, colEnd, rowEnd))) {
-        cfor(colStart)(_ < colEnd, _ + 1) { col =>
-          cfor(rowStart)(_ < rowEnd, _ + 1) { row =>
-            if (gridBounds.contains(col, row))
-              result.set(col - colMin, row - rowMin, get(col, row))
-          }
-        }
-      }
-    }
-    result
-  }
-
-
   def withNoData(noDataValue: Option[Double]): BitGeoTiffTile =
     new BitGeoTiffTile(segmentBytes, decompressor, segmentLayout, compression, cellType.withNoData(noDataValue))
 

raster/src/main/scala/geotrellis/raster/io/geotiff/ByteGeoTiffMultibandTile.scala

@@ -19,8 +19,6 @@ package geotrellis.raster.io.geotiff
 import geotrellis.raster._
 import geotrellis.raster.io.geotiff.compression._
 
-import spire.syntax.cfor._
-
 class ByteGeoTiffMultibandTile(
   compressedBytes: SegmentBytes,
   decompressor: Decompressor,

raster/src/main/scala/geotrellis/raster/io/geotiff/ByteGeoTiffSegmentCollection.scala

@@ -16,21 +16,18 @@
 
 package geotrellis.raster.io.geotiff
 
-import geotrellis.raster._
-import geotrellis.raster.io.geotiff.compression._
-
 trait ByteGeoTiffSegmentCollection extends GeoTiffSegmentCollection {
   type T = ByteGeoTiffSegment
 
   val bandType = ByteBandType
-  val noDataValue: Option[Byte]
+  def noDataValue: Option[Byte]
 
-  lazy val createSegment: Int => ByteGeoTiffSegment = noDataValue match {
+  lazy val decompressGeoTiffSegment = noDataValue match {
     case None =>
-      { i: Int => new ByteRawGeoTiffSegment(getDecompressedBytes(i)) }
-    case Some(nd) if (nd == Short.MinValue) =>
-      { i: Int => new ByteConstantNoDataCellTypeGeoTiffSegment(getDecompressedBytes(i)) }
+      (i: Int, bytes: Array[Byte]) => new ByteRawGeoTiffSegment(decompressor.decompress(bytes, i))
+    case Some(nd) if nd == Byte.MinValue =>
+      (i: Int, bytes: Array[Byte]) => new ByteConstantNoDataCellTypeGeoTiffSegment(decompressor.decompress(bytes, i))
     case Some(nd) =>
-      { i: Int => new ByteUserDefinedNoDataGeoTiffSegment(getDecompressedBytes(i), nd) }
-    }
+      (i: Int, bytes: Array[Byte]) => new ByteUserDefinedNoDataGeoTiffSegment(decompressor.decompress(bytes, i), nd)
+  }
 }

raster/src/main/scala/geotrellis/raster/io/geotiff/ByteGeoTiffTile.scala

@@ -18,7 +18,6 @@ package geotrellis.raster.io.geotiff
 
 import geotrellis.raster._
 import geotrellis.raster.io.geotiff.compression._
-import spire.syntax.cfor._
 
 class ByteGeoTiffTile(
   val segmentBytes: SegmentBytes,
@@ -34,82 +33,6 @@ class ByteGeoTiffTile(
     case ByteUserDefinedNoDataCellType(nd) => Some(nd)
   }
 
-  def mutable: MutableArrayTile = {
-    val arr = Array.ofDim[Byte](cols * rows)
-
-    if(segmentLayout.isStriped) {
-      var i = 0
-      cfor(0)(_ < segmentCount, _ + 1) { segmentIndex =>
-        val segment =
-          getSegment(segmentIndex)
-        val size = segment.bytes.size
-        System.arraycopy(segment.bytes, 0, arr, i, size)
-        i += size
-      }
-    } else {
-      cfor(0)(_ < segmentCount, _ + 1) { segmentIndex =>
-        val segment =
-          getSegment(segmentIndex)
-
-        val segmentTransform = segmentLayout.getSegmentTransform(segmentIndex)
-        val width = segmentTransform.segmentCols
-        val tileWidth = segmentLayout.tileLayout.tileCols
-
-        cfor(0)(_ < tileWidth * segmentTransform.segmentRows, _ + tileWidth) { i =>
-          val col = segmentTransform.indexToCol(i)
-          val row = segmentTransform.indexToRow(i)
-          val j = (row * cols) + col
-          System.arraycopy(segment.bytes, i, arr, j, width)
-        }
-      }
-    }
-    ByteArrayTile.fromBytes(arr, cols, rows, cellType)
-  }
-
-  def crop(gridBounds: GridBounds): MutableArrayTile = {
-    val arr = Array.ofDim[Byte](gridBounds.size)
-    var counter = 0
-
-    if (segmentLayout.isStriped) {
-      cfor(0)(_ < segmentCount, _ + 1) { i =>
-        val segmentGridBounds = segmentLayout.getGridBounds(i)
-        if (gridBounds.intersects(segmentGridBounds)) {
-          val segment = getSegment(i)
-
-          val result =
-            gridBounds.intersection(segmentGridBounds).get
-          val intersection =
-            Intersection(segmentGridBounds, result, segmentLayout)
-
-          cfor(intersection.start)(_ < intersection.end, _ + cols) { i =>
-            System.arraycopy(segment.bytes, i, arr, counter, result.width)
-            counter += result.width
-          }
-        }
-      }
-    } else {
-      cfor(0)(_ < segmentCount, _ + 1) {i =>
-        val segmentGridBounds = segmentLayout.getGridBounds(i)
-        if (gridBounds.intersects(segmentGridBounds)) {
-          val segment = getSegment(i)
-          val segmentTransform = segmentLayout.getSegmentTransform(i)
-
-          val result = gridBounds.intersection(segmentGridBounds).get
-          val intersection = Intersection(segmentGridBounds, result, segmentLayout)
-
-          cfor(intersection.start)(_ < intersection.end, _ + intersection.tileWidth) { i =>
-            val col = segmentTransform.indexToCol(i)
-            val row = segmentTransform.indexToRow(i)
-            val j = (row - gridBounds.rowMin) * gridBounds.width + (col - gridBounds.colMin)
-            System.arraycopy(segment.bytes, i, arr, j, result.width)
-          }
-        }
-      }
-    }
-    ByteArrayTile.fromBytes(arr, gridBounds.width, gridBounds.height, cellType)
-  }
-
-
   def withNoData(noDataValue: Option[Double]): ByteGeoTiffTile =
     new ByteGeoTiffTile(segmentBytes, decompressor, segmentLayout, compression, cellType.withNoData(noDataValue))