Fixes for #2177

raster-test/src/test/scala/geotrellis/raster/io/geotiff/writer/GeoTiffWriterTests.scala

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2016 Azavea
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package geotrellis.raster.io.geotiff.writer
+
+import geotrellis.raster.io.geotiff._
+import geotrellis.raster.io.geotiff.compression._
+import geotrellis.raster.testkit._
+import geotrellis.util._
+
+import org.scalatest._
+
+import java.io._
+
+class GeoTiffWriterTests extends FunSuite
+    with BeforeAndAfterAll
+    with RasterMatchers
+    with GeoTiffTestUtils {
+
+  override def afterAll = purge
+
+  test("Writing out an LZW raster from a streaming reader, and compressed (#2177)") {
+    /** This issue arose from immediately writing a compressed GeoTiff, without ever uncompressing it.
+      * We don't have support for writing LZW from uncompressed values, but we allow LZW to be used
+      * if the original compressed GeoTiff segments exist. There was an issue with Little Endian byte order
+      * with a passed through LZW compressor. Also, the predictor tag was not written. The streaming problem of
+      * the original issue was solved by removing the assumption that a SegmentBytes, when iterated over,
+      * would return the segment bytes in segment index order.
+      */
+    val temp = File.createTempFile("geotiff-writer", ".tif")
+    val path = temp.getPath
+
+    addToPurge(path)
+
+    val p = geoTiffPath("lzw-streaming-bug-2177.tif")
+    val rr = FileRangeReader(p)
+    val reader = StreamingByteReader(rr)
+
+    val gt1 = MultibandGeoTiff(reader)
+    val gt2 = MultibandGeoTiff.streaming(reader)
+    val gt3 = MultibandGeoTiff.compressed(p)
+
+    withClue("Assumption failed: Reading GeoTiff two ways didn't match") {
+      assertEqual(gt2.tile, gt1.tile)
+    }
+
+    withClue("Assumption failed: Reading GeoTiff compressed doesn't work") {
+      assertEqual(gt3.tile, gt1.tile)
+    }
+
+    gt3.write(path)
+
+    val resultComp = MultibandGeoTiff(path)
+    withClue("Writing from a compressed read produced incorrect GeoTiff.") {
+      assertEqual(resultComp.tile, gt1.tile)
+    }
+
+    gt2.write(path)
+
+    val result = MultibandGeoTiff(path)
+    withClue("Writing from a streaming read produced incorrect GeoTiff.") {
+      assertEqual(result.tile, gt1.tile)
+    }
+  }
+}

raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala

@@ -42,6 +42,8 @@ class ArraySegmentBytes(compressedBytes: Array[Array[Byte]]) extends SegmentByte
     */
   def getSegment(i: Int) = compressedBytes(i)
 
+  def getSegmentByteCount(i: Int): Int = compressedBytes(i).length
+
   def getSegments(indices: Traversable[Int]): Iterator[(Int, Array[Byte])] =
     indices.toIterator
       .map { i => i -> compressedBytes(i) }

raster/src/main/scala/geotrellis/raster/io/geotiff/LazySegmentBytes.scala

@@ -54,6 +54,8 @@ class LazySegmentBytes(
       (tileOffsets.get, tileByteCounts.get)
     }
 
+  def getSegmentByteCount(i: Int): Int = segmentByteCounts(i).toInt
+
   /** These are chunked segments in the order they appear in Image Data */
   protected def chunkSegments(segmentIds: Traversable[Int]): List[List[Segment]]  = {
     {for { id <- segmentIds } yield {

raster/src/main/scala/geotrellis/raster/io/geotiff/SegmentBytes.scala

@@ -27,7 +27,11 @@ trait SegmentBytes extends Seq[Array[Byte]] {
 
   def getSegments(indices: Traversable[Int]): Iterator[(Int, Array[Byte])]
 
+  def getSegmentByteCount(i: Int): Int
+
   def apply(idx: Int): Array[Byte] = getSegment(idx)
 
-  def iterator: Iterator[Array[Byte]] = getSegments(0 until length).map(_._2)
-}
+  /** Provides and iterator for segments. Do not assume the segments are in a particular order. */
+  def iterator: Iterator[Array[Byte]] =
+    getSegments(0 until length).map(_._2)
+}

raster/src/main/scala/geotrellis/raster/io/geotiff/SinglebandGeoTiff.scala

@@ -123,4 +123,4 @@ object SinglebandGeoTiff {
 
   def streaming(byteReader: ByteReader): SinglebandGeoTiff =
     GeoTiffReader.readSingleband(byteReader, false, true)
-}
+}

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/Decompressor.scala

@@ -23,6 +23,7 @@ import spire.syntax.cfor._
 
 trait Decompressor extends Serializable {
   def code: Int
+  def predictorCode: Int = Predictor.PREDICTOR_NONE
 
   def byteOrder: ByteOrder = ByteOrder.BIG_ENDIAN
 
@@ -31,9 +32,10 @@ trait Decompressor extends Serializable {
   /** Internally, we use the ByteBuffer's default byte ordering (BigEndian).
     * If the decompressed bytes are LittleEndian, flip 'em.
     */
-  def flipEndian(bytesPerFlip: Int): Decompressor = 
+  def flipEndian(bytesPerFlip: Int): Decompressor =
     new Decompressor {
       def code = Decompressor.this.code
+      override def predictorCode = Decompressor.this.predictorCode
 
       override
       def byteOrder = ByteOrder.LITTLE_ENDIAN // Since we have to flip, image data is in Little Endian
@@ -63,6 +65,8 @@ trait Decompressor extends Serializable {
   def withPredictor(predictor: Predictor): Decompressor =
     new Decompressor {
       def code = Decompressor.this.code
+      override def predictorCode = predictor.code
+      override def byteOrder = Decompressor.this.byteOrder
 
       def decompress(bytes: Array[Byte], segmentIndex: Int): Array[Byte] =
         predictor(Decompressor.this.decompress(bytes, segmentIndex), segmentIndex)
@@ -103,13 +107,13 @@ object Decompressor {
     }
 
     tiffTags.compression match {
-      case Uncompressed => 
+      case Uncompressed =>
         checkEndian(NoCompression)
-      case LZWCoded => 
+      case LZWCoded =>
         checkPredictor(LZWDecompressor(segmentSizes))
-      case ZLibCoded | PkZipCoded => 
+      case ZLibCoded | PkZipCoded =>
         checkPredictor(DeflateCompression.createDecompressor(segmentSizes))
-      case PackBitsCoded => 
+      case PackBitsCoded =>
         checkEndian(PackBitsDecompressor(segmentSizes))
 
       // Unsupported compression types

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/FloatingPointPredictor.scala

@@ -23,7 +23,7 @@ import spire.syntax.cfor._
 
 /** See TIFF Technical Note 3 */
 object FloatingPointPredictor {
-  def apply(tiffTags: TiffTags): FloatingPointPredictor = {
+  def apply(tiffTags: TiffTags): Predictor = {
     val colsPerRow = tiffTags.rowSize
     val rowsInSegment: (Int => Int) = { i => tiffTags.rowsInSegment(i) }
 
@@ -35,37 +35,38 @@ object FloatingPointPredictor {
       new FloatingPointPredictor(colsPerRow, rowsInSegment, bandType, 1)
     }
   }
-}
 
-class FloatingPointPredictor(colsPerRow: Int, rowsInSegment: Int => Int, bandType: BandType, bandCount: Int) extends Predictor {
-  val checkEndian = false
+  private class FloatingPointPredictor(colsPerRow: Int, rowsInSegment: Int => Int, bandType: BandType, bandCount: Int) extends Predictor {
+    val code = Predictor.PREDICTOR_FLOATINGPOINT
+    val checkEndian = false
 
-  def apply(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
-    val rows = rowsInSegment(segmentIndex)
-    val stride = bandCount
-    val bytesPerSample = bandType.bytesPerSample
+    def apply(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
+      val rows = rowsInSegment(segmentIndex)
+      val stride = bandCount
+      val bytesPerSample = bandType.bytesPerSample
 
-    val colValuesPerRow = colsPerRow * bandCount
-    val bytesPerRow = colValuesPerRow * bytesPerSample
-    cfor(0)(_ < rows, _ + 1) { row =>
-      // Undo the byte differencing
-      val rowByteIndex = row * bytesPerRow
+      val colValuesPerRow = colsPerRow * bandCount
+      val bytesPerRow = colValuesPerRow * bytesPerSample
+      cfor(0)(_ < rows, _ + 1) { row =>
+        // Undo the byte differencing
+        val rowByteIndex = row * bytesPerRow
 
-      val limit = (row + 1) * bytesPerRow
-      cfor(rowByteIndex + bandCount)(_ < limit, _ + 1) { i =>
-        bytes(i) = (bytes(i) + bytes(i - bandCount)).toByte
-      }
+        val limit = (row + 1) * bytesPerRow
+        cfor(rowByteIndex + bandCount)(_ < limit, _ + 1) { i =>
+          bytes(i) = (bytes(i) + bytes(i - bandCount)).toByte
+        }
 
-      val tmp = Array.ofDim[Byte](bytesPerRow)
-      System.arraycopy(bytes, rowByteIndex, tmp, 0, bytesPerRow)
+        val tmp = Array.ofDim[Byte](bytesPerRow)
+        System.arraycopy(bytes, rowByteIndex, tmp, 0, bytesPerRow)
 
-      cfor(0)(_ < colsPerRow * bandCount, _ + 1) { col =>
-        cfor(0)(_ < bytesPerSample, _ + 1) { byteIndex =>
-          bytes(rowByteIndex + (bytesPerSample * col + byteIndex)) =
-            tmp(byteIndex * colsPerRow + col)
+        cfor(0)(_ < colsPerRow * bandCount, _ + 1) { col =>
+          cfor(0)(_ < bytesPerSample, _ + 1) { byteIndex =>
+            bytes(rowByteIndex + (bytesPerSample * col + byteIndex)) =
+              tmp(byteIndex * colsPerRow + col)
+          }
         }
       }
+      bytes
     }
-    bytes
   }
 }

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/HorizontalPredictor.scala

@@ -40,67 +40,68 @@ object HorizontalPredictor {
 
     predictor.forBandType(bandType)
   }
-}
 
-class HorizontalPredictor(cols: Int, rowsInSegment: Int => Int, bandCount: Int) {
-  def forBandType(bandType: BandType): Predictor = {
-    val applyFunc: (Array[Byte], Int) => Array[Byte] =
-      bandType.bitsPerSample match {
-        case  8 => apply8 _
-        case 16 => apply16 _
-        case 32 => apply32 _
-        case _ =>
-          throw new MalformedGeoTiffException(s"""Horizontal differencing "Predictor" not supported with ${bandType.bitsPerSample} bits per sample""")
+  private class HorizontalPredictor(cols: Int, rowsInSegment: Int => Int, bandCount: Int) {
+    def forBandType(bandType: BandType): Predictor = {
+      val applyFunc: (Array[Byte], Int) => Array[Byte] =
+        bandType.bitsPerSample match {
+          case  8 => apply8 _
+          case 16 => apply16 _
+          case 32 => apply32 _
+          case _ =>
+            throw new MalformedGeoTiffException(s"""Horizontal differencing "Predictor" not supported with ${bandType.bitsPerSample} bits per sample""")
+        }
+
+      new Predictor {
+        val code = Predictor.PREDICTOR_HORIZONTAL
+        val checkEndian = true
+        def apply(bytes: Array[Byte], segmentIndex: Int): Array[Byte] =
+          applyFunc(bytes, segmentIndex)
       }
-
-    new Predictor {
-      val checkEndian = true
-      def apply(bytes: Array[Byte], segmentIndex: Int): Array[Byte] =
-        applyFunc(bytes, segmentIndex)
     }
-  }
 
-  def apply8(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
-    val rows = rowsInSegment(segmentIndex)
+    def apply8(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
+      val rows = rowsInSegment(segmentIndex)
 
-    cfor(0)(_ < rows, _ + 1) { row =>
-      var count = bandCount * (row * cols + 1)
-      cfor(bandCount)({ k => k < cols * bandCount && k < bytes.length }, _ + 1) { k =>
-        bytes(count) = (bytes(count) + bytes(count - bandCount)).toByte
-        count += 1
+      cfor(0)(_ < rows, _ + 1) { row =>
+        var count = bandCount * (row * cols + 1)
+        cfor(bandCount)({ k => k < cols * bandCount && k < bytes.length }, _ + 1) { k =>
+          bytes(count) = (bytes(count) + bytes(count - bandCount)).toByte
+          count += 1
+        }
       }
-    }
 
-    bytes
-  }
+      bytes
+    }
 
-  def apply16(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
-    val buffer = ByteBuffer.wrap(bytes).asShortBuffer
-    val rows = rowsInSegment(segmentIndex)
+    def apply16(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
+      val buffer = ByteBuffer.wrap(bytes).asShortBuffer
+      val rows = rowsInSegment(segmentIndex)
 
-    cfor(0)(_ < rows, _ + 1) { row =>
-      var count = bandCount * (row * cols + 1)
-      cfor(bandCount)(_ < cols * bandCount, _ + 1) { k =>
-        buffer.put(count, (buffer.get(count) + buffer.get(count - bandCount)).toShort)
-        count += 1
+      cfor(0)(_ < rows, _ + 1) { row =>
+        var count = bandCount * (row * cols + 1)
+        cfor(bandCount)(_ < cols * bandCount, _ + 1) { k =>
+          buffer.put(count, (buffer.get(count) + buffer.get(count - bandCount)).toShort)
+          count += 1
+        }
       }
-    }
 
-    bytes
-  }
+      bytes
+    }
 
-  def apply32(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
-    val buffer = ByteBuffer.wrap(bytes).asIntBuffer
-    val rows = rowsInSegment(segmentIndex)
+    def apply32(bytes: Array[Byte], segmentIndex: Int): Array[Byte] = {
+      val buffer = ByteBuffer.wrap(bytes).asIntBuffer
+      val rows = rowsInSegment(segmentIndex)
 
-    cfor(0)(_ < rows, _ + 1) { row =>
-      var count = bandCount * (row * cols + 1)
-      cfor(bandCount)(_ < cols * bandCount, _ + 1) { k =>
-        buffer.put(count, buffer.get(count) + buffer.get(count - bandCount))
-        count += 1
+      cfor(0)(_ < rows, _ + 1) { row =>
+        var count = bandCount * (row * cols + 1)
+        cfor(bandCount)(_ < cols * bandCount, _ + 1) { k =>
+          buffer.put(count, buffer.get(count) + buffer.get(count - bandCount))
+          count += 1
+        }
       }
-    }
 
-    bytes
+      bytes
+    }
   }
 }

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/LZWDecompression.scala

@@ -108,7 +108,7 @@ class LZWDecompressor(segmentSizes: Array[Int]) extends Decompressor {
         if (code == EoICode) {
           break = true
         } else {
-          outputArrayIndex = 
+          outputArrayIndex =
             TokenTable.writeToOutput(tokenTable(code), outputArray, outputArrayIndex)
         }
       } else if (code < tokenTableIndex) {

raster/src/main/scala/geotrellis/raster/io/geotiff/compression/Predictor.scala

@@ -32,6 +32,7 @@ object Predictor {
     ) match {
       case None | Some(PREDICTOR_NONE) =>
         new Predictor {
+          val code = PREDICTOR_NONE
           val checkEndian = true
           def apply(bytes: Array[Byte], segmentIndex: Int) = bytes
         }
@@ -46,7 +47,10 @@ object Predictor {
 }
 
 trait Predictor {
+  /** True if this predictor needs to check if the endian requires flipping */
   def checkEndian: Boolean
+  /** GeoTiff tag value for this predictor */
+  def code: Int
 
   def apply(bytes: Array[Byte], segmentIndex: Int): Array[Byte]
 }