diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala index 95b32c0def..6fd79cb6d2 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala @@ -1,5 +1,6 @@ package geotrellis.raster.io.geotiff +import geotrellis.util.ByteReader import geotrellis.vector.Extent import geotrellis.raster._ import geotrellis.raster.io.geotiff._ @@ -41,7 +42,7 @@ object ArraySegmentBytes { * @tiffTags: The [[TiffTags]] of the GeoTiff * @return A new instance of ArraySegmentBytes */ - def apply(byteBuffer: ByteBuffer, tiffTags: TiffTags): ArraySegmentBytes = { + def apply(byteBuffer: ByteReader, tiffTags: TiffTags): ArraySegmentBytes = { val compressedBytes: Array[Array[Byte]] = { def readSections(offsets: Array[Int], diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala index 26e5444c1a..db06b4707f 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala @@ -1,5 +1,6 @@ package geotrellis.raster.io.geotiff +import geotrellis.util.ByteReader import geotrellis.vector.Extent import geotrellis.raster._ import geotrellis.raster.io.geotiff._ @@ -19,7 +20,7 @@ import spire.syntax.cfor._ * @param tifftags: The [[TiffTags]] of the GeoTiff * @return A new instance of BufferSegmentBytes */ -case class BufferSegmentBytes(byteBuffer: ByteBuffer, tiffTags: TiffTags) extends SegmentBytes { +case class BufferSegmentBytes(byteBuffer: ByteReader, tiffTags: TiffTags) extends SegmentBytes { val (offsets, byteCounts) = if (tiffTags.hasStripStorage) { diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala index ce73abf8d6..2c5b7f8005 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala @@ -23,7 +23,7 @@ import geotrellis.raster.io.geotiff.util._ import geotrellis.raster.io.geotiff.tags._ import geotrellis.vector.Extent import geotrellis.proj4.CRS -import geotrellis.util.Filesystem +import geotrellis.util.{Filesystem, ByteReader} import monocle.syntax.apply._ @@ -83,7 +83,7 @@ object GeoTiffReader { /* Read a single band GeoTIFF file. * If there is more than one band in the GeoTiff, read the first band only. */ - def readSingleband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = { + def readSingleband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = { val info = readGeoTiffInfo(byteBuffer, decompress, streaming) val geoTiffTile = @@ -149,7 +149,7 @@ object GeoTiffReader { streaming: Boolean = false): MultibandGeoTiff = readMultiband(ByteBuffer.wrap(bytes), decompress, streaming) - def readMultiband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = { + def readMultiband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = { val info = readGeoTiffInfo(byteBuffer, decompress, streaming) val geoTiffTile = @@ -243,7 +243,7 @@ object GeoTiffReader { } } - private def readGeoTiffInfo(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): GeoTiffInfo = { + private def readGeoTiffInfo(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): GeoTiffInfo = { // set byte ordering (byteBuffer.get.toChar, byteBuffer.get.toChar) match { case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN) diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala index e5593f7e7f..b768a24f3c 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala @@ -4,24 +4,24 @@ import geotrellis.raster.io.geotiff.tags._ import geotrellis.raster.io.geotiff.tags.codes._ import TagCodes._ import TiffFieldType._ -import geotrellis.util.Filesystem import geotrellis.raster.io.geotiff.util._ +import geotrellis.util.{Filesystem, ByteReader} import spire.syntax.cfor._ import monocle.syntax.apply._ +import scala.language.implicitConversions import java.nio.{ ByteBuffer, ByteOrder } - object TiffTagsReader { def read(path: String): TiffTags = read(Filesystem.toMappedByteBuffer(path)) def read(bytes: Array[Byte]): TiffTags = - read(ByteBuffer.wrap(bytes, 0, bytes.size)) + read(ByteBuffer.wrap(bytes)) + + def read(byteBuffer: ByteReader): TiffTags = { - def read(byteBuffer: ByteBuffer): TiffTags = { - // set byte ordering (byteBuffer.get.toChar, byteBuffer.get.toChar) match { case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN) case ('M', 'M') => byteBuffer.order(ByteOrder.BIG_ENDIAN) @@ -32,15 +32,15 @@ object TiffTagsReader { val geoTiffIdNumber = byteBuffer.getChar if ( geoTiffIdNumber != 42) throw new MalformedGeoTiffException(s"bad identification number (must be 42, was $geoTiffIdNumber)") - val tagsStartPosition = byteBuffer.getInt read(byteBuffer, tagsStartPosition) } - def read(byteBuffer: ByteBuffer, tagsStartPosition: Int): TiffTags = { - byteBuffer.position(tagsStartPosition) + def read(byteBuffer: ByteReader, tagsStartPosition: Int): TiffTags = { + byteBuffer.position(tagsStartPosition) + val tagCount = byteBuffer.getShort // Read the tags. @@ -58,12 +58,15 @@ object TiffTagsReader { byteBuffer.getInt // Offset ) - if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag) + if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag) { geoTags = Some(tagMetadata) - else + } else { tiffTags = readTag(byteBuffer, tiffTags, tagMetadata) + } } + println(s"the gotags is: $geoTags") //, and the tifftags are: $tiffTags") + geoTags match { case Some(t) => tiffTags = readTag(byteBuffer, tiffTags, t) case None => @@ -72,7 +75,7 @@ object TiffTagsReader { tiffTags } - def readTag(byteBuffer: ByteBuffer, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags = + def readTag(byteBuffer: ByteReader, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags = (tagMetadata.tag, tagMetadata.fieldType) match { case (ModelPixelScaleTag, _) => byteBuffer.readModelPixelScaleTag(tiffTags, tagMetadata) @@ -106,7 +109,7 @@ object TiffTagsReader { byteBuffer.readDoublesTag(tiffTags, tagMetadata) } - implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteBuffer) extends AnyVal { + implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteReader) extends AnyVal { def readModelPixelScaleTag(tiffTags: TiffTags, tagMetadata: TiffTagMetadata) = { @@ -117,7 +120,7 @@ object TiffTagsReader { val scaleX = byteBuffer.getDouble val scaleY = byteBuffer.getDouble val scaleZ = byteBuffer.getDouble - + byteBuffer.position(oldPos) (tiffTags &|-> @@ -132,8 +135,6 @@ object TiffTagsReader { val numberOfPoints = tagMetadata.length / 6 - byteBuffer.position(tagMetadata.offset) - val points = Array.ofDim[(Pixel3D, Pixel3D)](numberOfPoints) cfor(0)(_ < numberOfPoints, _ + 1) { i => points(i) = @@ -165,16 +166,30 @@ object TiffTagsReader { byteBuffer.position(tagMetadata.offset) + println("ByteBuffer position before reading in keydirectorymetadata is", byteBuffer.getByteBuffer.position) + + val o = byteBuffer.getByteBuffer.position + + cfor(0)(_ < 25, _ + 1) { i => + println(byteBuffer.getByteBuffer.get) + } + byteBuffer.position(o) val version = byteBuffer.getShort val keyRevision = byteBuffer.getShort val minorRevision = byteBuffer.getShort val numberOfKeys = byteBuffer.getShort + //println(byteBuffer.getByteBuffer.position) + //println(version, keyRevision, minorRevision, numberOfKeys) val keyDirectoryMetadata = GeoKeyDirectoryMetadata(version, keyRevision, minorRevision, numberOfKeys) + + //println(byteBuffer.getByteBuffer.position) + //println(keyDirectoryMetadata) val geoKeyDirectory = GeoKeyReader.read(byteBuffer, tiffTags, GeoKeyDirectory(count = numberOfKeys)) + //println(byteBuffer.getByteBuffer.position) byteBuffer.position(oldPos) diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala index 9081b223d2..23477386c6 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala @@ -21,14 +21,16 @@ import codes.TagCodes._ import GeoKeys._ +import geotrellis.util.ByteReader import java.nio.ByteBuffer import monocle.syntax.apply._ import monocle.macros.Lenses +import scala.language.implicitConversions object GeoKeyReader { - def read(byteBuffer: ByteBuffer, imageDirectory: TiffTags, + def read(byteBuffer: ByteReader, imageDirectory: TiffTags, geoKeyDirectory: GeoKeyDirectory, index: Int = 0 ): GeoKeyDirectory = { diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala index eef852f087..8067adec3d 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala @@ -16,13 +16,15 @@ package geotrellis.raster.io.geotiff.util +import geotrellis.util.ByteReader import java.nio.ByteBuffer +import scala.language.implicitConversions + import spire.syntax.cfor._ trait ByteBufferExtensions { - - implicit class ByteBufferUtilities(byteBuffer: ByteBuffer) { + implicit class ByteBufferUtilities(byteBuffer: ByteReader) { @inline final private def ub2s(byte: Byte): Short = diff --git a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala index 519bb1483e..c9c674d23d 100644 --- a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala +++ b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala @@ -1,56 +1,131 @@ package geotrellis.spark.io.s3.util +import geotrellis.util._ import geotrellis.spark.io.s3._ import com.amazonaws.services.s3.model._ -class S3ByteBuffer(val request: GetObjectRequest, val client: AmazonS3Client) - extends S3InputStreamReader with S3Queue { - private var filePosition = 0 - private var arrayPosition = 0 - private var startingChunk = getMapArray - private var (offset, arr) = startingChunk.head +import java.nio.{ByteOrder, ByteBuffer, Buffer} +import scala.language.implicitConversions - def position = filePosition +class S3BytesByteReader( + val request: GetObjectRequest, + val client: AmazonS3Client) + extends S3Bytes { - def get: Byte = { - val value = - if (arrayPosition < arr.length - 1) { - arr(arrayPosition) - } else { - arrayPosition = 0 + private var chunk = getMappedArray + private def offset = chunk.head._1 + private def array = chunk.head._2 + private def length = array.length - saveChunk(Map(offset -> arr)) - offset = getMapArray.head._1 - arr = getMapArray.head._2 + private var byteBuffer = getByteBuffer - arr(arrayPosition) + private val byteOrder: ByteOrder = { + val order = + (get.toChar, get.toChar) match { + case ('I', 'I') => ByteOrder.LITTLE_ENDIAN + case ('M', 'M') => ByteOrder.BIG_ENDIAN + case _ => throw new Exception("incorrect byte order") } - arrayPosition += 1 - filePosition += 1 - value + byteBuffer.position(0) + order } - def get(array: Array[Byte], start: Int, end: Int): Array[Byte] = { - if (isContained(start, end, startingChunk)) { - System.arraycopy(arr, start, array, 0, end - start) - arrayPosition = start + end + def position = (offset + byteBuffer.position).toInt + + def position(newPoint: Int): Buffer = { + if (isContained(newPoint)) { + byteBuffer.position(newPoint) } else { - val mappedArray = getMapArray(start, end) - val bytes = mappedArray.head._2 + chunk = getMappedArray(newPoint) + byteBuffer = getByteBuffer + byteBuffer.position(0) + } + } - System.arraycopy(bytes, 0, array, 0, end - start) - saveChunk(mappedArray) + def get: Byte = { + if (!(byteBuffer.position + 1 <= byteBuffer.capacity)) { + chunk = getMappedArray(position + 1) + byteBuffer = getByteBuffer } - filePosition = start + end - array + byteBuffer.get + } + + def getChar: Char = { + if (!(byteBuffer.position + 2 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 2) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getChar } -} + def getShort: Short = { + if (!(byteBuffer.position + 2 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 2) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getShort + } + + def getInt: Int = { + if (!(byteBuffer.position + 4 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 4) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getInt + } -object S3ByteBuffer { - def apply(request: GetObjectRequest, client: AmazonS3Client): S3ByteBuffer = - new S3ByteBuffer(request, client) + def getFloat: Float = { + if (!(byteBuffer.position + 4 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 4) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getFloat + } + + def getDouble: Double = { + if (!(byteBuffer.position + 8 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 8) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getDouble + } + + def getByteBuffer: ByteBuffer = ByteBuffer.wrap(array).order(byteOrder) + + def getByteBuffer(arr: Array[Byte]) = ByteBuffer.wrap(arr).order(byteOrder) - def apply(bucket: String, key: String, client: AmazonS3Client): S3ByteBuffer = - new S3ByteBuffer(new GetObjectRequest(bucket, key), client) + def isContained(newPosition: Int): Boolean = + if (newPosition >= offset && newPosition <= offset + array.length) true else false +} + +object S3BytesByteReader { + def apply(bucket: String, key: String, client: AmazonS3Client): S3BytesByteReader = + new S3BytesByteReader(new GetObjectRequest(bucket, key), client) + + def apply(request: GetObjectRequest, client: AmazonS3Client): S3BytesByteReader = + new S3BytesByteReader(request, client) + + @inline implicit def toByteReader(s3BBR: S3BytesByteReader): ByteReader = { + new ByteReader() { + override def get = s3BBR.get + override def getChar = s3BBR.getChar + override def getShort = s3BBR.getShort + override def getInt = s3BBR.getInt + override def getFloat = s3BBR.getFloat + override def getDouble = s3BBR.getDouble + override def position: Int = s3BBR.position + override def position(i: Int): Buffer = s3BBR.position(i) + override def getByteBuffer = s3BBR.byteBuffer + } + } } diff --git a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala index e084d7a879..d45621529c 100644 --- a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala +++ b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala @@ -4,63 +4,66 @@ import geotrellis.spark.io.s3._ import scala.collection.mutable.Queue import com.amazonaws.services.s3.model._ +import java.nio.ByteBuffer -trait S3InputStreamReader { - def chunkSize = 256000 - - def request: GetObjectRequest +trait S3Bytes { def client: AmazonS3Client - - var location = 0 + def request: GetObjectRequest + + private val chunkSize = 256000 + private var streamPosition = 0 - private def metadata = + private val metadata = client.getObjectMetadata(request.getBucketName, request.getKey) - def objectLength = metadata.getContentLength + + private val objectLength = metadata.getContentLength private def pastLength(size: Int): Boolean = - if (location + size > objectLength) true else false + if (size > objectLength) true else false - private def getStream(start: Int, end: Int): S3ObjectInputStream = { + private def readStream(start: Int, end: Int): S3ObjectInputStream = { val obj = client.readRange(start, end, request) obj.getObjectContent } private def getArray: Array[Byte] = - getArray(location, chunkSize + location) + getArray(streamPosition) + + private def getArray(start: Int): Array[Byte] = + getArray(start, chunkSize) private def getArray(start: Int, length: Int): Array[Byte] = { val chunk = - if (!pastLength(length - start)) - length - start + if (!pastLength(length + start)) + length else - (objectLength - (length - start)).toInt + (objectLength - start).toInt val arr = Array.ofDim[Byte](chunk) - val stream = getStream(start, length) + val stream = readStream(start, chunk) stream.read(arr, 0, chunk) - stream.close() + streamPosition = start + length - location = length + start arr } - def getMapArray: Map[Long, Array[Byte]] = - getMapArray(location, chunkSize + location) + def getMappedArray: Map[Long, Array[Byte]] = + getMappedArray(streamPosition, chunkSize) - def getMapArray(start: Int, length: Int): Map[Long, Array[Byte]] = - Map(start.toLong -> getArray(start, length)) + def getMappedArray(start: Int): Map[Long, Array[Byte]] = + getMappedArray(start, chunkSize) - def isContained(start: Int, end: Int, map: Map[Long, Array[Byte]]): Boolean = { - val (offset, arr) = map.head - if (offset >= start && offset + arr.length <= end) true else false - } + def getMappedArray(start: Int, length: Int): Map[Long, Array[Byte]] = + Map(start.toLong -> getArray(start, length)) } -trait S3Queue { - private def mapQueue = Queue[Map[Long, Array[Byte]]]() +object S3Queue { + private val mapQueue = Queue[Map[Long, Array[Byte]]]() def size: Int = mapQueue.length + + def isEmpty: Boolean = mapQueue.isEmpty def saveChunk(chunk: Map[Long, Array[Byte]]): Unit = mapQueue.enqueue(chunk) diff --git a/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala b/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala index bf4941f7d4..68d809776a 100644 --- a/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala +++ b/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala @@ -1,44 +1,56 @@ package geotrellis.spark.io.s3.util import geotrellis.spark.io.s3._ +import geotrellis.util._ import com.amazonaws.services.s3.model._ import spire.syntax.cfor._ import org.scalatest._ -class S3ByteBufferSpec extends FunSpec with Matchers { +class S3BytesByteReaderSpec extends FunSpec with Matchers { val client = S3Client.default val bucket = "gt-rasters" val k = "nlcd/2011/tiles/nlcd_2011_01_01.tif" + val byteBuffer = Filesystem.toMappedByteBuffer("../../nlcd_2011_01_01.tif") - describe("MappedS3ByteBuffer") { + describe("MappedS3BytesByteReaderByteReader") { it("read in the first 4 bytes through get") { - val bb = S3ByteBuffer(bucket, k, client) + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) cfor(0)(_ < 4, _ + 1){ i => - bb.get + s3BytesByteReader.get } } + + /* it("should continue to read to the next MappedArray") { - val bb = S3ByteBuffer(bucket, k, client) + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) cfor(0)(_ < 256002, _ + 1) { i => - bb.get + s3BytesByteReader.get } } - - it("read in the first 4 bytes through bulk get") { - val bb = S3ByteBuffer(bucket, k, client) - val arr = Array.ofDim[Byte](4) - bb.get(arr, 0, arr.length) - } - it("should return the correct position") { - val bb = S3ByteBuffer(bucket, k, client) - val arr = Array.ofDim[Byte](950) - bb.get(arr, 789, arr.length) + it("should be able to read at various parts of the file") { + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) + val a = s3BytesByteReader.arr + cfor(0)(_ < 25, _ + 1) { i => + s3BytesByteReader.get + } + val arr = Array.ofDim[Byte](980) + s3BytesByteReader.position(400000) + byteBuffer.position(4000000) + val a2 = s3BytesByteReader.arr + + (a take 25).foreach(println) + (a2 take 25).foreach(println) + val expected = s3BytesByteReader.get + val actual = byteBuffer.get + + byteBuffer.position(0) - assert(bb.location == bb.position) + assert(actual == expected) } + */ } } diff --git a/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala b/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala new file mode 100644 index 0000000000..a5440e1e21 --- /dev/null +++ b/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala @@ -0,0 +1,114 @@ +package geotrellis.spark.io.s3.util + +import geotrellis.util.Filesystem +import geotrellis.spark.io.s3._ +import geotrellis.raster.io.geotiff.tags._ +import geotrellis.raster.io.geotiff.reader._ + +import org.scalatest._ +import com.amazonaws.services.s3.model._ +import spire.syntax.cfor._ + +class S3TiffTagsReaderSpec extends FunSpec { + + describe("tifftags reader") { + val client = S3Client.default + val bucket = "gt-rasters" + val k = "nlcd/2011/tiles/nlcd_2011_01_01.tif" + val s3ByteBuffer = S3BytesByteReader(bucket, k, client) + val fromLocal = TiffTagsReader.read("../../nlcd_2011_01_01.tif") + val fromServer = TiffTagsReader.read(s3ByteBuffer) + + /* + it("should read the same basic tags") { + val expected = fromServer.basicTags + val actual = fromLocal.basicTags + + assert(expected == actual) + } + + it("should read the same colimetry tags") { + val expected = fromServer.colimetryTags + val actual = fromLocal.colimetryTags + + assert(expected == actual) + } + + it("should read the same cmyk tags") { + val expected = fromServer.cmykTags + val actual = fromLocal.cmykTags + + assert(expected == actual) + } + + it("should read the same dataSampleForamt tags") { + val expected = fromServer.dataSampleFormatTags + val actual = fromLocal.dataSampleFormatTags + + assert(expected == actual) + } + + it("should read the same documentation tags") { + val expected = fromServer.documentationTags + val actual = fromLocal.documentationTags + + assert(expected == actual) + } + + it("should read the same geoTiffTags tags") { + val expected = fromServer.geoTiffTags + val actual = fromLocal.geoTiffTags + + assert(expected == actual) + } + + it("should read the same jpegTags tags") { + val expected = fromServer.jpegTags + val actual = fromLocal.jpegTags + + assert(expected == actual) + } + it("should read the same metadata tags") { + val expected = fromServer.metadataTags + val actual = fromLocal.metadataTags + + assert(expected == actual) + } + + it("should read the same nonBasic tags") { + val expected = fromServer.nonBasicTags + val actual = fromLocal.nonBasicTags + + assert(expected == actual) + } + + it("should read the same nonStandardized tags") { + val expected = fromServer.nonStandardizedTags + val actual = fromLocal.nonStandardizedTags + + assert(expected == actual) + } + + it("should read the same tags tags") { + val expected = fromServer.tags + val actual = fromLocal.tags + + assert(expected == actual) + } + + it("should read the same tile tags") { + val expected = fromServer.tileTags + val actual = fromLocal.tileTags + + assert(expected == actual) + } + + it("should read the same ycbcr tags") { + val expected = fromServer.yCbCrTags + val actual = fromLocal.yCbCrTags + + assert(expected == actual) + } + */ + } +} diff --git a/util/src/main/scala/geotrellis/util/ByteReader.scala b/util/src/main/scala/geotrellis/util/ByteReader.scala new file mode 100644 index 0000000000..5ac55e43ca --- /dev/null +++ b/util/src/main/scala/geotrellis/util/ByteReader.scala @@ -0,0 +1,33 @@ +package geotrellis.util + +import java.nio.{Buffer, ByteBuffer} +import scala.language.implicitConversions + +trait ByteReader { + def get: Byte + def getChar: Char + def getShort: Short + def getInt: Int + def getFloat: Float + def getDouble: Double + def position: Int + def position(i: Int): Buffer + def getByteBuffer: ByteBuffer +} + +object ByteReader { + implicit def byteBuffer2ByteReader(byteBuffer: ByteBuffer): ByteReader = { + new ByteReader() { + def get = byteBuffer.get + def getChar = byteBuffer.getChar + def getShort = byteBuffer.getShort + def getInt = byteBuffer.getInt + def getFloat = byteBuffer.getFloat + def getDouble = byteBuffer.getDouble + def position: Int = byteBuffer.position + def position(i: Int): Buffer = byteBuffer.position(i) + def getByteBuffer = byteBuffer + } + } + implicit def toByteBuffer(br: ByteReader): ByteBuffer = br.getByteBuffer +}