From 1297b44979192d15cb1e24dd9a9bd5a493684644 Mon Sep 17 00:00:00 2001 From: jbouffard Date: Fri, 9 Sep 2016 08:57:57 -0400 Subject: [PATCH] Added ByteReader made the code in the s3 util less confusing Changed how ByteReader works for S3 and ByteBuffer Continued work on S3 reading Continuing to search for problems in code --- .../raster/io/geotiff/ArraySegmentBytes.scala | 3 +- .../io/geotiff/BufferSegmentBytes.scala | 3 +- .../io/geotiff/reader/GeoTiffReader.scala | 8 +- .../io/geotiff/reader/TiffTagsReader.scala | 45 ++++-- .../raster/io/geotiff/tags/GeoKeyReader.scala | 4 +- .../geotiff/util/ByteBufferExtensions.scala | 6 +- .../spark/io/s3/util/S3ByteBuffer.scala | 147 +++++++++++++----- .../io/s3/util/S3InputStreamReader.scala | 57 +++---- .../spark/io/s3/S3ByteBufferSpec.scala | 46 ++++-- .../spark/io/s3/S3TiffTagsReaderSpec.scala | 114 ++++++++++++++ .../scala/geotrellis/util/ByteReader.scala | 33 ++++ 11 files changed, 362 insertions(+), 104 deletions(-) create mode 100644 s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala create mode 100644 util/src/main/scala/geotrellis/util/ByteReader.scala diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala index 95b32c0def..6fd79cb6d2 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala @@ -1,5 +1,6 @@ package geotrellis.raster.io.geotiff +import geotrellis.util.ByteReader import geotrellis.vector.Extent import geotrellis.raster._ import geotrellis.raster.io.geotiff._ @@ -41,7 +42,7 @@ object ArraySegmentBytes { * @tiffTags: The [[TiffTags]] of the GeoTiff * @return A new instance of ArraySegmentBytes */ - def apply(byteBuffer: ByteBuffer, tiffTags: TiffTags): ArraySegmentBytes = { + def apply(byteBuffer: ByteReader, tiffTags: TiffTags): ArraySegmentBytes = { val compressedBytes: Array[Array[Byte]] = { def readSections(offsets: Array[Int], diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala index 26e5444c1a..db06b4707f 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala @@ -1,5 +1,6 @@ package geotrellis.raster.io.geotiff +import geotrellis.util.ByteReader import geotrellis.vector.Extent import geotrellis.raster._ import geotrellis.raster.io.geotiff._ @@ -19,7 +20,7 @@ import spire.syntax.cfor._ * @param tifftags: The [[TiffTags]] of the GeoTiff * @return A new instance of BufferSegmentBytes */ -case class BufferSegmentBytes(byteBuffer: ByteBuffer, tiffTags: TiffTags) extends SegmentBytes { +case class BufferSegmentBytes(byteBuffer: ByteReader, tiffTags: TiffTags) extends SegmentBytes { val (offsets, byteCounts) = if (tiffTags.hasStripStorage) { diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala index ce73abf8d6..2c5b7f8005 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala @@ -23,7 +23,7 @@ import geotrellis.raster.io.geotiff.util._ import geotrellis.raster.io.geotiff.tags._ import geotrellis.vector.Extent import geotrellis.proj4.CRS -import geotrellis.util.Filesystem +import geotrellis.util.{Filesystem, ByteReader} import monocle.syntax.apply._ @@ -83,7 +83,7 @@ object GeoTiffReader { /* Read a single band GeoTIFF file. * If there is more than one band in the GeoTiff, read the first band only. */ - def readSingleband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = { + def readSingleband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = { val info = readGeoTiffInfo(byteBuffer, decompress, streaming) val geoTiffTile = @@ -149,7 +149,7 @@ object GeoTiffReader { streaming: Boolean = false): MultibandGeoTiff = readMultiband(ByteBuffer.wrap(bytes), decompress, streaming) - def readMultiband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = { + def readMultiband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = { val info = readGeoTiffInfo(byteBuffer, decompress, streaming) val geoTiffTile = @@ -243,7 +243,7 @@ object GeoTiffReader { } } - private def readGeoTiffInfo(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): GeoTiffInfo = { + private def readGeoTiffInfo(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): GeoTiffInfo = { // set byte ordering (byteBuffer.get.toChar, byteBuffer.get.toChar) match { case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN) diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala index e5593f7e7f..b768a24f3c 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala @@ -4,24 +4,24 @@ import geotrellis.raster.io.geotiff.tags._ import geotrellis.raster.io.geotiff.tags.codes._ import TagCodes._ import TiffFieldType._ -import geotrellis.util.Filesystem import geotrellis.raster.io.geotiff.util._ +import geotrellis.util.{Filesystem, ByteReader} import spire.syntax.cfor._ import monocle.syntax.apply._ +import scala.language.implicitConversions import java.nio.{ ByteBuffer, ByteOrder } - object TiffTagsReader { def read(path: String): TiffTags = read(Filesystem.toMappedByteBuffer(path)) def read(bytes: Array[Byte]): TiffTags = - read(ByteBuffer.wrap(bytes, 0, bytes.size)) + read(ByteBuffer.wrap(bytes)) + + def read(byteBuffer: ByteReader): TiffTags = { - def read(byteBuffer: ByteBuffer): TiffTags = { - // set byte ordering (byteBuffer.get.toChar, byteBuffer.get.toChar) match { case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN) case ('M', 'M') => byteBuffer.order(ByteOrder.BIG_ENDIAN) @@ -32,15 +32,15 @@ object TiffTagsReader { val geoTiffIdNumber = byteBuffer.getChar if ( geoTiffIdNumber != 42) throw new MalformedGeoTiffException(s"bad identification number (must be 42, was $geoTiffIdNumber)") - val tagsStartPosition = byteBuffer.getInt read(byteBuffer, tagsStartPosition) } - def read(byteBuffer: ByteBuffer, tagsStartPosition: Int): TiffTags = { - byteBuffer.position(tagsStartPosition) + def read(byteBuffer: ByteReader, tagsStartPosition: Int): TiffTags = { + byteBuffer.position(tagsStartPosition) + val tagCount = byteBuffer.getShort // Read the tags. @@ -58,12 +58,15 @@ object TiffTagsReader { byteBuffer.getInt // Offset ) - if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag) + if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag) { geoTags = Some(tagMetadata) - else + } else { tiffTags = readTag(byteBuffer, tiffTags, tagMetadata) + } } + println(s"the gotags is: $geoTags") //, and the tifftags are: $tiffTags") + geoTags match { case Some(t) => tiffTags = readTag(byteBuffer, tiffTags, t) case None => @@ -72,7 +75,7 @@ object TiffTagsReader { tiffTags } - def readTag(byteBuffer: ByteBuffer, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags = + def readTag(byteBuffer: ByteReader, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags = (tagMetadata.tag, tagMetadata.fieldType) match { case (ModelPixelScaleTag, _) => byteBuffer.readModelPixelScaleTag(tiffTags, tagMetadata) @@ -106,7 +109,7 @@ object TiffTagsReader { byteBuffer.readDoublesTag(tiffTags, tagMetadata) } - implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteBuffer) extends AnyVal { + implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteReader) extends AnyVal { def readModelPixelScaleTag(tiffTags: TiffTags, tagMetadata: TiffTagMetadata) = { @@ -117,7 +120,7 @@ object TiffTagsReader { val scaleX = byteBuffer.getDouble val scaleY = byteBuffer.getDouble val scaleZ = byteBuffer.getDouble - + byteBuffer.position(oldPos) (tiffTags &|-> @@ -132,8 +135,6 @@ object TiffTagsReader { val numberOfPoints = tagMetadata.length / 6 - byteBuffer.position(tagMetadata.offset) - val points = Array.ofDim[(Pixel3D, Pixel3D)](numberOfPoints) cfor(0)(_ < numberOfPoints, _ + 1) { i => points(i) = @@ -165,16 +166,30 @@ object TiffTagsReader { byteBuffer.position(tagMetadata.offset) + println("ByteBuffer position before reading in keydirectorymetadata is", byteBuffer.getByteBuffer.position) + + val o = byteBuffer.getByteBuffer.position + + cfor(0)(_ < 25, _ + 1) { i => + println(byteBuffer.getByteBuffer.get) + } + byteBuffer.position(o) val version = byteBuffer.getShort val keyRevision = byteBuffer.getShort val minorRevision = byteBuffer.getShort val numberOfKeys = byteBuffer.getShort + //println(byteBuffer.getByteBuffer.position) + //println(version, keyRevision, minorRevision, numberOfKeys) val keyDirectoryMetadata = GeoKeyDirectoryMetadata(version, keyRevision, minorRevision, numberOfKeys) + + //println(byteBuffer.getByteBuffer.position) + //println(keyDirectoryMetadata) val geoKeyDirectory = GeoKeyReader.read(byteBuffer, tiffTags, GeoKeyDirectory(count = numberOfKeys)) + //println(byteBuffer.getByteBuffer.position) byteBuffer.position(oldPos) diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala index 9081b223d2..23477386c6 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala @@ -21,14 +21,16 @@ import codes.TagCodes._ import GeoKeys._ +import geotrellis.util.ByteReader import java.nio.ByteBuffer import monocle.syntax.apply._ import monocle.macros.Lenses +import scala.language.implicitConversions object GeoKeyReader { - def read(byteBuffer: ByteBuffer, imageDirectory: TiffTags, + def read(byteBuffer: ByteReader, imageDirectory: TiffTags, geoKeyDirectory: GeoKeyDirectory, index: Int = 0 ): GeoKeyDirectory = { diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala index eef852f087..8067adec3d 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala @@ -16,13 +16,15 @@ package geotrellis.raster.io.geotiff.util +import geotrellis.util.ByteReader import java.nio.ByteBuffer +import scala.language.implicitConversions + import spire.syntax.cfor._ trait ByteBufferExtensions { - - implicit class ByteBufferUtilities(byteBuffer: ByteBuffer) { + implicit class ByteBufferUtilities(byteBuffer: ByteReader) { @inline final private def ub2s(byte: Byte): Short = diff --git a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala index 519bb1483e..c9c674d23d 100644 --- a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala +++ b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3ByteBuffer.scala @@ -1,56 +1,131 @@ package geotrellis.spark.io.s3.util +import geotrellis.util._ import geotrellis.spark.io.s3._ import com.amazonaws.services.s3.model._ -class S3ByteBuffer(val request: GetObjectRequest, val client: AmazonS3Client) - extends S3InputStreamReader with S3Queue { - private var filePosition = 0 - private var arrayPosition = 0 - private var startingChunk = getMapArray - private var (offset, arr) = startingChunk.head +import java.nio.{ByteOrder, ByteBuffer, Buffer} +import scala.language.implicitConversions - def position = filePosition +class S3BytesByteReader( + val request: GetObjectRequest, + val client: AmazonS3Client) + extends S3Bytes { - def get: Byte = { - val value = - if (arrayPosition < arr.length - 1) { - arr(arrayPosition) - } else { - arrayPosition = 0 + private var chunk = getMappedArray + private def offset = chunk.head._1 + private def array = chunk.head._2 + private def length = array.length - saveChunk(Map(offset -> arr)) - offset = getMapArray.head._1 - arr = getMapArray.head._2 + private var byteBuffer = getByteBuffer - arr(arrayPosition) + private val byteOrder: ByteOrder = { + val order = + (get.toChar, get.toChar) match { + case ('I', 'I') => ByteOrder.LITTLE_ENDIAN + case ('M', 'M') => ByteOrder.BIG_ENDIAN + case _ => throw new Exception("incorrect byte order") } - arrayPosition += 1 - filePosition += 1 - value + byteBuffer.position(0) + order } - def get(array: Array[Byte], start: Int, end: Int): Array[Byte] = { - if (isContained(start, end, startingChunk)) { - System.arraycopy(arr, start, array, 0, end - start) - arrayPosition = start + end + def position = (offset + byteBuffer.position).toInt + + def position(newPoint: Int): Buffer = { + if (isContained(newPoint)) { + byteBuffer.position(newPoint) } else { - val mappedArray = getMapArray(start, end) - val bytes = mappedArray.head._2 + chunk = getMappedArray(newPoint) + byteBuffer = getByteBuffer + byteBuffer.position(0) + } + } - System.arraycopy(bytes, 0, array, 0, end - start) - saveChunk(mappedArray) + def get: Byte = { + if (!(byteBuffer.position + 1 <= byteBuffer.capacity)) { + chunk = getMappedArray(position + 1) + byteBuffer = getByteBuffer } - filePosition = start + end - array + byteBuffer.get + } + + def getChar: Char = { + if (!(byteBuffer.position + 2 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 2) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getChar } -} + def getShort: Short = { + if (!(byteBuffer.position + 2 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 2) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getShort + } + + def getInt: Int = { + if (!(byteBuffer.position + 4 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 4) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getInt + } -object S3ByteBuffer { - def apply(request: GetObjectRequest, client: AmazonS3Client): S3ByteBuffer = - new S3ByteBuffer(request, client) + def getFloat: Float = { + if (!(byteBuffer.position + 4 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 4) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getFloat + } + + def getDouble: Double = { + if (!(byteBuffer.position + 8 <= byteBuffer.capacity)) { + val remaining = byteBuffer.slice.array + chunk = getMappedArray(position + 8) + val newArray = remaining ++ array + byteBuffer = getByteBuffer(newArray) + } + byteBuffer.getDouble + } + + def getByteBuffer: ByteBuffer = ByteBuffer.wrap(array).order(byteOrder) + + def getByteBuffer(arr: Array[Byte]) = ByteBuffer.wrap(arr).order(byteOrder) - def apply(bucket: String, key: String, client: AmazonS3Client): S3ByteBuffer = - new S3ByteBuffer(new GetObjectRequest(bucket, key), client) + def isContained(newPosition: Int): Boolean = + if (newPosition >= offset && newPosition <= offset + array.length) true else false +} + +object S3BytesByteReader { + def apply(bucket: String, key: String, client: AmazonS3Client): S3BytesByteReader = + new S3BytesByteReader(new GetObjectRequest(bucket, key), client) + + def apply(request: GetObjectRequest, client: AmazonS3Client): S3BytesByteReader = + new S3BytesByteReader(request, client) + + @inline implicit def toByteReader(s3BBR: S3BytesByteReader): ByteReader = { + new ByteReader() { + override def get = s3BBR.get + override def getChar = s3BBR.getChar + override def getShort = s3BBR.getShort + override def getInt = s3BBR.getInt + override def getFloat = s3BBR.getFloat + override def getDouble = s3BBR.getDouble + override def position: Int = s3BBR.position + override def position(i: Int): Buffer = s3BBR.position(i) + override def getByteBuffer = s3BBR.byteBuffer + } + } } diff --git a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala index e084d7a879..d45621529c 100644 --- a/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala +++ b/s3/src/main/scala/geotrellis/spark/io/s3/util/S3InputStreamReader.scala @@ -4,63 +4,66 @@ import geotrellis.spark.io.s3._ import scala.collection.mutable.Queue import com.amazonaws.services.s3.model._ +import java.nio.ByteBuffer -trait S3InputStreamReader { - def chunkSize = 256000 - - def request: GetObjectRequest +trait S3Bytes { def client: AmazonS3Client - - var location = 0 + def request: GetObjectRequest + + private val chunkSize = 256000 + private var streamPosition = 0 - private def metadata = + private val metadata = client.getObjectMetadata(request.getBucketName, request.getKey) - def objectLength = metadata.getContentLength + + private val objectLength = metadata.getContentLength private def pastLength(size: Int): Boolean = - if (location + size > objectLength) true else false + if (size > objectLength) true else false - private def getStream(start: Int, end: Int): S3ObjectInputStream = { + private def readStream(start: Int, end: Int): S3ObjectInputStream = { val obj = client.readRange(start, end, request) obj.getObjectContent } private def getArray: Array[Byte] = - getArray(location, chunkSize + location) + getArray(streamPosition) + + private def getArray(start: Int): Array[Byte] = + getArray(start, chunkSize) private def getArray(start: Int, length: Int): Array[Byte] = { val chunk = - if (!pastLength(length - start)) - length - start + if (!pastLength(length + start)) + length else - (objectLength - (length - start)).toInt + (objectLength - start).toInt val arr = Array.ofDim[Byte](chunk) - val stream = getStream(start, length) + val stream = readStream(start, chunk) stream.read(arr, 0, chunk) - stream.close() + streamPosition = start + length - location = length + start arr } - def getMapArray: Map[Long, Array[Byte]] = - getMapArray(location, chunkSize + location) + def getMappedArray: Map[Long, Array[Byte]] = + getMappedArray(streamPosition, chunkSize) - def getMapArray(start: Int, length: Int): Map[Long, Array[Byte]] = - Map(start.toLong -> getArray(start, length)) + def getMappedArray(start: Int): Map[Long, Array[Byte]] = + getMappedArray(start, chunkSize) - def isContained(start: Int, end: Int, map: Map[Long, Array[Byte]]): Boolean = { - val (offset, arr) = map.head - if (offset >= start && offset + arr.length <= end) true else false - } + def getMappedArray(start: Int, length: Int): Map[Long, Array[Byte]] = + Map(start.toLong -> getArray(start, length)) } -trait S3Queue { - private def mapQueue = Queue[Map[Long, Array[Byte]]]() +object S3Queue { + private val mapQueue = Queue[Map[Long, Array[Byte]]]() def size: Int = mapQueue.length + + def isEmpty: Boolean = mapQueue.isEmpty def saveChunk(chunk: Map[Long, Array[Byte]]): Unit = mapQueue.enqueue(chunk) diff --git a/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala b/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala index bf4941f7d4..68d809776a 100644 --- a/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala +++ b/s3/src/test/scala/geotrellis/spark/io/s3/S3ByteBufferSpec.scala @@ -1,44 +1,56 @@ package geotrellis.spark.io.s3.util import geotrellis.spark.io.s3._ +import geotrellis.util._ import com.amazonaws.services.s3.model._ import spire.syntax.cfor._ import org.scalatest._ -class S3ByteBufferSpec extends FunSpec with Matchers { +class S3BytesByteReaderSpec extends FunSpec with Matchers { val client = S3Client.default val bucket = "gt-rasters" val k = "nlcd/2011/tiles/nlcd_2011_01_01.tif" + val byteBuffer = Filesystem.toMappedByteBuffer("../../nlcd_2011_01_01.tif") - describe("MappedS3ByteBuffer") { + describe("MappedS3BytesByteReaderByteReader") { it("read in the first 4 bytes through get") { - val bb = S3ByteBuffer(bucket, k, client) + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) cfor(0)(_ < 4, _ + 1){ i => - bb.get + s3BytesByteReader.get } } + + /* it("should continue to read to the next MappedArray") { - val bb = S3ByteBuffer(bucket, k, client) + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) cfor(0)(_ < 256002, _ + 1) { i => - bb.get + s3BytesByteReader.get } } - - it("read in the first 4 bytes through bulk get") { - val bb = S3ByteBuffer(bucket, k, client) - val arr = Array.ofDim[Byte](4) - bb.get(arr, 0, arr.length) - } - it("should return the correct position") { - val bb = S3ByteBuffer(bucket, k, client) - val arr = Array.ofDim[Byte](950) - bb.get(arr, 789, arr.length) + it("should be able to read at various parts of the file") { + val s3BytesByteReader = S3BytesByteReader(bucket, k, client) + val a = s3BytesByteReader.arr + cfor(0)(_ < 25, _ + 1) { i => + s3BytesByteReader.get + } + val arr = Array.ofDim[Byte](980) + s3BytesByteReader.position(400000) + byteBuffer.position(4000000) + val a2 = s3BytesByteReader.arr + + (a take 25).foreach(println) + (a2 take 25).foreach(println) + val expected = s3BytesByteReader.get + val actual = byteBuffer.get + + byteBuffer.position(0) - assert(bb.location == bb.position) + assert(actual == expected) } + */ } } diff --git a/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala b/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala new file mode 100644 index 0000000000..a5440e1e21 --- /dev/null +++ b/s3/src/test/scala/geotrellis/spark/io/s3/S3TiffTagsReaderSpec.scala @@ -0,0 +1,114 @@ +package geotrellis.spark.io.s3.util + +import geotrellis.util.Filesystem +import geotrellis.spark.io.s3._ +import geotrellis.raster.io.geotiff.tags._ +import geotrellis.raster.io.geotiff.reader._ + +import org.scalatest._ +import com.amazonaws.services.s3.model._ +import spire.syntax.cfor._ + +class S3TiffTagsReaderSpec extends FunSpec { + + describe("tifftags reader") { + val client = S3Client.default + val bucket = "gt-rasters" + val k = "nlcd/2011/tiles/nlcd_2011_01_01.tif" + val s3ByteBuffer = S3BytesByteReader(bucket, k, client) + val fromLocal = TiffTagsReader.read("../../nlcd_2011_01_01.tif") + val fromServer = TiffTagsReader.read(s3ByteBuffer) + + /* + it("should read the same basic tags") { + val expected = fromServer.basicTags + val actual = fromLocal.basicTags + + assert(expected == actual) + } + + it("should read the same colimetry tags") { + val expected = fromServer.colimetryTags + val actual = fromLocal.colimetryTags + + assert(expected == actual) + } + + it("should read the same cmyk tags") { + val expected = fromServer.cmykTags + val actual = fromLocal.cmykTags + + assert(expected == actual) + } + + it("should read the same dataSampleForamt tags") { + val expected = fromServer.dataSampleFormatTags + val actual = fromLocal.dataSampleFormatTags + + assert(expected == actual) + } + + it("should read the same documentation tags") { + val expected = fromServer.documentationTags + val actual = fromLocal.documentationTags + + assert(expected == actual) + } + + it("should read the same geoTiffTags tags") { + val expected = fromServer.geoTiffTags + val actual = fromLocal.geoTiffTags + + assert(expected == actual) + } + + it("should read the same jpegTags tags") { + val expected = fromServer.jpegTags + val actual = fromLocal.jpegTags + + assert(expected == actual) + } + it("should read the same metadata tags") { + val expected = fromServer.metadataTags + val actual = fromLocal.metadataTags + + assert(expected == actual) + } + + it("should read the same nonBasic tags") { + val expected = fromServer.nonBasicTags + val actual = fromLocal.nonBasicTags + + assert(expected == actual) + } + + it("should read the same nonStandardized tags") { + val expected = fromServer.nonStandardizedTags + val actual = fromLocal.nonStandardizedTags + + assert(expected == actual) + } + + it("should read the same tags tags") { + val expected = fromServer.tags + val actual = fromLocal.tags + + assert(expected == actual) + } + + it("should read the same tile tags") { + val expected = fromServer.tileTags + val actual = fromLocal.tileTags + + assert(expected == actual) + } + + it("should read the same ycbcr tags") { + val expected = fromServer.yCbCrTags + val actual = fromLocal.yCbCrTags + + assert(expected == actual) + } + */ + } +} diff --git a/util/src/main/scala/geotrellis/util/ByteReader.scala b/util/src/main/scala/geotrellis/util/ByteReader.scala new file mode 100644 index 0000000000..5ac55e43ca --- /dev/null +++ b/util/src/main/scala/geotrellis/util/ByteReader.scala @@ -0,0 +1,33 @@ +package geotrellis.util + +import java.nio.{Buffer, ByteBuffer} +import scala.language.implicitConversions + +trait ByteReader { + def get: Byte + def getChar: Char + def getShort: Short + def getInt: Int + def getFloat: Float + def getDouble: Double + def position: Int + def position(i: Int): Buffer + def getByteBuffer: ByteBuffer +} + +object ByteReader { + implicit def byteBuffer2ByteReader(byteBuffer: ByteBuffer): ByteReader = { + new ByteReader() { + def get = byteBuffer.get + def getChar = byteBuffer.getChar + def getShort = byteBuffer.getShort + def getInt = byteBuffer.getInt + def getFloat = byteBuffer.getFloat + def getDouble = byteBuffer.getDouble + def position: Int = byteBuffer.position + def position(i: Int): Buffer = byteBuffer.position(i) + def getByteBuffer = byteBuffer + } + } + implicit def toByteBuffer(br: ByteReader): ByteBuffer = br.getByteBuffer +}