Skip to content

Commit

Permalink
Added ByteReader
Browse files Browse the repository at this point in the history 
made the code in the s3 util less confusing

Changed how ByteReader works for S3 and ByteBuffer

Continued work on S3 reading

Continuing to search for problems in code
  • Loading branch information
@jbouffard
jbouffard committed Oct 13, 2016
1 parent bd06172 commit 1297b44
Show file tree
Hide file tree
Showing 11 changed files with 362 additions and 104 deletions.
3 changes: 2 additions & 1 deletion raster/src/main/scala/geotrellis/raster/io/geotiff/ArraySegmentBytes.scala
View file
@@ -1,5 +1,6 @@
package geotrellis.raster.io.geotiff

import geotrellis.util.ByteReader
import geotrellis.vector.Extent
import geotrellis.raster._
import geotrellis.raster.io.geotiff._
Expand Down Expand Up @@ -41,7 +42,7 @@ object ArraySegmentBytes {
* @tiffTags: The [[TiffTags]] of the GeoTiff
* @return A new instance of ArraySegmentBytes
*/
def apply(byteBuffer: ByteBuffer, tiffTags: TiffTags): ArraySegmentBytes = {
def apply(byteBuffer: ByteReader, tiffTags: TiffTags): ArraySegmentBytes = {

val compressedBytes: Array[Array[Byte]] = {
def readSections(offsets: Array[Int],
Expand Down
3 changes: 2 additions & 1 deletion raster/src/main/scala/geotrellis/raster/io/geotiff/BufferSegmentBytes.scala
View file
@@ -1,5 +1,6 @@
package geotrellis.raster.io.geotiff

import geotrellis.util.ByteReader
import geotrellis.vector.Extent
import geotrellis.raster._
import geotrellis.raster.io.geotiff._
Expand All @@ -19,7 +20,7 @@ import spire.syntax.cfor._
* @param tifftags: The [[TiffTags]] of the GeoTiff
* @return A new instance of BufferSegmentBytes
*/
case class BufferSegmentBytes(byteBuffer: ByteBuffer, tiffTags: TiffTags) extends SegmentBytes {
case class BufferSegmentBytes(byteBuffer: ByteReader, tiffTags: TiffTags) extends SegmentBytes {

val (offsets, byteCounts) =
if (tiffTags.hasStripStorage) {
Expand Down
8 changes: 4 additions & 4 deletions raster/src/main/scala/geotrellis/raster/io/geotiff/reader/GeoTiffReader.scala
View file
Expand Up @@ -23,7 +23,7 @@ import geotrellis.raster.io.geotiff.util._
import geotrellis.raster.io.geotiff.tags._
import geotrellis.vector.Extent
import geotrellis.proj4.CRS
import geotrellis.util.Filesystem
import geotrellis.util.{Filesystem, ByteReader}

import monocle.syntax.apply._

Expand Down Expand Up @@ -83,7 +83,7 @@ object GeoTiffReader {
/* Read a single band GeoTIFF file.
* If there is more than one band in the GeoTiff, read the first band only.
*/
def readSingleband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = {
def readSingleband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): SinglebandGeoTiff = {
val info = readGeoTiffInfo(byteBuffer, decompress, streaming)

val geoTiffTile =
Expand Down Expand Up @@ -149,7 +149,7 @@ object GeoTiffReader {
streaming: Boolean = false): MultibandGeoTiff =
readMultiband(ByteBuffer.wrap(bytes), decompress, streaming)

def readMultiband(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = {
def readMultiband(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): MultibandGeoTiff = {
val info = readGeoTiffInfo(byteBuffer, decompress, streaming)

val geoTiffTile =
Expand Down Expand Up @@ -243,7 +243,7 @@ object GeoTiffReader {
}
}

private def readGeoTiffInfo(byteBuffer: ByteBuffer, decompress: Boolean, streaming: Boolean): GeoTiffInfo = {
private def readGeoTiffInfo(byteBuffer: ByteReader, decompress: Boolean, streaming: Boolean): GeoTiffInfo = {
// set byte ordering
(byteBuffer.get.toChar, byteBuffer.get.toChar) match {
case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN)
Expand Down
45 changes: 30 additions & 15 deletions raster/src/main/scala/geotrellis/raster/io/geotiff/reader/TiffTagsReader.scala
View file
Expand Up @@ -4,24 +4,24 @@ import geotrellis.raster.io.geotiff.tags._
import geotrellis.raster.io.geotiff.tags.codes._
import TagCodes._
import TiffFieldType._
import geotrellis.util.Filesystem

import geotrellis.raster.io.geotiff.util._
import geotrellis.util.{Filesystem, ByteReader}
import spire.syntax.cfor._
import monocle.syntax.apply._
import scala.language.implicitConversions

import java.nio.{ ByteBuffer, ByteOrder }


object TiffTagsReader {
def read(path: String): TiffTags =
read(Filesystem.toMappedByteBuffer(path))

def read(bytes: Array[Byte]): TiffTags =
read(ByteBuffer.wrap(bytes, 0, bytes.size))
read(ByteBuffer.wrap(bytes))

def read(byteBuffer: ByteReader): TiffTags = {

def read(byteBuffer: ByteBuffer): TiffTags = {
// set byte ordering
(byteBuffer.get.toChar, byteBuffer.get.toChar) match {
case ('I', 'I') => byteBuffer.order(ByteOrder.LITTLE_ENDIAN)
case ('M', 'M') => byteBuffer.order(ByteOrder.BIG_ENDIAN)
Expand All @@ -32,15 +32,15 @@ object TiffTagsReader {
val geoTiffIdNumber = byteBuffer.getChar
if ( geoTiffIdNumber != 42)
throw new MalformedGeoTiffException(s"bad identification number (must be 42, was $geoTiffIdNumber)")

val tagsStartPosition = byteBuffer.getInt

read(byteBuffer, tagsStartPosition)
}

def read(byteBuffer: ByteBuffer, tagsStartPosition: Int): TiffTags = {
byteBuffer.position(tagsStartPosition)
def read(byteBuffer: ByteReader, tagsStartPosition: Int): TiffTags = {

byteBuffer.position(tagsStartPosition)

val tagCount = byteBuffer.getShort

// Read the tags.
Expand All @@ -58,12 +58,15 @@ object TiffTagsReader {
byteBuffer.getInt // Offset
)

if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag)
if (tagMetadata.tag == codes.TagCodes.GeoKeyDirectoryTag) {
geoTags = Some(tagMetadata)
else
} else {
tiffTags = readTag(byteBuffer, tiffTags, tagMetadata)
}
}

println(s"the gotags is: $geoTags") //, and the tifftags are: $tiffTags")

geoTags match {
case Some(t) => tiffTags = readTag(byteBuffer, tiffTags, t)
case None =>
Expand All @@ -72,7 +75,7 @@ object TiffTagsReader {
tiffTags
}

def readTag(byteBuffer: ByteBuffer, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags =
def readTag(byteBuffer: ByteReader, tiffTags: TiffTags, tagMetadata: TiffTagMetadata): TiffTags =
(tagMetadata.tag, tagMetadata.fieldType) match {
case (ModelPixelScaleTag, _) =>
byteBuffer.readModelPixelScaleTag(tiffTags, tagMetadata)
Expand Down Expand Up @@ -106,7 +109,7 @@ object TiffTagsReader {
byteBuffer.readDoublesTag(tiffTags, tagMetadata)
}

implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteBuffer) extends AnyVal {
implicit class ByteBufferTagReaderWrapper(val byteBuffer: ByteReader) extends AnyVal {
def readModelPixelScaleTag(tiffTags: TiffTags,
tagMetadata: TiffTagMetadata) = {

Expand All @@ -117,7 +120,7 @@ object TiffTagsReader {
val scaleX = byteBuffer.getDouble
val scaleY = byteBuffer.getDouble
val scaleZ = byteBuffer.getDouble

byteBuffer.position(oldPos)

(tiffTags &|->
Expand All @@ -132,8 +135,6 @@ object TiffTagsReader {

val numberOfPoints = tagMetadata.length / 6

byteBuffer.position(tagMetadata.offset)

val points = Array.ofDim[(Pixel3D, Pixel3D)](numberOfPoints)
cfor(0)(_ < numberOfPoints, _ + 1) { i =>
points(i) =
Expand Down Expand Up @@ -165,16 +166,30 @@ object TiffTagsReader {

byteBuffer.position(tagMetadata.offset)

println("ByteBuffer position before reading in keydirectorymetadata is", byteBuffer.getByteBuffer.position)

val o = byteBuffer.getByteBuffer.position

cfor(0)(_ < 25, _ + 1) { i =>
println(byteBuffer.getByteBuffer.get)
}
byteBuffer.position(o)
val version = byteBuffer.getShort
val keyRevision = byteBuffer.getShort
val minorRevision = byteBuffer.getShort
val numberOfKeys = byteBuffer.getShort
//println(byteBuffer.getByteBuffer.position)

//println(version, keyRevision, minorRevision, numberOfKeys)
val keyDirectoryMetadata = GeoKeyDirectoryMetadata(version, keyRevision,
minorRevision, numberOfKeys)

//println(byteBuffer.getByteBuffer.position)
//println(keyDirectoryMetadata)

val geoKeyDirectory = GeoKeyReader.read(byteBuffer,
tiffTags, GeoKeyDirectory(count = numberOfKeys))
//println(byteBuffer.getByteBuffer.position)

byteBuffer.position(oldPos)

Expand Down
4 changes: 3 additions & 1 deletion raster/src/main/scala/geotrellis/raster/io/geotiff/tags/GeoKeyReader.scala
View file
Expand Up @@ -21,14 +21,16 @@ import codes.TagCodes._

import GeoKeys._

import geotrellis.util.ByteReader
import java.nio.ByteBuffer

import monocle.syntax.apply._
import monocle.macros.Lenses
import scala.language.implicitConversions

object GeoKeyReader {

def read(byteBuffer: ByteBuffer, imageDirectory: TiffTags,
def read(byteBuffer: ByteReader, imageDirectory: TiffTags,
geoKeyDirectory: GeoKeyDirectory, index: Int = 0
): GeoKeyDirectory = {

Expand Down
6 changes: 4 additions & 2 deletions raster/src/main/scala/geotrellis/raster/io/geotiff/util/ByteBufferExtensions.scala
View file
Expand Up @@ -16,13 +16,15 @@

package geotrellis.raster.io.geotiff.util

import geotrellis.util.ByteReader
import java.nio.ByteBuffer

import scala.language.implicitConversions

import spire.syntax.cfor._

trait ByteBufferExtensions {

implicit class ByteBufferUtilities(byteBuffer: ByteBuffer) {
implicit class ByteBufferUtilities(byteBuffer: ByteReader) {

@inline
final private def ub2s(byte: Byte): Short =
Expand Down

0 comments on commit 1297b44

Please sign in to comment.