Skip to content

Commit

Permalink
reprojectAsPolygon fix and specs refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@pomadchin
pomadchin committed Jul 10, 2019
1 parent 680bdda commit fc6ed2e
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 146 deletions.
5 changes: 3 additions & 2 deletions vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,9 @@ case class GeoTiffRasterSource(
override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
val geoTiffTile = tiff.tile.asInstanceOf[GeoTiffMultibandTile]
val intersectingBounds: Seq[GridBounds[Int]] =
bounds.flatMap(_.intersection(this.gridBounds)).
toSeq.map(b => b.toGridType[Int])
bounds
.flatMap(_.intersection(this.gridBounds)).toSeq
.map(b => b.toGridType[Int])

geoTiffTile.crop(intersectingBounds, bands.toArray).map { case (gb, tile) =>
convertRaster(Raster(tile, gridExtent.extentFor(gb.toGridType[Long], clamp = true)))
Expand Down
19 changes: 10 additions & 9 deletions vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffReprojectRasterSource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,9 @@ case class GeoTiffReprojectRasterSource(
protected lazy val backTransform = Transform(crs, baseCRS)

override lazy val gridExtent: GridExtent[Long] = reprojectOptions.targetRasterExtent match {
case Some(targetRasterExtent) =>
targetRasterExtent.toGridType[Long]

case None =>
ReprojectRasterExtent(baseGridExtent, transform, reprojectOptions)
}
case Some(targetRasterExtent) => targetRasterExtent.toGridType[Long]
case None => ReprojectRasterExtent(baseGridExtent, transform, reprojectOptions)
}

lazy val resolutions: List[GridExtent[Long]] =
gridExtent :: tiff.overviews.map(ovr => ReprojectRasterExtent(ovr.rasterExtent.toGridType[Long], transform))
Expand Down Expand Up @@ -79,7 +76,7 @@ case class GeoTiffReprojectRasterSource(
def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = {
val it = readBounds(List(bounds), bands)

tiff.synchronized { if (it.hasNext) Some(it.next) else None }
closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None }
}

override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
Expand All @@ -97,8 +94,12 @@ case class GeoTiffReprojectRasterSource(
val targetRasterExtent = RasterExtent(
extent = gridExtent.extentFor(targetPixelBounds, clamp = true),
cols = targetPixelBounds.width.toInt,
rows = targetPixelBounds.height.toInt)
val sourceExtent = targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).envelope
rows = targetPixelBounds.height.toInt
)

// A tmp workaround for https://github.com/locationtech/proj4j/pull/29
// Stacktrace details: https://github.com/geotrellis/geotrellis-contrib/pull/206#pullrequestreview-260115791
val sourceExtent = Proj4Transform.synchronized(targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).envelope)
val sourcePixelBounds = closestTiffOverview.rasterExtent.gridBoundsFor(sourceExtent, clamp = true)
(sourcePixelBounds, targetRasterExtent)
}}.toMap
Expand Down
14 changes: 7 additions & 7 deletions vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffResampleRasterSource.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ case class GeoTiffResampleRasterSource(

override lazy val gridExtent: GridExtent[Long] = resampleGrid(tiff.rasterExtent.toGridType[Long])
lazy val resolutions: List[GridExtent[Long]] = {
val ratio = gridExtent.cellSize.resolution / tiff.rasterExtent.cellSize.resolution
gridExtent :: tiff.overviews.map { ovr =>
val re = ovr.rasterExtent
val CellSize(cw, ch) = re.cellSize
new GridExtent[Long](re.extent, CellSize(cw * ratio, ch * ratio))
}
val ratio = gridExtent.cellSize.resolution / tiff.rasterExtent.cellSize.resolution
gridExtent :: tiff.overviews.map { ovr =>
val re = ovr.rasterExtent
val CellSize(cw, ch) = re.cellSize
new GridExtent[Long](re.extent, CellSize(cw * ratio, ch * ratio))
}
}

@transient protected lazy val closestTiffOverview: GeoTiff[MultibandTile] =
tiff.getClosestOverview(gridExtent.cellSize, strategy)
Expand Down Expand Up @@ -77,7 +77,7 @@ case class GeoTiffResampleRasterSource(
def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = {
val it = readBounds(List(bounds), bands)

tiff.synchronized { if (it.hasNext) Some(it.next) else None }
closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None }
}

override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
Expand Down
155 changes: 29 additions & 126 deletions ...src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,152 +16,55 @@

package geotrellis.contrib.vlm.geotiff

import geotrellis.contrib.vlm.Resource

import geotrellis.contrib.vlm.{RasterSource, Resource}
import geotrellis.proj4.CRS
import geotrellis.raster._
import geotrellis.raster.resample._
import geotrellis.util._

import org.scalatest.AsyncFunSpec

import scala.concurrent.{Future, ExecutionContext}
import scala.concurrent.duration.Duration
import cats.instances.future._
import cats.instances.list._
import cats.syntax.traverse._
import org.scalatest.{AsyncFunSpec, Matchers}

import scala.concurrent.{ExecutionContext, Future}

class GeoTiffRasterSourceMultiThreadingSpec extends AsyncFunSpec {
class GeoTiffRasterSourceMultiThreadingSpec extends AsyncFunSpec with Matchers {
val url = Resource.path("img/aspect-tiled.tif")
val source: GeoTiffRasterSource = new GeoTiffRasterSource(url)
val source = GeoTiffRasterSource(url)

implicit val ec = ExecutionContext.global

describe("GeoTiffRasterSource should be threadsafe") {

val iterations = (0 to 100).toList

/**
* readBounds and readExtends are not covered by these tests since these methods return an [[Iterator]].
* Due to the [[Iterator]] lazy nature, the lock in this case should be done on the user side.
* */
def testMultithreading(rs: RasterSource): Unit = {
it("read") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
source.read()
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
val res = iterations.map { _ => Future { rs.read() } }.sequence.map(_.flatten)
res.map { rasters => rasters.length shouldBe iterations.length }
}

it("readBounds - Option") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
source.read(source.gridBounds, 0 until source.bandCount toSeq)
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)
val res =
iterations
.map { _ => Future { rs.read(rs.gridBounds, 0 until rs.bandCount) } }
.sequence
.map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
res.map { rasters => rasters.length shouldBe iterations.length }
}
}

ignore("readBounds - List") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
source.readBounds(List(source.gridBounds), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

ignore("readExtents") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
source.readExtents(List(source.extent), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}
describe("GeoTiffRasterSource should be threadsafe") {
testMultithreading(source)
}

describe("GeoTiffRasterReprojectSource should be threadsafe") {
val reprojected = source.reproject(CRS.fromEpsgCode(4326))

it("read") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
reprojected.read()
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

it("readBounds - Option") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
reprojected.read(reprojected.gridBounds, 0 until source.bandCount toSeq)
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

ignore("readBounds - List") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
reprojected.readBounds(List(reprojected.gridBounds), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

ignore("readExtents") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
reprojected.readExtents(List(reprojected.extent), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}
testMultithreading(source.reproject(CRS.fromEpsgCode(4326)))
}

describe("GeoTiffRasterResampleSource should be threadsafe") {
val resampled = source.resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor)

it("read") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
resampled.read()
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

it("readBounds - Option") {
val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
resampled.read(resampled.gridBounds, 0 until source.bandCount toSeq)
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

ignore("readBounds - List") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
resampled.readBounds(List(resampled.gridBounds), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}

ignore("readExtents") {
val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future {
resampled.readExtents(List(resampled.extent), 0 until source.bandCount toSeq).toList
} }

val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten)

fres.map { rasters => assert(rasters.size == 101) }
}
testMultithreading(source.resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor))
}
}
3 changes: 1 addition & 2 deletions vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceSpec.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package geotrellis.contrib.vlm.geotiff

import geotrellis.contrib.vlm._
import geotrellis.proj4._
import geotrellis.raster._
import geotrellis.raster.io.geotiff.reader.GeoTiffReader
import geotrellis.raster.resample._
Expand All @@ -26,8 +25,8 @@ import geotrellis.vector._
import geotrellis.spark._
import geotrellis.spark.tiling._
import geotrellis.util._
import org.scalatest._

import org.scalatest._

class GeoTiffRasterSourceSpec extends FunSpec with RasterMatchers with BetterRasterMatchers with GivenWhenThen {
lazy val url = Resource.path("img/aspect-tiled.tif")
Expand Down

0 comments on commit fc6ed2e

Please sign in to comment.