diff --git a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSource.scala b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSource.scala index b0750467..8f98dd21 100644 --- a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSource.scala +++ b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSource.scala @@ -79,8 +79,9 @@ case class GeoTiffRasterSource( override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { val geoTiffTile = tiff.tile.asInstanceOf[GeoTiffMultibandTile] val intersectingBounds: Seq[GridBounds[Int]] = - bounds.flatMap(_.intersection(this.gridBounds)). - toSeq.map(b => b.toGridType[Int]) + bounds + .flatMap(_.intersection(this.gridBounds)).toSeq + .map(b => b.toGridType[Int]) geoTiffTile.crop(intersectingBounds, bands.toArray).map { case (gb, tile) => convertRaster(Raster(tile, gridExtent.extentFor(gb.toGridType[Long], clamp = true))) diff --git a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffReprojectRasterSource.scala b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffReprojectRasterSource.scala index 480b847b..f630abd7 100644 --- a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffReprojectRasterSource.scala +++ b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffReprojectRasterSource.scala @@ -44,12 +44,9 @@ case class GeoTiffReprojectRasterSource( protected lazy val backTransform = Transform(crs, baseCRS) override lazy val gridExtent: GridExtent[Long] = reprojectOptions.targetRasterExtent match { - case Some(targetRasterExtent) => - targetRasterExtent.toGridType[Long] - - case None => - ReprojectRasterExtent(baseGridExtent, transform, reprojectOptions) - } + case Some(targetRasterExtent) => targetRasterExtent.toGridType[Long] + case None => ReprojectRasterExtent(baseGridExtent, transform, reprojectOptions) + } lazy val resolutions: List[GridExtent[Long]] = gridExtent :: tiff.overviews.map(ovr => ReprojectRasterExtent(ovr.rasterExtent.toGridType[Long], transform)) @@ -79,7 +76,7 @@ case class GeoTiffReprojectRasterSource( def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { val it = readBounds(List(bounds), bands) - tiff.synchronized { if (it.hasNext) Some(it.next) else None } + closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None } } override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { @@ -97,8 +94,12 @@ case class GeoTiffReprojectRasterSource( val targetRasterExtent = RasterExtent( extent = gridExtent.extentFor(targetPixelBounds, clamp = true), cols = targetPixelBounds.width.toInt, - rows = targetPixelBounds.height.toInt) - val sourceExtent = targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).envelope + rows = targetPixelBounds.height.toInt + ) + + // A tmp workaround for https://github.com/locationtech/proj4j/pull/29 + // Stacktrace details: https://github.com/geotrellis/geotrellis-contrib/pull/206#pullrequestreview-260115791 + val sourceExtent = Proj4Transform.synchronized(targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).envelope) val sourcePixelBounds = closestTiffOverview.rasterExtent.gridBoundsFor(sourceExtent, clamp = true) (sourcePixelBounds, targetRasterExtent) }}.toMap diff --git a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffResampleRasterSource.scala b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffResampleRasterSource.scala index bda191fb..e5f22a4c 100644 --- a/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffResampleRasterSource.scala +++ b/vlm/src/main/scala/geotrellis/contrib/vlm/geotiff/GeoTiffResampleRasterSource.scala @@ -43,13 +43,13 @@ case class GeoTiffResampleRasterSource( override lazy val gridExtent: GridExtent[Long] = resampleGrid(tiff.rasterExtent.toGridType[Long]) lazy val resolutions: List[GridExtent[Long]] = { - val ratio = gridExtent.cellSize.resolution / tiff.rasterExtent.cellSize.resolution - gridExtent :: tiff.overviews.map { ovr => - val re = ovr.rasterExtent - val CellSize(cw, ch) = re.cellSize - new GridExtent[Long](re.extent, CellSize(cw * ratio, ch * ratio)) - } + val ratio = gridExtent.cellSize.resolution / tiff.rasterExtent.cellSize.resolution + gridExtent :: tiff.overviews.map { ovr => + val re = ovr.rasterExtent + val CellSize(cw, ch) = re.cellSize + new GridExtent[Long](re.extent, CellSize(cw * ratio, ch * ratio)) } + } @transient protected lazy val closestTiffOverview: GeoTiff[MultibandTile] = tiff.getClosestOverview(gridExtent.cellSize, strategy) @@ -77,7 +77,7 @@ case class GeoTiffResampleRasterSource( def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { val it = readBounds(List(bounds), bands) - tiff.synchronized { if (it.hasNext) Some(it.next) else None } + closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None } } override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { diff --git a/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala b/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala index 557a0fe6..ed18a9a1 100644 --- a/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala +++ b/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala @@ -16,152 +16,55 @@ package geotrellis.contrib.vlm.geotiff -import geotrellis.contrib.vlm.Resource - +import geotrellis.contrib.vlm.{RasterSource, Resource} import geotrellis.proj4.CRS -import geotrellis.raster._ import geotrellis.raster.resample._ -import geotrellis.util._ - -import org.scalatest.AsyncFunSpec -import scala.concurrent.{Future, ExecutionContext} -import scala.concurrent.duration.Duration +import cats.instances.future._ +import cats.instances.list._ +import cats.syntax.traverse._ +import org.scalatest.{AsyncFunSpec, Matchers} +import scala.concurrent.{ExecutionContext, Future} -class GeoTiffRasterSourceMultiThreadingSpec extends AsyncFunSpec { +class GeoTiffRasterSourceMultiThreadingSpec extends AsyncFunSpec with Matchers { val url = Resource.path("img/aspect-tiled.tif") - val source: GeoTiffRasterSource = new GeoTiffRasterSource(url) + val source = GeoTiffRasterSource(url) implicit val ec = ExecutionContext.global - - describe("GeoTiffRasterSource should be threadsafe") { + + val iterations = (0 to 100).toList + + /** + * readBounds and readExtends are not covered by these tests since these methods return an [[Iterator]]. + * Due to the [[Iterator]] lazy nature, the lock in this case should be done on the user side. + * */ + def testMultithreading(rs: RasterSource): Unit = { it("read") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - source.read() - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } + val res = iterations.map { _ => Future { rs.read() } }.sequence.map(_.flatten) + res.map { rasters => rasters.length shouldBe iterations.length } } it("readBounds - Option") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - source.read(source.gridBounds, 0 until source.bandCount toSeq) - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) + val res = + iterations + .map { _ => Future { rs.read(rs.gridBounds, 0 until rs.bandCount) } } + .sequence + .map(_.flatten) - fres.map { rasters => assert(rasters.size == 101) } + res.map { rasters => rasters.length shouldBe iterations.length } } + } - ignore("readBounds - List") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - source.readBounds(List(source.gridBounds), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - ignore("readExtents") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - source.readExtents(List(source.extent), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } + describe("GeoTiffRasterSource should be threadsafe") { + testMultithreading(source) } describe("GeoTiffRasterReprojectSource should be threadsafe") { - val reprojected = source.reproject(CRS.fromEpsgCode(4326)) - - it("read") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - reprojected.read() - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - it("readBounds - Option") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - reprojected.read(reprojected.gridBounds, 0 until source.bandCount toSeq) - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - ignore("readBounds - List") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - reprojected.readBounds(List(reprojected.gridBounds), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - ignore("readExtents") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - reprojected.readExtents(List(reprojected.extent), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } + testMultithreading(source.reproject(CRS.fromEpsgCode(4326))) } describe("GeoTiffRasterResampleSource should be threadsafe") { - val resampled = source.resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor) - - it("read") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - resampled.read() - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - it("readBounds - Option") { - val res: List[Future[Option[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - resampled.read(resampled.gridBounds, 0 until source.bandCount toSeq) - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - ignore("readBounds - List") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - resampled.readBounds(List(resampled.gridBounds), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } - - ignore("readExtents") { - val res: List[Future[List[Raster[MultibandTile]]]] = (0 to 100).toList.map { _ => Future { - resampled.readExtents(List(resampled.extent), 0 until source.bandCount toSeq).toList - } } - - val fres: Future[List[Raster[MultibandTile]]] = Future.sequence(res).map(_.flatten) - - fres.map { rasters => assert(rasters.size == 101) } - } + testMultithreading(source.resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor)) } } diff --git a/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceSpec.scala b/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceSpec.scala index 2165dc8f..f3847d69 100644 --- a/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceSpec.scala +++ b/vlm/src/test/scala/geotrellis/contrib/vlm/geotiff/GeoTiffRasterSourceSpec.scala @@ -17,7 +17,6 @@ package geotrellis.contrib.vlm.geotiff import geotrellis.contrib.vlm._ -import geotrellis.proj4._ import geotrellis.raster._ import geotrellis.raster.io.geotiff.reader.GeoTiffReader import geotrellis.raster.resample._ @@ -26,8 +25,8 @@ import geotrellis.vector._ import geotrellis.spark._ import geotrellis.spark.tiling._ import geotrellis.util._ -import org.scalatest._ +import org.scalatest._ class GeoTiffRasterSourceSpec extends FunSpec with RasterMatchers with BetterRasterMatchers with GivenWhenThen { lazy val url = Resource.path("img/aspect-tiled.tif")