RasterSourceF experimental implementation

[pomadchin]

Overview

This PR addresses #162 and #180

Demo

import cats.data.NonEmptyList
import cats.effect._
import cats.implicits._
import geotrellis.contrib.vlm.effect.MosaicRasterSource
import geotrellis.contrib.vlm.effect.geotiff._
import geotrellis.raster.{MultibandTile, Raster}
import geotrellis.vector.Extent

import scala.concurrent.ExecutionContext

val uri = "/tmp/aspect-tiled.tif"

object optcase {
  lazy val source: GeoTiffRasterSource[Option] = GeoTiffRasterSource[Option](uri)
  lazy val rasterSources = NonEmptyList(Option(GeoTiffRasterSource[Option](uri)), List(Option(GeoTiffRasterSource[Option](uri)), Option(GeoTiffRasterSource[Option](uri))))
  // lazy val msource = MosaicRasterSource[Option](rasterSources, source.crs)

  lazy val raster1: Option[Raster[MultibandTile]] = source.read(Extent(0, 0, 1, 1))
  lazy val raster2: Option[Raster[MultibandTile]] = source.read(Extent(630000.0, 215000.0, 639000.0, 219500.0))
  lazy val raster3: Option[Raster[MultibandTile]] = source.read()

  // lazy val mraster1: Option[Raster[MultibandTile]] = msource.read(Extent(0, 0, 1, 1))
  // lazy val mraster2: Option[Raster[MultibandTile]] = msource.read(Extent(630000.0, 215000.0, 639000.0, 219500.0))
  // lazy val mraster3: Option[Raster[MultibandTile]] = msource.read()

  def run = (raster1, raster2, raster3)

  // def mrun = (mraster1, mraster2, mraster3)

}

object iocase {
  implicit lazy val cs = IO.contextShift(ExecutionContext.global)

  // run in a main thread
  // implicit val cs = IO.contextShift(ExecutionContext.fromExecutor(new RasterSourceF.CurrentThreadExecutor))

  /*val i = 1000
  val n = 200
  val pool = Executors.newFixedThreadPool(n, new ThreadFactoryBuilder().setNameFormat("scala-sheets-%d").build())
  val ec = ExecutionContext.fromExecutor(pool)
  implicit val cs = IO.contextShift(ec)*/

  lazy val source: GeoTiffRasterSource[IO] = GeoTiffRasterSource[IO](uri)
  lazy val rasterSources = IO(NonEmptyList(GeoTiffRasterSource[IO](uri), List(GeoTiffRasterSource[IO](uri), GeoTiffRasterSource[IO](uri))))

  lazy val msource = MosaicRasterSource[IO](rasterSources, source.crs)

  lazy val raster1: IO[Raster[MultibandTile]] = source.read(Extent(0, 0, 1, 1))
  lazy val raster2: IO[Raster[MultibandTile]] = source.read(Extent(630000.0, 215000.0, 639000.0, 219500.0))
  lazy val raster3: IO[Raster[MultibandTile]] = source.read()

  lazy val mraster1: IO[Raster[MultibandTile]] = msource.read(Extent(0, 0, 1, 1))
  lazy val mraster2: IO[Raster[MultibandTile]] = msource.read(Extent(630000.0, 215000.0, 639000.0, 219500.0))
  lazy val mraster3: IO[Raster[MultibandTile]] = msource.read()

  println(s"${Thread.currentThread().getName}")

  lazy val result: Either[Throwable, List[Raster[MultibandTile]]] =
    List(
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3,
      raster2, raster3
    ).zipWithIndex
      .map { case (io, idx) => // add a thread printing
        IO.shift *> io.flatMap { v =>
          IO.shift *> IO {
            println(s"${idx}: ${Thread.currentThread().getName}")
            v
          }
        }
      }
      .parSequence
      .attempt
      .unsafeRunSync()

  lazy val mresult: Either[Throwable, List[Raster[MultibandTile]]] =
    List(
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3,
      mraster2, mraster3
    ).zipWithIndex
      .map { case (io, idx) => // add a thread printing
        IO.shift *> io.flatMap { v =>
          IO.shift *> IO {
            println(s"${idx}: ${Thread.currentThread().getName}")
            v
          }
        }
      }
      .parSequence
      .attempt
      .unsafeRunSync()

  def run = result
  def mrun = mresult
}

optcase.run
iocase.run
iocase.mrun

Checklist

• Add entry to CHANGELOG.md
• Improve multithreading specs and make the same coverage as in GeoTiffRasterSource Thread Safety #206

Closes #162 #180

[echeipesh]

@pomadchin I'm not hip enough to understand the implications of using F[_]: Par constraint and how the magic works there, I'll need to educate myself there. However, this PR has a lot of good changes that need to happen and the RasterSourceF code looks and smells good.

I'd like to merge this optimistically after addressing single comment above and spend more time groking the async implementations when this is not blocking other work.

[echeipesh]

LGTM, the rebase is good and the effect/MosairRasterSource is good enough to test out more deeply

[pomadchin]

Decreased the number of iterations to speed up an already slow tests set.

[pomadchin]

Merging!