diff --git a/.gitignore b/.gitignore index b8a0fb4e90..73287215a2 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,5 @@ derby.log metastore_db/ *.log +spark/src/test/resources/vlm/*catalog* + diff --git a/.locationtech/deploy-211.sh b/.locationtech/deploy-211.sh index 5eeee2fc47..a48f4d08f9 100755 --- a/.locationtech/deploy-211.sh +++ b/.locationtech/deploy-211.sh @@ -26,4 +26,5 @@ && ./sbt "project vectortile" publish -no-colors \ && ./sbt "project raster-testkit" publish -no-colors \ && ./sbt "project vector-testkit" publish -no-colors \ - && ./sbt "project spark-testkit" publish -no-colors + && ./sbt "project spark-testkit" publish -no-colors \ + && ./sbt "project gdal" publish -no-colors diff --git a/.locationtech/deploy-212.sh b/.locationtech/deploy-212.sh index 6271dad160..ed6cbf0183 100755 --- a/.locationtech/deploy-212.sh +++ b/.locationtech/deploy-212.sh @@ -26,4 +26,5 @@ && ./sbt -212 "project vectortile" publish -no-colors \ && ./sbt -212 "project raster-testkit" publish -no-colors \ && ./sbt -212 "project vector-testkit" publish -no-colors \ - && ./sbt -212 "project spark-testkit" publish -no-colors + && ./sbt -212 "project spark-testkit" publish -no-colors \ + && ./sbt -212 "project gdal" publish -no-colors diff --git a/.travis.yml b/.travis.yml index 893dc501d8..a071533757 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,7 @@ cache: - $HOME/downloads script: - - .travis/build-and-test.sh + - .travis/build-and-test-docker.sh notifications: email: diff --git a/.travis/build-and-test-docker.sh b/.travis/build-and-test-docker.sh new file mode 100755 index 0000000000..aca69f37d8 --- /dev/null +++ b/.travis/build-and-test-docker.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +docker run -it --net=host \ + -v $HOME/.ivy2:/root/.ivy2 \ + -v $HOME/.sbt:/root/.sbt \ + -v $TRAVIS_BUILD_DIR:/geotrellis \ + -e RUN_SET=$RUN_SET \ + -e TRAVIS_SCALA_VERSION=$TRAVIS_SCALA_VERSION \ + -e TRAVIS_COMMIT=$TRAVIS_COMMIT \ + -e TRAVIS_JDK_VERSION=$TRAVIS_JDK_VERSION quay.io/azavea/openjdk-gdal:2.4-jdk8-slim /bin/bash -c "cd /geotrellis; .travis/build-and-test.sh" \ No newline at end of file diff --git a/.travis/build-and-test-set-2.sh b/.travis/build-and-test-set-2.sh index 038e8e547a..8fe74344b5 100755 --- a/.travis/build-and-test-set-2.sh +++ b/.travis/build-and-test-set-2.sh @@ -2,6 +2,7 @@ ./sbt "++$TRAVIS_SCALA_VERSION" \ "project raster" test \ + "project gdal" test \ "project accumulo" test \ "project accumulo-spark" test \ "project s3" test \ diff --git a/.travis/build-and-test-set-3.sh b/.travis/build-and-test-set-3.sh index 06443bea98..a587e68117 100755 --- a/.travis/build-and-test-set-3.sh +++ b/.travis/build-and-test-set-3.sh @@ -2,4 +2,5 @@ ./sbt "++$TRAVIS_SCALA_VERSION" \ "project spark" test \ + "project gdal-spark" test \ "project spark-pipeline" test || { exit 1; } diff --git a/.travis/build-set-2.sh b/.travis/build-set-2.sh index 038e8e547a..8fe74344b5 100755 --- a/.travis/build-set-2.sh +++ b/.travis/build-set-2.sh @@ -2,6 +2,7 @@ ./sbt "++$TRAVIS_SCALA_VERSION" \ "project raster" test \ + "project gdal" test \ "project accumulo" test \ "project accumulo-spark" test \ "project s3" test \ diff --git a/.travis/build-set-3.sh b/.travis/build-set-3.sh index 06443bea98..a587e68117 100755 --- a/.travis/build-set-3.sh +++ b/.travis/build-set-3.sh @@ -2,4 +2,5 @@ ./sbt "++$TRAVIS_SCALA_VERSION" \ "project spark" test \ + "project gdal-spark" test \ "project spark-pipeline" test || { exit 1; } diff --git a/.travis/hbase-install.sh b/.travis/hbase-install.sh index 6710c6481b..090f9b55c3 100755 --- a/.travis/hbase-install.sh +++ b/.travis/hbase-install.sh @@ -1,6 +1,6 @@ #! /bin/bash -if [ ! -f $HOME/downloads/hbase-2.1.5-bin.tar.gz ]; then sudo wget -O $HOME/downloads/hbase-2.1.5-bin.tar.gz http://www-us.apache.org/dist/hbase/2.1.5/hbase-2.1.5-bin.tar.gz; fi -sudo mv $HOME/downloads/hbase-2.1.5-bin.tar.gz hbase-2.1.5-bin.tar.gz && tar xzf hbase-2.1.5-bin.tar.gz -sudo rm -f hbase-2.1.5/conf/hbase-site.xml && sudo mv .travis/hbase/hbase-site.xml hbase-2.1.5/conf -sudo hbase-2.1.5/bin/start-hbase.sh +if [ ! -f $HOME/downloads/hbase-2.1.6-bin.tar.gz ]; then sudo wget -O $HOME/downloads/hbase-2.1.6-bin.tar.gz http://www-us.apache.org/dist/hbase/2.1.6/hbase-2.1.6-bin.tar.gz; fi +sudo mv $HOME/downloads/hbase-2.1.6-bin.tar.gz hbase-2.1.6-bin.tar.gz && tar xzf hbase-2.1.6-bin.tar.gz +sudo rm -f hbase-2.1.6/conf/hbase-site.xml && sudo mv .travis/hbase/hbase-site.xml hbase-2.1.6/conf +sudo hbase-2.1.6/bin/start-hbase.sh diff --git a/build.sbt b/build.sbt index 8ad7780555..ef9d8d1ab3 100644 --- a/build.sbt +++ b/build.sbt @@ -58,7 +58,7 @@ lazy val commonSettings = Seq( .filter(_.asFile.canRead) .map(Credentials(_)), - addCompilerPlugin("org.spire-math" % "kind-projector" % "0.9.10" cross CrossVersion.binary), + addCompilerPlugin("org.typelevel" %% "kind-projector" % "0.10.3" cross CrossVersion.binary), addCompilerPlugin("org.scalamacros" %% "paradise" % "2.1.1" cross CrossVersion.full), pomExtra := ( @@ -105,23 +105,23 @@ lazy val commonSettings = Seq( updateOptions := updateOptions.value.withGigahorse(false) ) -lazy val root = Project("geotrellis", file(".")). - aggregate( - `accumulo`, +lazy val root = Project("geotrellis", file(".")) + .aggregate( + accumulo, `accumulo-spark`, - `cassandra`, + cassandra, `cassandra-spark`, `doc-examples`, geomesa, geotools, geowave, - `hbase`, + hbase, `hbase-spark`, macros, proj4, raster, `raster-testkit`, - `s3`, + s3, `s3-spark`, shapefile, spark, @@ -130,20 +130,13 @@ lazy val root = Project("geotrellis", file(".")). util, vector, `vector-testkit`, - vectortile - ). - settings(commonSettings: _*). - enablePlugins(ScalaUnidocPlugin). - settings( - initialCommands in console := - """ - import geotrellis.raster._ - import geotrellis.vector._ - import geotrellis.proj4._ - import geotrellis.spark._ - """ - ). - settings(unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(geowave)) + vectortile, + gdal, + `gdal-spark` + ) + .settings(commonSettings: _*) + .enablePlugins(ScalaUnidocPlugin) + .settings(unidocProjectFilter in (ScalaUnidoc, unidoc) := inAnyProject -- inProjects(geowave)) lazy val macros = project .settings(commonSettings) @@ -204,24 +197,24 @@ lazy val `spark-testkit` = project .settings(commonSettings) .settings(Settings.`spark-testkit`) -lazy val `s3` = project +lazy val s3 = project .dependsOn(store) .settings(commonSettings) - .settings(Settings.`s3`) + .settings(Settings.s3) lazy val `s3-spark` = project .dependsOn( spark % "compile->compile;test->test", // <-- spark-testkit update should simplify this - `s3`, + s3, `spark-testkit` % Test ) .settings(commonSettings) .settings(Settings.`s3-spark`) -lazy val `accumulo` = project +lazy val accumulo = project .dependsOn(store) .settings(commonSettings) - .settings(Settings.`accumulo`) + .settings(Settings.accumulo) lazy val `accumulo-spark` = project .dependsOn( @@ -232,29 +225,29 @@ lazy val `accumulo-spark` = project .settings(commonSettings) .settings(Settings.`accumulo-spark`) -lazy val `cassandra` = project +lazy val cassandra = project .dependsOn(store) .settings(commonSettings) - .settings(Settings.`cassandra`) + .settings(Settings.cassandra) lazy val `cassandra-spark` = project .dependsOn( - `cassandra`, + cassandra, spark % "compile->compile;test->test", // <-- spark-testkit update should simplify this `spark-testkit` % Test ) .settings(commonSettings) .settings(Settings.`cassandra-spark`) -lazy val `hbase` = project +lazy val hbase = project .dependsOn(store) .settings(commonSettings) // HBase depends on its own protobuf version - .settings(Settings.`hbase`) + .settings(Settings.hbase) .settings(projectDependencies := { Seq((projectID in layer).value.exclude("com.google.protobuf", "protobuf-java")) }) lazy val `hbase-spark` = project .dependsOn( - `hbase`, + hbase, spark % "compile->compile;test->test", // <-- spark-testkit update should simplify this `spark-testkit` % Test ) @@ -269,7 +262,7 @@ lazy val `spark-pipeline` = Project(id = "spark-pipeline", base = file("spark-pi lazy val geotools = project .dependsOn(raster, vector, proj4, `vector-testkit` % Test, `raster-testkit` % Test, - `raster` % "test->test" // <-- to get rid of this, move `GeoTiffTestUtils` to the testkit. + raster % "test->test" // <-- to get rid of this, move `GeoTiffTestUtils` to the testkit. ) .settings(commonSettings) .settings(Settings.geotools) @@ -310,7 +303,7 @@ lazy val bench = project .settings(Settings.bench) lazy val layer = project - .dependsOn(raster) + .dependsOn(raster, `raster-testkit` % Test) .settings(commonSettings) .settings(Settings.layer) @@ -318,3 +311,14 @@ lazy val store = project .dependsOn(layer) .settings(commonSettings) .settings(Settings.store) + +lazy val gdal = project + .dependsOn(raster, `raster-testkit` % Test) + .settings(commonSettings) + .settings(Settings.gdal) + +lazy val `gdal-spark` = project + .dependsOn(gdal, spark, `spark-testkit` % Test) + .settings(commonSettings) + .settings(publish / skip := true) // at this point we need this project only for tests + .settings(Settings.`gdal-spark`) diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 967a2e27df..250a574f1e 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -6,6 +6,8 @@ Changelog API Changes & Project structure changes +- **New:** Add RasterSources API (`#3053 `_). + - ``geotrellis-vector`` - **Change:** We are now favoring direct use of JTS geometries for improved interoperability with other projects. Scala wrapper classes for Geometry have been snuffed. Many submodules of ``geotrellis.vector`` have also been sacked in favor of direct usage of the corresponding JTS functionality. Extension methods and companion objects have been employed to maintain a crisp, candy shell around JTS to keep most interactions from messing up your fingers. Import ``geotrellis.vector._`` to access these niceties; if it is required, ``import org.locationtech.jts.{geom => jts}`` to prevent namespace collisions. In the REPL, geometries will need to be constructed via the duplicate definitions in the ``JTS`` object to avoid namespace clashes that appear to be buggy behavior on the part of the REPL (that is, use ``JTS.Point(0,0)`` to construct a point at the origin in interactive sessions, but in compiled code, ``Point(0,0)`` will suffice). @@ -106,7 +108,6 @@ API Changes & Project structure changes Fixes & Updates ^^^^^^^^^^^^^^^ -- Update pureconfig to version 0.10.2 (`#2882 `_). - Update dependencies (`#2904 `_). - Bump ScalaPB version up with some API enhancements (`#2898 `_). - Artifacts in Viewshed have been addressed, the pixels/meter calculation has also been improved (`#2917 `_). @@ -119,6 +120,7 @@ Fixes & Updates - ``S3RangeReader`` will now optionally read the HEADER of an object (`#3025 `_). - ``FileRangeReaderProvider`` can now handle more types of ``URI``\s (`#3034 `_). - Bump proj4 version to fix multiple performance issues (`#3039 `_). +- Update dependencies (`#3053 `_). 2.3.0 ----- diff --git a/gdal-spark/src/test/scala/geotrellis/GDALTestUtils.scala b/gdal-spark/src/test/scala/geotrellis/GDALTestUtils.scala new file mode 100644 index 0000000000..e0e476a6da --- /dev/null +++ b/gdal-spark/src/test/scala/geotrellis/GDALTestUtils.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis + +import java.io.File + +object GDALTestUtils { + def gdalGeoTiffPath(name: String): String = { + def baseDataPath = "../gdal/src/test/resources" + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } + + def sparkGeoTiffPath(name: String): String = { + def baseDataPath = "../spark/src/test/resources" + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } +} diff --git a/gdal-spark/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala b/gdal-spark/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala new file mode 100644 index 0000000000..c029368ac4 --- /dev/null +++ b/gdal-spark/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala @@ -0,0 +1,95 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.layer._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.resample._ +import geotrellis.vector.ProjectedExtent +import geotrellis.raster.testkit._ + +import org.scalatest._ + +class GDALRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + import geotrellis.GDALTestUtils._ + val uri = gdalGeoTiffPath("vlm/aspect-tiled.tif") + + describe("should perform a tileToLayout of a GeoTiffRasterSource") { + + // we are going to use this source for resampling into weird resolutions, let's check it + // usually we align pixels + lazy val source: GDALRasterSource = GDALRasterSource(uri, GDALWarpOptions(alignTargetPixels = false)) + + val cellSizes = { + val tiff = GeoTiffReader.readMultiband(uri) + (tiff +: tiff.overviews).map(_.rasterExtent.cellSize).map { case CellSize(w, h) => + CellSize(w + 1, h + 1) + } + } + + cellSizes.foreach { targetCellSize => + it(s"for cellSize: $targetCellSize") { + val pe = ProjectedExtent(source.extent, source.crs) + val scheme = FloatingLayoutScheme(256) + val layout = scheme.levelFor(pe.extent, targetCellSize).layout + val mapTransform = layout.mapTransform + val resampledSource = source.resampleToGrid(layout) + + val expected: List[(SpatialKey, MultibandTile)] = + mapTransform(pe.extent).coordsIter.map { case (col, row) => + val key = SpatialKey(col, row) + val ext = mapTransform.keyToExtent(key) + val raster = resampledSource.read(ext).get + val newTile = raster.tile.prototype(source.cellType, layout.tileCols, layout.tileRows) + key -> newTile.merge( + ext, + raster.extent, + raster.tile, + NearestNeighbor + ) + }.toList + + val layoutSource = source.tileToLayout(layout) + val actual: List[(SpatialKey, MultibandTile)] = layoutSource.readAll().toList + + withClue(s"actual.size: ${actual.size} expected.size: ${expected.size}") { + actual.size should be(expected.size) + } + + val sortedActual: List[Raster[MultibandTile]] = + actual + .sortBy { case (k, _) => (k.col, k.row) } + .map { case (k, v) => Raster(v, mapTransform.keyToExtent(k)) } + + val sortedExpected: List[Raster[MultibandTile]] = + expected + .sortBy { case (k, _) => (k.col, k.row) } + .map { case (k, v) => Raster(v, mapTransform.keyToExtent(k)) } + + val grouped: List[(Raster[MultibandTile], Raster[MultibandTile])] = + sortedActual.zip(sortedExpected) + + grouped.foreach { case (actualTile, expectedTile) => + withGeoTiffClue(actualTile, expectedTile, source.crs) { + assertRastersEqual(actualTile, expectedTile) + } + } + } + } + } +} diff --git a/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSourceRDDSpec.scala b/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSourceRDDSpec.scala new file mode 100644 index 0000000000..d4d66a5a9b --- /dev/null +++ b/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSourceRDDSpec.scala @@ -0,0 +1,404 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark.gdal + +import geotrellis.layer._ +import geotrellis.raster.gdal._ +import geotrellis.raster.geotiff._ +import geotrellis.raster.{RasterSource, ReadingSource} +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff._ +import geotrellis.spark._ +import geotrellis.spark.partition._ +import geotrellis.spark.store.hadoop._ +import geotrellis.spark.testkit._ +import geotrellis.store.hadoop._ + +import cats.effect.{ContextShift, IO} +import cats.implicits._ +import spire.syntax.cfor._ +import org.apache.spark.rdd.RDD +import org.scalatest.Inspectors._ +import org.scalatest._ + +import java.util.concurrent.Executors +import scala.concurrent.ExecutionContext + +class GDALRasterSourceRDDSpec extends FunSpec with TestEnvironment with BeforeAndAfterAll { + import geotrellis.GDALTestUtils._ + + val uri = gdalGeoTiffPath("vlm/aspect-tiled.tif") + def filePathByIndex(i: Int): String = sparkGeoTiffPath(s"vlm/aspect-tiled-$i.tif") + lazy val rasterSource = GeoTiffRasterSource(uri) + val targetCRS = CRS.fromEpsgCode(3857) + val scheme = ZoomedLayoutScheme(targetCRS) + lazy val layout = scheme.levelForZoom(13).layout + + lazy val reprojectedSource = rasterSource.reprojectToGrid(targetCRS, layout) + + describe("reading in GeoTiffs as RDDs") { + + it("should have the right number of tiles") { + val expectedKeys = + layout + .mapTransform + .keysForGeometry(reprojectedSource.extent.toPolygon) + .toSeq + .sortBy { key => (key.col, key.row) } + + info(s"RasterSource CRS: ${reprojectedSource.crs}") + + val rdd = RasterSourceRDD.spatial(reprojectedSource, layout) + + val actualKeys = rdd.keys.collect().sortBy { key => (key.col, key.row) } + + for ((actual, expected) <- actualKeys.zip(expectedKeys)) { + actual should be(expected) + } + } + + it("should read in the tiles as squares") { + val reprojectedRasterSource = rasterSource.reprojectToGrid(targetCRS, layout) + val rdd = RasterSourceRDD.spatial(reprojectedRasterSource, layout) + val rows = rdd.collect() + + forAll(rows) { case (key, tile) => + withClue(s"$key") { + tile should have( + // dimensions(256, 256), + cellType(rasterSource.cellType), + bandCount(rasterSource.bandCount) + ) + } + } + } + } + + describe("Match reprojection from HadoopGeoTiffRDD") { + val floatingLayout = FloatingLayoutScheme(256) + val geoTiffRDD = HadoopGeoTiffRDD.spatialMultiband(uri) + val md = geoTiffRDD.collectMetadata[SpatialKey](floatingLayout)._2 + + val reprojectedExpectedRDD: MultibandTileLayerRDD[SpatialKey] = + geoTiffRDD.tileToLayout(md).reproject(targetCRS, layout)._2.persist() + + def assertRDDLayersEqual( + expected: MultibandTileLayerRDD[SpatialKey], + actual: MultibandTileLayerRDD[SpatialKey], + matchRasters: Boolean = false + ): Unit = { + val joinedRDD = expected.filter { case (_, t) => !t.band(0).isNoDataTile }.leftOuterJoin(actual) + + joinedRDD.collect().foreach { case (key, (expected, actualTile)) => + actualTile match { + case Some(actual) => + /*writePngOutputTile( + actual, + name = "actual", + discriminator = s"-${key}" + ) + + writePngOutputTile( + expected, + name = "expected", + discriminator = s"-${key}" + )*/ + + // withGeoTiffClue(key, layout, actual, expected, targetCRS) { + withClue(s"$key:") { + expected.dimensions should be(actual.dimensions) + if (matchRasters) assertTilesEqual(expected, actual) + } + // } + + case None => + throw new Exception(s"$key does not exist in the rasterSourceRDD") + } + } + } + + it("should reproduce tileToLayout") { + // This should be the same as result of .tileToLayout(md.layout) + val rasterSourceRDD: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.spatial(rasterSource, md.layout) + + // Complete the reprojection + val reprojectedSource = rasterSourceRDD.reproject(targetCRS, layout)._2 + + assertRDDLayersEqual(reprojectedExpectedRDD, reprojectedSource) + } + + it("should reproduce tileToLayout followed by reproject") { + val reprojectedSourceRDD: MultibandTileLayerRDD[SpatialKey] = + RasterSourceRDD.spatial(rasterSource.reprojectToGrid(targetCRS, layout), layout) + + // geotrellis.raster.io.geotiff.GeoTiff(reprojectedExpectedRDD.stitch, targetCRS).write("/tmp/expected.tif") + // geotrellis.raster.io.geotiff.GeoTiff(reprojectedSourceRDD.stitch, targetCRS).write("/tmp/actual.tif") + + val actual = reprojectedSourceRDD.stitch.tile.band(0) + val expected = reprojectedExpectedRDD.stitch.tile.band(0) + + var (diff, pixels, mismatched) = (0d, 0d, 0) + cfor(0)(_ < math.min(actual.cols, expected.cols), _ + 1) { c => + cfor(0)(_ < math.min(actual.rows, expected.rows), _ + 1) { r => + pixels += 1d + if (math.abs(actual.get(c, r) - expected.get(c, r)) > 1e-6) + diff += 1d + if (isNoData(actual.get(c, r)) != isNoData(expected.get(c, r))) + mismatched += 1 + } + } + + assert(diff / pixels < 0.005) // half percent of pixels or less are not equal + assert(mismatched < 3) + } + + it("should reproduce tileToLayout when given an RDD[RasterSource]") { + val rasterSourceRDD: RDD[RasterSource] = sc.parallelize(Seq(rasterSource)) + + // Need to define these here or else a serialization error will occur + val targetLayout = layout + val crs = targetCRS + + val reprojectedRasterSourceRDD: RDD[RasterSource] = rasterSourceRDD.map { _.reprojectToGrid(crs, targetLayout) } + + val tiledSource: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.tiledLayerRDD(reprojectedRasterSourceRDD, targetLayout) + + assertRDDLayersEqual(reprojectedExpectedRDD, tiledSource) + } + + describe("GDALRasterSource") { + val expectedFilePath = gdalGeoTiffPath("vlm/aspect-tiled-near-merc-rdd.tif") + + it("should reproduce tileToLayout") { + val rasterSource = GDALRasterSource(uri) + + // This should be the same as result of .tileToLayout(md.layout) + val rasterSourceRDD: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.spatial(rasterSource, md.layout) + // Complete the reprojection + val reprojectedSource = rasterSourceRDD.reproject(targetCRS, layout)._2 + + assertRDDLayersEqual(reprojectedExpectedRDD, reprojectedSource, true) + } + + it("should reproduce tileToLayout followed by reproject GDAL") { + val expectedRasterSource = GDALRasterSource(expectedFilePath) + val reprojectedExpectedRDDGDAL: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.spatial(expectedRasterSource, layout) + val rasterSource = GDALRasterSource(uri) + val reprojectedRasterSource = rasterSource.reprojectToGrid(targetCRS, layout) + + // This should be the same as .tileToLayout(md.layout).reproject(crs, layout) + val reprojectedSourceRDD: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.spatial(reprojectedRasterSource, layout) + + assertRDDLayersEqual(reprojectedExpectedRDDGDAL, reprojectedSourceRDD, true) + } + + def parellSpec(n: Int = 1000)(implicit cs: ContextShift[IO]): List[RasterSource] = { + println(java.lang.Thread.activeCount()) + + /** Functions to trigger Datasets computation */ + def ltsWithDatasetsTriggered(lts: LayoutTileSource[SpatialKey]): LayoutTileSource[SpatialKey] = { rsWithDatasetsTriggered(lts.source); lts } + def rsWithDatasetsTriggered(rs: RasterSource): RasterSource = { + val brs = rs.asInstanceOf[GDALRasterSource] + brs.dataset.rasterExtent + brs.dataset.rasterExtent(GDALDataset.SOURCE) + rs + } + + /** Do smth usual with the original RasterSource to force VRTs allocation */ + def reprojRS(i: Int): LayoutTileSource[SpatialKey] = + ltsWithDatasetsTriggered( + rsWithDatasetsTriggered( + rsWithDatasetsTriggered(GDALRasterSource(filePathByIndex(i))) + .reprojectToGrid(targetCRS, layout) + ).tileToLayout(layout) + ) + + /** Simulate possible RF backsplash calls */ + def dirtyCalls(rs: RasterSource): RasterSource = { + val ds = rs.asInstanceOf[GDALRasterSource].dataset + ds.rasterExtent + ds.crs + ds.cellSize + ds.extent + rs + } + + val res = (1 to n).toList.flatMap { _ => + (0 to 4).flatMap { i => + List(IO { + // println(Thread.currentThread().getName()) + // Thread.sleep((Math.random() * 100).toLong) + val lts = reprojRS(i) + lts.readAll(lts.keys.take(10).toIterator) + reprojRS(i).source.resolutions + + dirtyCalls(reprojRS(i).source) + }, IO { + // println(Thread.currentThread().getName()) + // Thread.sleep((Math.random() * 100).toLong) + val lts = reprojRS(i) + lts.readAll(lts.keys.take(10).toIterator) + reprojRS(i).source.resolutions + + dirtyCalls(reprojRS(i).source) + }, IO { + // println(Thread.currentThread().getName()) + // Thread.sleep((Math.random() * 100).toLong) + val lts = reprojRS(i) + lts.readAll(lts.keys.take(10).toIterator) + reprojRS(i).source.resolutions + + dirtyCalls(reprojRS(i).source) + }) + } + }.parSequence.unsafeRunSync + + println(java.lang.Thread.activeCount()) + + res + } + + it(s"should not fail on parallelization with a fork join pool") { + val i = 1000 + implicit val cs = IO.contextShift(ExecutionContext.global) + + val res = parellSpec(i) + } + + it(s"should not fail on parallelization with a fixed thread pool") { + val i = 1000 + val n = 200 + val pool = Executors.newFixedThreadPool(n) + val ec = ExecutionContext.fromExecutor(pool) + implicit val cs = IO.contextShift(ec) + + val res = parellSpec(i) + } + } + } + + describe("RasterSourceRDD.read") { + + val floatingScheme = FloatingLayoutScheme(500, 270) + val floatingLayout = floatingScheme.levelFor(rasterSource.extent, rasterSource.cellSize).layout + + val cellType = rasterSource.cellType + + val multibandTilePath = sparkGeoTiffPath("vlm/aspect-tiled-0-1-2.tif") + + val noDataTile = ArrayTile.alloc(cellType, rasterSource.cols.toInt, rasterSource.rows.toInt).fill(NODATA).interpretAs(cellType) + + val paths: Seq[String] = 0 to 5 map { index => filePathByIndex(index) } + + lazy val expectedMultibandTile = { + val tiles = paths.map { MultibandGeoTiff(_, streaming = false).tile.band(0) } + MultibandTile(tiles) + } + + it("should read in singleband tiles") { + val readingSources: Seq[ReadingSource] = paths.zipWithIndex.map { case (path, index) => ReadingSource(GeoTiffRasterSource(path), 0, index) } + + val expected = expectedMultibandTile + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + expected.dimensions shouldBe actual.dimensions + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband tiles with missing bands") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(paths(0)), 0, 0), + ReadingSource(GeoTiffRasterSource(paths(2)), 0, 1), + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 3) + ) + + val expected = + MultibandTile( + expectedMultibandTile.band(0), + expectedMultibandTile.band(2), + noDataTile, + expectedMultibandTile.band(4) + ) + + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + expected.dimensions shouldBe actual.dimensions + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband and multiband tiles") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(multibandTilePath), 0, 0), + ReadingSource(GeoTiffRasterSource(paths(1)), 0, 1), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 2, 2), + ReadingSource(GeoTiffRasterSource(paths(3)), 0, 3), + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 4), + ReadingSource(GeoTiffRasterSource(paths(5)), 0, 5) + ) + + val expected = expectedMultibandTile + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + expected.dimensions shouldBe actual.dimensions + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband and multiband tiles with missing bands") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 5), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 1, 0), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 2, 1) + ) + + val expected = + MultibandTile( + expectedMultibandTile.band(1), + expectedMultibandTile.band(2), + noDataTile, + noDataTile, + noDataTile, + expectedMultibandTile.band(4) + ) + + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + expected.dimensions shouldBe actual.dimensions + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + } +} + diff --git a/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSummarySpec.scala b/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSummarySpec.scala new file mode 100644 index 0000000000..601ec7b1d1 --- /dev/null +++ b/gdal-spark/src/test/scala/geotrellis/spark/gdal/GDALRasterSummarySpec.scala @@ -0,0 +1,170 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark.gdal + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.gdal._ +import geotrellis.raster.resample.Bilinear +import geotrellis.spark._ +import geotrellis.spark.partition._ +import geotrellis.layer._ +import geotrellis.vector.Extent + +import geotrellis.spark.testkit._ + +import org.apache.spark.rdd._ +import spire.syntax.cfor._ + +import org.scalatest._ + +class GDALRasterSummarySpec extends FunSpec with TestEnvironment with GivenWhenThen { + import geotrellis.GDALTestUtils._ + + describe("Should collect GDALRasterSource RasterSummary correct") { + it("should collect summary for a raw source") { + val inputPath = gdalGeoTiffPath("vlm/aspect-tiled.tif") + val files = inputPath :: Nil + + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GDALRasterSource(uri): RasterSource) + .cache() + + val summary = RasterSummary.fromRDD(sourceRDD) + val rasterSource = GDALRasterSource(inputPath) + + rasterSource.crs shouldBe summary.crs + rasterSource.extent shouldBe summary.extent + rasterSource.cellSize shouldBe summary.cellSize + rasterSource.cellType shouldBe summary.cellType + rasterSource.size shouldBe summary.cells + files.length shouldBe summary.count + } + + // TODO: fix this test + it("should collect summary for a tiled to layout source GDAL") { + val inputPath = gdalGeoTiffPath("vlm/aspect-tiled.tif") + val files = inputPath :: Nil + val targetCRS = WebMercator + val method = Bilinear + val layoutScheme = ZoomedLayoutScheme(targetCRS, tileSize = 256) + + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GDALRasterSource(uri).reproject(targetCRS, method = method): RasterSource) + .cache() + + val summary = RasterSummary.fromRDD(sourceRDD) + val LayoutLevel(zoom, layout) = summary.levelFor(layoutScheme) + val tiledRDD = sourceRDD.map(_.tileToLayout(layout, method)) + + val summaryCollected = RasterSummary.fromRDD(tiledRDD.map(_.source)) + val summaryResampled = summary.resample(TargetGrid(layout)) + + val metadata = summary.toTileLayerMetadata(layout) + val metadataResampled = summaryResampled.toTileLayerMetadata(GlobalLayout(256, zoom, 0.1)) + + metadata shouldBe metadataResampled + + summaryCollected.crs shouldBe summaryResampled.crs + summaryCollected.cellType shouldBe summaryResampled.cellType + + val CellSize(widthCollected, heightCollected) = summaryCollected.cellSize + val CellSize(widthResampled, heightResampled) = summaryResampled.cellSize + + // the only weird place where cellSize is a bit different + widthCollected shouldBe (widthResampled +- 1e-7) + heightCollected shouldBe (heightResampled +- 1e-7) + + // TODO: investigate the reason of why this won't work here + // but probably this function should be removed in the future completely and nowhere used + // val Extent(xminc, yminc, xmaxc, ymaxc) = summaryCollected.extent + // val Extent(xminr, yminr, xmaxr, ymaxr) = summaryResampled.extent + + // extent probably can be calculated a bit different via GeoTrellis API + // xminc shouldBe xminr +- 1e-5 + // yminc shouldBe yminr +- 1e-5 + // xmaxc shouldBe xmaxr +- 1e-5 + // ymaxc shouldBe ymaxr +- 1e-5 + + // summaryCollected.cells shouldBe summaryResampled.cells + // summaryCollected.count shouldBe summaryResampled.count + } + } + + it("should create ContextRDD from RDD of GDALRasterSources") { + val inputPath = gdalGeoTiffPath("vlm/aspect-tiled.tif") + val files = inputPath :: Nil + val targetCRS = WebMercator + val method = Bilinear + val layoutScheme = ZoomedLayoutScheme(targetCRS, tileSize = 256) + + // read sources + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GDALRasterSource(uri).reproject(targetCRS, method = method): RasterSource) + .cache() + + // collect raster summary + val summary = RasterSummary.fromRDD(sourceRDD) + val LayoutLevel(_, layout) = summary.levelFor(layoutScheme) + + val tiledLayoutSource = sourceRDD.map(_.tileToLayout(layout, method)) + + // Create RDD of references, references contain information how to read rasters + val rasterRefRdd: RDD[(SpatialKey, RasterRegion)] = tiledLayoutSource.flatMap(_.keyedRasterRegions()) + val tileRDD: RDD[(SpatialKey, MultibandTile)] = + rasterRefRdd // group by keys and distribute raster references using SpatialPartitioner + .groupByKey(SpatialPartitioner(summary.estimatePartitionsNumber)) + .mapValues { iter => MultibandTile { + iter.flatMap { rr => rr.raster.toSeq.flatMap(_.tile.bands) } + } } // read rasters + + val metadata = summary.toTileLayerMetadata(layout) + val contextRDD: MultibandTileLayerRDD[SpatialKey] = ContextRDD(tileRDD, metadata) + + val res = contextRDD.collect() + res.foreach { case (_, v) => v.dimensions shouldBe layout.tileLayout.tileDimensions } + res.length shouldBe rasterRefRdd.count() + res.length shouldBe 72 + + contextRDD.stitch.tile.band(0).renderPng().write("/tmp/raster-source-contextrdd-gdal.png") + } + + it("Should cleanup GDAL Datasets by the end of the loop (10 iterations)") { + val inputPath = gdalGeoTiffPath("vlm/aspect-tiled.tif") + val targetCRS = WebMercator + val method = Bilinear + val layout = LayoutDefinition(GridExtent[Int](Extent(-2.0037508342789244E7, -2.0037508342789244E7, 2.0037508342789244E7, 2.0037508342789244E7), CellSize(9.554628535647032, 9.554628535647032)), 256) + val RasterExtent(Extent(exmin, eymin, exmax, eymax), ecw, ech, ecols, erows) = RasterExtent(Extent(-8769161.632988561, 4257685.794912352, -8750625.653629405, 4274482.8318780195), CellSize(9.554628535647412, 9.554628535646911)) + + cfor(0)(_ < 11, _ + 1) { _ => + val reference = GDALRasterSource(inputPath).reproject(targetCRS, method = method).tileToLayout(layout, method) + val RasterExtent(Extent(axmin, aymin, axmax, aymax), acw, ach, acols, arows) = reference.source.gridExtent.toRasterExtent + + axmin shouldBe exmin +- 1e-5 + aymin shouldBe eymin +- 1e-5 + axmax shouldBe exmax +- 1e-5 + aymax shouldBe eymax +- 1e-5 + acw shouldBe ecw +- 1e-5 + ach shouldBe ech +- 1e-5 + acols shouldBe ecols + arows shouldBe erows + } + } +} diff --git a/gdal/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider b/gdal/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider new file mode 100644 index 0000000000..f002981553 --- /dev/null +++ b/gdal/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider @@ -0,0 +1 @@ +geotrellis.raster.gdal.GDALRasterSourceProvider diff --git a/gdal/src/main/resources/reference.conf b/gdal/src/main/resources/reference.conf new file mode 100644 index 0000000000..ee2f966828 --- /dev/null +++ b/gdal/src/main/resources/reference.conf @@ -0,0 +1,4 @@ +geotrellis.raster.gdal { + acceptable-datasets = ["SOURCE", "WARPED"] + number-of-attempts = 1048576 +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataType.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataType.scala new file mode 100644 index 0000000000..2306742a7f --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataType.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import com.azavea.gdal.GDALWarp + +object GDALDataType { + val types = + List( + UnknownType, + TypeByte, + TypeUInt16, + TypeInt16, + TypeUInt32, + TypeInt32, + TypeFloat32, + TypeFloat64, + TypeCInt16, + TypeCInt32, + TypeCFloat32, + TypeCFloat64 + ) + + implicit def intToGDALDataType(i: Int): GDALDataType = + types.find(_.code == i) match { + case Some(dt) => dt + case None => sys.error(s"Invalid GDAL data type code: $i") + } + + implicit def GDALDataTypeToInt(typ: GDALDataType): Int = + typ.code +} + +abstract sealed class GDALDataType(val code: Int) { + override + def toString: String = code.toString +} + +// https://github.com/geotrellis/gdal-warp-bindings/blob/9d75e7c65c4c8a0c2c39175a75656bba458a46f0/src/main/java/com/azavea/gdal/GDALWarp.java#L26-L38 +case object UnknownType extends GDALDataType(GDALWarp.GDT_Unknown) +case object TypeByte extends GDALDataType(GDALWarp.GDT_Byte) +case object TypeUInt16 extends GDALDataType(GDALWarp.GDT_UInt16) +case object TypeInt16 extends GDALDataType(GDALWarp.GDT_Int16) +case object TypeUInt32 extends GDALDataType(GDALWarp.GDT_UInt32) +case object TypeInt32 extends GDALDataType(GDALWarp.GDT_Int32) +case object TypeFloat32 extends GDALDataType(GDALWarp.GDT_Float32) +case object TypeFloat64 extends GDALDataType(GDALWarp.GDT_Float64) +case object TypeCInt16 extends GDALDataType(GDALWarp.GDT_CInt16) +case object TypeCInt32 extends GDALDataType(GDALWarp.GDT_CInt32) +case object TypeCFloat32 extends GDALDataType(GDALWarp.GDT_CFloat32) +case object TypeCFloat64 extends GDALDataType(GDALWarp.GDT_CFloat64) diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataset.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataset.scala new file mode 100644 index 0000000000..e196e8b27e --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALDataset.scala @@ -0,0 +1,348 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.gdal.config.GDALOptionsConfig +import geotrellis.raster.gdal.GDALDataset.DatasetType +import geotrellis.raster._ +import geotrellis.proj4.{CRS, LatLng} +import geotrellis.vector.Extent + +import com.azavea.gdal.GDALWarp + +case class GDALDataset(token: Long) extends AnyVal { + def getAllMetadataFlatten(): Map[String, String] = getAllMetadataFlatten(GDALDataset.SOURCE) + + def getAllMetadataFlatten(datasetType: DatasetType): Map[String, String] = + (0 until bandCount(datasetType)).map(getAllMetadata(datasetType, _).flatMap(_._2)).reduce(_ ++ _) + + def getAllMetadata(band: Int): Map[GDALMetadataDomain, Map[String, String]] = + getAllMetadata(GDALDataset.SOURCE, band) + + def getAllMetadata(datasetType: DatasetType, band: Int): Map[GDALMetadataDomain, Map[String, String]] = + getMetadataDomainList(datasetType.value).map { domain => domain -> getMetadata(domain, band) }.filter(_._2.nonEmpty).toMap + + def getMetadataDomainList(band: Int): List[GDALMetadataDomain] = getMetadataDomainList(GDALDataset.SOURCE, band) + + /** https://github.com/OSGeo/gdal/blob/b1c9c12ad373e40b955162b45d704070d4ebf7b0/gdal/doc/source/development/rfc/rfc43_getmetadatadomainlist.rst */ + def getMetadataDomainList(datasetType: DatasetType, band: Int): List[GDALMetadataDomain] = { + val arr = Array.ofDim[Byte](100, 1 << 10) + val returnValue = GDALWarp.get_metadata_domain_list(token, datasetType.value, numberOfAttempts, band, arr) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedProjectionException( + s"Unable to get the metadata domain list. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + (GDALMetadataDomain.ALL ++ arr.map(new String(_, "UTF-8").trim).filter(_.nonEmpty).toList.map(UserDefinedDomain)).distinct + } + + def getMetadata(domains: List[GDALMetadataDomain], band: Int): Map[GDALMetadataDomain, Map[String, String]] = + getMetadata(GDALDataset.SOURCE, domains, band) + + def getMetadata(datasetType: DatasetType, domains: List[GDALMetadataDomain], band: Int): Map[GDALMetadataDomain, Map[String, String]] = + domains.map(domain => domain -> getMetadata(datasetType, domain, band)).filter(_._2.nonEmpty).toMap + + def getMetadata(domain: GDALMetadataDomain, band: Int): Map[String, String] = getMetadata(GDALDataset.SOURCE, domain, band) + + def getMetadata(datasetType: DatasetType, domain: GDALMetadataDomain, band: Int): Map[String, String] = { + val arr = Array.ofDim[Byte](100, 1 << 10) + val returnValue = GDALWarp.get_metadata(token, datasetType.value, numberOfAttempts, band, domain.name, arr) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedProjectionException( + s"Unable to get the metadata. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + arr + .map(new String(_, "UTF-8").trim) + .filter(_.nonEmpty) + .flatMap { str => + val arr = str.split("=") + if(arr.length == 2) { + val Array(key, value) = str.split("=") + Some(key -> value) + } else Some("" -> str) + }.toMap + } + + def getMetadataItem(key: String, domain: GDALMetadataDomain, band: Int): String = getMetadataItem(GDALDataset.WARPED, key, domain, band) + + def getMetadataItem(datasetType: DatasetType, key: String, domain: GDALMetadataDomain, band: Int): String = { + val arr = Array.ofDim[Byte](1 << 10) + val returnValue = GDALWarp.get_metadata_item(token, datasetType.value, numberOfAttempts, band, key, domain.name, arr) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedProjectionException( + s"Unable to get the metadata item. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + new String(arr, "UTF-8").trim + } + + def getProjection: Option[String] = getProjection(GDALDataset.WARPED) + + def getProjection(datasetType: DatasetType): Option[String] = { + require(acceptableDatasets contains datasetType) + val crs = Array.ofDim[Byte](1 << 10) + val returnValue = GDALWarp.get_crs_wkt(token, datasetType.value, numberOfAttempts, crs) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedProjectionException( + s"Unable to parse projection as WKT String. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + Some(new String(crs, "UTF-8")) + } + + def rasterExtent: RasterExtent = rasterExtent(GDALDataset.WARPED) + + def rasterExtent(datasetType: DatasetType): RasterExtent = { + require(acceptableDatasets contains datasetType) + val transform = Array.ofDim[Double](6) + val width_height = Array.ofDim[Int](2) + + val transformReturnValue = GDALWarp.get_transform(token, datasetType.value, numberOfAttempts, transform) + val dimensionReturnValue = GDALWarp.get_width_height(token, datasetType.value, numberOfAttempts, width_height) + + val returnValue = + if (transformReturnValue < 0) transformReturnValue + else if (dimensionReturnValue < 0) dimensionReturnValue + else 0 + + if (returnValue < 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataException( + s"Unable to construct a RasterExtent from the Transformation given. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + val x1 = transform(0) + val y1 = transform(3) + val x2 = x1 + transform(1) * width_height(0) + val y2 = y1 + transform(5) * width_height(1) + val e = Extent( + math.min(x1, x2), + math.min(y1, y2), + math.max(x1, x2), + math.max(y1, y2) + ) + + RasterExtent(e, math.abs(transform(1)), math.abs(transform(5)), width_height(0), width_height(1)) + } + + def resolutions(): List[RasterExtent] = resolutions(GDALDataset.WARPED) + + def resolutions(datasetType: DatasetType): List[RasterExtent] = { + require(acceptableDatasets contains datasetType) + val N = 1 << 8 + val widths = Array.ofDim[Int](N) + val heights = Array.ofDim[Int](N) + val extent = this.extent(datasetType) + + val returnValue = + GDALWarp.get_overview_widths_heights(token, datasetType.value, numberOfAttempts, 1, widths, heights) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataException( + s"Unable to construct the overview RasterExtents for the resample. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + widths.zip(heights).flatMap({ case (w, h) => + if (w > 0 && h > 0) Some(RasterExtent(extent, cols = w, rows = h)) + else None + }).toList + } + + def extent: Extent = extent(GDALDataset.WARPED) + + def extent(datasetType: DatasetType): Extent = { + require(acceptableDatasets contains datasetType) + this.rasterExtent(datasetType).extent + } + + def bandCount: Int = bandCount(GDALDataset.WARPED) + + def bandCount(datasetType: DatasetType): Int = { + require(acceptableDatasets contains datasetType) + val count = Array.ofDim[Int](1) + + val returnValue = GDALWarp.get_band_count(token, datasetType.value, numberOfAttempts, count) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataException( + s"A bandCount of <= 0 was found. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + count(0) + } + + def crs: CRS = crs(GDALDataset.WARPED) + + def crs(datasetType: DatasetType): CRS = { + require(acceptableDatasets contains datasetType) + val crs = Array.ofDim[Byte](1 << 16) + + val returnValue = GDALWarp.get_crs_proj4(token, datasetType.value, numberOfAttempts, crs) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedProjectionException( + s"Unable to parse projection as CRS. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + val proj4String: String = new String(crs, "UTF-8").trim + if (proj4String.length > 0) CRS.fromString(proj4String.trim) + else LatLng + } + + def noDataValue: Option[Double] = noDataValue(GDALDataset.WARPED) + + def noDataValue(datasetType: DatasetType): Option[Double] = { + require(acceptableDatasets contains datasetType) + val nodata = Array.ofDim[Double](1) + val success = Array.ofDim[Int](1) + + val returnValue = GDALWarp.get_band_nodata(token, datasetType.value, numberOfAttempts, 1, nodata, success) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataTypeException( + s"Unable to determine NoData value. GDAL Exception Code: $positiveValue", + positiveValue + ) + } + + if (success(0) == 0) None + else Some(nodata(0)) + } + + def dataType: Int = dataType(GDALDataset.WARPED) + + def dataType(datasetType: DatasetType): Int = { + require(acceptableDatasets contains datasetType) + val dataType = Array.ofDim[Int](1) + + val returnValue = GDALWarp.get_band_data_type(token, datasetType.value, numberOfAttempts, 1, dataType) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataTypeException( + s"Unable to determine DataType. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + dataType(0) + } + + def cellSize: CellSize = cellSize(GDALDataset.WARPED) + + def cellSize(datasetType: DatasetType): CellSize = { + require(acceptableDatasets contains datasetType) + val transform = Array.ofDim[Double](6) + GDALWarp.get_transform(token, datasetType.value, numberOfAttempts, transform) + CellSize(transform(1), transform(5)) + } + + def cellType: CellType = cellType(GDALDataset.WARPED) + + def cellType(datasetType: DatasetType): CellType = { + require(acceptableDatasets contains datasetType) + val nd = noDataValue(datasetType) + val dt = GDALDataType.intToGDALDataType(this.dataType(datasetType)) + val mm = { + val minmax = Array.ofDim[Double](2) + val success = Array.ofDim[Int](1) + + val returnValue = GDALWarp.get_band_min_max(token, datasetType.value, numberOfAttempts, 1, true, minmax, success) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new MalformedDataTypeException( + s"Unable to deterime the min/max values in order to calculate CellType. GDAL Error Code: $positiveValue", + positiveValue + ) + } + if (success(0) != 0) Some(minmax(0), minmax(1)) + else None + } + GDALUtils.dataTypeToCellType(datatype = dt, noDataValue = nd, minMaxValues = mm) + } + + def readTile(gb: GridBounds[Int] = rasterExtent.gridBounds, band: Int, datasetType: DatasetType = GDALDataset.WARPED): Tile = { + require(acceptableDatasets contains datasetType) + val GridBounds(xmin, ymin, xmax, ymax) = gb + val srcWindow: Array[Int] = Array(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1) + val dstWindow: Array[Int] = Array(srcWindow(2), srcWindow(3)) + val ct = this.cellType(datasetType) + val dt = this.dataType(datasetType) + val bytes = Array.ofDim[Byte](dstWindow(0) * dstWindow(1) * ct.bytes) + + val returnValue = GDALWarp.get_data(token, datasetType.value, numberOfAttempts, srcWindow, dstWindow, band, dt, bytes) + + if (returnValue <= 0) { + val positiveValue = math.abs(returnValue) + throw new GDALIOException( + s"Unable to read in data. GDAL Error Code: $positiveValue", + positiveValue + ) + } + + ArrayTile.fromBytes(bytes, ct, dstWindow(0), dstWindow(1)) + } + + def readMultibandTile(gb: GridBounds[Int] = rasterExtent.gridBounds, bands: Seq[Int] = 1 to bandCount, datasetType: DatasetType = GDALDataset.WARPED): MultibandTile = + MultibandTile(bands.map { readTile(gb, _, datasetType) }) + + def readMultibandRaster(gb: GridBounds[Int] = rasterExtent.gridBounds, bands: Seq[Int] = 1 to bandCount, datasetType: DatasetType = GDALDataset.WARPED): Raster[MultibandTile] = + Raster(readMultibandTile(gb, bands, datasetType), rasterExtent.rasterExtentFor(gb).extent) +} + +object GDALDataset { + /** ADTs to encode the Dataset source type. */ + sealed trait DatasetType { def value: Int } + /** [[SOURCE]] allows to access the source dataset of the warped dataset. */ + case object SOURCE extends DatasetType { val value: Int = GDALWarp.SOURCE } + /** [[WARPED]] allows to access the warped dataset. */ + case object WARPED extends DatasetType { val value: Int = GDALWarp.WARPED } + + GDALOptionsConfig.setOptions + def apply(uri: String, options: Array[String]): GDALDataset = GDALDataset(GDALWarp.get_token(uri, options)) + def apply(uri: String): GDALDataset = apply(uri, Array()) +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALException.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALException.scala new file mode 100644 index 0000000000..631f8ddc04 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALException.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + + +/** Base class for all Exceptions involving GDAL. */ +class GDALException(message: String, gdalErrorCode: Int) extends Exception(message) + +/** Exception thrown when data could not be read from data source. */ +class GDALIOException(message: String, gdalErrorCode: Int) extends GDALException(message, gdalErrorCode) + +/** Exception thrown when the attributes of a data source are found to be bad. */ +class MalformedDataException(message: String, gdalErrorCode: Int) extends GDALException(message, gdalErrorCode) + +/** Exception thrown when the DataType of a data source is found to be bad. */ +class MalformedDataTypeException(message: String, gdalErrorCode: Int) extends GDALException(message, gdalErrorCode) + +/** Exception thrown when the projection of a data source is found to be bad. */ +class MalformedProjectionException(message: String, gdalErrorCode: Int) extends GDALException(message, gdalErrorCode) diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadata.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadata.scala new file mode 100644 index 0000000000..836c75bba7 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadata.scala @@ -0,0 +1,60 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.{RasterMetadata, SourceName} +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, GridExtent} + +case class GDALMetadata( + name: SourceName, + crs: CRS, + bandCount: Int, + cellType: CellType, + gridExtent: GridExtent[Long], + resolutions: List[GridExtent[Long]], + /** GDAL per domain metadata */ + baseMetadata: Map[GDALMetadataDomain, Map[String, String]] = Map.empty, + /** GDAL per band per domain metadata */ + bandsMetadata: List[Map[GDALMetadataDomain, Map[String, String]]] = Nil +) extends RasterMetadata { + /** Returns the GDAL metadata merged into a single metadata domain. */ + def attributes: Map[String, String] = baseMetadata.flatMap(_._2) + /** Returns the GDAL per band metadata merged into a single metadata domain. */ + def attributesForBand(band: Int): Map[String, String] = bandsMetadata.map(_.flatMap(_._2)).lift(band).getOrElse(Map.empty) +} + +object GDALMetadata { + def apply(rasterMetadata: RasterMetadata, dataset: GDALDataset, domains: List[GDALMetadataDomain]): GDALMetadata = + domains match { + case Nil => + GDALMetadata(rasterMetadata.name, rasterMetadata.crs, rasterMetadata.bandCount, rasterMetadata.cellType, rasterMetadata.gridExtent, rasterMetadata.resolutions) + case _ => + GDALMetadata( + rasterMetadata.name, rasterMetadata.crs, rasterMetadata.bandCount, rasterMetadata.cellType, rasterMetadata.gridExtent, rasterMetadata.resolutions, + dataset.getMetadata(GDALDataset.SOURCE, domains, 0), + (1 until dataset.bandCount).toList.map(dataset.getMetadata(GDALDataset.SOURCE, domains, _)) + ) + } + + def apply(rasterMetadata: RasterMetadata, dataset: GDALDataset): GDALMetadata = + GDALMetadata( + rasterMetadata.name, rasterMetadata.crs, rasterMetadata.bandCount, rasterMetadata.cellType, rasterMetadata.gridExtent, rasterMetadata.resolutions, + dataset.getAllMetadata(GDALDataset.SOURCE, 0), + (1 until dataset.bandCount).toList.map(dataset.getAllMetadata(GDALDataset.SOURCE, _)) + ) +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadataDomain.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadataDomain.scala new file mode 100644 index 0000000000..8dd950b322 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALMetadataDomain.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +/** https://github.com/geosolutions-it/imageio-ext/blob/1.3.2/library/gdalframework/src/main/java/it/geosolutions/imageio/gdalframework/GDALUtilities.java#L68 */ +sealed trait GDALMetadataDomain { + def name: String + override def toString: String = name +} + +object GDALMetadataDomain { + def ALL = List(DefaultDomain, ImageStructureDomain, SubdatasetsDomain) +} + +case object DefaultDomain extends GDALMetadataDomain { + def name = "" +} + +/** https://github.com/OSGeo/gdal/blob/bed760bfc8479348bc263d790730ef7f96b7d332/gdal/doc/source/development/rfc/rfc14_imagestructure.rst **/ +case object ImageStructureDomain extends GDALMetadataDomain { + def name = "IMAGE_STRUCTURE" +} +/** https://github.com/OSGeo/gdal/blob/6417552c7b3ef874f8306f83e798f979eb37b309/gdal/doc/source/drivers/raster/eedai.rst#subdatasets */ +case object SubdatasetsDomain extends GDALMetadataDomain { + def name = "SUBDATASETS" +} + +case class UserDefinedDomain(name: String) extends GDALMetadataDomain diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALPath.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALPath.scala new file mode 100644 index 0000000000..2336539602 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALPath.scala @@ -0,0 +1,169 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.SourcePath + +import cats.syntax.option._ +import io.lemonlabs.uri._ +import io.lemonlabs.uri.encoding.PercentEncoder +import io.lemonlabs.uri.encoding.PercentEncoder.PATH_CHARS_TO_ENCODE +import java.net.MalformedURLException + +/** Represents and formats a path that points to a files to be read by GDAL. + * + * @param value Path to the file. This path can be formatted in the following + * styles: `VSI`, `URI`, or relative path if the file is local. In addition, + * this path can be prefixed with, '''gdal+''' to signify that the target GeoTiff + * is to be read in only by [[GDALRasterSource]]. + * @example "/vsizip//vsicurl/http://localhost:8000/files.zip" + * @example "s3://bucket/prefix/data.tif" + * @example "gdal+file:///tmp/data.tiff" + * @note Under normal usage, GDAL requires that all paths to be read be given in its + * `VSI Format`. Thus, if given another format type, this class will format it + * so that it can be read. + * + * @example "zip+s3://bucket/prefix/zipped-data.zip!data.tif" + */ +case class GDALPath(value: String) extends SourcePath + +object GDALPath { + val PREFIX = "gdal+" + + /* This object conatins the different schemes and filetypes one can pass into GDAL */ + object Schemes { + final val FTP = "ftp" + final val HTTP = "http" + final val HTTPS = "https" + final val TAR = "tar" + final val ZIP = "zip" + final val GZIP = "gzip" + final val GZ = "gz" + final val FILE = "file" + final val S3 = "s3" + final val GS = "gs" + final val WASB = "wasb" + final val WASBS = "wasbs" + final val HDFS = "hdfs" + final val TGZ = "tgz" + final val KMZ = "kmz" + final val ODS = "ods" + final val XLSX = "xlsx" + final val EMPTY = "" + + final val COMPRESSED_FILE_TYPES = Array(TAR, TGZ, ZIP, KMZ, ODS, XLSX, GZIP, GZ, KMZ) + final val URI_PROTOCOL_INCLUDE = Array(FTP, HTTP, HTTPS, HDFS) + final val URI_HOST_EXCLUDE = Array(WASB, WASBS) + + def isCompressed(schemes: String): Boolean = + COMPRESSED_FILE_TYPES.map(toVSIScheme).collect { case es if es.nonEmpty => schemes.contains(es) }.reduce(_ || _) + + def extraCompressionScheme(path: String): Option[String] = + COMPRESSED_FILE_TYPES + .flatMap { ext => if (path.contains(s".$ext")) Some(toVSIScheme(ext)) else None } + .lastOption + + def isVSIFormatted(path: String): Boolean = path.startsWith("/vsi") + + def toVSIScheme(scheme: String): String = scheme match { + case FTP | HTTP | HTTPS => "/vsicurl/" + case S3 => "/vsis3/" + case GS => "/vsigs/" + case WASB | WASBS => "/vsiaz/" + case HDFS => "/vsihdfs/" + case ZIP | KMZ => "/vsizip/" + case GZ | GZIP => "/vsigzip/" + case TAR | TGZ => "/vsitar/" + case _ => "" + } + } + + implicit def toGDALDataPath(path: String): GDALPath = GDALPath.parse(path) + + def parseOption( + path: String, + compressedFileDelimiter: Option[String] = "!".some, + percentEncoder: PercentEncoder = PercentEncoder(PATH_CHARS_TO_ENCODE ++ Set('%', '?', '#')) + ): Option[GDALPath] = { + import Schemes._ + + // Trying to read something locally on Windows matters + // because of how file paths on Windows are formatted. + // Therefore, we need to handle them differently. + val onLocalWindows = System.getProperty("os.name").toLowerCase == "win" + val upath = percentEncoder.encode(path, "UTF-8") + + val vsiPath: Option[String] = + if (isVSIFormatted(path)) path.some + else + UrlWithAuthority + .parseOption(upath) + // try to parse it, otherwise it is a path + .fold((Url().withPath(UrlPath.fromRaw(upath)): Url).some)(_.some) + .flatMap { url => + // authority is an optional thing and required only for Azure + val authority = + url match { + case url: UrlWithAuthority => url.authority.userInfo.user.getOrElse(EMPTY) + case _ => EMPTY + } + + // relative path, scheme and charecters should be percent decoded + val relativeUrl = url.toRelativeUrl.path.toStringRaw + + // it can also be the case that there is no scheme (the Path case) + url.schemeOption.fold(EMPTY.some)(_.some).map { scheme => + val schemesArray = scheme.split("\\+") + val schemes = schemesArray.map(toVSIScheme).mkString + + // reverse slashes are used on windows for zip files paths + val path = + (if (schemes.contains(FILE) && onLocalWindows) compressedFileDelimiter.map(relativeUrl.replace(_, """\\""")) + else compressedFileDelimiter.map(relativeUrl.replace(_, "/"))).getOrElse(relativeUrl) + + // check out the last .${extension}, probably we need auto add it into the vsipath construction + val extraScheme = extraCompressionScheme(path) + + // check out that we won't append a vsi path duplicate or other compression vsipath + val extraSchemeExists = extraScheme.exists { es => schemes.nonEmpty && (schemes.contains(es) || isCompressed(schemes)) } + + val extendedSchemes = extraScheme.fold(schemes) { + case _ if extraSchemeExists => schemes + case str => s"$str$schemes" + } + + // in some cases scheme:// should be added after the vsi path protocol, sometimes not + val webProtocol = schemesArray.collectFirst { case sch if URI_PROTOCOL_INCLUDE.contains(sch) => s"$sch://" }.getOrElse(EMPTY) + + url + .hostOption + .filterNot(_ => URI_HOST_EXCLUDE.map(schemesArray.contains).reduce(_ || _)) // filter the host out, for instance in the Azure case + .fold(s"$extendedSchemes$webProtocol$authority$path")(host => s"$extendedSchemes$webProtocol$authority$host$path") + } + } + + vsiPath.map(GDALPath(_)) + } + + def parse( + path: String, + compressedFileDelimiter: Option[String] = "!".some, + percentEncoder: PercentEncoder = PercentEncoder(PATH_CHARS_TO_ENCODE ++ Set('%', '?', '#')) + ): GDALPath = + parseOption(path, compressedFileDelimiter, percentEncoder) + .getOrElse(throw new MalformedURLException(s"Unable to parse GDALDataPath: $path")) +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSource.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSource.scala new file mode 100644 index 0000000000..f1bc8b5faa --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSource.scala @@ -0,0 +1,182 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.gdal.GDALDataset.DatasetType +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} +import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} +import geotrellis.vector._ + +class GDALRasterSource( + val dataPath: GDALPath, + val options: GDALWarpOptions = GDALWarpOptions.EMPTY, + private[raster] val targetCellType: Option[TargetCellType] = None +) extends RasterSource { + + /** + * All the information received from the JNI side should be cached on the JVM side, + * to minimize JNI calls. Too aggressive JNI functions usage can lead to a significant performance regression + * as the result of the often memory copy. + * + * For the each JNI call the proxy function will send arguments into the C bindings, + * on the C side the result would be computed and sent to the JVM side (memory copy happened two times). + * + * Since it would happen for each call, much safer and faster would be to remember once computed value in a JVM memory + * and interact only with it: it will allow to minimize JNI calls, speed up function calls and will allow to memoize some + * potentially sensitive data. + * + */ + + def name: GDALPath = dataPath + val path: String = dataPath.value + + lazy val datasetType: DatasetType = options.datasetType + + // current dataset + @transient lazy val dataset: GDALDataset = + GDALDataset(path, options.toWarpOptionsList.toArray) + + /** + * Fetches a default metadata from the default domain. + * If there is a need in some custom domain, use the metadataForDomain function. + */ + lazy val metadata: GDALMetadata = GDALMetadata(this, dataset, DefaultDomain :: Nil) + + /** + * Return the "base" metadata, usually it is a zero band metadata, + * a metadata that is valid for the entire source and for the zero band + */ + def attributes: Map[String, String] = metadata.attributes + + /** + * Return a per band metadata + */ + def attributesForBand(band: Int): Map[String, String] = metadata.attributesForBand(band) + + /** + * Fetches a metadata from the specified [[GDALMetadataDomain]] list. + */ + def metadataForDomain(domainList: List[GDALMetadataDomain]): GDALMetadata = GDALMetadata(this, dataset, domainList) + + /** + * Fetches a metadata from all domains. + */ + def metadataForAllDomains: GDALMetadata = GDALMetadata(this, dataset) + + lazy val bandCount: Int = dataset.bandCount + + lazy val crs: CRS = dataset.crs + + // noDataValue from the previous step + lazy val noDataValue: Option[Double] = dataset.noDataValue(GDALDataset.SOURCE) + + lazy val dataType: Int = dataset.dataType + + lazy val cellType: CellType = dstCellType.getOrElse(dataset.cellType) + + lazy val gridExtent: GridExtent[Long] = dataset.rasterExtent(datasetType).toGridType[Long] + + /** Resolutions of available overviews in GDAL Dataset + * + * These resolutions could represent actual overview as seen in source file + * or overviews of VRT that was created as result of resample operations. + */ + lazy val resolutions: List[GridExtent[Long]] = dataset.resolutions(datasetType).map(_.toGridType[Long]) + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + bounds + .toIterator + .flatMap { gb => gridBounds.intersection(gb) } + .map { gb => + val tile = dataset.readMultibandTile(gb.toGridType[Int], bands.map(_ + 1), datasetType) + val extent = this.gridExtent.extentFor(gb) + convertRaster(Raster(tile, extent)) + } + } + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + new GDALRasterSource(dataPath, options.reproject(gridExtent, crs, targetCRS, resampleGrid, method)) + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = + new GDALRasterSource(dataPath, options.resample(gridExtent, resampleGrid)) + + /** Converts the contents of the GDALRasterSource to the [[TargetCellType]]. + * + * Note: + * + * GDAL handles Byte data differently than GeoTrellis. Unlike GeoTrellis, + * GDAL treats all Byte data as Unsigned Bytes. Thus, the output from + * converting to a Signed Byte CellType can result in unexpected results. + * When given values to convert to Byte, GDAL takes the following steps: + * + * 1. Checks to see if the values falls in [0, 255]. + * 2. If the value falls outside of that range, it'll clamp it so that + * it falls within it. For example: -1 would become 0 and 275 would turn + * into 255. + * 3. If the value falls within that range and is a floating point, then + * GDAL will round it up. For example: 122.492 would become 122 and 64.1 + * would become 64. + * + * Thus, it is recommended that one avoids converting to Byte without first + * ensuring that no data will be lost. + * + * Note: + * + * It is not currently possible to convert to the [[BitCellType]] using GDAL. + * @group convert + */ + def convert(targetCellType: TargetCellType): RasterSource = { + /** To avoid incorrect warp cellSize transformation, we need explicitly set target dimensions. */ + new GDALRasterSource(dataPath, options.convert(targetCellType, noDataValue, Some(cols.toInt -> rows.toInt)), Some(targetCellType)) + } + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val bounds = gridExtent.gridBoundsFor(extent.buffer(- cellSize.width / 2, - cellSize.height / 2), clamp = false) + read(bounds, bands) + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val it = readBounds(List(bounds).flatMap(_.intersection(this.gridBounds)), bands) + if (it.hasNext) Some(it.next) else None + } + + override def readExtents(extents: Traversable[Extent]): Iterator[Raster[MultibandTile]] = { + val bounds = extents.map(gridExtent.gridBoundsFor(_, clamp = false)) + readBounds(bounds, 0 until bandCount) + } + + override def toString: String = { + s"GDALRasterSource(${dataPath.value},$options)" + } + + override def equals(other: Any): Boolean = { + other match { + case that: GDALRasterSource => + this.dataPath == that.dataPath && this.options == that.options && this.targetCellType == that.targetCellType + case _ => false + } + } + + override def hashCode(): Int = java.util.Objects.hash(dataPath, options, targetCellType) +} + +object GDALRasterSource { + def apply(dataPath: GDALPath, options: GDALWarpOptions = GDALWarpOptions.EMPTY, targetCellType: Option[TargetCellType] = None): GDALRasterSource = + new GDALRasterSource(dataPath, options, targetCellType) +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSourceProvider.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSourceProvider.scala new file mode 100644 index 0000000000..83b46e5b59 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALRasterSourceProvider.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.RasterSourceProvider +import geotrellis.raster.geotiff.GeoTiffPath + +class GDALRasterSourceProvider extends RasterSourceProvider { + def canProcess(path: String): Boolean = + (!path.startsWith(GeoTiffPath.PREFIX) && !path.startsWith("gt+")) && GDALPath.parseOption(path).nonEmpty && path.nonEmpty + + def rasterSource(path: String): GDALRasterSource = new GDALRasterSource(path) +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALUtils.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALUtils.scala new file mode 100644 index 0000000000..3956b035e3 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALUtils.scala @@ -0,0 +1,107 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster._ +import geotrellis.raster.io.geotiff._ +import geotrellis.raster.resample._ + +object GDALUtils { + def deriveResampleMethodString(method: ResampleMethod): String = + method match { + case NearestNeighbor => "near" + case Bilinear => "bilinear" + case CubicConvolution => "cubic" + case CubicSpline => "cubicspline" + case Lanczos => "lanczos" + case Average => "average" + case Mode => "mode" + case Max => "max" + case Min => "min" + case Median => "med" + case _ => throw new Exception(s"Could not find equivalent GDALResampleMethod for: $method") + } + + def dataTypeToCellType(datatype: GDALDataType, noDataValue: Option[Double] = None, typeSizeInBits: => Option[Int] = None, minMaxValues: => Option[(Double, Double)] = None): CellType = + datatype match { + case TypeByte => + typeSizeInBits match { + case Some(bits) if bits == 1 => BitCellType + case _ => + minMaxValues match { + case Some((mi, ma)) if (mi.toInt >= 0 && mi <= 255) && (ma.toInt >= 0 && ma <= 255) => + noDataValue match { + case Some(nd) if nd.toInt > 0 && nd <= 255 => UByteUserDefinedNoDataCellType(nd.toByte) + case Some(nd) if nd.toInt == 0 => UByteConstantNoDataCellType + case _ => UByteCellType + } + case _ => + noDataValue match { + case Some(nd) if nd.toInt > Byte.MinValue.toInt && nd <= Byte.MaxValue.toInt => ByteUserDefinedNoDataCellType(nd.toByte) + case Some(nd) if nd.toInt == Byte.MinValue.toInt => ByteConstantNoDataCellType + case _ => ByteCellType + } + } + } + case TypeUInt16 => + noDataValue match { + case Some(nd) if nd.toInt > 0 && nd <= 65535 => UShortUserDefinedNoDataCellType(nd.toShort) + case Some(nd) if nd.toInt == 0 => UShortConstantNoDataCellType + case _ => UShortCellType + } + case TypeInt16 => + noDataValue match { + case Some(nd) if nd > Short.MinValue.toDouble && nd <= Short.MaxValue.toDouble => ShortUserDefinedNoDataCellType(nd.toShort) + case Some(nd) if nd == Short.MinValue.toDouble => ShortConstantNoDataCellType + case _ => ShortCellType + } + case TypeUInt32 => + noDataValue match { + case Some(nd) if nd.toLong > 0l && nd.toLong <= 4294967295l => FloatUserDefinedNoDataCellType(nd.toFloat) + case Some(nd) if nd.toLong == 0l => FloatConstantNoDataCellType + case _ => FloatCellType + } + case TypeInt32 => + noDataValue match { + case Some(nd) if nd.toInt > Int.MinValue && nd.toInt <= Int.MaxValue => IntUserDefinedNoDataCellType(nd.toInt) + case Some(nd) if nd.toInt == Int.MinValue => IntConstantNoDataCellType + case _ => IntCellType + } + case TypeFloat32 => + noDataValue match { + case Some(nd) if isData(nd) && Float.MinValue.toDouble <= nd && Float.MaxValue.toDouble >= nd => FloatUserDefinedNoDataCellType(nd.toFloat) + case Some(nd) => FloatConstantNoDataCellType + case _ => FloatCellType + } + case TypeFloat64 => + noDataValue match { + case Some(nd) if isData(nd) => DoubleUserDefinedNoDataCellType(nd) + case Some(nd) => DoubleConstantNoDataCellType + case _ => DoubleCellType + } + case UnknownType => + throw new UnsupportedOperationException(s"Datatype ${datatype} is not supported.") + case TypeCInt16 | TypeCInt32 | TypeCFloat32 | TypeCFloat64 => + throw new UnsupportedOperationException("Complex datatypes are not supported.") + } + + def deriveOverviewStrategyString(strategy: OverviewStrategy): String = strategy match { + case Auto(n) => s"AUTO-$n" + case AutoHigherResolution => "AUTO" + case Base => "NONE" + } +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/GDALWarpOptions.scala b/gdal/src/main/scala/geotrellis/raster/gdal/GDALWarpOptions.scala new file mode 100644 index 0000000000..7412452a2d --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/GDALWarpOptions.scala @@ -0,0 +1,446 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.gdal.GDALDataset.DatasetType +import geotrellis.raster.{ConvertTargetCellType, TargetCellType} +import geotrellis.raster._ +import geotrellis.raster.resample._ +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} +import geotrellis.proj4.CRS +import geotrellis.vector.Extent + +import cats.Monad +import cats.instances.option._ +import cats.syntax.apply._ +import cats.syntax.option._ + +import scala.collection.JavaConverters._ + +/** + * GDALWarpOptions basically should cover https://www.gdal.org/gdalwarp.html + */ +case class GDALWarpOptions( + /** -of, Select the output format. The default is GeoTIFF (GTiff). Use the short format name. */ + outputFormat: Option[String] = Some("VRT"), + /** -r, Resampling method to use, visit https://www.gdal.org/gdalwarp.html for details. */ + resampleMethod: Option[ResampleMethod] = None, + /** -et, error threshold for transformation approximation */ + errorThreshold: Option[Double] = None, + /** -tr, set output file resolution (in target georeferenced units) */ + cellSize: Option[CellSize] = None, + /** -tap, aligns the coordinates of the extent of the output file to the values of the target resolution, + * such that the aligned extent includes the minimum extent. + * + * In terms of GeoTrellis it's very similar to [[GridExtent]].createAlignedGridExtent: + * + * newMinX = floor(envelop.minX / xRes) * xRes + * newMaxX = ceil(envelop.maxX / xRes) * xRes + * newMinY = floor(envelop.minY / yRes) * yRes + * newMaxY = ceil(envelop.maxY / yRes) * yRes + * + * if (xRes == 0) || (yRes == 0) than GDAL calculates it using the extent and the cellSize + * xRes = (maxX - minX) / cellSize.width + * yRes = (maxY - minY) / cellSize.height + * + * If -tap parameter is NOT set, GDAL increases extent by a half of a pixel, to avoid missing points on the border. + * + * The actual code reference: https://github.com/OSGeo/gdal/blob/v2.3.2/gdal/apps/gdal_rasterize_lib.cpp#L402-L461 + * The actual part with the -tap logic: https://github.com/OSGeo/gdal/blob/v2.3.2/gdal/apps/gdal_rasterize_lib.cpp#L455-L461 + * + * The initial PR that introduced that feature in GDAL 1.8.0: https://trac.osgeo.org/gdal/attachment/ticket/3772/gdal_tap.patch + * A discussion thread related to it: https://lists.osgeo.org/pipermail/gdal-dev/2010-October/thread.html#26209 + * + */ + alignTargetPixels: Boolean = true, + dimensions: Option[(Int, Int)] = None, // -ts + /** -s_srs, source spatial reference set */ + sourceCRS: Option[CRS] = None, + /** -t_srs, target spatial reference set */ + targetCRS: Option[CRS] = None, + /** -te, set georeferenced extents of output file to be created (with a CRS specified) */ + te: Option[Extent] = None, + teCRS: Option[CRS] = None, + /** -srcnodata, set nodata masking values for input bands (different values can be supplied for each band) */ + srcNoData: List[String] = Nil, + /** -dstnodata, set nodata masking values for output bands (different values can be supplied for each band) */ + dstNoData: List[String] = Nil, + /** -ovr, To specify which overview level of source files must be used. + * The default choice, AUTO, will select the overview level whose resolution is the closest to the target resolution. + * Specify an integer value (0-based, i.e. 0=1st overview level) to select a particular level. + * Specify AUTO-n where n is an integer greater or equal to 1, to select an overview level below the AUTO one. + * Or specify NONE to force the base resolution to be used (can be useful if overviews have been generated with a low quality resampling method, and the warping is done using a higher quality resampling method). + */ + ovr: Option[OverviewStrategy] = Some(AutoHigherResolution), + /** -to, set a transformer option suitable to pass to [GDALCreateGenImgProjTransformer2()](https://www.gdal.org/gdal__alg_8h.html#a94cd172f78dbc41d6f407d662914f2e3) */ + to: List[(String, String)] = Nil, + /** -novshiftgrid, Disable the use of vertical datum shift grids when one of the source or target SRS has an explicit vertical datum, and the input dataset is a single band dataset. */ + novShiftGrid: Boolean = false, + /** -order n, order of polynomial used for warping (1 to 3). The default is to select a polynomial order based on the number of GCPs.*/ + order: Option[Int] = None, + /** -tps, force use of thin plate spline transformer based on available GCPs. */ + tps: Boolean = false, + /** -rps, force use of RPCs. */ + rps: Boolean = false, + /** -geoloc, force use of Geolocation Arrays. */ + geoloc: Boolean = false, + /** -refine_gcps, refines the GCPs by automatically eliminating outliers. Outliers will be eliminated until minimum_gcps + * are left or when no outliers can be detected. The tolerance is passed to adjust when a GCP will be eliminated. + * Not that GCP refinement only works with polynomial interpolation. The tolerance is in pixel units if no projection is available, + * otherwise it is in SRS units. If minimum_gcps is not provided, the minimum GCPs according to the polynomial model is used. + */ + refineGCPs: Option[(Double, Int)] = None, + /** -wo, set a warp option. The GDALWarpOptions::papszWarpOptions docs show all options. Multiple -wo options may be listed. */ + wo: List[(String, String)] = Nil, + /** -ot, for the output bands to be of the indicated data type. */ + outputType: Option[String] = None, + /** -wt, working pixel data type. The data type of pixels in the source image and destination image buffers. */ + wt: Option[String] = None, + /** -srcalpha, force the last band of a source image to be considered as a source alpha band. */ + srcAlpha: Boolean = false, + /** -nosrcalpha, prevent the alpha band of a source image to be considered as such (it will be warped as a regular band). */ + noSrcAlpha: Boolean = false, + /** -dstalpha, create an output alpha band to identify nodata (unset/transparent) pixels. */ + dstAlpha: Boolean = false, + /** -wm, set the amount of memory (in megabytes) that the warp API is allowed to use for caching. */ + wm: Option[Int] = None, + /** -multi, Use multithreaded warping implementation. Two threads will be used to process chunks of image and perform input/output operation simultaneously. + * Note that computation is not multithreaded itself. To do that, you can use the -wo NUM_THREADS=val/ALL_CPUS option, which can be combined with -multi. + */ + multi: Boolean = false, + /** -q, be quiet. */ + q: Boolean = false, + /** -co, passes a creation option to the output format driver. Multiple -co options may be listed. https://www.gdal.org/formats_list.html */ + co: List[(String, String)] = Nil, + /** -cutline, enable use of a blend cutline from the name OGR support datasource. */ + cutline: Option[String] = None, + /** -cl, select the named layer from the cutline datasource. */ + cl: Option[String] = None, + /** -cwhere, restrict desired cutline features based on attribute query. */ + cwhere: Option[String] = None, + /** -csql, select cutline features using an SQL query instead of from a layer with -cl. */ + csql: Option[String] = None, + /** -cblend, set a blend distance to use to blend over cutlines (in pixels). */ + cblend: Option[String] = None, + /** -crop_to_cutline, crop the extent of the target dataset to the extent of the cutline. */ + cropToCutline: Boolean = false, + /** -overwrite, overwrite the target dataset if it already exists. */ + overwrite: Boolean = false, + /** -nomd, do not copy metadata. Without this option, dataset and band metadata (as well as some band information) + * will be copied from the first source dataset. Items that differ between source datasets will be set to * (see -cvmd option). + */ + nomd: Boolean = false, + /** -cvmd, value to set metadata items that conflict between source datasets (default is "*"). Use "" to remove conflicting items. */ + cvmd: Option[String] = None, + /** -setci, set the color interpretation of the bands of the target dataset from the source dataset. */ + setci: Boolean = false, + /** -oo, dataset open option (format specific). */ + oo: List[(String, String)] = Nil, + /** -doo, output dataset open option (format specific). */ + doo: List[(String, String)] = Nil, + /** -srcfile, the source file name(s). */ + srcFile: List[String] = Nil, + /** -dstfile, the destination file name. */ + dstFile: Option[String] = None +) { + lazy val name: String = toWarpOptionsList.map(_.toLowerCase).mkString("_") + + def toWarpOptionsList: List[String] = { + outputFormat.toList.flatMap { of => List("-of", of) } ::: + resampleMethod.toList.flatMap { method => List("-r", s"${GDALUtils.deriveResampleMethodString(method)}") } ::: + errorThreshold.toList.flatMap { et => List("-et", s"${et}") } ::: + cellSize.toList.flatMap { cz => + // the -tap parameter can only be set if -tr is set as well + val tr = List("-tr", s"${cz.width}", s"${cz.height}") + if (alignTargetPixels) "-tap" +: tr else tr + } ::: dimensions.toList.flatMap { case (c, r) => List("-ts", s"$c", s"$r") } ::: + sourceCRS.toList.flatMap { source => List("-s_srs", source.toProj4String) } ::: + targetCRS.toList.flatMap { target => List("-t_srs", target.toProj4String) } ::: + ovr.toList.flatMap { o => List("-ovr", GDALUtils.deriveOverviewStrategyString(o)) } ::: + te.toList.flatMap { ext => + List("-te", s"${ext.xmin}", s"${ext.ymin}", s"${ext.xmax}", s"${ext.ymax}") ::: + teCRS.orElse(targetCRS).toList.flatMap { tcrs => List("-te_srs", s"${tcrs.toProj4String}") } + } ::: { if(srcNoData.nonEmpty) "-srcnodata" +: srcNoData else Nil } ::: + { if(dstNoData.nonEmpty) "-dstnodata" +: dstNoData else Nil } ::: + { if(to.nonEmpty) { "-to" +: to.map { case (k, v) => s"$k=$v" } } else Nil } ::: + { if(novShiftGrid) List("-novshiftgrid") else Nil } ::: + order.toList.flatMap { n => List("-order", s"$n") } ::: + { if(tps) List("-tps") else Nil } ::: { if(rps) List("-rps") else Nil } ::: + { if(geoloc) List("-geoloc") else Nil } ::: refineGCPs.toList.flatMap { case (tolerance, minimumGCPs) => + List("-refine_gcps", s"$tolerance", s"$minimumGCPs") + } ::: { if(wo.nonEmpty) { "-wo" +: wo.map { case (k, v) => s"$k=$v" } } else Nil } ::: + outputType.toList.flatMap { ot => List("-ot", s"$ot") } ::: wt.toList.flatMap { wt => List("-wt", s"$wt") } ::: + { if(srcAlpha) List("-srcalpha") else Nil } ::: { if(noSrcAlpha) List("-nosrcalpha") else Nil } ::: + { if(dstAlpha) List("-dstalpha") else Nil } ::: wm.toList.flatMap { wm => List("-wm", s"$wm") } ::: + { if(multi) List("-multi") else Nil } ::: { if(q) List("-q") else Nil } ::: + { if(co.nonEmpty) { "-co" +: co.map { case (k, v) => s"$k=$v" } } else Nil } ::: + cutline.toList.flatMap { cutline => List("-cutline", s"$cutline") } ::: + cl.toList.flatMap { cl => List("-cl", s"$cl") } ::: cwhere.toList.flatMap { cw => List("-cwhere", s"$cw") } ::: + csql.toList.flatMap { csql => List("-csql", s"$csql") } ::: cblend.toList.flatMap { cblend => List("-cblend", s"$cblend") } ::: + { if(cropToCutline) List("-crop_to_cutline") else Nil } ::: { if(overwrite) List("-overwrite") else Nil } ::: + { if(nomd) List("-nomd") else Nil } ::: cvmd.toList.flatMap { cvmd => List("-cvmd", s"$cvmd") } ::: + { if(setci) List("-setci") else Nil } ::: { if(oo.nonEmpty) { "-oo" +: oo.map { case (k, v) => s"$k=$v" } } else Nil } ::: + { if(doo.nonEmpty) { "-doo" +: doo.map { case (k, v) => s"$k=$v" } } else Nil } ::: + { if(srcFile.nonEmpty) { "-srcfile" +: srcFile } else Nil } ::: dstFile.toList.flatMap { df => List("-dstfile", s"$df") } + } + + def combine(that: GDALWarpOptions): GDALWarpOptions = { + if (that == this) this + else this.copy( + outputFormat orElse that.outputFormat, + resampleMethod orElse that.resampleMethod, + errorThreshold orElse that.errorThreshold, + cellSize orElse that.cellSize, + alignTargetPixels, + dimensions orElse that.dimensions, + sourceCRS orElse that.sourceCRS, + targetCRS orElse that.targetCRS, + te orElse that.te, + teCRS orElse that.teCRS, + { if(srcNoData.isEmpty) that.srcNoData else srcNoData }, + { if(dstNoData.isEmpty) that.dstNoData else dstNoData }, + ovr orElse that.ovr, + { if (to.isEmpty) that.to else to }, + novShiftGrid, + order orElse that.order, + tps, + rps, + geoloc, + refineGCPs orElse that.refineGCPs, + { if(wo.isEmpty) that.wo else wo }, + outputType orElse that.outputType, + wt orElse that.wt, + srcAlpha, + noSrcAlpha, + dstAlpha, + wm orElse that.wm, + multi, + q, + { if(co.isEmpty) that.co else co }, + cutline orElse that.cutline, + cl orElse that.cl, + cwhere orElse that.cwhere, + csql orElse that.csql, + cblend orElse that.cblend, + cropToCutline, + overwrite, + nomd, + cvmd orElse that.cvmd, + setci, + { if(oo.isEmpty) that.oo else oo }, + { if(doo.isEmpty) that.doo else doo }, + { if(srcFile.isEmpty) that.srcFile else srcFile }, + dstFile orElse dstFile + ) + } + + def isEmpty: Boolean = this == GDALWarpOptions.EMPTY + def datasetType: DatasetType = if(isEmpty) GDALDataset.SOURCE else GDALDataset.WARPED + + /** Adjust GDAL options to represents reprojection with following parameters. + * This call matches semantics and arguments of {@see RasterSource#reproject} + */ + def reproject(rasterExtent: GridExtent[Long], sourceCRS: CRS, targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, resampleMethod: ResampleMethod = NearestNeighbor): GDALWarpOptions = { + val reprojectOptions = ResampleGrid.toReprojectOptions[Long](rasterExtent, resampleGrid, resampleMethod) + val re = rasterExtent.reproject(sourceCRS, targetCRS, reprojectOptions) + + this.copy( + cellSize = re.cellSize.some, + targetCRS = targetCRS.some, + sourceCRS = sourceCRS.some, + resampleMethod = reprojectOptions.method.some + ) + } + + /** Adjust GDAL options to represents resampling with following parameters . + * This call matches semantics and arguments of {@see RasterSource#resample} + */ + def resample(gridExtent: => GridExtent[Long], resampleGrid: ResampleGrid[Long]): GDALWarpOptions = { + resampleGrid match { + case Dimensions(cols, rows) => + this.copy(te = gridExtent.extent.some, cellSize = None, dimensions = (cols.toInt, rows.toInt).some) + + case _ => + val re = { + val targetRasterExtent = resampleGrid(gridExtent).toRasterExtent + if(this.alignTargetPixels) targetRasterExtent.alignTargetPixels else targetRasterExtent + } + + this.copy(te = re.extent.some, cellSize = re.cellSize.some) + } + } + /** Adjust GDAL options to represents conversion to desired cell type. + * This call matches semantics and arguments of {@see RasterSource#convert} + */ + def convert(targetCellType: TargetCellType, noDataValue: Option[Double], dimensions: Option[(Int, Int)]): GDALWarpOptions = { + val convertOptions = + GDALWarpOptions + .createConvertOptions(targetCellType, noDataValue) + .map(_.copy(dimensions = this.cellSize.fold(dimensions)(_ => None))) + .toList + + (convertOptions :+ this).reduce(_ combine _) + } + + override def toString: String = s"GDALWarpOptions(${toWarpOptionsList.mkString(" ")})" +} + +object GDALWarpOptions { + val EMPTY = GDALWarpOptions() + + implicit def lift2Monad[F[_]: Monad](options: GDALWarpOptions): F[GDALWarpOptions] = Monad[F].pure(options) + + def createConvertOptions(targetCellType: TargetCellType, noDataValue: Option[Double]): Option[GDALWarpOptions] = targetCellType match { + case ConvertTargetCellType(target) => + target match { + case BitCellType => throw new Exception("Cannot convert GDALRasterSource to the BitCellType") + case ByteConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List(Byte.MinValue.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case ByteCellType => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List("None"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case ByteUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case UByteConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List(0.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case UByteCellType => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List("none"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case UByteUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Byte"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case ShortConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("Int16"), + dstNoData = List(Short.MinValue.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case ShortCellType => + GDALWarpOptions( + outputType = Some("Int16"), + dstNoData = List("None"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case ShortUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Int16"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case UShortConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("UInt16"), + dstNoData = List(0.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case UShortCellType => + GDALWarpOptions( + outputType = Some("UInt16"), + dstNoData = List("None"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case UShortUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("UInt16"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case IntConstantNoDataCellType => + Option(GDALWarpOptions( + outputType = Some("Int32"), + dstNoData = List(Int.MinValue.toString), + srcNoData = noDataValue.map(_.toString).toList + )) + case IntCellType => + GDALWarpOptions( + outputType = Some("Int32"), + dstNoData = List("None"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case IntUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Int32"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case FloatConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("Float32"), + dstNoData = List(Float.NaN.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case FloatCellType => + GDALWarpOptions( + outputType = Some("Float32"), + dstNoData = List("NaN"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case FloatUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Float32"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + + case DoubleConstantNoDataCellType => + GDALWarpOptions( + outputType = Some("Float64"), + dstNoData = List(Double.NaN.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + case DoubleCellType => + GDALWarpOptions( + outputType = Some("Float64"), + dstNoData = List("NaN"), + srcNoData = noDataValue.map(_.toString).toList + ).some + case DoubleUserDefinedNoDataCellType(value) => + GDALWarpOptions( + outputType = Some("Float64"), + dstNoData = List(value.toString), + srcNoData = noDataValue.map(_.toString).toList + ).some + } + case _ => None + } +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/config/GDALOptionsConfig.scala b/gdal/src/main/scala/geotrellis/raster/gdal/config/GDALOptionsConfig.scala new file mode 100644 index 0000000000..e71797f127 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/config/GDALOptionsConfig.scala @@ -0,0 +1,56 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal.config + +import com.azavea.gdal.GDALWarp +import geotrellis.raster.gdal.GDALDataset +import geotrellis.raster.gdal.GDALDataset.DatasetType +import pureconfig.generic.auto._ + +import scala.collection.concurrent.TrieMap + +case class GDALOptionsConfig(options: Map[String, String] = Map.empty, acceptableDatasets: List[String] = List("SOURCE", "WARPED"), numberOfAttempts: Int = 1 << 20) { + def set: Unit = { + // register first config options from the conf file + options.foreach { case (key, value) => GDALWarp.set_config_option(key, value) } + // register programmatically set options + GDALOptionsConfig.setRegistryOptions + } + + def getAcceptableDatasets: Set[DatasetType] = { + val res = acceptableDatasets.collect { + case "SOURCE" => GDALDataset.SOURCE + case "WARPED" => GDALDataset.WARPED + } + + if(res.nonEmpty) res.toSet else Set(GDALDataset.SOURCE, GDALDataset.WARPED) + } + + def getNumberOfAttempts: Int = if(numberOfAttempts > 0) numberOfAttempts else 1 << 20 +} + +object GDALOptionsConfig extends Serializable { + private val optionsRegistry = TrieMap[String, String]() + + def registerOption(key: String, value: String): Unit = optionsRegistry += (key -> value) + def registerOptions(seq: (String, String)*): Unit = seq.foreach(optionsRegistry += _) + def setRegistryOptions: Unit = optionsRegistry.foreach { case (key, value) => GDALWarp.set_config_option(key, value) } + def setOptions: Unit = { conf.set; setRegistryOptions } + + lazy val conf: GDALOptionsConfig = pureconfig.loadConfigOrThrow[GDALOptionsConfig]("geotrellis.raster.gdal") + implicit def gdalOptionsConfig(obj: GDALOptionsConfig.type): GDALOptionsConfig = conf +} diff --git a/gdal/src/main/scala/geotrellis/raster/gdal/package.scala b/gdal/src/main/scala/geotrellis/raster/gdal/package.scala new file mode 100644 index 0000000000..a5c49b53e4 --- /dev/null +++ b/gdal/src/main/scala/geotrellis/raster/gdal/package.scala @@ -0,0 +1,25 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.gdal.config.GDALOptionsConfig +import geotrellis.raster.gdal.GDALDataset.DatasetType + +package object gdal { + val acceptableDatasets: Set[DatasetType] = GDALOptionsConfig.getAcceptableDatasets + val numberOfAttempts: Int = GDALOptionsConfig.getNumberOfAttempts +} diff --git a/gdal/src/test/resources/application.conf b/gdal/src/test/resources/application.conf new file mode 100644 index 0000000000..e9a9455a3e --- /dev/null +++ b/gdal/src/test/resources/application.conf @@ -0,0 +1,4 @@ +geotrellis.raster.gdal.options { + GDAL_DISABLE_READDIR_ON_OPEN = "YES" + CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif" +} diff --git a/gdal/src/test/resources/log4j.properties b/gdal/src/test/resources/log4j.properties new file mode 100644 index 0000000000..b07e8b7eff --- /dev/null +++ b/gdal/src/test/resources/log4j.properties @@ -0,0 +1,18 @@ +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.out +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss} %c{1}: %m%n + +log4j.logger.geotrellis.spark=INFO + +# Settings to quiet third party logs that are too verbose +log4j.logger.org.eclipse.jetty=WARN +log4j.logger.org.apache.spark=WARN +log4j.logger.org.apache.hadoop=WARN +log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=WARN +log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=WARN + +log4j.logger.org.spark-project.jetty=WARN +org.spark-project.jetty.LEVEL=WARN diff --git a/gdal/src/test/resources/vlm/all-ones.tif b/gdal/src/test/resources/vlm/all-ones.tif new file mode 100644 index 0000000000..e83db55256 Binary files /dev/null and b/gdal/src/test/resources/vlm/all-ones.tif differ diff --git a/gdal/src/test/resources/vlm/aspect-tiled-bilinear-linux.tif b/gdal/src/test/resources/vlm/aspect-tiled-bilinear-linux.tif new file mode 100644 index 0000000000..864d1538dd Binary files /dev/null and b/gdal/src/test/resources/vlm/aspect-tiled-bilinear-linux.tif differ diff --git a/gdal/src/test/resources/vlm/aspect-tiled-bilinear.tif b/gdal/src/test/resources/vlm/aspect-tiled-bilinear.tif new file mode 100644 index 0000000000..b257b2bdc5 Binary files /dev/null and b/gdal/src/test/resources/vlm/aspect-tiled-bilinear.tif differ diff --git a/gdal/src/test/resources/vlm/aspect-tiled-bilinear.vrt b/gdal/src/test/resources/vlm/aspect-tiled-bilinear.vrt new file mode 100644 index 0000000000..5db1b8cd63 --- /dev/null +++ b/gdal/src/test/resources/vlm/aspect-tiled-bilinear.vrt @@ -0,0 +1,48 @@ + + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] + -7.8774664346142089e+01, 1.1205739005262146e-04, 0.0000000000000000e+00, 3.5809619852165454e+01, 0.0000000000000000e+00, -1.1205739005261982e-04 + + Area + + + -9999 + Gray + + 512 + 128 + + 6.71089e+07 + Bilinear + Float32 + + + aspect-tiled.tif + + + 0.125 + + + 630000,10,0,228500,0,-10 + -63000,0.100000000000000006,0,22850,0,-0.100000000000000006 + -78.7746643461420888,0.000112057390052621456,0,35.8096198521654543,0,-0.000112057390052619816 + 702985,8923.99867184490176,0,319565,0,-8923.99867184503273 + + + PROJCS["unnamed",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",36.16666666666666],PARAMETER["standard_parallel_2",34.33333333333334],PARAMETER["latitude_of_origin",33.75],PARAMETER["central_meridian",-79],PARAMETER["false_easting",609601.22],PARAMETER["false_northing",0],UNIT["Meter",1]] + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] + + + + + + + + + -9999 + 0 + -9999 + 0 + + + + diff --git a/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.tif b/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.tif new file mode 100644 index 0000000000..b92b929668 Binary files /dev/null and b/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.tif differ diff --git a/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.vrt b/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.vrt new file mode 100644 index 0000000000..a3fef38e94 --- /dev/null +++ b/gdal/src/test/resources/vlm/aspect-tiled-near-merc-rdd.vrt @@ -0,0 +1,48 @@ + + PROJCS["unnamed",GEOGCS["unnamed ellipse",DATUM["unknown",SPHEROID["unnamed",6378137,0],EXTENSION["PROJ4_GRIDS","@null"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Mercator_2SP"],PARAMETER["standard_parallel_1",0],PARAMETER["central_meridian",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1],EXTENSION["PROJ4","+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs"]] + -8.7691616329885609e+06, 1.9109257071294063e+01, 0.0000000000000000e+00, 4.2744732772494843e+06, 0.0000000000000000e+00, -1.9109257071294063e+01 + + Area + + + -9999 + Gray + + 512 + 128 + + 6.71089e+07 + NearestNeighbour + Float32 + + + aspect-tiled.tif + + + 0.125 + + + 630000,10,0,228500,0,-10 + -63000,0.100000000000000006,0,22850,0,-0.100000000000000006 + -8769161.63298856094,19.1092570712940635,0,4274473.27724948432,0,-19.1092570712940635 + 458896,0.0523306581867172896,0,223686.000000000029,0,-0.0523306581867172896 + + + PROJCS["unnamed",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",36.16666666666666],PARAMETER["standard_parallel_2",34.33333333333334],PARAMETER["latitude_of_origin",33.75],PARAMETER["central_meridian",-79],PARAMETER["false_easting",609601.22],PARAMETER["false_northing",0],UNIT["Meter",1]] + PROJCS["unnamed",GEOGCS["unnamed ellipse",DATUM["unknown",SPHEROID["unnamed",6378137,0],EXTENSION["PROJ4_GRIDS","@null"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Mercator_2SP"],PARAMETER["standard_parallel_1",0],PARAMETER["central_meridian",0],PARAMETER["false_easting",0],PARAMETER["false_northing",0],UNIT["Meter",1],EXTENSION["PROJ4","+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs"]] + + + + + + + + + -9999 + 0 + -9999 + 0 + + + + diff --git a/gdal/src/test/resources/vlm/aspect-tiled-near.tif b/gdal/src/test/resources/vlm/aspect-tiled-near.tif new file mode 100644 index 0000000000..7ec6b256fa Binary files /dev/null and b/gdal/src/test/resources/vlm/aspect-tiled-near.tif differ diff --git a/gdal/src/test/resources/vlm/aspect-tiled-near.vrt b/gdal/src/test/resources/vlm/aspect-tiled-near.vrt new file mode 100644 index 0000000000..191813f738 --- /dev/null +++ b/gdal/src/test/resources/vlm/aspect-tiled-near.vrt @@ -0,0 +1,48 @@ + + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] + -7.8774664346142089e+01, 1.1205739005262146e-04, 0.0000000000000000e+00, 3.5809619852165454e+01, 0.0000000000000000e+00, -1.1205739005261982e-04 + + Area + + + -9999 + Gray + + 512 + 128 + + 6.71089e+07 + NearestNeighbour + Float32 + + + aspect-tiled.tif + + + 0.125 + + + 630000,10,0,228500,0,-10 + -63000,0.100000000000000006,0,22850,0,-0.100000000000000006 + -78.7746643461420888,0.000112057390052621456,0,35.8096198521654543,0,-0.000112057390052619816 + 702985,8923.99867184490176,0,319565,0,-8923.99867184503273 + + + PROJCS["unnamed",GEOGCS["NAD83",DATUM["North_American_Datum_1983",SPHEROID["GRS 1980",6378137,298.257222101,AUTHORITY["EPSG","7019"]],TOWGS84[0,0,0,0,0,0,0],AUTHORITY["EPSG","6269"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4269"]],PROJECTION["Lambert_Conformal_Conic_2SP"],PARAMETER["standard_parallel_1",36.16666666666666],PARAMETER["standard_parallel_2",34.33333333333334],PARAMETER["latitude_of_origin",33.75],PARAMETER["central_meridian",-79],PARAMETER["false_easting",609601.22],PARAMETER["false_northing",0],UNIT["Meter",1]] + GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] + + + + + + + + + -9999 + 0 + -9999 + 0 + + + + diff --git a/gdal/src/test/resources/vlm/aspect-tiled.tif b/gdal/src/test/resources/vlm/aspect-tiled.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/gdal/src/test/resources/vlm/aspect-tiled.tif differ diff --git a/gdal/src/test/resources/vlm/badnodata.tif b/gdal/src/test/resources/vlm/badnodata.tif new file mode 100644 index 0000000000..097353e32c Binary files /dev/null and b/gdal/src/test/resources/vlm/badnodata.tif differ diff --git a/gdal/src/test/resources/vlm/byte-tile.tiff b/gdal/src/test/resources/vlm/byte-tile.tiff new file mode 100644 index 0000000000..eaac4b2243 Binary files /dev/null and b/gdal/src/test/resources/vlm/byte-tile.tiff differ diff --git a/gdal/src/test/resources/vlm/c41078a1.tif b/gdal/src/test/resources/vlm/c41078a1.tif new file mode 100644 index 0000000000..7329bd6107 Binary files /dev/null and b/gdal/src/test/resources/vlm/c41078a1.tif differ diff --git a/gdal/src/test/resources/vlm/extent-bug.tif b/gdal/src/test/resources/vlm/extent-bug.tif new file mode 100644 index 0000000000..0d5a644eaf Binary files /dev/null and b/gdal/src/test/resources/vlm/extent-bug.tif differ diff --git a/gdal/src/test/resources/vlm/jpeg2000-test-files/jpegTiff.tif b/gdal/src/test/resources/vlm/jpeg2000-test-files/jpegTiff.tif new file mode 100644 index 0000000000..7002e48e08 Binary files /dev/null and b/gdal/src/test/resources/vlm/jpeg2000-test-files/jpegTiff.tif differ diff --git a/gdal/src/test/resources/vlm/jpeg2000-test-files/testJpeg2000.jp2 b/gdal/src/test/resources/vlm/jpeg2000-test-files/testJpeg2000.jp2 new file mode 100644 index 0000000000..5e115b0449 Binary files /dev/null and b/gdal/src/test/resources/vlm/jpeg2000-test-files/testJpeg2000.jp2 differ diff --git a/gdal/src/test/resources/vlm/slope.tif b/gdal/src/test/resources/vlm/slope.tif new file mode 100644 index 0000000000..22be57b60b Binary files /dev/null and b/gdal/src/test/resources/vlm/slope.tif differ diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALConvertedRasterSourceSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALConvertedRasterSourceSpec.scala new file mode 100644 index 0000000000..a7604560ff --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALConvertedRasterSourceSpec.scala @@ -0,0 +1,223 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.testkit._ + +import org.scalatest._ + +class GDALConvertedRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + + val url = Resource.path("vlm/aspect-tiled.tif") + val uri = s"file://$url" + + val url2 = Resource.path("vlm/byte-tile.tiff") + val uri2 = s"file://$url2" + + lazy val source: GDALRasterSource = GDALRasterSource(url) + lazy val byteSource: GDALRasterSource = GDALRasterSource(url2) + + lazy val expectedRaster: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(url, streaming = false) + .raster + + lazy val expectedRaster2: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(url2, streaming = false) + .raster + + lazy val targetExtent = expectedRaster.extent + lazy val targetExtent2 = expectedRaster2.extent + + /** Note: + * Many of these tests have a threshold of 1.0. The reason for this large value + * is due to the difference in how GeoTrellis and GDAL convert floating point + * to non-floating point values. In GeoTrellis, all floaint point values are + * rounded down. Thus, 17.1 -> 17.0, 256.981 -> 256, etc. Whereas GDAL always + * rounds the values up. Therefore, 17.1 -> 18, 256.981 -> 257, etc. This means + * that we need to account for this when comparing certain conversion results. + */ + + describe("Converting to a different CellType") { + + describe("Byte CellType") { + it("should convert to: ByteConstantNoDataCellType") { + val actual = byteSource.convert(ByteConstantNoDataCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(ByteConstantNoDataCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: ByteUserDefinedNoDataCellType(-10)") { + val actual = byteSource.convert(ByteUserDefinedNoDataCellType(-10)).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(ByteUserDefinedNoDataCellType(-10)) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: ByteCellType") { + val actual = byteSource.convert(ByteCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(ByteCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + } + + describe("UByte CellType") { + it("should convert to: UByteConstantNoDataCellType") { + + val actual = byteSource.convert(UByteConstantNoDataCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UByteConstantNoDataCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: UByteUserDefinedNoDataCellType(10)") { + val actual = byteSource.convert(UByteUserDefinedNoDataCellType(10)).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UByteUserDefinedNoDataCellType(10)) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: UByteCellType") { + val actual = byteSource.convert(UByteCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UByteCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + } + + describe("Short CellType") { + it("should convert to: ShortConstantNoDataCellType") { + val actual = source.convert(ShortConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortConstantNoDataCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: ShortUserDefinedNoDataCellType(-1)") { + val actual = source.convert(ShortUserDefinedNoDataCellType(-1)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortUserDefinedNoDataCellType(-1)) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: ShortCellType") { + val actual = source.convert(ShortCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + } + + describe("UShort CellType") { + it("should convert to: UShortConstantNoDataCellType") { + val actual = byteSource.convert(UShortConstantNoDataCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UShortConstantNoDataCellType) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: UShortUserDefinedNoDataCellType(-1)") { + val actual = byteSource.convert(UShortUserDefinedNoDataCellType(-1)).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UShortUserDefinedNoDataCellType(-1)) } + + assertRastersEqual(actual, expected, 1.0) + } + + it("should convert to: UShortCellType") { + val actual = byteSource.convert(UShortCellType).read(targetExtent2).get + val expected = byteSource.read(targetExtent2).get.mapTile { _.convert(UShortCellType) } + + assertRastersEqual(actual, expected) + } + } + + describe("Int CellType") { + it("should convert to: IntConstantNoDataCellType") { + val actual = source.convert(IntConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntConstantNoDataCellType) } + + assertRastersEqual(actual, expected, 1) + } + + it("should convert to: IntUserDefinedNoDataCellType(-100)") { + val actual = source.convert(IntUserDefinedNoDataCellType(-100)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntUserDefinedNoDataCellType(-100)) } + + assertRastersEqual(actual, expected, 1) + } + + it("should convert to: IntCellType") { + val actual = source.convert(IntCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntCellType) } + + assertRastersEqual(actual, expected, 1) + } + } + + describe("Float CellType") { + it("should convert to: FloatConstantNoDataCellType") { + val actual = source.convert(FloatConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatConstantNoDataCellType) } + + assertRastersEqual(actual, expected) + } + + it("should convert to: FloatUserDefinedNoDataCellType(0)") { + val actual = source.convert(FloatUserDefinedNoDataCellType(0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatUserDefinedNoDataCellType(0)) } + + assertRastersEqual(actual, expected) + } + + it("should convert to: FloatCellType") { + val actual = source.convert(FloatCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatCellType) } + + assertRastersEqual(actual, expected) + } + } + + describe("Double CellType") { + it("should convert to: DoubleConstantNoDataCellType") { + val actual = source.convert(DoubleConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleConstantNoDataCellType) } + + assertRastersEqual(actual, expected) + } + + it("should convert to: DoubleUserDefinedNoDataCellType(1.0)") { + val actual = source.convert(DoubleUserDefinedNoDataCellType(1.0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleUserDefinedNoDataCellType(1.0)) } + + assertRastersEqual(actual, expected) + } + + it("should convert to: DoubleCellType") { + val actual = source.convert(DoubleCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleCellType) } + + assertRastersEqual(actual, expected) + } + } + } +} + diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALDataPathSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALDataPathSpec.scala new file mode 100644 index 0000000000..a0e988db1f --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALDataPathSpec.scala @@ -0,0 +1,382 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import org.scalatest._ + +class GDALDataPathSpec extends FunSpec with Matchers { + val fileName = "file-1.tiff" + + describe("Formatting the given uris") { + describe("http") { + it("http url") { + val filePath = "www.radomdata.com/test-files/file-1.tiff" + val url = s"http://$filePath" + val expectedPath = s"/vsicurl/$url" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("http that points to gzip url") { + val filePath = "www.radomdata.com/test-files/data.gzip" + val url = s"http://$filePath" + val expectedPath = s"/vsigzip//vsicurl/$url" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("http that points to gzip with ! url") { + val filePath = "www.radomdata.com/test-files/data.gzip" + val url = s"http://$filePath!$fileName" + val expectedPath = s"/vsigzip//vsicurl/http://$filePath/$fileName" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("http that points to gz url") { + val filePath = "www.radomdata.com/test-files/data.gz" + val url = s"http://$filePath" + val expectedPath = s"/vsigzip//vsicurl/$url" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("http that points to gz with ! url") { + val filePath = "www.radomdata.com/test-files/data.gz" + val url = s"http://$filePath!$fileName" + val expectedPath = s"/vsigzip//vsicurl/http://$filePath/$fileName" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("zip+http url") { + val filePath = "www.radomdata.com/test-files/data.zip" + val url = s"zip+http://$filePath" + val expectedPath = s"/vsizip//vsicurl/http://$filePath" + + GDALPath.parse(url).value should be (expectedPath) + } + + it("zip+http with ! url") { + val filePath = "www.radomdata.com/test-files/data.zip" + val url = s"zip+http://$filePath!$fileName" + val expectedPath = s"/vsizip//vsicurl/http://$filePath/$fileName" + + GDALPath.parse(url).value should be (expectedPath) + } + } + + describe("file") { + it("file uri") { + val filePath = "/home/jake/Documents/test-files/file-1.tiff" + val uri = s"file://$filePath" + val expectedPath = filePath + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("file that points to zip uri") { + val filePath = "/home/jake/Documents/test-files/files.zip" + val uri = s"file://$filePath" + val expectedPath = s"/vsizip/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("file that points to zip with ! uri") { + val filePath = "/home/jake/Documents/test-files/files.zip" + val uri = s"file://$filePath!$fileName" + val expectedPath = s"/vsizip/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("zip+file uri") { + val path = "/tmp/some/data/data.zip" + val uri = s"zip+file://$path" + val expectedPath = "/vsizip//tmp/some/data/data.zip" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("zip+file with ! uri") { + val path = "/tmp/some/data/data.zip" + val uri = s"zip+file://$path!$fileName" + val expectedPath = s"/vsizip/$path/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("force zip+file with ! uri") { + val path = "/tmp/some/data/data.gz" + val uri = s"zip+file://$path!$fileName" + val expectedPath = s"/vsizip/$path/$fileName" + + println(s"GDALDataPath.parse(uri): ${GDALPath.parse(uri)}") + println(s"expectedPath: ${expectedPath}") + + GDALPath.parse(uri).value should be (expectedPath) + } + } + + describe("s3") { + it("s3 uri") { + val filePath = "test-files/nlcd/data/tiff-0.tiff" + val uri = s"s3://$filePath" + val expectedPath = s"/vsis3/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("s3 that points to gzip uri") { + val filePath = "test-files/nlcd/data/data.gzip" + val uri = s"s3://$filePath" + val expectedPath = s"/vsigzip//vsis3/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("s3 that points to gzip with uri") { + val filePath = "test-files/nlcd/data/data.gzip" + val uri = s"s3://$filePath!$fileName" + val expectedPath = s"/vsigzip//vsis3/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("gzip+s3 uri") { + val path = "some/bucket/data/data.gzip" + val uri = s"gzip+s3://$path" + val expectedPath = s"/vsigzip//vsis3/$path" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("gzip+s3 uri with !") { + val path = "some/bucket/data/data.gzip" + val uri = s"gzip+s3://$path!$fileName" + val expectedPath = s"/vsigzip//vsis3/$path/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("s3 that points to gz uri") { + val filePath = "test-files/nlcd/data/data.gz" + val uri = s"s3://$filePath" + val expectedPath = s"/vsigzip//vsis3/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("s3 that points to gz with uri") { + val filePath = "test-files/nlcd/data/data.gz" + val uri = s"s3://$filePath!$fileName" + val expectedPath = s"/vsigzip//vsis3/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("gzip+s3 uri for a .gz ext") { + val path = "some/bucket/data/data.gz" + val uri = s"gzip+s3://$path" + val expectedPath = s"/vsigzip//vsis3/$path" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("gzip+s3 uri with ! for a .gz ext") { + val path = "some/bucket/data/data.gz" + val uri = s"gzip+s3://$path!$fileName" + val expectedPath = s"/vsigzip//vsis3/$path/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + } + + describe("hdfs") { + it("hdfs uri") { + val filePath = "test-files/nlcd/data/tiff-0.tiff" + val uri = s"hdfs://$filePath" + val expectedPath = s"/vsihdfs/$uri" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("hdfs that points to tgz uri") { + val filePath = "test-files/nlcd/data/my_data.tgz" + val uri = s"hdfs://$filePath" + val expectedPath = s"/vsitar//vsihdfs/$uri" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("hdfs that points to tgz with ! uri") { + val filePath = "test-files/nlcd/data/my_data.tgz" + val uri = s"hdfs://$filePath!$fileName" + val expectedPath = s"/vsitar//vsihdfs/hdfs://$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("zip+hdfs uri") { + val filePath = "hdfs://test-files/nlcd/data/data.zip" + val uri = s"zip+$filePath" + val expectedPath = s"/vsizip//vsihdfs/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("zip+hdfs with ! uri") { + val filePath = "hdfs://test-files/nlcd/data/data.zip" + val uri = s"zip+$filePath!$fileName" + val expectedPath = s"/vsizip//vsihdfs/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + } + + describe("Google Cloud Storage") { + it("Google Cloud Storage uri") { + val filePath = "test-files/nlcd/data/tiff-0.tiff" + val uri = s"gs://$filePath" + val expectedPath = s"/vsigs/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("Google Cloud Storage that points to tar uri") { + val filePath = "test-files/nlcd/data/data.tar" + val uri = s"gs://$filePath" + val expectedPath = s"/vsitar//vsigs/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("Google Cloud Storage that points to tar with ! uri") { + val filePath = "test-files/nlcd/data/data.tar" + val uri = s"gs://$filePath!$fileName" + val expectedPath = s"/vsitar//vsigs/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("tar+gs uri") { + val filePath = "test-files/nlcd/data/data.tar" + val uri = s"tar+gs://$filePath" + val expectedPath = s"/vsitar//vsigs/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("tar+gs with ! uri") { + val filePath = "test-files/nlcd/data/data.tar" + val uri = s"tar+gs://$filePath!$fileName" + val expectedPath = s"/vsitar//vsigs/$filePath/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + } + + describe("Azure") { + it("Azure uri") { + val uri = "wasb://test-files@myaccount.blah.core.net/nlcd/data/tiff-0.tiff" + val expectedPath = "/vsiaz/test-files/nlcd/data/tiff-0.tiff" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("Azure that points to kmz uri") { + val uri = "wasb://test-files@myaccount.blah.core.net/nlcd/data/info.kmz" + val expectedPath = "/vsizip//vsiaz/test-files/nlcd/data/info.kmz" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("Azure that points to kmz with ! uri") { + val uri = s"wasb://test-files@myaccount.blah.core.net/nlcd/data/info.kmz!$fileName" + val expectedPath = s"/vsizip//vsiaz/test-files/nlcd/data/info.kmz/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("wasb+zip uri") { + val uri = "zip+wasb://test-files@myaccount.blah.core.net/nlcd/data/info.zip" + val expectedPath = "/vsizip//vsiaz/test-files/nlcd/data/info.zip" + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("wasb+zip with ! uri") { + val path = "zip+wasb://test-files@myaccount.blah.core.net/nlcd/data/info.zip" + val uri = s"$path!$fileName" + val expectedPath = s"/vsizip//vsiaz/test-files/nlcd/data/info.zip/$fileName" + + GDALPath.parse(uri).value should be (expectedPath) + } + } + + describe("relative path") { + it("relative path uri") { + val filePath = "../../test-files/file-1.tiff" + val uri = filePath + val expectedPath = filePath + + GDALPath.parse(uri).value should be (expectedPath) + } + + it("relative path that points to zip uri") { + val filePath = "../../test-files/data.zip" + val uri = filePath + val expectedPath = s"/vsizip/$filePath" + + GDALPath.parse(uri).value should be (expectedPath) + } + } + } + + describe("Formatting VSI paths") { + it("should parse a VSI path") { + val filePath = "/vsihdfs/hdfs://data/my-data/data.tif" + + GDALPath.parse(filePath).value should be (filePath) + } + + it("should parse a chained VSI path") { + val filePath = "/vsizip//vsis3/data/my-data/data.zip" + + GDALPath.parse(filePath).value should be (filePath) + } + } + + describe("Formatting the given uris - edge cases") { + it("should parse a path with uncommon characters") { + val filePath = """data/jake__user--data!@#$%^&*()`~{}[]\|=+,?';<>;/files/my-data.tif""" + val uri = s"s3://$filePath" + val expectedPath = s"/vsis3/$filePath" + + GDALPath.parse(uri, None).value should be (expectedPath) + } + + it("should parse a targeted compressed file with a differenct delimiter") { + val filePath = "data/my-data/data!.zip" + val uri = s"zip+s3://$filePath/$fileName" + val expectedPath = s"/vsizip//vsis3/$filePath/$fileName" + + GDALPath.parse(uri, Some("/")).value should be (expectedPath) + } + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALErrorSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALErrorSpec.scala new file mode 100644 index 0000000000..2e5224b7dd --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALErrorSpec.scala @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import com.azavea.gdal.GDALWarp +import geotrellis.raster.testkit.Resource + +import org.scalatest._ + +class GDALErrorSpec extends FunSpec { + val uri = Resource.path("vlm/c41078a1.tif") + val token = GDALWarp.get_token(uri, Array()) + val dataset: GDALDataset.DatasetType = GDALWarpOptions.EMPTY.datasetType + + val sourceWindow: Array[Int] = Array(1000000, 1000000, 5000000, 5000000) + val destWindow: Array[Int] = Array(500, 500) + val buffer = Array.ofDim[Byte](500 * 500) + + describe("GDALErrors") { + it("should return the ObjectNull error code") { + val result = GDALWarp.get_data(token, dataset.value, 1, sourceWindow, destWindow, 42, 1, buffer) + + assert(math.abs(result) == 10) + } + + it("should return the IllegalArg error code") { + val result = GDALWarp.get_data(token, dataset.value, 1, sourceWindow, destWindow, 1, 1, buffer) + + assert(math.abs(result) == 5) + } + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceProviderSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceProviderSpec.scala new file mode 100644 index 0000000000..4fdb259b94 --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceProviderSpec.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.RasterSource + +import org.scalatest._ + +class GDALRasterSourceProviderSpec extends FunSpec { + describe("GDALRasterSourceProvider") { + val provider = new GDALRasterSourceProvider() + + it("should process a VSI path") { + assert(provider.canProcess("/vsicurl/https://path/to/some/file.tiff")) + } + + it("should process a non-prefixed string") { + assert(provider.canProcess("s3://bucket/thing/something/file.jp2")) + } + + it("should process a prefixed string") { + assert(provider.canProcess("zip+s3://bucket/thing/something/more-data.zip")) + } + + it("should not be able to process a GeoTrellis catalog path") { + assert(!provider.canProcess("gt+s3://path/to/my/fav/catalog?layer=fav&zoom=3")) + } + + it("should not be able to process a GeoTiff prefixed path") { + assert(!provider.canProcess("gtiff+file:///tmp/temp-file.tif")) + } + + it("should produce a GDALRasterSource from a string") { + assert(RasterSource("file:///tmp/dumping-ground/random/file.zip").isInstanceOf[GDALRasterSource]) + } + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala new file mode 100644 index 0000000000..597e0b0a1b --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALRasterSourceSpec.scala @@ -0,0 +1,113 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.raster.geotiff.GeoTiffRasterSource +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.resample._ +import geotrellis.raster.testkit._ + +import org.scalatest._ + +class GDALRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + + val uri = Resource.path("vlm/aspect-tiled.tif") + + describe("GDALRasterSource") { + + // we are going to use this source for resampling into weird resolutions, let's check it + // usually we align pixels + lazy val source: GDALRasterSource = GDALRasterSource(uri, GDALWarpOptions(alignTargetPixels = false)) + + it("should be able to read upper left corner") { + val bounds = GridBounds(0, 0, 10, 10).toGridType[Long] + val chip: Raster[MultibandTile] = source.read(bounds).get + chip should have ( + // dimensions (bounds.width, bounds.height), + cellType (source.cellType) + ) + } + + // no resampling is implemented there + it("should be able to resample") { + // read in the whole file and resample the pixels in memory + val expected: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(uri, streaming = false) + .raster + .resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor) + // resample to 0.9 so we RasterSource picks the base layer and not an overview + + val resampledSource = + source.resample(expected.tile.cols, expected.tile.rows, NearestNeighbor) + + // resampledSource should have (dimensions (expected.tile.dimensions)) + + info(s"Source CellSize: ${source.cellSize}") + info(s"Target CellSize: ${resampledSource.cellSize}") + + // calculated expected resolutions of overviews + // it's a rough approximation there as we're not calculating resolutions like GDAL + val ratio = resampledSource.cellSize.resolution / source.cellSize.resolution + resampledSource.resolutions.zip (source.resolutions.map { re => + val CellSize(cw, ch) = re.cellSize + RasterExtent(re.extent, CellSize(cw * ratio, ch * ratio)) + }).map { case (rea, ree) => rea.cellSize.resolution shouldBe ree.cellSize.resolution +- 3e-1 } + + val actual: Raster[MultibandTile] = + resampledSource.read(GridBounds(0, 0, resampledSource.cols - 1, resampledSource.rows - 1)).get + + withGeoTiffClue(actual, expected, resampledSource.crs) { + assertRastersEqual(actual, expected) + } + } + + it("should not read past file edges") { + Given("bounds larger than raster") + val bounds = GridBounds(0, 0, source.cols + 100, source.rows + 100) + When("reading by pixel bounds") + val chip = source.read(bounds).get + val expected = source.read(source.extent) + + Then("return only pixels that exist") + // chip.tile should have (dimensions (source.dimensions)) + + // check also that the tile is valid + withGeoTiffClue(chip, expected.get, source.crs) { + assertRastersEqual(chip, expected.get) + } + } + + it("should derive a consistent extent") { + GDALRasterSource(uri).extent should be (GeoTiffRasterSource(uri).extent) + + val p = Resource.path("vlm/extent-bug.tif") + GDALRasterSource(GDALPath(p)).extent should be (GeoTiffRasterSource(p).extent) + } + + it("should not fail on creation of the GDALRasterSource on a 'malformed URI', since we don't know if it is a path or it is a scheme") { + val result = GDALRasterSource("file:/random/path/here/N49W155.hgt.gz") + result.path shouldBe "/vsigzip/file:/random/path/here/N49W155.hgt.gz" + } + + it("should read the same metadata as GeoTiffRasterSource") { + lazy val tsource = GeoTiffRasterSource(uri) + source.metadata.attributes.mapValues(_.toUpperCase) shouldBe tsource.metadata.attributes.mapValues(_.toUpperCase) + } + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALReprojectRasterSourceSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALReprojectRasterSourceSpec.scala new file mode 100644 index 0000000000..66c50b7ee7 --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALReprojectRasterSourceSpec.scala @@ -0,0 +1,111 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.resample._ +import geotrellis.raster.testkit._ + +import org.scalatest._ + +import java.io.File + +class GDALReprojectRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + + /** + * Pipeline to generate test dataset from the aspect-tiled.tif. + * Example below is for output format VRT, however at the moment we use in memory rasters (-of MEM), + * basically this line can be generated by printing GDALWarpOptions: + * gdalwarp -of VRT -r bilinear -et 0.125 -tap -tr 1.1205739005262034E-4 1.1205739005262034E-4 -s_srs '+proj=lcc +lat_1=36.16666666666666 +lat_2=34.33333333333334 +lat_0=33.75 +lon_0=-79 +x_0=609601.22 +y_0=0 +datum=NAD83 +units=m +no_defs' -t_srs '+proj=longlat +datum=WGS84 +no_defs' + * + * The line above will produce a VRT file (all files are present there at the moment). + * To translate it into tiff use the next command: + * gdal_translate in.vrt out.tif + * + * The output file would be a warped result. + * */ + + describe("Reprojecting a RasterSource") { + + val uri = Resource.path("vlm/aspect-tiled.tif") + + /** + * For some reasons, the Pipeline described above is OS specific, + * and Bilinear interpolation behaves differently. + * To make tests pass there was generated one bilinear version under mac and anther inside a linux container. + * + * TODO: investigate the nature of this bug later + * */ + + val expectedUri = Map[ResampleMethod, String]( + Bilinear -> { + if(System.getProperty("os.name").toLowerCase().startsWith("mac")) + Resource.path("vlm/aspect-tiled-bilinear.tif") + else + Resource.path("vlm/aspect-tiled-bilinear-linux.tif") + }, + NearestNeighbor -> Resource.path("vlm/aspect-tiled-near.tif") + ) + + def testReprojection(method: ResampleMethod) = { + val rasterSource = GDALRasterSource(uri) + val expectedRasterSource = GDALRasterSource(expectedUri(method)) + val expectedRasterExtent = expectedRasterSource.gridExtent.toRasterExtent + val warpRasterSource = rasterSource.reprojectToRegion(LatLng, expectedRasterExtent, method) + val testBounds = GridBounds(0, 0, expectedRasterExtent.cols, expectedRasterExtent.rows).split(64,64).toSeq + + warpRasterSource.resolutions.size shouldBe rasterSource.resolutions.size + + for (bound <- testBounds) yield { + withClue(s"Read window ${bound}: ") { + val targetExtent = expectedRasterExtent.extentFor(bound) + val testRasterExtent = RasterExtent( + extent = targetExtent, + cellwidth = expectedRasterExtent.cellwidth, + cellheight = expectedRasterExtent.cellheight, + cols = bound.width, + rows = bound.height + ) + + // due to a bit different logic used by GDAL working with different output formats + // there can be a difference around +-1e-11 + val expected = Utils.roundRaster(expectedRasterSource.read(testRasterExtent.extent).get) + val actual = Utils.roundRaster(warpRasterSource.read(bound.toGridType[Long]).get) + + actual.extent.covers(expected.extent) should be (true) // -- doesn't work due to a precision issue + actual.rasterExtent.extent.xmin should be (expected.rasterExtent.extent.xmin +- 1e-5) + actual.rasterExtent.extent.ymax should be (expected.rasterExtent.extent.ymax +- 1e-5) + actual.rasterExtent.cellwidth should be (expected.rasterExtent.cellwidth +- 1e-5) + actual.rasterExtent.cellheight should be (expected.rasterExtent.cellheight +- 1e-5) + + withGeoTiffClue(actual, expected, LatLng) { + assertRastersEqual(actual, expected) + } + } + } + } + + it("should reproject using NearestNeighbor") { + testReprojection(NearestNeighbor) + } + + it("should reproject using Bilinear") { + testReprojection(Bilinear) + } + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpOptionsSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpOptionsSpec.scala new file mode 100644 index 0000000000..2d9bcc8db8 --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpOptionsSpec.scala @@ -0,0 +1,191 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.AutoHigherResolution +import geotrellis.raster.resample._ +import geotrellis.vector.Extent +import geotrellis.raster.testkit._ + +import org.gdal.gdal._ + +import scala.collection.JavaConverters._ + +import org.scalatest._ + +class GDALWarpOptionsSpec extends FunSpec with RasterMatchers with GivenWhenThen { + import GDALWarpOptionsSpec._ + + org.gdal.gdal.gdal.AllRegister() + + val filePath = Resource.path("vlm/aspect-tiled.tif") + def filePathByIndex(i: Int): String = Resource.path(s"vlm/aspect-tiled-$i.tif") + + val reprojectOptions: GDALWarpOptions = + generateWarpOptions( + sourceCRS = Some(CRS.fromString("+proj=lcc +lat_1=36.16666666666666 +lat_2=34.33333333333334 +lat_0=33.75 +lon_0=-79 +x_0=609601.22 +y_0=0 +datum=NAD83 +units=m +no_defs ")), + targetCRS = Some(WebMercator), + cellSize = Some(CellSize(10, 10)) + ) + + val resampleOptions: GDALWarpOptions = + generateWarpOptions( + et = None, + cellSize = Some(CellSize(22, 22)), + sourceCRS = None, + targetCRS = None + ) + + def rasterSourceFromUriOptions(uri: String, options: GDALWarpOptions): GDALRasterSource = GDALRasterSource(uri, options) + + def dsreprojectOpt(uri: String): GDALRasterSource = { + val opts = + GDALWarpOptions + .EMPTY + .reproject( + rasterExtent = GridExtent(Extent(630000.0, 215000.0, 645000.0, 228500.0), 10, 10), + CRS.fromString("+proj=lcc +lat_1=36.16666666666666 +lat_2=34.33333333333334 +lat_0=33.75 +lon_0=-79 +x_0=609601.22 +y_0=0 +datum=NAD83 +units=m +no_defs "), + WebMercator, + TargetCellSize[Long](CellSize(10, 10)) + ) + rasterSourceFromUriOptions(uri, opts) + } + + def dsresampleOpt(uri: String): GDALRasterSource = { + val opts = + GDALWarpOptions + .EMPTY + .reproject( + rasterExtent = GridExtent(Extent(630000.0, 215000.0, 645000.0, 228500.0), 10, 10), + CRS.fromString("+proj=lcc +lat_1=36.16666666666666 +lat_2=34.33333333333334 +lat_0=33.75 +lon_0=-79 +x_0=609601.22 +y_0=0 +datum=NAD83 +units=m +no_defs "), + WebMercator, + TargetCellSize[Long](CellSize(10, 10)) + ) + .resample( + GridExtent(Extent(-8769160.0, 4257700.0, -8750630.0, 4274460.0), CellSize(10, 10)), + TargetRegion(GridExtent(Extent(-8769160.0, 4257700.0, -8750630.0, 4274460.0), CellSize(22, 22))) + ) + rasterSourceFromUriOptions(uri, opts) + } + + describe("GDALWarp transformations") { + def datasetToRasterExtent(ds: Dataset): RasterExtent = { + val transform = ds.GetGeoTransform + val width = ds.GetRasterXSize + val height = ds.GetRasterYSize + val x1 = transform(0) + val y1 = transform(3) + val x2 = x1 + transform(1) * width + val y2 = y1 + transform(5) * height + val e = Extent( + math.min(x1, x2), + math.min(y1, y2), + math.max(x1, x2), + math.max(y1, y2) + ) + + RasterExtent(e, math.abs(transform(1)), math.abs(transform(5)), width, height) + } + + it("optimized transformation should behave in a same way as a list of warp applications") { + val base = filePath + + val optimizedReproject = dsreprojectOpt(base) + val optimizedResample = dsresampleOpt(base) + + val reprojectWarpAppOptions = new WarpOptions(new java.util.Vector(reprojectOptions.toWarpOptionsList.asJava)) + val resampleWarpAppOptions = new WarpOptions(new java.util.Vector(resampleOptions.toWarpOptionsList.asJava)) + val underlying = org.gdal.gdal.gdal.Open(filePath, org.gdal.gdalconst.gdalconstConstants.GA_ReadOnly) + val originalReproject = org.gdal.gdal.gdal.Warp("/dev/null", Array(underlying), reprojectWarpAppOptions) + val originalResample = org.gdal.gdal.gdal.Warp("/dev/null", Array(originalReproject), resampleWarpAppOptions) + + datasetToRasterExtent(originalReproject) shouldBe optimizedReproject.gridExtent.toRasterExtent + datasetToRasterExtent(originalResample) shouldBe optimizedResample.gridExtent.toRasterExtent + + // cleanup JNI objects + originalResample.delete() + originalReproject.delete() + underlying.delete() + resampleWarpAppOptions.delete() + reprojectWarpAppOptions.delete() + } + + it("raster sources optimized transformations should behave in a same way as a single warp application") { + val base = filePath + + val optimizedRawResample = dsresampleOpt(base) + + val reprojectWarpAppOptions = new WarpOptions(new java.util.Vector(reprojectOptions.toWarpOptionsList.asJava)) + val resampleWarpAppOptions = new WarpOptions(new java.util.Vector(resampleOptions.toWarpOptionsList.asJava)) + val underlying = org.gdal.gdal.gdal.Open(filePath, org.gdal.gdalconst.gdalconstConstants.GA_ReadOnly) + val reprojected = org.gdal.gdal.gdal.Warp("/dev/null", Array(underlying), reprojectWarpAppOptions) + + val originalRawResample = org.gdal.gdal.gdal.Warp("/dev/null", Array(reprojected), resampleWarpAppOptions) + + val rs = + GDALRasterSource(filePath) + .reproject( + targetCRS = WebMercator, + resampleGrid = TargetCellSize[Long](CellSize(10, 10)), + strategy = AutoHigherResolution + ) + .resampleToRegion( + region = GridExtent(Extent(-8769160.0, 4257700.0, -8750630.0, 4274460.0), CellSize(22, 22)) + ) + + optimizedRawResample.gridExtent shouldBe rs.gridExtent + datasetToRasterExtent(originalRawResample) shouldBe rs.gridExtent.toRasterExtent + + // cleanup JNI objects + originalRawResample.delete() + reprojected.delete() + underlying.delete() + resampleWarpAppOptions.delete() + reprojectWarpAppOptions.delete() + } + } + +} + +object GDALWarpOptionsSpec { + def generateWarpOptions( + et: Option[Double] = Some(0.125), + cellSize: Option[CellSize] = Some(CellSize(19.1, 19.1)), + sourceCRS: Option[CRS] = Some(CRS.fromString("+proj=lcc +lat_1=36.16666666666666 +lat_2=34.33333333333334 +lat_0=33.75 +lon_0=-79 +x_0=609601.22 +y_0=0 +datum=NAD83 +units=m +no_defs ")), + targetCRS: Option[CRS] = Some(WebMercator) + ): GDALWarpOptions = { + GDALWarpOptions( + Some("VRT"), + Some(NearestNeighbor), + et, + cellSize, + true, + None, + sourceCRS, + targetCRS, + None, + None, + List("-9999.0"), + Nil, + Some(AutoHigherResolution), + Nil, false, None, false, false, false, None, Nil, None, None, false, false, + false, None, false, false, Nil, None, None, None, None, None, false, false, false, None, false, Nil, Nil, Nil, None + ) + } +} diff --git a/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpReadTileSpec.scala b/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpReadTileSpec.scala new file mode 100644 index 0000000000..e95cd00b26 --- /dev/null +++ b/gdal/src/test/scala/geotrellis/raster/gdal/GDALWarpReadTileSpec.scala @@ -0,0 +1,138 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.gdal + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.testkit._ +import geotrellis.raster.io.geotiff._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.vector.Extent + +import org.scalatest._ +import java.io.File + +class GDALWarpReadTileSpec extends FunSpec with RasterMatchers { + val path = Resource.path("vlm/slope.tif") + + describe("reading a GeoTiff") { + it("should read full raster correct") { + val filePath = Resource.path("vlm/aspect-tiled.tif") + val dataset = GDALDataset(filePath) + val gdalTile = dataset.readMultibandTile() + val gtTile = GeoTiffReader.readMultiband(filePath).tile.toArrayTile + + gdalTile.cellType shouldBe gtTile.cellType + assertEqual(gdalTile, gtTile) + } + + it("should read a raster with bad nodata value set correct") { + val filePath = Resource.path("vlm/badnodata.tif") + // using a small extent to make tests work faster + val ext = Extent(680138.59203, 4904905.667, 680189.7, 4904955.9) + val dataset = GDALDataset(filePath) + val gdalTile = dataset.readMultibandTile(dataset.rasterExtent.gridBoundsFor(ext, clamp = false)) + val gtTile = GeoTiffReader.readMultiband(filePath, ext).tile.toArrayTile + + gdalTile.cellType shouldBe gtTile.cellType + assertEqual(gdalTile, gtTile) + } + + it("should match one read with GeoTools") { + println("Reading with GDAL...") + val dataset = GDALDataset(path) + val raster = dataset.readMultibandRaster() + val gdRaster = raster.tile.band(0) + val gdExt = raster.extent + println("Reading with GeoTools....") + val Raster(gtRaster, gtExt) = SinglebandGeoTiff(path).raster + println("Done.") + + gdExt.xmin should be(gtExt.xmin +- 0.00001) + gdExt.xmax should be(gtExt.xmax +- 0.00001) + gdExt.ymin should be(gtExt.ymin +- 0.00001) + gdExt.ymax should be(gtExt.ymax +- 0.00001) + + gdRaster.cols should be(gtRaster.cols) + gdRaster.rows should be(gtRaster.rows) + + gdRaster.cellType.toString.take(7) should be(gtRaster.cellType.toString.take(7)) + + println("Comparing rasters...") + for (col <- 0 until gdRaster.cols) { + for (row <- 0 until gdRaster.rows) { + val actual = gdRaster.getDouble(col, row) + val expected = gtRaster.getDouble(col, row) + withClue(s"At ($col, $row): GDAL - $actual GeoTools - $expected") { + isNoData(actual) should be(isNoData(expected)) + if (isData(actual)) actual should be(expected) + } + } + } + } + + it("should do window reads") { + val dataset = GDALDataset(path) + val gtiff = MultibandGeoTiff(path) + val gridBounds = dataset.rasterExtent.gridBounds.split(15, 21) + + gridBounds.foreach { gb => + val actualTile = dataset.readMultibandTile(gb) + val expectedTile = gtiff.tile.crop(gb) + + assertEqual(actualTile, expectedTile) + } + } + + it("should read CRS from file") { + val dataset = GDALDataset(Resource.path("vlm/all-ones.tif"), Array()) + dataset.crs.epsgCode should equal(LatLng.epsgCode) + } + } + + describe("reading a JPEG2000") { + val lengthExpected = 100 + type TypeExpected = UShortCells + val jpeg2000Path = Resource.path("vlm/jpeg2000-test-files/testJpeg2000.jp2") + val jpegTiffPath = Resource.path("vlm/jpeg2000-test-files/jpegTiff.tif") + + val jpegDataset = GDALDataset(jpeg2000Path) + val tiffDataset = GDALDataset(jpegTiffPath) + + val gridBounds: Iterator[GridBounds[Int]] = + jpegDataset.rasterExtent.gridBounds.split(20, 15) + + it("should read a JPEG2000 from a file") { + val raster = Raster(jpegDataset.readMultibandTile(), jpegDataset.rasterExtent.extent) + val tile = raster.tile + val extent = raster.rasterExtent + + extent.cols should be(lengthExpected) + extent.rows should be(lengthExpected) + tile.cellType shouldBe a[TypeExpected] + } + + it("should do window reads") { + gridBounds.foreach { gb => + val actualTile = jpegDataset.readMultibandTile(gb) + val expectedTile = tiffDataset.readMultibandTile(gb) + + assertEqual(actualTile, expectedTile) + } + } + } +} \ No newline at end of file diff --git a/layer/src/main/scala/geotrellis/layer/Boundable.scala b/layer/src/main/scala/geotrellis/layer/Boundable.scala index ab929b4c4c..de2f07efd1 100644 --- a/layer/src/main/scala/geotrellis/layer/Boundable.scala +++ b/layer/src/main/scala/geotrellis/layer/Boundable.scala @@ -16,6 +16,9 @@ package geotrellis.layer +import jp.ne.opt.chronoscala.Imports._ +import java.time.ZonedDateTime + /** This type class marks K as point that can be bounded in space. * It is used to construct bounding hypercube for a set of Ks. * @@ -61,3 +64,15 @@ trait Boundable[K] extends Serializable { intersect(b1,b2).isDefined } } + +object Boundable { + implicit val zonedDateTimeBoundable = new Boundable[ZonedDateTime] { + def minBound(p1: ZonedDateTime, p2: ZonedDateTime): ZonedDateTime = if(p1 <= p2) p1 else p2 + def maxBound(p1: ZonedDateTime, p2: ZonedDateTime): ZonedDateTime = if(p1 > p2) p1 else p2 + } + + implicit val unitBoundable = new Boundable[Unit] { + def minBound(p1: Unit, p2: Unit): Unit = p1 + def maxBound(p1: Unit, p2: Unit): Unit = p1 + } +} diff --git a/layer/src/main/scala/geotrellis/layer/Implicits.scala b/layer/src/main/scala/geotrellis/layer/Implicits.scala index 772533a134..1fb3078201 100644 --- a/layer/src/main/scala/geotrellis/layer/Implicits.scala +++ b/layer/src/main/scala/geotrellis/layer/Implicits.scala @@ -16,9 +16,11 @@ package geotrellis.layer -import geotrellis.raster.CellGrid +import geotrellis.raster.{CellGrid, RasterSource, ResampleMethod} import geotrellis.util._ import java.time.Instant + +import geotrellis.raster.resample.NearestNeighbor import geotrellis.vector.io.json.CrsFormats @@ -53,4 +55,19 @@ trait Implicits extends merge.Implicits implicit class withCellGridLayoutCollectionMethods[K: SpatialComponent, V <: CellGrid[Int], M: GetComponent[?, LayoutDefinition]](val self: Seq[(K, V)] with Metadata[M]) extends CellGridLayoutCollectionMethods[K, V, M] + + implicit class TileToLayoutOps(val self: RasterSource) { + def tileToLayout[K: SpatialComponent]( + layout: LayoutDefinition, + tileKeyTransform: SpatialKey => K, + resampleMethod: ResampleMethod = NearestNeighbor + ): LayoutTileSource[K] = + LayoutTileSource(self.resampleToGrid(layout, resampleMethod), layout, tileKeyTransform) + + def tileToLayout(layout: LayoutDefinition, resampleMethod: ResampleMethod): LayoutTileSource[SpatialKey] = + tileToLayout(layout, identity, resampleMethod) + + def tileToLayout(layout: LayoutDefinition): LayoutTileSource[SpatialKey] = + tileToLayout(layout, NearestNeighbor) + } } diff --git a/layer/src/main/scala/geotrellis/layer/KeyExtractor.scala b/layer/src/main/scala/geotrellis/layer/KeyExtractor.scala new file mode 100644 index 0000000000..5f0cc52049 --- /dev/null +++ b/layer/src/main/scala/geotrellis/layer/KeyExtractor.scala @@ -0,0 +1,50 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.layer + +import geotrellis.raster.{RasterMetadata, SourceName} + +import java.time.ZonedDateTime + +trait KeyExtractor[K] extends Serializable { + type M + def getMetadata(rs: RasterMetadata): M + def getKey(metadata: M, spatialKey: SpatialKey): K +} + +object KeyExtractor { + /** Aux pattern is used to help compiler proving that the M = RealWorld metadata */ + type Aux[K, M0] = KeyExtractor[K] { type M = M0 } + + val spatialKeyExtractor: KeyExtractor.Aux[SpatialKey, Unit] = new KeyExtractor[SpatialKey] { + type M = Unit + def getMetadata(rs: RasterMetadata): Unit = () + def getKey(metadata: Unit, spatialKey: SpatialKey): SpatialKey = spatialKey + } +} + +trait TemporalKeyExtractor extends KeyExtractor[SpaceTimeKey] { + type M = ZonedDateTime + def getMetadata(rs: RasterMetadata): M + def getKey(metadata: ZonedDateTime, spatialKey: SpatialKey): SpaceTimeKey = SpaceTimeKey(spatialKey, TemporalKey(metadata.toInstant.toEpochMilli)) +} + +object TemporalKeyExtractor { + def fromPath(parseTime: SourceName => ZonedDateTime): KeyExtractor.Aux[SpaceTimeKey, ZonedDateTime] = new TemporalKeyExtractor { + def getMetadata(rs: RasterMetadata): ZonedDateTime = parseTime(rs.name) + } +} diff --git a/layer/src/main/scala/geotrellis/layer/LayoutTileSource.scala b/layer/src/main/scala/geotrellis/layer/LayoutTileSource.scala new file mode 100644 index 0000000000..cf8ade0a87 --- /dev/null +++ b/layer/src/main/scala/geotrellis/layer/LayoutTileSource.scala @@ -0,0 +1,222 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.layer + +import geotrellis.raster._ +import geotrellis.layer._ +import geotrellis.util._ + +/** Reads tiles by key from a [[RasterSource]] as keyed by a [[LayoutDefinition]] + * @note It is required that the [[RasterSource]] is pixel aligned with the [[LayoutDefinition]] + * + * @param source raster source that can be queried by bounding box + * @param layout definition of a tile grid over the pixel grid + * @param tileKeyTransform defines the key transformation you want to apply to the spatially tiled data + */ +class LayoutTileSource[K: SpatialComponent]( + val source: RasterSource, + val layout: LayoutDefinition, + val tileKeyTransform: SpatialKey => K +) { + LayoutTileSource.requireGridAligned(source.gridExtent, layout) + + def sourceColOffset: Long = ((source.extent.xmin - layout.extent.xmin) / layout.cellwidth).toLong + def sourceRowOffset: Long = ((layout.extent.ymax - source.extent.ymax) / layout.cellheight).toLong + + def rasterRegionForKey(key: K): Option[RasterRegion] = { + val spatialComponent = key.getComponent[SpatialKey] + val col = spatialComponent.col.toLong + val row = spatialComponent.row.toLong + /** + * We need to do this manually instead of using RasterExtent.gridBoundsFor because + * the target pixel area isn't always square. + */ + val sourcePixelBounds = GridBounds[Long]( + colMin = col * layout.tileCols - sourceColOffset, + rowMin = row * layout.tileRows - sourceRowOffset, + colMax = (col+1) * layout.tileCols - 1 - sourceColOffset, + rowMax = (row+1) * layout.tileRows - 1 - sourceRowOffset + ) + + if (source.gridBounds.intersects(sourcePixelBounds)) + Some(RasterRegion(source, sourcePixelBounds)) + else + None + } + + def read(key: K): Option[MultibandTile] = + read(key, 0 until source.bandCount) + + /** Read tile according to key. + * If tile area intersects source partially the non-intersecting pixels will be filled with NODATA. + * If tile area does not intersect source None will be returned. + */ + def read(key: K, bands: Seq[Int]): Option[MultibandTile] = { + val spatialComponent = key.getComponent[SpatialKey] + val col = spatialComponent.col.toLong + val row = spatialComponent.row.toLong + val sourcePixelBounds = GridBounds( + colMin = col * layout.tileCols - sourceColOffset, + rowMin = row * layout.tileRows - sourceRowOffset, + colMax = (col+1) * layout.tileCols - 1 - sourceColOffset, + rowMax = (row+1) * layout.tileRows - 1 - sourceRowOffset + ) + + for { + bounds <- sourcePixelBounds.intersection(source.gridBounds) + raster <- source.read(bounds, bands) + } yield { + if (raster.tile.cols == layout.tileCols && raster.tile.rows == layout.tileRows) { + raster.tile + } else { + // raster is smaller but not bigger than I think ... + // its offset is relative to the raster we wished we had + val colOffset = bounds.colMin - sourcePixelBounds.colMin + val rowOffset = bounds.rowMin - sourcePixelBounds.rowMin + raster.tile.mapBands { (_, band) => + PaddedTile(band, colOffset.toInt, rowOffset.toInt, layout.tileCols, layout.tileRows) + } + } + } + } + + /** Read multiple tiles according to key. + * If each tile area intersects source partially the non-intersecting pixels will be filled with NODATA. + * If tile area does not intersect source it will be excluded from result iterator. + */ + def readAll(keys: Iterator[K], bands: Seq[Int]): Iterator[(K, MultibandTile)] = + for { + key <- keys + spatialComponent = key.getComponent[SpatialKey] + col = spatialComponent.col.toLong + row = spatialComponent.row.toLong + sourcePixelBounds = GridBounds( + colMin = col * layout.tileCols - sourceColOffset, + rowMin = row * layout.tileRows - sourceRowOffset, + colMax = (col+1) * layout.tileCols - 1 - sourceColOffset, + rowMax = (row+1) * layout.tileRows - 1 - sourceRowOffset + ) + bounds <- sourcePixelBounds.intersection(source.gridBounds) + raster <- source.read(bounds, bands) + } yield { + val tile = + if (raster.tile.cols == layout.tileCols && raster.tile.rows == layout.tileRows) { + raster.tile + } else { + // raster is smaller but not bigger than I think ... + // its offset is relative to the raster we wished we had + val colOffset = bounds.colMin - sourcePixelBounds.colMin + val rowOffset = bounds.rowMin - sourcePixelBounds.rowMin + raster.tile.mapBands { (_, band) => + PaddedTile(band, colOffset.toInt, rowOffset.toInt, layout.tileCols, layout.tileRows) + } + } + (key, tile) + } + + def readAll(keys: Iterator[K]): Iterator[(K, MultibandTile)] = + readAll(keys, 0 until source.bandCount) + + /** Read all available tiles */ + def readAll(): Iterator[(K, MultibandTile)] = + readAll(keys.toIterator) + + /** Set of keys that can be read from this tile source */ + def keys: Set[K] = { + lazy val buffX = layout.cellSize.width * -0.25 + lazy val buffY = layout.cellSize.height * -0.25 + + layout.extent.intersection(source.extent) match { + case Some(intersection) => + /** + * Buffered down by a quarter of a pixel size in order to + * avoid floating point errors that can occur during + * key generation. + */ + val buffered = intersection.copy( + intersection.xmin - buffX, + intersection.ymin - buffY, + intersection.xmax + buffX, + intersection.ymax + buffY + ) + + layout.mapTransform.keysForGeometry(buffered.toPolygon).map(tileKeyTransform) + case None => + Set.empty[K] + } + } + + /** All intersecting RasterRegions with their respective keys */ + def keyedRasterRegions(): Iterator[(K, RasterRegion)] = + keys + .toIterator + .flatMap { key => + val result = rasterRegionForKey(key) + result.map { region => (key, region) } + } +} + +object LayoutTileSource { + def apply[K: SpatialComponent](source: RasterSource, layout: LayoutDefinition, tileKeyTransform: SpatialKey => K): LayoutTileSource[K] = + new LayoutTileSource(source, layout, tileKeyTransform) + + def spatial(source: RasterSource, layout: LayoutDefinition): LayoutTileSource[SpatialKey] = + new LayoutTileSource(source, layout, identity) + + def temporal(source: RasterSource, layout: LayoutDefinition, tileKeyTransform: SpatialKey => SpaceTimeKey): LayoutTileSource[SpaceTimeKey] = + new LayoutTileSource(source, layout, tileKeyTransform) + + private def requireGridAligned(a: GridExtent[Long], b: GridExtent[Long]): Unit = { + import org.scalactic._ + import TripleEquals._ + import Tolerance._ + + val epsX: Double = math.min(a.cellwidth, b.cellwidth) * 0.01 + val epsY: Double = math.min(a.cellheight, b.cellheight) * 0.01 + + require((a.cellwidth === b.cellwidth +- epsX) && (a.cellheight === b.cellheight +- epsY), + s"CellSize differs: ${a.cellSize}, ${b.cellSize}") + + @inline def offset(a: Double, b: Double, w: Double): Double = { + val cols = (a - b) / w + cols - math.floor(cols) + } + + + val deltaX = math.round((a.extent.xmin - b.extent.xmin) / b.cellwidth) + val deltaY = math.round((a.extent.ymin - b.extent.ymin) / b.cellheight) + + /** + * resultX and resultY represent the pixel bounds of b that is + * closest to the a.extent.xmin and a.extent.ymin. + */ + + val resultX = deltaX * b.cellwidth + b.extent.xmin + val resultY = deltaY * b.cellheight + b.extent.ymin + + /** + * TODO: This is ignored at the moment to make it soft and to make GDAL work, + * we need to reconsider these things to be softer (?) + */ + + require(a.extent.xmin === resultX +- epsX, + s"x-aligned: offset by ${a.cellSize} ${offset(a.extent.xmin, resultX, a.cellwidth)}") + + require(a.extent.ymin === resultY +- epsY, + s"y-aligned: offset by ${a.cellSize} ${offset(a.extent.ymin, resultY, a.cellheight)}") + } +} diff --git a/layer/src/main/scala/geotrellis/layer/LayoutType.scala b/layer/src/main/scala/geotrellis/layer/LayoutType.scala new file mode 100644 index 0000000000..37fac0863e --- /dev/null +++ b/layer/src/main/scala/geotrellis/layer/LayoutType.scala @@ -0,0 +1,59 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.layer + +import geotrellis.proj4.CRS +import geotrellis.raster.CellSize +import geotrellis.vector.Extent + +/** Strategy for selecting LayoutScheme before metadata is collected */ +sealed trait LayoutType { + /** Produce the [[LayoutDefinition]] and zoom level, if applicable, for given raster */ + def layoutDefinitionWithZoom(crs: CRS, extent: Extent, cellSize: CellSize): (LayoutDefinition, Option[Int]) + + /** Produce the [[LayoutDefinition]] for given raster */ + def layoutDefinition(crs: CRS, extent: Extent, cellSize: CellSize): LayoutDefinition = + layoutDefinitionWithZoom(crs, extent, cellSize)._1 +} + +/** @see [[geotrellis.layer.ZoomedLayoutScheme]] */ +case class GlobalLayout(tileSize: Int, zoom: Int, threshold: Double) extends LayoutType { + def layoutDefinitionWithZoom(crs: CRS, extent: Extent, cellSize: CellSize) = { + val scheme = new ZoomedLayoutScheme(crs, tileSize, threshold) + Option(zoom) match { + case Some(zoom) => + scheme.levelForZoom(zoom).layout -> Some(zoom) + case None => + val LayoutLevel(zoom, ld) = scheme.levelFor(extent, cellSize) + ld -> Some(zoom) + } + } +} + +/** @see [[geotrellis.layer.FloatingLayoutScheme]] */ +case class LocalLayout(tileCols: Int, tileRows: Int) extends LayoutType { + def layoutDefinitionWithZoom(crs: CRS, extent: Extent, cellSize: CellSize) = { + val scheme = new FloatingLayoutScheme(tileCols, tileRows) + scheme.levelFor(extent, cellSize).layout -> None + } +} + + +object LocalLayout { + def apply(tileSize: Int): LocalLayout = + LocalLayout(tileSize, tileSize) +} diff --git a/layer/src/main/scala/geotrellis/layer/package.scala b/layer/src/main/scala/geotrellis/layer/package.scala index 69d6a588aa..ae5d0b55f7 100644 --- a/layer/src/main/scala/geotrellis/layer/package.scala +++ b/layer/src/main/scala/geotrellis/layer/package.scala @@ -16,18 +16,15 @@ package geotrellis - -import geotrellis.layer.{KeyBounds, MapKeyTransform} import geotrellis.raster._ import geotrellis.raster.io.geotiff.reader.GeoTiffReader import geotrellis.vector.Extent import geotrellis.proj4._ import geotrellis.util._ -import org.locationtech.proj4j.UnsupportedParameterException -import scala.util.{Failure, Success, Try} +import org.locationtech.proj4j.UnsupportedParameterException -package object layer extends Implicits { +package object layer extends layer.Implicits { type TileBounds = GridBounds[Int] type SpatialComponent[K] = Component[K, SpatialKey] type TemporalComponent[K] = Component[K, TemporalKey] diff --git a/layer/src/test/resources/vlm/issue-116-cog.tif b/layer/src/test/resources/vlm/issue-116-cog.tif new file mode 100644 index 0000000000..4c190273c0 Binary files /dev/null and b/layer/src/test/resources/vlm/issue-116-cog.tif differ diff --git a/layer/src/test/resources/vlm/multiband.tif b/layer/src/test/resources/vlm/multiband.tif new file mode 100644 index 0000000000..f9d2f3f1d5 Binary files /dev/null and b/layer/src/test/resources/vlm/multiband.tif differ diff --git a/layer/src/test/scala/geotrellis/layer/LayoutTileSourceSpec.scala b/layer/src/test/scala/geotrellis/layer/LayoutTileSourceSpec.scala new file mode 100644 index 0000000000..27e38c23d3 --- /dev/null +++ b/layer/src/test/scala/geotrellis/layer/LayoutTileSourceSpec.scala @@ -0,0 +1,210 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.layer + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.testkit.{RasterMatchers, Resource} +import geotrellis.raster.geotiff._ +import geotrellis.raster.io.geotiff.reader._ +import geotrellis.raster.resample.NearestNeighbor +import geotrellis.vector.ProjectedExtent + +import org.scalatest._ + +import java.io.File + +class LayoutTileSourceSpec extends FunSpec with RasterMatchers { + /** Function to get paths from the raster/data dir. */ + def rasterGeoTiffPath(name: String): String = { + def baseDataPath = "raster/data" + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } + + val testFile = rasterGeoTiffPath("vlm/aspect-tiled.tif") + lazy val tiff = GeoTiffReader.readMultiband(testFile, streaming = false) + + lazy val rasterSource = GeoTiffRasterSource(testFile) + val scheme = FloatingLayoutScheme(256) + lazy val layout = scheme.levelFor(rasterSource.extent, rasterSource.cellSize).layout + lazy val source = LayoutTileSource.spatial(rasterSource, layout) + + val mbTestFile = Resource.path("vlm/multiband.tif") + lazy val mbTiff = GeoTiffReader.readMultiband(mbTestFile, streaming = false) + lazy val mbRasterSource = GeoTiffRasterSource(mbTestFile) + lazy val mbLayout = scheme.levelFor(mbRasterSource.extent, mbRasterSource.cellSize).layout + lazy val mbSource = LayoutTileSource.spatial(mbRasterSource, mbLayout) + + it("should read all the keys") { + val keys = source.layout + .mapTransform + .extentToBounds(rasterSource.extent) + .coordsIter.toList + + for ((col, row) <- keys ) { + val key = SpatialKey(col, row) + val re = RasterExtent( + extent = layout.mapTransform.keyToExtent(key), + cellwidth = layout.cellwidth, + cellheight = layout.cellheight, + cols = layout.tileCols, + rows = layout.tileRows) + + withClue(s"$key:") { + val tile = source.read(key).get + val actual = Raster(tile, re.extent) + val expected = tiff.crop(rasterExtent = re) + + withGeoTiffClue(actual, expected, source.source.crs) { + assertRastersEqual(actual, expected) + } + } + } + } + + it("should subset bands if requested") { + val coord = mbSource.layout + .mapTransform + .extentToBounds(mbRasterSource.extent) + .coordsIter.toList + .head + + val key = SpatialKey(coord._1, coord._2) + val re = RasterExtent( + extent = mbLayout.mapTransform.keyToExtent(key), + cellwidth = mbLayout.cellwidth, + cellheight = mbLayout.cellheight, + cols = mbLayout.tileCols, + rows = mbLayout.tileRows + ) + + withClue(s"$key:") { + val tile = mbSource.read(key, Seq(1, 2)).get + val actual = Raster(tile, re.extent) + val expected = Raster( + mbTiff + .crop(rasterExtent = re.copy(extent = re.extent.buffer(re.cellSize.resolution / 4))) + .tile + .subsetBands(1, 2), + re.extent + ) + withGeoTiffClue(actual, expected, mbSource.source.crs) { + assertRastersEqual(actual, expected) + } + } + } + + /** https://github.com/geotrellis/geotrellis-contrib/issues/116 */ + describe("should read by key on high zoom levels properly // see issue-116 in a geotrellis-contrib repo") { + val crs: CRS = WebMercator + + val tmsLevels: Array[LayoutDefinition] = { + val scheme = ZoomedLayoutScheme(crs, 256) + for (zoom <- 0 to 64) yield scheme.levelForZoom(zoom).layout + }.toArray + + val path = Resource.path("vlm/issue-116-cog.tif") + val subsetBands = List(0, 1, 2) + + val layoutDefinition = tmsLevels(22) + + for(c <- 1249656 to 1249658; r <- 1520655 to 1520658) { + it(s"reading 22/$c/$r") { + val result = + GeoTiffRasterSource(path) + .reproject(WebMercator) + .tileToLayout(layoutDefinition) + .read(SpatialKey(c, r), subsetBands) + .get + + + result.bands.foreach { band => + (1 until band.rows).foreach { r => + band.getDouble(band.cols - 1, r) shouldNot be(0d) + } + } + } + } + } + + describe("should perform a tileToLayout of a GeoTiffRasterSource") { + lazy val url = testFile + + lazy val source: GeoTiffRasterSource = GeoTiffRasterSource(url) + + val cellSizes = { + val tiff = GeoTiffReader.readMultiband(url) + (tiff +: tiff.overviews).map(_.rasterExtent.cellSize).map { case CellSize(w, h) => + CellSize(w + 1, h + 1) + } + } + + cellSizes.foreach { targetCellSize => + it(s"for cellSize: ${targetCellSize}") { + val pe = ProjectedExtent(source.extent, source.crs) + val scheme = FloatingLayoutScheme(256) + val layout = scheme.levelFor(pe.extent, targetCellSize).layout + val mapTransform = layout.mapTransform + val resampledSource = source.resampleToGrid(layout) + + val expected: List[(SpatialKey, MultibandTile)] = + mapTransform(pe.extent).coordsIter.map { case (col, row) => + val key = SpatialKey(col, row) + val ext = mapTransform.keyToExtent(key) + val raster = resampledSource.read(ext).get + val newTile = raster.tile.prototype(source.cellType, layout.tileCols, layout.tileRows) + key -> newTile.merge( + ext, + raster.extent, + raster.tile, + NearestNeighbor + ) + }.toList + + val layoutSource = source.tileToLayout(layout) + val actual: List[(SpatialKey, MultibandTile)] = layoutSource.readAll().toList + + withClue(s"actual.size: ${actual.size} expected.size: ${expected.size}") { + actual.size should be(expected.size) + } + + val sortedActual: List[Raster[MultibandTile]] = + actual + .sortBy { case (k, _) => (k.col, k.row) } + .map { case (k, v) => Raster(v, mapTransform.keyToExtent(k)) } + + val sortedExpected: List[Raster[MultibandTile]] = + expected + .sortBy { case (k, _) => (k.col, k.row) } + .map { case (k, v) => Raster(v, mapTransform.keyToExtent(k)) } + + val grouped: List[(Raster[MultibandTile], Raster[MultibandTile])] = + sortedActual.zip(sortedExpected) + + grouped.foreach { case (actualTile, expectedTile) => + withGeoTiffClue(actualTile, expectedTile, source.crs) { + assertRastersEqual(actualTile, expectedTile) + } + } + + layoutSource.source + } + } + } +} diff --git a/layer/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala b/layer/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala new file mode 100644 index 0000000000..c05d8e3b23 --- /dev/null +++ b/layer/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala @@ -0,0 +1,82 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.layer._ +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.reproject._ +import geotrellis.raster.testkit.RasterMatchers + +import org.scalatest._ + +import java.io.File + +class GeoTiffReprojectRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + def rasterGeoTiffPath(name: String): String = { + def baseDataPath = "raster/data" + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } + + lazy val uri = rasterGeoTiffPath("vlm/aspect-tiled.tif") + + lazy val rasterSource = GeoTiffRasterSource(uri) + lazy val sourceTiff = GeoTiffReader.readMultiband(uri) + + lazy val expectedRasterExtent = { + val re = ReprojectRasterExtent(rasterSource.gridExtent, Transform(rasterSource.crs, LatLng)) + // stretch target raster extent slightly to avoid default case in ReprojectRasterExtent + RasterExtent(re.extent, CellSize(re.cellheight * 1.1, re.cellwidth * 1.1)) + } + describe("Reprojecting a RasterSource") { + it("should select correct overview to sample from with a GeoTiffReprojectRasterSource") { + // we choose LatLng to switch scales, the source projection is in meters + val baseReproject = rasterSource.reproject(LatLng).asInstanceOf[GeoTiffReprojectRasterSource] + // known good start, CellSize(10, 10) is the base resolution of source + baseReproject.closestTiffOverview.cellSize shouldBe CellSize(10, 10) + + info(s"lcc resolutions: ${rasterSource.resolutions.map(_.cellSize)}") + val twiceFuzzyLayout = { + val CellSize(width, height) = baseReproject.cellSize + LayoutDefinition(RasterExtent(LatLng.worldExtent, CellSize(width*2.1, height*2.1)), tileSize = 256) + } + + val twiceFuzzySource = rasterSource.reprojectToGrid(LatLng, twiceFuzzyLayout).asInstanceOf[GeoTiffReprojectRasterSource] + twiceFuzzySource.closestTiffOverview.cellSize shouldBe CellSize(20,20) + + val thriceFuzzyLayout = { + val CellSize(width, height) = baseReproject.cellSize + LayoutDefinition(RasterExtent(LatLng.worldExtent, CellSize(width*3.5, height*3.5)), tileSize = 256) + } + + val thriceFuzzySource = rasterSource.reprojectToGrid(LatLng, thriceFuzzyLayout).asInstanceOf[GeoTiffReprojectRasterSource] + thriceFuzzySource.closestTiffOverview.cellSize shouldBe CellSize(20,20) + + val quatroFuzzyLayout = { + val CellSize(width, height) = baseReproject.cellSize + LayoutDefinition(RasterExtent(LatLng.worldExtent, CellSize(width*4.1, height*4.1)), tileSize = 256) + } + + val quatroTimesFuzzySource = rasterSource.reprojectToGrid(LatLng, quatroFuzzyLayout).asInstanceOf[GeoTiffReprojectRasterSource] + quatroTimesFuzzySource.closestTiffOverview.cellSize shouldBe CellSize(40.0,39.94082840236686) + + } + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5639cf62e3..e22a7e5ac7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,9 +17,9 @@ import sbt._ object Dependencies { - val pureconfig = "com.github.pureconfig" %% "pureconfig" % "0.10.2" - val logging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0" - val scalatest = "org.scalatest" %% "scalatest" % "3.0.7" + val pureconfig = "com.github.pureconfig" %% "pureconfig" % "0.11.1" + val logging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2" + val scalatest = "org.scalatest" %% "scalatest" % "3.0.8" val scalacheck = "org.scalacheck" %% "scalacheck" % "1.14.0" val scalaXml = "org.scala-lang.modules" %% "scala-xml" % "1.2.0" val jts = "org.locationtech.jts" % "jts-core" % "1.16.1" @@ -28,7 +28,7 @@ object Dependencies { val monocleCore = "com.github.julien-truffaut" %% "monocle-core" % Version.monocle val monocleMacro = "com.github.julien-truffaut" %% "monocle-macro" % Version.monocle - val openCSV = "com.opencsv" % "opencsv" % "4.5" + val openCSV = "com.opencsv" % "opencsv" % "4.6" val spire = "org.spire-math" %% "spire" % Version.spire val spireMacro = "org.spire-math" %% "spire-macros" % Version.spire @@ -37,15 +37,15 @@ object Dependencies { val apacheMath = "org.apache.commons" % "commons-math3" % "3.6.1" - val chronoscala = "jp.ne.opt" %% "chronoscala" % "0.3.0" + val chronoscala = "jp.ne.opt" %% "chronoscala" % "0.3.2" - val awsSdkS3 = "software.amazon.awssdk" % "s3" % "2.5.29" + val awsSdkS3 = "software.amazon.awssdk" % "s3" % "2.7.32" - val catsCore = "org.typelevel" %% "cats-core" % "1.6.0" - val catsEffect = "org.typelevel" %% "cats-effect" % "1.2.0" + val catsCore = "org.typelevel" %% "cats-core" % "1.6.1" + val catsEffect = "org.typelevel" %% "cats-effect" % "1.3.1" - val fs2Core = "co.fs2" %% "fs2-core" % "1.0.4" - val fs2Io = "co.fs2" %% "fs2-io" % "1.0.4" + val fs2Core = "co.fs2" %% "fs2-core" % "1.0.5" + val fs2Io = "co.fs2" %% "fs2-io" % "1.0.5" val sparkCore = "org.apache.spark" %% "spark-core" % Version.spark val sparkSQL = "org.apache.spark" %% "spark-sql" % Version.spark @@ -58,9 +58,7 @@ object Dependencies { val jsonSchemaValidator = "com.networknt" % "json-schema-validator" % "0.1.23" - val scaffeine = "com.github.blemale" %% "scaffeine" % "2.6.0" - - val simulacrum = "com.github.mpilquist" %% "simulacrum" % "0.17.0" + val scaffeine = "com.github.blemale" %% "scaffeine" % "3.1.0" val circeCore = "io.circe" %% "circe-core" % Version.circe val circeGeneric = "io.circe" %% "circe-generic" % Version.circe @@ -112,9 +110,19 @@ object Dependencies { val uzaygezenCore = "com.google.uzaygezen" % "uzaygezen-core" % "0.2" - val scalaj = "org.scalaj" %% "scalaj-http" % "2.4.1" + val scalaj = "org.scalaj" %% "scalaj-http" % "2.4.2" val scalapbRuntime = "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion val squants = "org.typelevel" %% "squants" % "1.4.0" + val scalactic = "org.scalactic" %% "scalactic" % "3.0.8" + val scalaURI = "io.lemonlabs" %% "scala-uri" % "1.4.10" + + val gdalBindings = "org.gdal" % "gdal" % Version.gdal + val gdalWarp = "com.azavea.gdal" % "gdal-warp-bindings" % Version.gdalWarp + + val jacksonCore = "com.fasterxml.jackson.core" % "jackson-core" % "2.6.7" + val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7" + val jacksonAnnotations = "com.fasterxml.jackson.core" % "jackson-annotations" % "2.6.7" + val jacksonModuleScala = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.6.7" } diff --git a/project/Environment.scala b/project/Environment.scala index fef3bf355b..8ced47c758 100644 --- a/project/Environment.scala +++ b/project/Environment.scala @@ -21,6 +21,6 @@ object Environment { Properties.envOrElse(environmentVariable, default) lazy val hadoopVersion = either("SPARK_HADOOP_VERSION", "2.8.5") - lazy val sparkVersion = either("SPARK_VERSION", "2.4.1") + lazy val sparkVersion = either("SPARK_VERSION", "2.4.3") lazy val versionSuffix = either("GEOTRELLIS_VERSION_SUFFIX", "-SNAPSHOT") } diff --git a/project/Settings.scala b/project/Settings.scala index c3a6771cc5..4769d2d7f7 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -36,8 +36,8 @@ object Settings { val geowaveSnapshot = "geowave-snapshot" at "http://geowave-maven.s3.amazonaws.com/snapshot" val ivy2Local = Resolver.file("local", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns) val mavenLocal = Resolver.mavenLocal - val local = Seq(ivy2Local, mavenLocal) + val azaveaBintray = Resolver.bintrayRepo("azavea", "geotrellis") } lazy val noForkInTests = Seq( @@ -45,7 +45,7 @@ object Settings { parallelExecution in Test := false ) - lazy val `accumulo` = Seq( + lazy val accumulo = Seq( name := "geotrellis-accumulo", libraryDependencies ++= Seq( accumuloCore @@ -96,7 +96,7 @@ object Settings { // jmhExtraOptions := Some("-jvmArgsAppend -prof geotrellis.bench.GeotrellisFlightRecordingProfiler") ) - lazy val `cassandra` = Seq( + lazy val cassandra = Seq( name := "geotrellis-cassandra", libraryDependencies ++= Seq( cassandraDriverCore @@ -294,7 +294,7 @@ object Settings { """ ) ++ noForkInTests - lazy val `hbase` = Seq( + lazy val hbase = Seq( name := "geotrellis-hbase", libraryDependencies ++= Seq( hbaseCommon exclude("javax.servlet", "servlet-api"), @@ -414,6 +414,7 @@ object Settings { monocleCore, monocleMacro, scalaXml, + scalaURI, scalatest % Test, scalacheck % Test ), @@ -443,7 +444,7 @@ object Settings { libraryDependencies += scalatest ) - lazy val `s3` = Seq( + lazy val s3 = Seq( name := "geotrellis-s3", libraryDependencies ++= Seq( awsSdkS3, @@ -623,7 +624,6 @@ object Settings { circeGenericExtras, circeParser, apacheMath, - simulacrum, spire, scalatest % Test, scalacheck % Test @@ -665,6 +665,7 @@ object Settings { scaffeine, uzaygezenCore, pureconfig, + scalactic, scalatest % Test ), initialCommands in console := @@ -698,4 +699,46 @@ object Settings { scalatest % Test ) ) + + lazy val gdal = Seq( + name := "geotrellis-gdal", + libraryDependencies ++= Seq( + gdalWarp, + scalatest % Test, + gdalBindings % Test + ), + resolvers += Repositories.azaveaBintray, + Test / fork := true, + Test / parallelExecution := false, + Test / testOptions += Tests.Argument("-oDF"), + javaOptions ++= Seq("-Djava.library.path=/usr/local/lib") + ) + + lazy val `gdal-spark` = Seq( + name := "geotrellis-gdal-spark", + libraryDependencies ++= Seq( + gdalWarp, + sparkCore % Provided, + sparkSQL % Test, + scalatest % Test + ), + // caused by the AWS SDK v2 + dependencyOverrides ++= { + val deps = Seq( + jacksonCore, + jacksonDatabind, + jacksonAnnotations + ) + CrossVersion.partialVersion(scalaVersion.value) match { + // if Scala 2.12+ is used + case Some((2, scalaMajor)) if scalaMajor >= 12 => deps + case _ => deps :+ jacksonModuleScala + } + }, + resolvers += Repositories.azaveaBintray, + Test / fork := true, + Test / parallelExecution := false, + Test / testOptions += Tests.Argument("-oDF"), + javaOptions ++= Seq("-Djava.library.path=/usr/local/lib") + ) } diff --git a/project/Version.scala b/project/Version.scala index d88d1f1dba..c9c3575584 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -18,16 +18,20 @@ object Version { val geotrellis = "3.0.0" + Environment.versionSuffix val scala = "2.11.12" val crossScala = Seq(scala, "2.12.7") - val geotools = "21.0" + val geotools = "21.2" val monocle = "1.5.1-cats" val spire = "0.13.0" val accumulo = "1.9.3" - val cassandra = "3.7.1" - val hbase = "2.1.4" - val geomesa = "2.3.0" + val cassandra = "3.7.2" + val hbase = "2.2.0" + val geomesa = "2.3.1" val geowave = "0.9.3" val circe = "0.11.1" val previousVersion = "2.2.0" + + val gdal = "2.4.0" + val gdalWarp = "33.61199eb" + lazy val hadoop = Environment.hadoopVersion lazy val spark = Environment.sparkVersion } diff --git a/project/plugins.sbt b/project/plugins.sbt index 4db141e97a..3ccfae7940 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,10 +1,10 @@ -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.0") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") +addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.2") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.4.0") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.2.0") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.4") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.3.7") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") -addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.3.0") -addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.20") -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.9.0-RC1" +addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.6.0") +addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.23") +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.9.0" diff --git a/raster-testkit/src/main/scala/geotrellis/raster/testkit/RasterMatchers.scala b/raster-testkit/src/main/scala/geotrellis/raster/testkit/RasterMatchers.scala index b6d00e2c8e..ff245d445b 100644 --- a/raster-testkit/src/main/scala/geotrellis/raster/testkit/RasterMatchers.scala +++ b/raster-testkit/src/main/scala/geotrellis/raster/testkit/RasterMatchers.scala @@ -16,13 +16,26 @@ package geotrellis.raster.testkit -import org.scalatest._ +import geotrellis.proj4.CRS import geotrellis.raster._ +import geotrellis.raster.io.geotiff.GeoTiff +import geotrellis.raster.render.ascii.AsciiArtEncoder +import geotrellis.raster.render.png.{PngColorEncoding, RgbaPngEncoding} +import spire.implicits.cfor +import spire.math.Integral +import org.scalatest._ +import org.scalatest.matchers.{HavePropertyMatchResult, HavePropertyMatcher} +import org.scalatest.tools.BetterPrinters -import spire.syntax.cfor._ +import scala.reflect.ClassTag +import java.nio.file.{Files, Paths} +import geotrellis.vector.Extent + +import scala.util.Random trait RasterMatchers extends Matchers { + import RasterMatchers._ val Eps = 1e-3 @@ -184,4 +197,232 @@ trait RasterMatchers extends Matchers { } } } + + private def dimsToString[T <: Grid[N], N: Integral](t: T): String = + s"""(${t.cols}, ${t.rows})""" + + + // def dimensions(dims: (Int, Int)) = HavePropertyMatcher[CellGrid[Int], (Int, Int)] { grid => + // HavePropertyMatchResult(grid.dimensions == dims, "dimensions", dims, grid.dimensions) + // } + + def cellType[T<: CellGrid[_]: ClassTag] (ct: CellType) = HavePropertyMatcher[T, CellType] { grid => + HavePropertyMatchResult(grid.cellType == ct, "cellType", ct, grid.cellType) + } + + def bandCount(count: Int) = HavePropertyMatcher[MultibandTile, Int] { tile => + HavePropertyMatchResult(tile.bandCount == count, "bandCount", count, tile.bandCount) + } + + def assertTilesEqual(actual: Tile, expected: Tile): Unit = + assertTilesEqual(actual, expected, Eps) + + def assertTilesEqual(actual: Tile, expected: Tile, threshold: Double): Unit = + assertTilesEqual(MultibandTile(actual), MultibandTile(expected), threshold: Double) + + def assertTilesEqual(actual: MultibandTile, expected: MultibandTile): Unit = + assertTilesEqual(actual, expected, Eps) + + def assertTilesEqual(actual: MultibandTile, expected: MultibandTile, threshold: Double): Unit = { + actual should have ( + cellType (expected.cellType), + // dimensions (expected.dimensions), + bandCount (expected.bandCount) + ) + + withAsciiDiffClue(actual, expected){ + assertEqual(actual, expected, threshold) + } + } + + def assertRastersEqual(actual: Raster[Tile], expected: Raster[MultibandTile])(implicit d: DummyImplicit): Unit = + assertRastersEqual(actual.mapTile(MultibandTile(_)), expected) + + def assertRastersEqual(actual: Raster[Tile], expected: Raster[MultibandTile], threshold: Double)(implicit d: DummyImplicit): Unit = + assertRastersEqual(actual.mapTile(MultibandTile(_)), expected, threshold) + + def assertRastersEqual(actual: Raster[Tile], expected: Raster[MultibandTile], threshold: Double, thresholdExtent: Double)(implicit d: DummyImplicit): Unit = + assertRastersEqual(actual.mapTile(MultibandTile(_)), expected, threshold, thresholdExtent) + + def assertRastersEqual(actual: Raster[MultibandTile], expected: Raster[MultibandTile]): Unit = + assertRastersEqual(actual: Raster[MultibandTile], expected: Raster[MultibandTile], Eps) + + def assertRastersEqual(actual: Raster[MultibandTile], expected: Raster[MultibandTile], threshold: Double): Unit = + assertRastersEqual(actual: Raster[MultibandTile], expected: Raster[MultibandTile], threshold: Double, Eps) + + def assertRastersEqual(actual: Raster[MultibandTile], expected: Raster[MultibandTile], threshold: Double, thresholdExtent: Double): Unit = { + actual.extent.xmin shouldBe expected.extent.xmin +- thresholdExtent + actual.extent.ymin shouldBe expected.extent.ymin +- thresholdExtent + actual.extent.xmax shouldBe expected.extent.xmax +- thresholdExtent + actual.extent.ymax shouldBe expected.extent.ymax +- thresholdExtent + + actual.tile should have ( + cellType (expected.cellType), + // dimensions (expected.dimensions), + bandCount (expected.tile.bandCount) + ) + + withAsciiDiffClue(actual.tile, expected.tile){ + assertEqual(actual.tile, expected.tile, threshold) + } + } + + /** Renders scaled diff tiles as a clue */ + def withAsciiDiffClue[T]( + actual: MultibandTile, + expect: MultibandTile, + mode: DiffMode = DiffMode.DiffSum, + eps: Double = 0, + palette: AsciiArtEncoder.Palette = AsciiArtEncoder.Palette(" â–‘â–’â–“â–ˆ"), + size: Int = 24 + )(fun: => T) = withClue({ + require(actual.bandCount == expect.bandCount, s"Band count doesn't match: ${actual.bandCount} != ${expect.bandCount}") + val diffs = for (b <- 0 until actual.bandCount) yield + scaledDiff(actual.band(b), expect.band(b), mode = mode, maxDim = size, eps = eps) + + val asciiDiffs = diffs.map(_.renderAscii(palette)) + + val joinedDiffs: String = asciiDiffs + .map(_.lines.toSeq) + .transpose + .map(_.mkString("\t")) + .mkString("\n") + + val bandList = (0 until actual.bandCount).mkString(",") + val scale = s"1 char == ${actual.rows / diffs(0).rows} rows == ${actual.cols / diffs(0).cols} cols" + s""" + |+ Diff: band(${bandList}) @ ($scale) + |${joinedDiffs} + | + """.stripMargin + })(fun) + + def withGeoTiffClue[T]( + actual: Raster[MultibandTile], + expect: Raster[MultibandTile], + crs: CRS + )(fun: => T): T = withClue({ + val tmpDir = Files.createTempDirectory(getClass.getSimpleName) + val actualFile = tmpDir.resolve("actual.tiff") + val expectFile = tmpDir.resolve("expect.tiff") + var diffFile = tmpDir.resolve("diff.tiff") + GeoTiff(actual, crs).write(actualFile.toString, optimizedOrder = true) + GeoTiff(expect, crs).write(expectFile.toString, optimizedOrder = true) + + if ((actual.tile.bandCount == expect.tile.bandCount) && (actual.dimensions == expect.dimensions)) { + val diff = actual.tile.bands.zip(expect.tile.bands).map { case (l, r) => l - r }.toArray + GeoTiff(ArrayMultibandTile(diff), actual.extent, crs).write(diffFile.toString, optimizedOrder = true) + } else { + diffFile = null + } + + s""" + |+ actual: ${actualFile} + |+ expect: ${expectFile} + |+ diff : ${Option(diffFile).getOrElse("--")} + """stripMargin + })(fun) + + def writePngOutputTile( + tile: MultibandTile, + colorEncoding: PngColorEncoding = RgbaPngEncoding, + band: Int = 0, + name: String = "output", + discriminator: String = "", + outputDir: Option[String] = None + ): MultibandTile = { + val tmpDir = outputDir.fold(Files.createTempDirectory(getClass.getSimpleName))(Paths.get(_)) + val outputFile = tmpDir.resolve(s"${name}${discriminator}.png") + tile.band(band).renderPng().write(outputFile.toString) + + val msg = s""" + |+ png output path : ${outputFile} + """stripMargin + + BetterPrinters.printAnsiGreen(msg) + tile + } + + def writePngOutputRaster( + raster: Raster[MultibandTile], + colorEncoding: PngColorEncoding = RgbaPngEncoding, + band: Int = 0, + name: String = "output", + discriminator: String = "", + outputDir: Option[String] = None + ): Raster[MultibandTile] = + raster.mapTile(writePngOutputTile(_, colorEncoding, band, name, discriminator, outputDir)) + + def randomExtentWithin(extent: Extent, sampleScale: Double = 0.10): Extent = { + assert(sampleScale > 0 && sampleScale <= 1) + val extentWidth = extent.xmax - extent.xmin + val extentHeight = extent.ymax - extent.ymin + + val sampleWidth = extentWidth * sampleScale + val sampleHeight = extentHeight * sampleScale + + val testRandom = Random.nextDouble() + val subsetXMin = (testRandom * (extentWidth - sampleWidth)) + extent.xmin + val subsetYMin = (Random.nextDouble() * (extentHeight - sampleHeight)) + extent.ymin + + Extent(subsetXMin, subsetYMin, subsetXMin + sampleWidth, subsetYMin + sampleHeight) + } +} + + +sealed trait DiffMode { + def apply(acc: Double, next: Double): Double +} + +object DiffMode { + case object DiffCount extends DiffMode { + def apply(acc: Double, next: Double) = if (isNoData(acc)) 1 else acc + 1 + } + case object DiffSum extends DiffMode { + def apply(acc: Double, next: Double) = if (isNoData(acc)) next else acc + next + } + case object DiffMax extends DiffMode{ + def apply(acc: Double, next: Double) = if (isNoData(acc)) next else math.max(acc, next) + } + case object DiffMin extends DiffMode{ + def apply(acc: Double, next: Double) = if (isNoData(acc)) next else math.min(acc, next) + } +} + +object RasterMatchers { + def scaledDiff(actual: Tile, expect: Tile, maxDim: Int, mode: DiffMode = DiffMode.DiffSum, eps: Double = 0): Tile = { + require(actual.dimensions == expect.dimensions, + s"dimensions mismatch: ${actual.dimensions}, ${expect.dimensions}") + + val cols = actual.cols + val rows = actual.rows + val scale: Double = maxDim / math.max(cols, rows).toDouble + val diff = ArrayTile.empty(FloatConstantNoDataCellType, (cols * scale).toInt, (rows * scale).toInt) + val colScale: Double = diff.cols.toDouble / actual.cols.toDouble + val rowScale: Double = diff.rows.toDouble / actual.rows.toDouble + var diffs = 0 + cfor(0)(_ < cols, _ + 1) { col => + cfor(0)(_ < rows, _ + 1) { row => + val v1 = actual.getDouble(col, row) + val v2 = expect.getDouble(col, row) + val vd: Double = + if (isNoData(v1) && isNoData(v2)) Double.NaN + else if (isData(v1) && isNoData(v2)) math.abs(v1) + else if (isNoData(v1) && isData(v2)) math.abs(v2) + else math.abs(v1 - v2) + + if (isData(vd) && (vd > eps)) { + val dcol = (colScale * col).toInt + val drow = (rowScale * row).toInt + val ac = diff.getDouble(dcol, drow) + if (isData(ac)) { + diff.setDouble(dcol, drow, ac + vd) + } else + diff.setDouble(dcol, drow, vd) + diffs += 1 + } + } + } + diff + } } diff --git a/raster-testkit/src/main/scala/geotrellis/raster/testkit/Resource.scala b/raster-testkit/src/main/scala/geotrellis/raster/testkit/Resource.scala new file mode 100644 index 0000000000..f8977c7f98 --- /dev/null +++ b/raster-testkit/src/main/scala/geotrellis/raster/testkit/Resource.scala @@ -0,0 +1,39 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.testkit + +import java.io._ +import java.net.{URI, URL} + +object Resource { + def apply(name: String): String = { + val stream: InputStream = getClass.getResourceAsStream(s"/$name") + try { scala.io.Source.fromInputStream( stream ).getLines.mkString(" ") } finally { stream.close() } + } + + def url(name: String): URL = { + getClass.getResource(s"/$name") + } + + def uri(name: String): URI = { + getClass.getResource(s"/$name").toURI + } + + def path(name: String): String = { + getClass.getResource(s"/$name").getFile + } +} \ No newline at end of file diff --git a/raster-testkit/src/main/scala/geotrellis/raster/testkit/Utils.scala b/raster-testkit/src/main/scala/geotrellis/raster/testkit/Utils.scala new file mode 100644 index 0000000000..801e366cb2 --- /dev/null +++ b/raster-testkit/src/main/scala/geotrellis/raster/testkit/Utils.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.testkit + +import geotrellis.raster.{CellGrid, Raster} +import geotrellis.vector.Extent + +import scala.util.Try + +object Utils { + def roundRaster[T <: CellGrid[Int]](raster: Raster[T], scale: Int = 11): Raster[T] = + raster.copy(extent = roundExtent(raster.extent, scale)) + + def roundExtent(extent: Extent, scale: Int = 11): Extent = { + val Extent(xmin, ymin, xmax, ymax) = extent + Extent( + BigDecimal(xmin).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble, + BigDecimal(ymin).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble, + BigDecimal(xmax).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble, + BigDecimal(ymax).setScale(scale, BigDecimal.RoundingMode.HALF_UP).toDouble + ) + } + + /** A dirty reflection function to modify object vals */ + def modifyField(obj: AnyRef, name: String, value: Any) { + def impl(clazz: Class[_]) { + Try(clazz.getDeclaredField(name)).toOption match { + case Some(field) => + field.setAccessible(true) + clazz.getMethod(name).invoke(obj) // force init in case it's a lazy val + field.set(obj, value) // overwrite value + case None => + if (clazz.getSuperclass != null) { + impl(clazz.getSuperclass) + } + } + } + + impl(obj.getClass) + } +} diff --git a/raster-testkit/src/main/scala/org/scalatest/tools/BetterPrinters.scala b/raster-testkit/src/main/scala/org/scalatest/tools/BetterPrinters.scala new file mode 100644 index 0000000000..eed763a4c8 --- /dev/null +++ b/raster-testkit/src/main/scala/org/scalatest/tools/BetterPrinters.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.scalatest.tools + +object BetterPrinters { + def printText(text: String, color: AnsiColor, presentInColor: Boolean): Unit = + print(Fragment(text, color).toPossiblyColoredText(presentInColor)) + + def printAnsiGreen(text: String): Unit = printText(text, AnsiGreen, true) + def printAnsiRed(text: String): Unit = printText(text, AnsiRed, true) + def printAnsiYellow(text: String): Unit = printText(text, AnsiYellow, true) + def printAnsiCyan(text: String): Unit = printText(text, AnsiCyan, true) +} diff --git a/raster/data/vlm/0_to_99.tif b/raster/data/vlm/0_to_99.tif new file mode 100644 index 0000000000..be3fa28670 Binary files /dev/null and b/raster/data/vlm/0_to_99.tif differ diff --git a/raster/data/vlm/aspect-tiled.tif b/raster/data/vlm/aspect-tiled.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/raster/data/vlm/aspect-tiled.tif differ diff --git a/raster/data/vlm/geotiff-at-origin.tif b/raster/data/vlm/geotiff-at-origin.tif new file mode 100644 index 0000000000..707385585b Binary files /dev/null and b/raster/data/vlm/geotiff-at-origin.tif differ diff --git a/raster/data/vlm/geotiff-off-origin.tif b/raster/data/vlm/geotiff-off-origin.tif new file mode 100644 index 0000000000..1e767c2254 Binary files /dev/null and b/raster/data/vlm/geotiff-off-origin.tif differ diff --git a/raster/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider b/raster/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider new file mode 100644 index 0000000000..218c768049 --- /dev/null +++ b/raster/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider @@ -0,0 +1 @@ +geotrellis.raster.geotiff.GeoTiffRasterSourceProvider diff --git a/raster/src/main/scala/geotrellis/raster/ArrayTile.scala b/raster/src/main/scala/geotrellis/raster/ArrayTile.scala index e255e35b69..896f462ae9 100644 --- a/raster/src/main/scala/geotrellis/raster/ArrayTile.scala +++ b/raster/src/main/scala/geotrellis/raster/ArrayTile.scala @@ -42,9 +42,6 @@ trait ArrayTile extends Tile with Serializable { def convert(targetCellType: CellType): ArrayTile = { val tile = ArrayTile.alloc(targetCellType, cols, rows) - if(targetCellType.isFloatingPoint != cellType.isFloatingPoint) - logger.debug(s"Conversion from $cellType to $targetCellType may lead to data loss.") - if(!cellType.isFloatingPoint) { cfor(0)(_ < rows, _ + 1) { row => cfor(0)(_ < cols, _ + 1) { col => diff --git a/raster/src/main/scala/geotrellis/raster/CompositeTile.scala b/raster/src/main/scala/geotrellis/raster/CompositeTile.scala index 5d695ebc73..fa11134558 100644 --- a/raster/src/main/scala/geotrellis/raster/CompositeTile.scala +++ b/raster/src/main/scala/geotrellis/raster/CompositeTile.scala @@ -166,9 +166,6 @@ case class CompositeTile(tiles: Seq[Tile], if (cols.toLong * rows.toLong > Int.MaxValue.toLong) { sys.error("This tiled raster is too big to convert into an array.") } else { - if(targetCellType.isFloatingPoint != cellType.isFloatingPoint) - logger.warn(s"Conversion from $cellType to $targetCellType may lead to data loss.") - val tile = ArrayTile.alloc(targetCellType, cols, rows) val len = cols * rows val layoutCols = tileLayout.layoutCols diff --git a/raster/src/main/scala/geotrellis/raster/ConstantTile.scala b/raster/src/main/scala/geotrellis/raster/ConstantTile.scala index 8f554cad46..c0f063fa91 100644 --- a/raster/src/main/scala/geotrellis/raster/ConstantTile.scala +++ b/raster/src/main/scala/geotrellis/raster/ConstantTile.scala @@ -75,9 +75,6 @@ trait ConstantTile extends Tile { * @return The new Tile */ def convert(newType: CellType): Tile = { - if(newType.isFloatingPoint != cellType.isFloatingPoint) - logger.warn(s"Conversion from $cellType to $newType may lead to data loss.") - newType match { case BitCellType => new BitConstantTile(if (iVal == 0) false else true, cols, rows) case ct: ByteCells => ByteConstantTile(iVal.toByte, cols, rows, ct) diff --git a/raster/src/main/scala/geotrellis/raster/CroppedTile.scala b/raster/src/main/scala/geotrellis/raster/CroppedTile.scala index edebcf308c..9caa8de707 100644 --- a/raster/src/main/scala/geotrellis/raster/CroppedTile.scala +++ b/raster/src/main/scala/geotrellis/raster/CroppedTile.scala @@ -138,9 +138,6 @@ case class CroppedTile(sourceTile: Tile, * @return An MutableArrayTile */ def mutable(targetCellType: CellType): MutableArrayTile = { - if(targetCellType.isFloatingPoint != cellType.isFloatingPoint) - logger.warn(s"Conversion from $cellType to $targetCellType may lead to data loss.") - val tile = ArrayTile.alloc(targetCellType, cols, rows) if(!cellType.isFloatingPoint) { diff --git a/raster/src/main/scala/geotrellis/raster/GridExtent.scala b/raster/src/main/scala/geotrellis/raster/GridExtent.scala index e769bdd8ee..801549cc5e 100644 --- a/raster/src/main/scala/geotrellis/raster/GridExtent.scala +++ b/raster/src/main/scala/geotrellis/raster/GridExtent.scala @@ -16,10 +16,13 @@ package geotrellis.raster +import geotrellis.proj4.{CRS, Transform} +import geotrellis.raster.reproject.Reproject.Options +import geotrellis.raster.reproject.ReprojectRasterExtent import geotrellis.vector._ -import scala.math.{min, max, ceil} -import spire.math.{Integral} +import scala.math.{ceil, max, min} +import spire.math.Integral import spire.implicits._ /** @@ -280,6 +283,28 @@ class GridExtent[@specialized(Int, Long) N: Integral]( def createAlignedRasterExtent(targetExtent: Extent): RasterExtent = createAlignedGridExtent(targetExtent).toRasterExtent + /** + * This method copies gdalwarp -tap logic: + * + * The actual code reference: https://github.com/OSGeo/gdal/blob/v2.3.2/gdal/apps/gdal_rasterize_lib.cpp#L402-L461 + * The actual part with the -tap logic: https://github.com/OSGeo/gdal/blob/v2.3.2/gdal/apps/gdal_rasterize_lib.cpp#L455-L461 + * + * The initial PR that introduced that feature in GDAL 1.8.0: https://trac.osgeo.org/gdal/attachment/ticket/3772/gdal_tap.patch + * A discussion thread related to it: https://lists.osgeo.org/pipermail/gdal-dev/2010-October/thread.html#26209 + * + */ + def alignTargetPixels: GridExtent[N] = { + val extent = this.extent + val cellSize @ CellSize(width, height) = this.cellSize + + GridExtent[N](Extent( + xmin = math.floor(extent.xmin / width) * width, + ymin = math.floor(extent.ymin / height) * height, + xmax = math.ceil(extent.xmax / width) * width, + ymax = math.ceil(extent.ymax / height) * height + ), cellSize) + } + /** * Gets the Extent that matches the grid bounds passed in, aligned * with this RasterExtent. @@ -391,4 +416,18 @@ object GridExtent { if (math.abs(value - roundedValue) < GridExtent.epsilon) roundedValue else math.floor(value) } + + implicit class gridExtentMethods[N: Integral](self: GridExtent[N]) { + def reproject(src: CRS, dest: CRS, options: Options): GridExtent[N] = + if(src == dest) self + else { + val transform = Transform(src, dest) + options + .targetRasterExtent + .map(_.toGridType[N]) + .getOrElse(ReprojectRasterExtent(self, transform, options = options)) + } + + def reproject(src: CRS, dest: CRS): GridExtent[N] = reproject(src, dest, Options.DEFAULT) + } } diff --git a/raster/src/main/scala/geotrellis/raster/MosaicMetadata.scala b/raster/src/main/scala/geotrellis/raster/MosaicMetadata.scala new file mode 100644 index 0000000000..ea9f0bd796 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/MosaicMetadata.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.proj4.CRS + +import cats.data.NonEmptyList + +case class MosaicMetadata( + name: SourceName, + crs: CRS, + bandCount: Int, + cellType: CellType, + gridExtent: GridExtent[Long], + resolutions: List[GridExtent[Long]], + list: NonEmptyList[RasterMetadata] +) extends RasterMetadata { + /** Mosaic metadata usually doesn't contain a metadata that is common for all RasterSources */ + def attributes: Map[String, String] = Map.empty + def attributesForBand(band: Int): Map[String, String] = Map.empty +} diff --git a/raster/src/main/scala/geotrellis/raster/MosaicRasterSource.scala b/raster/src/main/scala/geotrellis/raster/MosaicRasterSource.scala new file mode 100644 index 0000000000..c42ea348d0 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/MosaicRasterSource.scala @@ -0,0 +1,182 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.vector._ +import geotrellis.raster._ +import geotrellis.raster.resample._ +import geotrellis.raster.reproject.Reproject +import geotrellis.proj4.{CRS, WebMercator} +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} + +import cats.Semigroup +import cats.implicits._ +import cats.data.NonEmptyList +import spire.math.Integral + +/** + * Single threaded instance of a reader for reading windows out of collections + * of rasters + * + * @param sources The underlying [[RasterSource]]s that you'll use for data access + * @param crs The crs to reproject all [[RasterSource]]s to anytime we need information about their data + * Since MosaicRasterSources represent collections of [[RasterSource]]s, we don't know in advance + * whether they'll have the same CRS. crs allows specifying the CRS on read instead of + * having to make sure at compile time that you're threading CRSes through everywhere correctly. + */ +trait MosaicRasterSource extends RasterSource { + + val sources: NonEmptyList[RasterSource] + val crs: CRS + def gridExtent: GridExtent[Long] + + import MosaicRasterSource._ + + val targetCellType = None + + /** + * The bandCount of the first [[RasterSource]] in sources + * + * If this value is larger than the bandCount of later [[RasterSource]]s in sources, + * reads of all bands will fail. It is a client's responsibility to construct + * mosaics that can be read. + */ + def bandCount: Int = sources.head.bandCount + + def cellType: CellType = { + val cellTypes = sources map { _.cellType } + cellTypes.tail.foldLeft(cellTypes.head)(_ union _) + } + + /** All available RasterSources metadata. */ + def metadata: MosaicMetadata = MosaicMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, sources) + + def attributes: Map[String, String] = Map.empty + + def attributesForBand(band: Int): Map[String, String] = Map.empty + + /** + * All available resolutions for all RasterSources in this MosaicRasterSource + * + * @see [[geotrellis.contrib.vlm.RasterSource.resolutions]] + */ + def resolutions: List[GridExtent[Long]] = sources.map { _.resolutions }.reduce + + /** Create a new MosaicRasterSource with sources transformed according to the provided + * crs, options, and strategy, and a new crs + * + * @see [[geotrellis.contrib.vlm.RasterSource.reproject]] + */ + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + MosaicRasterSource( + sources map { _.reproject(targetCRS, resampleGrid, method, strategy) }, + crs, + gridExtent.reproject(this.crs, targetCRS, Reproject.Options.DEFAULT.copy(method = method)), + name + ) + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val rasters = sources map { _.read(extent, bands) } + rasters.reduce + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val rasters = sources map { _.read(bounds, bands) } + rasters.reduce + } + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = MosaicRasterSource( + sources map { _.resample(resampleGrid, method, strategy) }, crs, name) + + def convert(targetCellType: TargetCellType): RasterSource = + MosaicRasterSource(sources map { _.convert(targetCellType) }, crs, name) +} + +object MosaicRasterSource { + // Orphan instance for semigroups for rasters, so we can combine + // Option[Raster[_]]s later + implicit val rasterSemigroup: Semigroup[Raster[MultibandTile]] = + new Semigroup[Raster[MultibandTile]] { + def combine(l: Raster[MultibandTile], r: Raster[MultibandTile]) = { + val targetRE = RasterExtent( + l.rasterExtent.extent combine r.rasterExtent.extent, + List(l.rasterExtent.cellSize, r.rasterExtent.cellSize).minBy(_.resolution)) + val result = l.resample(targetRE) merge r.resample(targetRE) + result + } + } + + implicit def gridExtentSemigroup[N: Integral]: Semigroup[GridExtent[N]] = + new Semigroup[GridExtent[N]] { + def combine(l: GridExtent[N], r: GridExtent[N]): GridExtent[N] = { + if (l.cellwidth != r.cellwidth) + throw GeoAttrsError(s"illegal cellwidths: ${l.cellwidth} and ${r.cellwidth}") + if (l.cellheight != r.cellheight) + throw GeoAttrsError(s"illegal cellheights: ${l.cellheight} and ${r.cellheight}") + + val newExtent = l.extent.combine(r.extent) + val newRows = Integral[N].fromDouble(math.round(newExtent.height / l.cellheight)) + val newCols = Integral[N].fromDouble(math.round(newExtent.width / l.cellwidth)) + new GridExtent[N](newExtent, l.cellwidth, l.cellheight, newCols, newRows) + } + } + + def apply(sourcesList: NonEmptyList[RasterSource], targetCRS: CRS, targetGridExtent: GridExtent[Long]): MosaicRasterSource = + apply(sourcesList, targetCRS, targetGridExtent, EmptyName) + + def apply(sourcesList: NonEmptyList[RasterSource], targetCRS: CRS, targetGridExtent: GridExtent[Long], rasterSourceName: SourceName): MosaicRasterSource = { + new MosaicRasterSource { + val name = rasterSourceName + val sources = sourcesList map { _.reprojectToGrid(targetCRS, gridExtent) } + val crs = targetCRS + + def gridExtent: GridExtent[Long] = targetGridExtent + } + } + + def apply(sourcesList: NonEmptyList[RasterSource], targetCRS: CRS): MosaicRasterSource = + apply(sourcesList, targetCRS, EmptyName) + + def apply(sourcesList: NonEmptyList[RasterSource], targetCRS: CRS, rasterSourceName: SourceName): MosaicRasterSource = { + new MosaicRasterSource { + val name = rasterSourceName + val sources = sourcesList map { _.reprojectToGrid(targetCRS, sourcesList.head.gridExtent) } + val crs = targetCRS + def gridExtent: GridExtent[Long] = { + val reprojectedExtents = + sourcesList map { source => source.gridExtent.reproject(source.crs, targetCRS) } + val minCellSize: CellSize = reprojectedExtents.toList map { rasterExtent => + CellSize(rasterExtent.cellwidth, rasterExtent.cellheight) + } minBy { _.resolution } + reprojectedExtents.toList.reduce( + (re1: GridExtent[Long], re2: GridExtent[Long]) => { + re1.withResolution(minCellSize) combine re2.withResolution(minCellSize) + } + ) + } + } + } + + @SuppressWarnings(Array("TraversableHead", "TraversableTail")) + def unsafeFromList(sourcesList: List[RasterSource], targetCRS: CRS = WebMercator, targetGridExtent: Option[GridExtent[Long]], rasterSourceName: SourceName = EmptyName): MosaicRasterSource = + new MosaicRasterSource { + val name = rasterSourceName + val sources = NonEmptyList(sourcesList.head, sourcesList.tail) + val crs = targetCRS + def gridExtent: GridExtent[Long] = targetGridExtent getOrElse { sourcesList.head.gridExtent} + } +} diff --git a/raster/src/main/scala/geotrellis/raster/PaddedTile.scala b/raster/src/main/scala/geotrellis/raster/PaddedTile.scala new file mode 100644 index 0000000000..c8fbfb9f3a --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/PaddedTile.scala @@ -0,0 +1,259 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster._ +import spire.syntax.cfor._ + +case class PaddedTile(chunk: Tile, colOffset: Int, rowOffset: Int, cols: Int, rows: Int) extends Tile { + private val chunkBounds = GridBounds( + colMin = colOffset, + rowMin = rowOffset, + colMax = colOffset + chunk.cols - 1, + rowMax = rowOffset + chunk.rows - 1 + ) + + require(colOffset >= 0 && rowOffset >= 0 && colOffset < cols && rowOffset < rows, + s"chunk offset out of bounds: $colOffset, $rowOffset") + + require((chunk.cols + colOffset <= cols) && (chunk.rows + rowOffset <= rows), + s"chunk at $chunkBounds exceeds tile boundary at ($cols, $rows)") + + def cellType = chunk.cellType + + def convert(cellType: CellType): Tile = + copy(chunk = chunk.convert(cellType)) + + def withNoData(noDataValue: Option[Double]): Tile = + copy(chunk = chunk.withNoData(noDataValue)) + + def interpretAs(newCellType: CellType): Tile = + copy(chunk = chunk.interpretAs(newCellType)) + + def get(col: Int, row: Int): Int = { + if (chunkBounds.contains(col, row)) + chunk.get(col - colOffset, row - rowOffset) + else NODATA + } + + def getDouble(col: Int, row: Int): Double = { + if (chunkBounds.contains(col, row)) + chunk.getDouble(col - colOffset, row - rowOffset) + else Double.NaN + } + + def map(f: Int => Int): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row))) + } + } + tile + } + + def mapDouble(f: Double => Double): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row))) + } + } + tile + } + + def foreach(f: Int => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + if (row < rowOffset || row > (rowOffset + rows - 1)) { + cfor(0)(_ < cols, _ + 1) { _ => + f(NODATA) + } + } else { + cfor(0)(_ < colOffset, _ + 1) { _ => + f(NODATA) + } + cfor(0)(_ < chunk.cols, _ + 1) { col => + f(chunk.get(col, row - rowOffset)) + } + cfor(colOffset + chunk.cols)(_ < cols, _ + 1) { _ => + f(NODATA) + } + } + } + } + + def foreachDouble(f: Double => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + if (row < rowOffset || row > (rowOffset + chunk.rows) - 1) { + cfor(0)(_ < cols, _ + 1) { _ => + f(Double.NaN) + } + } else { + cfor(0)(_ < colOffset, _ + 1) { _ => + f(Double.NaN) + } + cfor(0)(_ < chunk.cols, _ + 1) { col => + f(chunk.getDouble(col, row - rowOffset)) + } + cfor(colOffset + chunk.cols)(_ < cols, _ + 1) { _ => + f(Double.NaN) + } + } + } + } + + def mutable(): MutableArrayTile = + mutable(cellType) + + def mutable(targetCellType: CellType): MutableArrayTile = { + val tile = ArrayTile.alloc(targetCellType, cols, rows) + + if(!cellType.isFloatingPoint) { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, get(col, row)) + } + } + } else { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, getDouble(col, row)) + } + } + } + + tile + } + + + def toArrayTile(): ArrayTile = mutable + + def toArray(): Array[Int] = { + val arr = Array.ofDim[Int](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = get(col, row) + i += 1 + } + } + + arr + } + + def toArrayDouble(): Array[Double] = { + val arr = Array.ofDim[Double](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = getDouble(col, row) + i += 1 + } + } + + arr + } + + def toBytes(): Array[Byte] = toArrayTile.toBytes + + + def combine(other: Tile)(f: (Int, Int) => Int): Tile = { + (this, other).assertEqualDimensions + + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row), other.get(col, row))) + } + } + + tile + } + + def combineDouble(other: Tile)(f: (Double, Double) => Double): Tile = { + (this, other).assertEqualDimensions + + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row), other.getDouble(col, row))) + } + } + + tile + } + + def foreachIntVisitor(visitor: IntTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + if (row < rowOffset || row > (rowOffset + rows - 1)) { + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, NODATA) + } + } else { + cfor(0)(_ < colOffset, _ + 1) { col => + visitor(col, row, NODATA) + } + cfor(0)(_ < chunk.cols, _ + 1) { col => + visitor(col + colOffset, row, chunk.get(col, row - rowOffset)) + } + cfor(colOffset + chunk.cols)(_ < cols, _ + 1) { col => + visitor(col, row, NODATA) + } + } + } + } + + def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + if (row < rowOffset || row > (rowOffset + rows - 1)) { + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, Double.NaN) + } + } else { + cfor(0)(_ < colOffset, _ + 1) { col => + visitor(col, row, Double.NaN) + } + cfor(0)(_ < chunk.cols, _ + 1) { col => + visitor(col + colOffset, row, chunk.getDouble(col, row - rowOffset)) + } + cfor(colOffset + chunk.cols)(_ < cols, _ + 1) { col => + visitor(col, row, Double.NaN) + } + } + } + } + + def mapIntMapper(mapper: IntTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + chunk.foreach { (col, row, z) => + tile.set(colOffset + col, rowOffset + row, mapper(col, row, z)) + } + + tile + } + + def mapDoubleMapper(mapper: DoubleTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + chunk.foreachDouble { (col, row, z) => + tile.setDouble(colOffset + col, rowOffset + row, mapper(col, row, z)) + } + + tile + } +} diff --git a/raster/src/main/scala/geotrellis/raster/ProjectedRasterLike.scala b/raster/src/main/scala/geotrellis/raster/ProjectedRasterLike.scala new file mode 100644 index 0000000000..229bc187f5 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/ProjectedRasterLike.scala @@ -0,0 +1,28 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.proj4.CRS +import geotrellis.vector.Extent + +/** + * Conformance interface for entities that are tile-like with a projected extent. + */ +trait ProjectedRasterLike extends CellGrid[Int] { + def crs: CRS + def extent: Extent +} diff --git a/raster/src/main/scala/geotrellis/raster/RasterMetadata.scala b/raster/src/main/scala/geotrellis/raster/RasterMetadata.scala new file mode 100644 index 0000000000..ec729d51d5 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/RasterMetadata.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.proj4.CRS +import geotrellis.vector.Extent + +trait RasterMetadata extends Serializable { + /** Source name, that can be a path or any name that is associated with Raster */ + def name: SourceName + def crs: CRS + def bandCount: Int + def cellType: CellType + + /** Cell size at which rasters will be read when using this [[RasterSource]] + * + * Note: some re-sampling of underlying raster data may be required to produce this cell size. + */ + def cellSize: CellSize = gridExtent.cellSize + + def gridExtent: GridExtent[Long] + + /** All available resolutions for this raster source + * + *
  • For base [[RasterSource]] instance this will be resolutions of available overviews. + *
  • For reprojected [[RasterSource]] these resolutions represent an estimate where + * each cell in target CRS has ''approximately'' the same geographic coverage as a cell in the source CRS. + * + * When reading raster data the underlying implementation will have to sample from one of these resolutions. + * It is possible that a read request for a small bounding box will results in significant IO request when the target + * cell size is much larger than closest available resolution. + * + * __Note__: It is expected but not guaranteed that the extent each [[RasterExtent]] in this list will be the same. + */ + def resolutions: List[GridExtent[Long]] + + def extent: Extent = gridExtent.extent + + /** Raster pixel column count */ + def cols: Long = gridExtent.cols + + /** Raster pixel row count */ + def rows: Long = gridExtent.rows + + /** + * Return the "base" metadata, usually it is a zero band metadata, + * a metadata that is valid for the entire source and for the zero band + */ + def attributes: Map[String, String] + /** + * Return a per band metadata + */ + def attributesForBand(band: Int): Map[String, String] +} diff --git a/raster/src/main/scala/geotrellis/raster/RasterRegion.scala b/raster/src/main/scala/geotrellis/raster/RasterRegion.scala new file mode 100644 index 0000000000..fbcce6ba11 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/RasterRegion.scala @@ -0,0 +1,62 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.proj4.CRS +import geotrellis.vector.Extent + +/** + * Reference to a pixel region in a [[RasterSource]] that may be read at a later time. + */ +trait RasterRegion extends ProjectedRasterLike with Serializable { + def raster: Option[Raster[MultibandTile]] +} + +object RasterRegion { + /** Reference to a pixel region in a [[RasterSource]] that may be read at a later time. + * @note It is required that the [[RasterSource]] intersects with the given [[GridBounds]]. + * + * @param source raster source that can be used to read this region. + * @param bounds pixel bounds relative to the source, maybe not be fully contained by the source bounds. + */ + def apply(source: RasterSource, bounds: GridBounds[Long]): RasterRegion = + GridBoundsRasterRegion(source, bounds) + + case class GridBoundsRasterRegion(source: RasterSource, bounds: GridBounds[Long]) extends RasterRegion { + require(bounds.intersects(source.gridBounds), s"The given bounds: $bounds must intersect the given source: $source") + @transient lazy val raster: Option[Raster[MultibandTile]] = + for { + intersection <- source.gridBounds.intersection(bounds) + raster <- source.read(intersection) + } yield { + if (raster.tile.cols == cols && raster.tile.rows == rows) + raster + else { + val colOffset = math.abs(bounds.colMin - intersection.colMin) + val rowOffset = math.abs(bounds.rowMin - intersection.rowMin) + require(colOffset <= Int.MaxValue && rowOffset <= Int.MaxValue, "Computed offsets are outside of RasterBounds") + raster.mapTile { _.mapBands { (_, band) => PaddedTile(band, colOffset.toInt, rowOffset.toInt, cols, rows) } } + } + } + + override def cols: Int = bounds.width.toInt + override def rows: Int = bounds.height.toInt + override def extent: Extent = source.gridExtent.extentFor(bounds, clamp = false) + override def crs: CRS = source.crs + override def cellType: CellType = source.cellType + } +} diff --git a/raster/src/main/scala/geotrellis/raster/RasterSource.scala b/raster/src/main/scala/geotrellis/raster/RasterSource.scala new file mode 100644 index 0000000000..72070454a5 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/RasterSource.scala @@ -0,0 +1,214 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.vector._ +import geotrellis.raster._ +import geotrellis.raster.resample._ +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} +import geotrellis.util.GetComponent + +import java.util.ServiceLoader + +/** + * Single threaded instance of a reader that is able to read windows from larger raster. + * Some initilization step is expected to provide metadata about source raster + * + * @groupname read Read + * @groupdesc read Functions to read windows of data from a raster source. + * @groupprio read 0 + + * @groupname resample Resample + * @groupdesc resample Functions to resample raster data in native projection. + * @groupprio resample 1 + * + * @groupname reproject Reproject + * @groupdesc reproject Functions to resample raster data in target projection. + * @groupprio reproject 2 + */ +trait RasterSource extends CellGrid[Long] with RasterMetadata { + /** All available RasterSource metadata */ + def metadata: RasterMetadata + + protected def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource + + /** Reproject to different CRS with explicit sampling reprojectOptions. + * @see [[geotrellis.raster.reproject.Reproject]] + * @group reproject + */ + def reproject(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + if (targetCRS == this.crs) this + else reprojection(targetCRS, resampleGrid, method, strategy) + + + /** Sampling grid and resolution is defined by given [[GridExtent]]. + * Resulting extent is the extent of the minimum enclosing pixel region + * of the data footprint in the target grid. + * @group reproject a + */ + def reprojectToGrid(targetCRS: CRS, grid: GridExtent[Long], method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + if (targetCRS == this.crs && grid == this.gridExtent) this + else if (targetCRS == this.crs) resampleToGrid(grid, method) + else reprojection(targetCRS, TargetGrid[Long](grid), method, strategy) + + /** Sampling grid and resolution is defined by given [[RasterExtent]] region. + * The extent of the result is also taken from given [[RasterExtent]], + * this region may be larger or smaller than the footprint of the data + * @group reproject + */ + def reprojectToRegion(targetCRS: CRS, region: RasterExtent, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + if (targetCRS == this.crs && region == this.gridExtent) this + else if (targetCRS == this.crs) resampleToRegion(region.asInstanceOf[GridExtent[Long]], method) + else reprojection(targetCRS, TargetRegion[Long](region.toGridType[Long]), method, strategy) + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource + + /** Sampling grid is defined of the footprint of the data with resolution implied by column and row count. + * @group resample + */ + def resample(targetCols: Long, targetRows: Long, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + resample(Dimensions(targetCols, targetRows), method, strategy) + + /** Sampling grid and resolution is defined by given [[GridExtent]]. + * Resulting extent is the extent of the minimum enclosing pixel region + * of the data footprint in the target grid. + * @group resample + */ + def resampleToGrid(grid: GridExtent[Long], method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + resample(TargetGrid[Long](grid), method, strategy) + + /** Sampling grid and resolution is defined by given [[RasterExtent]] region. + * The extent of the result is also taken from given [[RasterExtent]], + * this region may be larger or smaller than the footprint of the data + * @group resample + */ + def resampleToRegion(region: GridExtent[Long], method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + resample(TargetRegion[Long](region), method, strategy) + + /** Reads a window for the extent. + * Return extent may be smaller than requested extent around raster edges. + * May return None if the requested extent does not overlap the raster extent. + * @group read + */ + @throws[IndexOutOfBoundsException]("if requested bands do not exist") + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] + + /** Reads a window for pixel bounds. + * Return extent may be smaller than requested extent around raster edges. + * May return None if the requested extent does not overlap the raster extent. + * @group read + */ + @throws[IndexOutOfBoundsException]("if requested bands do not exist") + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] + + /** + * @group read + */ + def read(extent: Extent): Option[Raster[MultibandTile]] = + read(extent, (0 until bandCount)) + + /** + * @group read + */ + def read(bounds: GridBounds[Long]): Option[Raster[MultibandTile]] = + read(bounds, (0 until bandCount)) + + /** + * @group read + */ + def read(): Option[Raster[MultibandTile]] = + read(extent, (0 until bandCount)) + + /** + * @group read + */ + def read(bands: Seq[Int]): Option[Raster[MultibandTile]] = + read(extent, bands) + + /** + * @group read + */ + def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + extents.toIterator.flatMap(read(_, bands).toIterator) + + /** + * @group read + */ + def readExtents(extents: Traversable[Extent]): Iterator[Raster[MultibandTile]] = + readExtents(extents, (0 until bandCount)) + /** + * @group read + */ + def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + bounds.toIterator.flatMap(read(_, bands).toIterator) + + /** + * @group read + */ + def readBounds(bounds: Traversable[GridBounds[Long]]): Iterator[Raster[MultibandTile]] = + bounds.toIterator.flatMap(read(_, (0 until bandCount)).toIterator) + + private[raster] def targetCellType: Option[TargetCellType] + + protected lazy val dstCellType: Option[CellType] = + targetCellType match { + case Some(target) => Some(target.cellType) + case None => None + } + + protected lazy val convertRaster: Raster[MultibandTile] => Raster[MultibandTile] = + targetCellType match { + case Some(target: TargetCellType) => + (raster: Raster[MultibandTile]) => target(raster) + case _ => + (raster: Raster[MultibandTile]) => raster + } + + def convert(targetCellType: TargetCellType): RasterSource + + /** Converts the values within the RasterSource from one [[CellType]] to another. + * + * Note: + * + * [[GDALRasterSource]] differs in how it converts data from the other RasterSources. + * Please see the convert docs for [[GDALRasterSource]] for more information. + * @group convert + */ + def convert(targetCellType: CellType): RasterSource = + convert(ConvertTargetCellType(targetCellType)) + + def interpretAs(targetCellType: CellType): RasterSource = + convert(InterpretAsTargetCellType(targetCellType)) +} + +object RasterSource { + implicit def projectedExtentComponent[T <: RasterSource]: GetComponent[T, ProjectedExtent] = + GetComponent(rs => ProjectedExtent(rs.extent, rs.crs)) + + def apply(path: String): RasterSource = { + import scala.collection.JavaConverters._ + + ServiceLoader + .load(classOf[RasterSourceProvider]) + .iterator() + .asScala + .find(_.canProcess(path)) + .getOrElse(throw new RuntimeException(s"Unable to find RasterSource for $path")) + .rasterSource(path) + } +} diff --git a/raster/src/main/scala/geotrellis/raster/RasterSourceProvider.scala b/raster/src/main/scala/geotrellis/raster/RasterSourceProvider.scala new file mode 100644 index 0000000000..e757eeeb8c --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/RasterSourceProvider.scala @@ -0,0 +1,23 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +trait RasterSourceProvider { + def canProcess(path: String): Boolean + + def rasterSource(path: String): RasterSource +} diff --git a/raster/src/main/scala/geotrellis/raster/ReadingSource.scala b/raster/src/main/scala/geotrellis/raster/ReadingSource.scala new file mode 100644 index 0000000000..71937bf0c8 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/ReadingSource.scala @@ -0,0 +1,30 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +case class ReadingSource( + source: RasterSource, + sourceToTargetBand: Map[Int, Int] +) extends Serializable + +object ReadingSource { + def apply(source: RasterSource, sourceBand: Int, targetBand: Int): ReadingSource = + ReadingSource(source, Map(sourceBand -> targetBand)) + + def apply(source: RasterSource, targetBand: Int): ReadingSource = + apply(source, 0, targetBand) +} diff --git a/raster/src/main/scala/geotrellis/raster/ResampleGrid.scala b/raster/src/main/scala/geotrellis/raster/ResampleGrid.scala new file mode 100644 index 0000000000..4f6ca9a174 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/ResampleGrid.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.reproject.Reproject + +import spire.math.Integral +import spire.implicits._ + +sealed trait ResampleGrid[N] { + // this is a by name parameter, as we don't need to call the source in all ResampleGrid types + def apply(source: => GridExtent[N]): GridExtent[N] +} + +case class Dimensions[N: Integral](cols: N, rows: N) extends ResampleGrid[N] { + def apply(source: => GridExtent[N]): GridExtent[N] = + new GridExtent(source.extent, cols, rows) +} + +case class TargetGrid[N: Integral](grid: GridExtent[Long]) extends ResampleGrid[N] { + def apply(source: => GridExtent[N]): GridExtent[N] = + grid.createAlignedGridExtent(source.extent).toGridType[N] +} + +case class TargetRegion[N: Integral](region: GridExtent[N]) extends ResampleGrid[N] { + def apply(source: => GridExtent[N]): GridExtent[N] = + region +} + +case class TargetCellSize[N: Integral](cellSize: CellSize) extends ResampleGrid[N] { + def apply(source: => GridExtent[N]): GridExtent[N] = + source.withResolution(cellSize) +} + +case object IdentityResampleGrid extends ResampleGrid[Long] { + def apply(source: => GridExtent[Long]): GridExtent[Long] = source +} + + +object ResampleGrid { + /** Used when reprojecting to original RasterSource CRS, pick-out the grid */ + private[geotrellis] def fromReprojectOptions(options: Reproject.Options): ResampleGrid[Long] ={ + if (options.targetRasterExtent.isDefined) { + TargetRegion(options.targetRasterExtent.get.toGridType[Long]) + } else if (options.parentGridExtent.isDefined) { + TargetGrid(options.parentGridExtent.get) + } else if (options.targetCellSize.isDefined) { + ??? // TODO: convert from CellSize to Column count based on ... something + } else { + IdentityResampleGrid + } + } + + /** Used when resampling on already reprojected RasterSource */ + private[geotrellis] def toReprojectOptions[N: Integral]( + current: GridExtent[Long], + resampleGrid: ResampleGrid[N], + resampleMethod: ResampleMethod + ): Reproject.Options = { + resampleGrid match { + case Dimensions(cols, rows) => + val updated = current.withDimensions(cols.toLong, rows.toLong).toGridType[Int] + Reproject.Options(method = resampleMethod, targetRasterExtent = Some(updated.toRasterExtent)) + + case TargetGrid(grid) => + Reproject.Options(method = resampleMethod, parentGridExtent = Some(grid.toGridType[Long])) + + case TargetRegion(region) => + Reproject.Options(method = resampleMethod, targetRasterExtent = Some(region.toGridType[Int].toRasterExtent)) + + case TargetCellSize(cellSize) => + Reproject.Options(method = resampleMethod, targetCellSize = Some(cellSize)) + + case IdentityResampleGrid => + Reproject.Options.DEFAULT.copy(method = resampleMethod) + } + } +} \ No newline at end of file diff --git a/raster/src/main/scala/geotrellis/raster/SourceName.scala b/raster/src/main/scala/geotrellis/raster/SourceName.scala new file mode 100644 index 0000000000..0bf8f8d9fb --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/SourceName.scala @@ -0,0 +1,29 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +/** + * Represents the path to data or name of the data that is to be read. + */ +trait SourceName extends Serializable + +case class StringName(value: String) extends SourceName +case object EmptyName extends SourceName + +object SourceName { + implicit def stringToDataName(str: String): SourceName = if(str.isEmpty) EmptyName else StringName(str) +} diff --git a/raster/src/main/scala/geotrellis/raster/SourcePath.scala b/raster/src/main/scala/geotrellis/raster/SourcePath.scala new file mode 100644 index 0000000000..9238aed10b --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/SourcePath.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import java.net.URI + +/** + * Represents the path to data that is to be read. + */ +trait SourcePath extends SourceName { + + /** + * The given path to the data. This can be formatted in a number of different + * ways depending on which [[RasterSource]] is to be used. For more information + * on the different ways of formatting this string, see the docs on the + * DataPath for that given soure. + */ + def value: String + + override def toString: String = value + + def toURI: URI = new URI(value) +} diff --git a/raster/src/main/scala/geotrellis/raster/TargetCellType.scala b/raster/src/main/scala/geotrellis/raster/TargetCellType.scala new file mode 100644 index 0000000000..43095b291d --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/TargetCellType.scala @@ -0,0 +1,33 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +sealed trait TargetCellType { + def cellType: CellType + + def apply(output: => Raster[MultibandTile]): Raster[MultibandTile] +} + +case class ConvertTargetCellType(cellType: CellType) extends TargetCellType { + def apply(output: => Raster[MultibandTile]): Raster[MultibandTile] = + output.mapTile { _.convert(cellType) } +} + +case class InterpretAsTargetCellType(cellType: CellType) extends TargetCellType { + def apply(output: => Raster[MultibandTile]): Raster[MultibandTile] = + output.mapTile { _.interpretAs(cellType) } +} diff --git a/raster/src/main/scala/geotrellis/raster/Tile.scala b/raster/src/main/scala/geotrellis/raster/Tile.scala index 92885d0f4d..7ba800775f 100644 --- a/raster/src/main/scala/geotrellis/raster/Tile.scala +++ b/raster/src/main/scala/geotrellis/raster/Tile.scala @@ -16,12 +16,10 @@ package geotrellis.raster -import com.typesafe.scalalogging.LazyLogging - /** * Base trait for a Tile. */ -trait Tile extends CellGrid[Int] with IterableTile with MappableTile[Tile] with LazyLogging { +trait Tile extends CellGrid[Int] with IterableTile with MappableTile[Tile] { /** * Execute a function at each pixel of a [[Tile]]. Two functions * are given: an integer version which is used if the tile is an diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffMetadata.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffMetadata.scala new file mode 100644 index 0000000000..42c64e08c4 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffMetadata.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster.{RasterMetadata, SourceName} +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, GridExtent} +import geotrellis.raster.io.geotiff.Tags + +case class GeoTiffMetadata( + name: SourceName, + crs: CRS, + bandCount: Int, + cellType: CellType, + gridExtent: GridExtent[Long], + resolutions: List[GridExtent[Long]], + tags: Tags +) extends RasterMetadata { + /** Returns the GeoTiff head tags. */ + def attributes: Map[String, String] = tags.headTags + /** Returns the GeoTiff per band tags. */ + def attributesForBand(band: Int): Map[String, String] = tags.bandTags.lift(band).getOrElse(Map.empty) +} \ No newline at end of file diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffPath.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffPath.scala new file mode 100644 index 0000000000..5c4bc71c87 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffPath.scala @@ -0,0 +1,63 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster.SourcePath + +import cats.syntax.option._ +import io.lemonlabs.uri.Uri +import io.lemonlabs.uri.encoding.PercentEncoder +import io.lemonlabs.uri.encoding.PercentEncoder.PATH_CHARS_TO_ENCODE + +import java.net.MalformedURLException + +/** Represents a VALID path that points to a GeoTiff to be read. + * @note The target file must have a file extension. + * + * @param value Path to a GeoTiff. There are two ways to format this `String`: either + * in the `URI` format, or as a relative path if the file is local. In addition, + * this path can be prefixed with, '''gtiff+''' to signify that the target GeoTiff + * is to be read in only by [[GeoTiffRasterSource]]. + * @example "data/my-data.tiff" + * @example "s3://bucket/prefix/data.tif" + * @example "gtiff+file:///tmp/data.tiff" + * + * @note Capitalization of the extension is not regarded. + */ +case class GeoTiffPath(value: String) extends SourcePath { + /** Function that points to the file with external overviews. */ + def externalOverviews: String = s"$value.ovr" +} + +object GeoTiffPath { + val PREFIX = "gtiff+" + + implicit def toGeoTiffDataPath(path: String): GeoTiffPath = parse(path) + + def parseOption(path: String, percentEncoder: PercentEncoder = PercentEncoder(PATH_CHARS_TO_ENCODE ++ Set('%', '?', '#'))): Option[GeoTiffPath] = { + val upath = percentEncoder.encode(path, "UTF-8") + Uri.parseOption(upath).fold(Option.empty[GeoTiffPath]) { uri => + GeoTiffPath(uri.schemeOption.fold(uri.toStringRaw) { scheme => + uri.withScheme(scheme.split("\\+").last).toStringRaw + }).some + } + } + + def parse(path: String, percentEncoder: PercentEncoder = PercentEncoder(PATH_CHARS_TO_ENCODE ++ Set('%', '?', '#'))): GeoTiffPath = + parseOption(path, percentEncoder).getOrElse(throw new MalformedURLException(s"Unable to parse GeoTiffDataPath: $path")) +} + diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSource.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSource.scala new file mode 100644 index 0000000000..8c8697fc80 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSource.scala @@ -0,0 +1,107 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.vector._ +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} +import geotrellis.raster.io.geotiff._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.util.RangeReader + +class GeoTiffRasterSource( + val dataPath: GeoTiffPath, + private[raster] val targetCellType: Option[TargetCellType] = None, + @transient private[raster] val baseTiff: Option[MultibandGeoTiff] = None +) extends RasterSource { + def name: GeoTiffPath = dataPath + + @transient lazy val tiff: MultibandGeoTiff = + Option(baseTiff) + .flatten + .getOrElse(GeoTiffReader.readMultiband( + RangeReader(dataPath.value), + streaming = true, withOverviews = true, + RangeReader.validated(dataPath.externalOverviews) + )) + + def bandCount: Int = tiff.bandCount + def cellType: CellType = dstCellType.getOrElse(tiff.cellType) + def tags: Tags = tiff.tags + def metadata: GeoTiffMetadata = GeoTiffMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, tags) + + /** Returns the GeoTiff head tags. */ + def attributes: Map[String, String] = tags.headTags + /** Returns the GeoTiff per band tags. */ + def attributesForBand(band: Int): Map[String, String] = tags.bandTags.lift(band).getOrElse(Map.empty) + + def crs: CRS = tiff.crs + + lazy val gridExtent: GridExtent[Long] = tiff.rasterExtent.toGridType[Long] + lazy val resolutions: List[GridExtent[Long]] = gridExtent :: tiff.overviews.map(_.rasterExtent.toGridType[Long]) + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): GeoTiffReprojectRasterSource = + GeoTiffReprojectRasterSource(dataPath, targetCRS, resampleGrid, method, strategy, targetCellType = targetCellType, baseTiff = Some(tiff)) + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): GeoTiffResampleRasterSource = + GeoTiffResampleRasterSource(dataPath, resampleGrid, method, strategy, targetCellType, Some(tiff)) + + def convert(targetCellType: TargetCellType): GeoTiffRasterSource = + GeoTiffRasterSource(dataPath, Some(targetCellType), Some(tiff)) + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val bounds = gridExtent.gridBoundsFor(extent, clamp = false).toGridType[Int] + val geoTiffTile = tiff.tile.asInstanceOf[GeoTiffMultibandTile] + val it = geoTiffTile.crop(List(bounds), bands.toArray).map { case (gb, tile) => + // TODO: shouldn't GridExtent give me Extent for types other than N ? + Raster(tile, gridExtent.extentFor(gb.toGridType[Long], clamp = false)) + } + + // We want to use this tiff in different `RasterSource`s, so we + // need to lock it in order to garuntee the state of tiff when + // it's being accessed by a thread. + tiff.synchronized { if (it.hasNext) Some(convertRaster(it.next)) else None } + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val it = readBounds(List(bounds), bands) + + tiff.synchronized { if (it.hasNext) Some(it.next) else None } + } + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val bounds = extents.map(gridExtent.gridBoundsFor(_, clamp = true)) + + readBounds(bounds, bands) + } + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val geoTiffTile = tiff.tile.asInstanceOf[GeoTiffMultibandTile] + val intersectingBounds: Seq[GridBounds[Int]] = + bounds.flatMap(_.intersection(this.gridBounds)).toSeq.map(_.toGridType[Int]) + + geoTiffTile.crop(intersectingBounds, bands.toArray).map { case (gb, tile) => + convertRaster(Raster(tile, gridExtent.extentFor(gb.toGridType[Long], clamp = true))) + } + } +} + +object GeoTiffRasterSource { + def apply(dataPath: GeoTiffPath, targetCellType: Option[TargetCellType] = None, baseTiff: Option[MultibandGeoTiff] = None): GeoTiffRasterSource = + new GeoTiffRasterSource(dataPath, targetCellType, baseTiff) +} diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProvider.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProvider.scala new file mode 100644 index 0000000000..95d2af1df5 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProvider.scala @@ -0,0 +1,26 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster.{RasterSource, RasterSourceProvider} + +class GeoTiffRasterSourceProvider extends RasterSourceProvider { + def canProcess(path: String): Boolean = + (!path.startsWith("gt+") && !path.startsWith("gdal+")) && path.nonEmpty && GeoTiffPath.parseOption(path).nonEmpty + + def rasterSource(path: String): RasterSource = GeoTiffRasterSource(path) +} diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSource.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSource.scala new file mode 100644 index 0000000000..f2b2221177 --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSource.scala @@ -0,0 +1,170 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.vector.Extent +import geotrellis.raster._ +import geotrellis.raster.reproject._ +import geotrellis.raster.resample._ +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.io.geotiff.{AutoHigherResolution, GeoTiff, GeoTiffMultibandTile, MultibandGeoTiff, OverviewStrategy, Tags} +import geotrellis.util.RangeReader + +class GeoTiffReprojectRasterSource( + val dataPath: GeoTiffPath, + val crs: CRS, + val targetResampleGrid: ResampleGrid[Long] = IdentityResampleGrid, + val resampleMethod: ResampleMethod = NearestNeighbor, + val strategy: OverviewStrategy = AutoHigherResolution, + val errorThreshold: Double = 0.125, + private[raster] val targetCellType: Option[TargetCellType] = None, + @transient private[raster] val baseTiff: Option[MultibandGeoTiff] = None +) extends RasterSource { + def name: GeoTiffPath = dataPath + + @transient lazy val tiff: MultibandGeoTiff = + Option(baseTiff) + .flatten + .getOrElse(GeoTiffReader.readMultiband( + RangeReader(dataPath.value), + streaming = true, withOverviews = true, + RangeReader.validated(dataPath.externalOverviews) + )) + + def bandCount: Int = tiff.bandCount + def cellType: CellType = dstCellType.getOrElse(tiff.cellType) + def tags: Tags = tiff.tags + def metadata: GeoTiffMetadata = GeoTiffMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, tags) + + /** Returns the GeoTiff head tags. */ + def attributes: Map[String, String] = tags.headTags + /** Returns the GeoTiff per band tags. */ + def attributesForBand(band: Int): Map[String, String] = tags.bandTags.lift(band).getOrElse(Map.empty) + + protected lazy val baseCRS: CRS = tiff.crs + protected lazy val baseGridExtent: GridExtent[Long] = tiff.rasterExtent.toGridType[Long] + + // TODO: remove transient notation with Proj4 1.1 release + @transient protected lazy val transform = Transform(baseCRS, crs) + @transient protected lazy val backTransform = Transform(crs, baseCRS) + + override lazy val gridExtent: GridExtent[Long] = { + lazy val reprojectedRasterExtent = + ReprojectRasterExtent( + baseGridExtent, + transform, + Reproject.Options.DEFAULT.copy(method = resampleMethod, errorThreshold = errorThreshold) + ) + + targetResampleGrid match { + case targetRegion: TargetRegion[Long] => targetRegion.region + case targetGrid: TargetGrid[Long] => targetGrid(reprojectedRasterExtent) + case dimensions: Dimensions[Long] => dimensions(reprojectedRasterExtent) + case targetCellSize: TargetCellSize[Long] => targetCellSize(reprojectedRasterExtent) + case _ => reprojectedRasterExtent + } + } + + lazy val resolutions: List[GridExtent[Long]] = + gridExtent :: tiff.overviews.map(ovr => ReprojectRasterExtent(ovr.rasterExtent.toGridType[Long], transform)) + + @transient private[raster] lazy val closestTiffOverview: GeoTiff[MultibandTile] = { + targetResampleGrid match { + case IdentityResampleGrid => tiff.getClosestOverview(baseGridExtent.cellSize, strategy) + case _ => + // we're asked to match specific target resolution, estimate what resolution we need in source to sample it + val estimatedSource = ReprojectRasterExtent(gridExtent, backTransform) + tiff.getClosestOverview(estimatedSource.cellSize, strategy) + } + } + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val bounds = gridExtent.gridBoundsFor(extent, clamp = false) + + read(bounds, bands) + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val it = readBounds(List(bounds), bands) + + closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None } + } + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val bounds = extents.map(gridExtent.gridBoundsFor(_, clamp = true)) + + readBounds(bounds, bands) + } + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val geoTiffTile = closestTiffOverview.tile.asInstanceOf[GeoTiffMultibandTile] + val intersectingWindows = { for { + queryPixelBounds <- bounds + targetPixelBounds <- queryPixelBounds.intersection(this.gridBounds) + } yield { + val targetRasterExtent = RasterExtent( + extent = gridExtent.extentFor(targetPixelBounds, clamp = true), + cols = targetPixelBounds.width.toInt, + rows = targetPixelBounds.height.toInt + ) + + // A tmp workaround for https://github.com/locationtech/proj4j/pull/29 + // Stacktrace details: https://github.com/geotrellis/geotrellis-contrib/pull/206#pullrequestreview-260115791 + val sourceExtent = Proj4Transform.synchronized(targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).getEnvelopeInternal) + val sourcePixelBounds = closestTiffOverview.rasterExtent.gridBoundsFor(sourceExtent, clamp = true) + (sourcePixelBounds, targetRasterExtent) + }}.toMap + + geoTiffTile.crop(intersectingWindows.keys.toSeq, bands.toArray).map { case (sourcePixelBounds, tile) => + val targetRasterExtent = intersectingWindows(sourcePixelBounds) + val sourceRaster = Raster(tile, closestTiffOverview.rasterExtent.extentFor(sourcePixelBounds, clamp = true)) + val rr = implicitly[RasterRegionReproject[MultibandTile]] + rr.regionReproject( + sourceRaster, + baseCRS, + crs, + targetRasterExtent, + targetRasterExtent.extent.toPolygon, + resampleMethod, + errorThreshold + ) + }.map { convertRaster } + } + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = + GeoTiffReprojectRasterSource(dataPath, targetCRS, resampleGrid, method, strategy, targetCellType = targetCellType, baseTiff = Some(tiff)) + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = + GeoTiffReprojectRasterSource(dataPath, crs, resampleGrid, method, strategy, targetCellType = targetCellType, baseTiff = Some(tiff)) + + def convert(targetCellType: TargetCellType): RasterSource = + GeoTiffReprojectRasterSource(dataPath, crs, targetResampleGrid, resampleMethod, strategy, targetCellType = Some(targetCellType)) +} + +object GeoTiffReprojectRasterSource { + def apply( + dataPath: GeoTiffPath, + crs: CRS, + targetResampleGrid: ResampleGrid[Long] = IdentityResampleGrid, + resampleMethod: ResampleMethod = NearestNeighbor, + strategy: OverviewStrategy = AutoHigherResolution, + errorThreshold: Double = 0.125, + targetCellType: Option[TargetCellType] = None, + baseTiff: Option[MultibandGeoTiff] = None + ): GeoTiffReprojectRasterSource = new GeoTiffReprojectRasterSource(dataPath, crs, targetResampleGrid, resampleMethod, strategy, errorThreshold, targetCellType, baseTiff) +} diff --git a/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffResampleRasterSource.scala b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffResampleRasterSource.scala new file mode 100644 index 0000000000..611fd9440a --- /dev/null +++ b/raster/src/main/scala/geotrellis/raster/geotiff/GeoTiffResampleRasterSource.scala @@ -0,0 +1,150 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.vector.Extent +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.reproject.{Reproject, ReprojectRasterExtent} +import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} +import geotrellis.raster.io.geotiff.{AutoHigherResolution, GeoTiff, GeoTiffMultibandTile, MultibandGeoTiff, OverviewStrategy, Tags} +import geotrellis.util.RangeReader + +class GeoTiffResampleRasterSource( + val dataPath: GeoTiffPath, + val resampleGrid: ResampleGrid[Long], + val method: ResampleMethod = NearestNeighbor, + val strategy: OverviewStrategy = AutoHigherResolution, + private[raster] val targetCellType: Option[TargetCellType] = None, + @transient private[raster] val baseTiff: Option[MultibandGeoTiff] = None +) extends RasterSource { + def resampleMethod: Option[ResampleMethod] = Some(method) + def name: GeoTiffPath = dataPath + + @transient lazy val tiff: MultibandGeoTiff = + Option(baseTiff) + .flatten + .getOrElse(GeoTiffReader.readMultiband( + RangeReader(dataPath.value), + streaming = true, withOverviews = true, + RangeReader.validated(dataPath.externalOverviews) + )) + + def bandCount: Int = tiff.bandCount + def cellType: CellType = dstCellType.getOrElse(tiff.cellType) + def tags: Tags = tiff.tags + def metadata: GeoTiffMetadata = GeoTiffMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, tags) + + /** Returns the GeoTiff head tags. */ + def attributes: Map[String, String] = tags.headTags + /** Returns the GeoTiff per band tags. */ + def attributesForBand(band: Int): Map[String, String] = tags.bandTags.lift(band).getOrElse(Map.empty) + + def crs: CRS = tiff.crs + + override lazy val gridExtent: GridExtent[Long] = resampleGrid(tiff.rasterExtent.toGridType[Long]) + lazy val resolutions: List[GridExtent[Long]] = { + val ratio = gridExtent.cellSize.resolution / tiff.rasterExtent.cellSize.resolution + gridExtent :: tiff.overviews.map { ovr => + val re = ovr.rasterExtent + val CellSize(cw, ch) = re.cellSize + new GridExtent[Long](re.extent, CellSize(cw * ratio, ch * ratio)) + } + } + + @transient private[raster] lazy val closestTiffOverview: GeoTiff[MultibandTile] = + tiff.getClosestOverview(gridExtent.cellSize, strategy) + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): GeoTiffReprojectRasterSource = + new GeoTiffReprojectRasterSource(dataPath, targetCRS, resampleGrid, method, strategy, targetCellType = targetCellType) { + override lazy val gridExtent: GridExtent[Long] = { + val reprojectedRasterExtent = + ReprojectRasterExtent( + baseGridExtent, + transform, + Reproject.Options.DEFAULT.copy(method = resampleMethod, errorThreshold = errorThreshold) + ) + + targetResampleGrid match { + case targetRegion: TargetRegion[Long] => targetRegion.region + case targetGrid: TargetGrid[Long] => targetGrid(reprojectedRasterExtent) + case dimensions: Dimensions[Long] => dimensions(reprojectedRasterExtent) + case targetCellSize: TargetCellSize[Long] => targetCellSize(reprojectedRasterExtent) + case _ => reprojectedRasterExtent + } + } + } + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = + GeoTiffResampleRasterSource(dataPath, resampleGrid, method, strategy, targetCellType, Some(tiff)) + + def convert(targetCellType: TargetCellType): RasterSource = + GeoTiffResampleRasterSource(dataPath, resampleGrid, method, strategy, Some(targetCellType), Some(tiff)) + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val bounds = gridExtent.gridBoundsFor(extent, clamp = false) + + read(bounds, bands) + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val it = readBounds(List(bounds), bands) + + closestTiffOverview.synchronized { if (it.hasNext) Some(it.next) else None } + } + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val targetPixelBounds = extents.map(gridExtent.gridBoundsFor(_)) + // result extents may actually expand to cover pixels at our resolution + // TODO: verify the logic here, should the sourcePixelBounds be calculated from input or expanded extent? + readBounds(targetPixelBounds, bands) + } + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = { + val geoTiffTile = closestTiffOverview.tile.asInstanceOf[GeoTiffMultibandTile] + + val windows = { for { + queryPixelBounds <- bounds + targetPixelBounds <- queryPixelBounds.intersection(this.gridBounds) + } yield { + val targetExtent = gridExtent.extentFor(targetPixelBounds) + val sourcePixelBounds = closestTiffOverview.rasterExtent.gridBoundsFor(targetExtent, clamp = true) + val targetRasterExtent = RasterExtent(targetExtent, targetPixelBounds.width.toInt, targetPixelBounds.height.toInt) + (sourcePixelBounds, targetRasterExtent) + }}.toMap + + geoTiffTile.crop(windows.keys.toSeq, bands.toArray).map { case (gb, tile) => + val targetRasterExtent = windows(gb) + Raster( + tile = tile, + extent = targetRasterExtent.extent + ).resample(targetRasterExtent.cols, targetRasterExtent.rows, method) + } + } +} + +object GeoTiffResampleRasterSource { + def apply( + dataPath: GeoTiffPath, + resampleGrid: ResampleGrid[Long], + method: ResampleMethod = NearestNeighbor, + strategy: OverviewStrategy = AutoHigherResolution, + targetCellType: Option[TargetCellType] = None, + baseTiff: Option[MultibandGeoTiff] = None + ): GeoTiffResampleRasterSource = new GeoTiffResampleRasterSource(dataPath, resampleGrid, method, strategy, targetCellType, baseTiff) +} diff --git a/raster/src/main/scala/geotrellis/raster/io/geotiff/GeoTiffTile.scala b/raster/src/main/scala/geotrellis/raster/io/geotiff/GeoTiffTile.scala index 0686b19f7d..400f203f05 100644 --- a/raster/src/main/scala/geotrellis/raster/io/geotiff/GeoTiffTile.scala +++ b/raster/src/main/scala/geotrellis/raster/io/geotiff/GeoTiffTile.scala @@ -317,9 +317,6 @@ abstract class GeoTiffTile( * @return A new [[Tile]] that contains the new CellTypes */ def convert(newCellType: CellType): GeoTiffTile = { - if(newCellType.isFloatingPoint != cellType.isFloatingPoint) - logger.warn(s"Conversion from $cellType to $newCellType may lead to data loss.") - val arr = Array.ofDim[Array[Byte]](segmentCount) val compressor = compression.createCompressor(segmentCount) diff --git a/raster/src/test/scala/geotrellis/raster/MosaicRasterSourceSpec.scala b/raster/src/test/scala/geotrellis/raster/MosaicRasterSourceSpec.scala new file mode 100644 index 0000000000..2ec23b539c --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/MosaicRasterSourceSpec.scala @@ -0,0 +1,116 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.geotiff._ +import geotrellis.raster.io.geotiff.GeoTiffTestUtils +import geotrellis.proj4.{LatLng, WebMercator} +import geotrellis.vector.Extent + +import geotrellis.raster.testkit.RasterMatchers +import cats.data.NonEmptyList + +import org.scalatest._ + +class MosaicRasterSourceSpec extends FunSpec with RasterMatchers with GeoTiffTestUtils { + + describe("union operations") { + // With Extent(0, 0, 1, 1) + val inputPath1 = baseGeoTiffPath("vlm/geotiff-at-origin.tif") + // With Extent(1, 0, 2, 1) + val inputPath2 = baseGeoTiffPath("vlm/geotiff-off-origin.tif") + + val gtRasterSource1 = GeoTiffRasterSource(inputPath1) + val gtRasterSource2 = GeoTiffRasterSource(inputPath2) + + val mosaicRasterSource = MosaicRasterSource( + NonEmptyList(gtRasterSource1, List(gtRasterSource2)), LatLng, + gtRasterSource1.gridExtent combine gtRasterSource2.gridExtent) + + it("should understand its bounds") { + mosaicRasterSource.cols shouldBe 8 + mosaicRasterSource.rows shouldBe 4 + } + + it("should union extents of its sources") { + mosaicRasterSource.gridExtent shouldBe ( + gtRasterSource1.gridExtent combine gtRasterSource2.gridExtent + ) + } + + it("should union extents with reprojection") { + mosaicRasterSource.reproject(WebMercator).gridExtent shouldBe mosaicRasterSource.gridExtent.reproject(LatLng, WebMercator) + } + + it("the extent read should match the extent requested") { + val extentRead = Extent(0, 0, 3, 3) + val mosaicRasterSource1 = MosaicRasterSource( + NonEmptyList(gtRasterSource1, List()), + LatLng, + gtRasterSource1.gridExtent + ) + assertEqual( + mosaicRasterSource1.read(extentRead, Seq(0)).get, + gtRasterSource1.read(extentRead, Seq(0)).get + ) + } + + it("should return the whole tiles from the whole tiles' extents") { + val extentRead1 = Extent(0, 0, 1, 1) + val extentRead2 = Extent(1, 0, 2, 1) + + assertEqual( + mosaicRasterSource.read(extentRead1, Seq(0)).get, + gtRasterSource1.read(gtRasterSource1.gridExtent.extent, Seq(0)).get + ) + assertEqual( + mosaicRasterSource.read(extentRead2, Seq(0)).get, + gtRasterSource2.read(gtRasterSource2.gridExtent.extent, Seq(0)).get + ) + } + + it("should read an extent overlapping both tiles") { + val extentRead = Extent(0, 0, 1.5, 1) + val expectation = Raster( + MultibandTile( + IntConstantNoDataArrayTile(Array(1, 2, 3, 4, 1, 2, + 5, 6, 7, 8, 5, 6, + 9, 10, 11, 12, 9, 10, + 13, 14, 15, 16, 13, 14), + 6, 4)), + extentRead + ) + val result = mosaicRasterSource.read(extentRead, Seq(0)).get + result shouldEqual expectation + } + + it("should get the expected tile from a gridbounds-based read") { + val expectation = Raster( + MultibandTile( + IntConstantNoDataArrayTile(Array(1, 2, 3, 4, 1, 2, 3, 4, + 5, 6, 7, 8, 5, 6, 7, 8, + 9, 10, 11, 12, 9, 10, 11, 12, + 13, 14, 15, 16, 13, 14, 15, 16), + 8, 4)), + mosaicRasterSource.gridExtent.extent + ) + val result = mosaicRasterSource.read(mosaicRasterSource.gridBounds, Seq(0)).get + result shouldEqual expectation + result.extent shouldEqual expectation.extent + } + } +} diff --git a/raster/src/test/scala/geotrellis/raster/PaddedTileSpec.scala b/raster/src/test/scala/geotrellis/raster/PaddedTileSpec.scala new file mode 100644 index 0000000000..d3ded13a85 --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/PaddedTileSpec.scala @@ -0,0 +1,118 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.testkit.RasterMatchers +import org.scalatest._ + +import spire.syntax.cfor._ + +class PaddedTileSpec extends FunSpec with Matchers with RasterMatchers { + val padded = PaddedTile(chunk = IntArrayTile.fill(1, cols = 8, rows = 8), colOffset = 8, rowOffset = 8, cols = 16, rows = 16) + + val expected = { + val tile = IntArrayTile.empty(16, 16) + cfor(8)(_ < 16, _ + 1) { col => + cfor(8)(_ < 16, _ + 1) { row => + tile.set(col, row, 1) + } + } + tile + } + + it("foreach should iterate correct cell count") { + var noDataCount = 0 + var dataCount = 0 + padded.foreach { z => if (isNoData(z)) noDataCount +=1 else dataCount += 1} + + dataCount shouldBe 8*8 + (dataCount + noDataCount) should be (16 * 16) + } + + it("foreachDouble should iterate correct cell count") { + var noDataCount = 0 + var dataCount = 0 + padded.foreachDouble { z => if (isNoData(z)) noDataCount +=1 else dataCount += 1} + + dataCount shouldBe 8*8 + (dataCount + noDataCount) should be (16 * 16) + } + + it("should implement col,row,value visitor for Int") { + padded.foreach((col, row, z) => + withClue(s"col = $col, row = $row") { + z shouldBe expected.get(col, row) + } + ) + } + + it("should implement col,row,value visitor for Double") { + padded.foreachDouble((col, row, z) => + withClue(s"col = $col, row = $row") { + val x = expected.getDouble(col, row) + // (Double.NaN == Double.NaN) == false + if (isData(z) || isData(x)) z shouldBe x + } + ) + } + + it("should implement row-major foreach") { + val copy = IntArrayTile.empty(16, 16) + var i = 0 + padded.foreach{ z => + copy.update(i, z) + i += 1 + } + copy shouldBe expected + } + + it("should implement row-major foreachDouble") { + val copy = IntArrayTile.empty(16, 16) + var i = 0 + padded.foreachDouble{ z => + copy.updateDouble(i, z) + i += 1 + } + copy shouldBe expected + } + + it("should go through all of the rows of the chunk") { + val padded = PaddedTile( + chunk = IntArrayTile.fill(1, cols = 8, rows = 8), + colOffset = 0, rowOffset = 0, cols = 16, rows = 16) + + val expected = { + val tile = IntArrayTile.empty(16, 16) + cfor(0)(_ < 8, _ + 1) { col => + cfor(0)(_ < 8, _ + 1) { row => + tile.set(col, row, 1) + } + } + tile + } + + val copy = IntArrayTile.empty(16, 16) + var i = 0 + + padded.foreachDouble{ z => + copy.updateDouble(i, z) + i += 1 + } + + assertEqual(copy, expected) + } +} diff --git a/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffConvertedRasterSourceSpec.scala b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffConvertedRasterSourceSpec.scala new file mode 100644 index 0000000000..fffefbeaac --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffConvertedRasterSourceSpec.scala @@ -0,0 +1,237 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.GeoTiffTestUtils +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.testkit._ +import geotrellis.vector._ + +import org.scalatest._ + +class GeoTiffConvertedRasterSourceSpec extends FunSpec with RasterMatchers with GeoTiffTestUtils with GivenWhenThen { + lazy val url = baseGeoTiffPath("vlm/aspect-tiled.tif") + + lazy val source: GeoTiffRasterSource = GeoTiffRasterSource(url) + + lazy val expectedRaster: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(url, streaming = false) + .raster + + describe("Converting to a different CellType") { + lazy val targetExtent = expectedRaster.extent + + describe("Bit CellType") { + it("should convert to: ByteCellType") { + val actual = source.convert(BitCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(BitCellType) } + + assertEqual(actual, expected) + } + } + + describe("Byte CellType") { + it("should convert to: ByteConstantNoDataCellType") { + val actual = source.convert(ByteConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: ByteUserDefinedNoDataCellType(10)") { + val actual = source.convert(ByteUserDefinedNoDataCellType(10)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteUserDefinedNoDataCellType(10)) } + + assertEqual(actual, expected) + } + + it("should convert to: ByteCellType") { + val actual = source.convert(ByteCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteCellType) } + + assertEqual(actual, expected) + } + } + + describe("UByte CellType") { + it("should convert to: UByteConstantNoDataCellType") { + val actual = source.convert(UByteConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: UByteUserDefinedNoDataCellType(10)") { + val actual = source.convert(UByteUserDefinedNoDataCellType(10)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteUserDefinedNoDataCellType(10)) } + + assertEqual(actual, expected) + } + + it("should convert to: UByteCellType") { + val actual = source.convert(UByteCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteCellType) } + + assertEqual(actual, expected) + } + } + + describe("Short CellType") { + it("should convert to: ShortConstantNoDataCellType") { + val actual = source.convert(ShortConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: ShortUserDefinedNoDataCellType(-1)") { + val actual = source.convert(ShortUserDefinedNoDataCellType(-1)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortUserDefinedNoDataCellType(-1)) } + + assertEqual(actual, expected) + } + + it("should convert to: ShortCellType") { + val actual = source.convert(ShortCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortCellType) } + + assertEqual(actual, expected) + } + } + + describe("UShort CellType") { + it("should convert to: UShortConstantNoDataCellType") { + val actual = source.convert(UShortConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: UShortUserDefinedNoDataCellType(-1)") { + val actual = source.convert(UShortUserDefinedNoDataCellType(-1)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortUserDefinedNoDataCellType(-1)) } + + assertEqual(actual, expected) + } + + it("should convert to: UShortCellType") { + val actual = source.convert(UShortCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortCellType) } + + assertEqual(actual, expected) + } + } + + describe("Int CellType") { + it("should convert to: IntConstantNoDataCellType") { + val actual = source.convert(IntConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: IntUserDefinedNoDataCellType(-100)") { + val actual = source.convert(IntUserDefinedNoDataCellType(-100)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntUserDefinedNoDataCellType(-100)) } + + assertEqual(actual, expected) + } + + it("should convert to: IntCellType") { + val actual = source.convert(IntCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntCellType) } + + assertEqual(actual, expected) + } + } + + describe("Float CellType") { + it("should convert to: FloatConstantNoDataCellType") { + val actual = source.convert(FloatConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: FloatUserDefinedNoDataCellType(0)") { + val actual = source.convert(FloatUserDefinedNoDataCellType(0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatUserDefinedNoDataCellType(0)) } + + assertEqual(actual, expected) + } + + it("should convert to: FloatCellType") { + val actual = source.convert(FloatCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatCellType) } + + assertEqual(actual, expected) + } + } + + describe("Double CellType") { + it("should convert to: DoubleConstantNoDataCellType") { + val actual = source.convert(DoubleConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: DoubleUserDefinedNoDataCellType(1.0)") { + val actual = source.convert(DoubleUserDefinedNoDataCellType(1.0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleUserDefinedNoDataCellType(1.0)) } + + assertEqual(actual, expected) + } + + it("should convert to: DoubleCellType") { + val actual = source.convert(DoubleCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleCellType) } + + assertEqual(actual, expected) + } + } + } + + describe("Chaining together operations") { + val targetCellType = DoubleConstantNoDataCellType + + val targetExtent = expectedRaster.extent.reproject(source.crs, WebMercator) + + val expectedTile: MultibandTile = expectedRaster.tile + + it("should have the correct CellType after reproject") { + val actual = source.convert(targetCellType).reproject(WebMercator).read(targetExtent).get.cellType + + actual should be (targetCellType) + } + + it("should have the correct CellType after multiple conversions") { + val actual = + source + .convert(FloatUserDefinedNoDataCellType(0)) + .reproject(WebMercator) + .convert(targetCellType) + .read(targetExtent).get.cellType + + actual should be (targetCellType) + } + } +} + diff --git a/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala new file mode 100644 index 0000000000..537d2bf015 --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceMultiThreadingSpec.scala @@ -0,0 +1,72 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster.RasterSource +import geotrellis.proj4.CRS +import geotrellis.raster.resample._ +import geotrellis.raster.io.geotiff.GeoTiffTestUtils + +import cats.instances.future._ +import cats.instances.list._ +import cats.syntax.traverse._ + +import org.scalatest._ + +import scala.concurrent.{ExecutionContext, Future} + +class GeoTiffRasterSourceMultiThreadingSpec extends AsyncFunSpec with GeoTiffTestUtils { + lazy val url = baseGeoTiffPath("vlm/aspect-tiled.tif") + val source = GeoTiffRasterSource(url) + + implicit val ec = ExecutionContext.global + + val iterations = (0 to 30).toList + + /** + * readBounds and readExtends are not covered by these tests since these methods return an [[Iterator]]. + * Due to the [[Iterator]] lazy nature, the lock in these cases should be done on the user side. + * */ + def testMultithreading(rs: RasterSource): Unit = { + it("read") { + val res = iterations.map { _ => Future { rs.read() } }.sequence.map(_.flatten) + res.map { rasters => rasters.length shouldBe iterations.length } + } + + it("readBounds") { + val res = + iterations + .map { _ => Future { rs.read(rs.gridBounds, 0 until rs.bandCount) } } + .sequence + .map(_.flatten) + + res.map { rasters => rasters.length shouldBe iterations.length } + } + } + + describe("GeoTiffRasterSource should be threadsafe") { + testMultithreading(source) + } + + describe("GeoTiffRasterReprojectSource should be threadsafe") { + testMultithreading(source.reproject(CRS.fromEpsgCode(4326))) + } + + describe("GeoTiffRasterResampleSource should be threadsafe") { + testMultithreading(source.resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor)) + } +} diff --git a/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProviderSpec.scala b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProviderSpec.scala new file mode 100644 index 0000000000..0f7a2c1419 --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceProviderSpec.scala @@ -0,0 +1,51 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster.RasterSource + +import org.scalatest._ + +class GeoTiffRasterSourceProviderSpec extends FunSpec { + describe("GeoTiffRasterSourceProvider") { + val provider = new GeoTiffRasterSourceProvider() + + it("should process a non-prefixed string") { + assert(provider.canProcess("file:///tmp/path/to/random/file.tiff")) + } + + it("should process a prefixed string") { + assert(provider.canProcess("gtiff+s3://my-files/tiffs/big-tiff.TIFF")) + } + + it("should process a non-prefixed relative path") { + assert(provider.canProcess("../../my-file.tif")) + } + + it("should process a prefixed relative path") { + assert(provider.canProcess("gtiff+../../my-file.tif")) + } + + it("should not be able to process a GDAL prefixed path") { + assert(!provider.canProcess("gdal+file:///tmp/temp-file.tif")) + } + + it("should produce a GeoTiffRasterSource from a string") { + assert(RasterSource("file://dumping-ground/part-2/random/file.tiff").isInstanceOf[GeoTiffRasterSource]) + } + } +} diff --git a/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceSpec.scala b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceSpec.scala new file mode 100644 index 0000000000..b356871ac8 --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffRasterSourceSpec.scala @@ -0,0 +1,87 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.GeoTiffTestUtils +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.resample._ +import geotrellis.raster.testkit._ +import geotrellis.vector._ + +import org.scalatest._ + +class GeoTiffRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen with GeoTiffTestUtils { + lazy val url = baseGeoTiffPath("vlm/aspect-tiled.tif") + + lazy val source: GeoTiffRasterSource = GeoTiffRasterSource(url) + + it("should convert celltypes correctly") { + val rs = GeoTiffRasterSource(baseGeoTiffPath("vlm/0_to_99.tif")) + val raster = rs.read(Extent(1,1,100,100)).get + val interpreted = rs.interpretAs(IntUserDefinedNoDataCellType(0)).read(Extent(1,1,100,100)).get + interpreted.tile.band(0).get(0,0) should be (Int.MinValue) + } + + it("should be able to read upper left corner") { + val bounds = GridBounds(0, 0, 10, 10).toGridType[Long] + val chip: Raster[MultibandTile] = source.read(bounds).get + chip should have ( + // dimensions (bounds.width, bounds.height), + cellType (source.cellType) + ) + } + + it("should not read past file edges") { + Given("bounds larger than raster") + val bounds = GridBounds(0, 0, source.cols + 100, source.rows + 100) + When("reading by pixel bounds") + val chip = source.read(bounds).get + Then("return only pixels that exist") + // chip.tile should have (dimensions (source.dimensions)) + } + + it("should be able to resample") { + // read in the whole file and resample the pixels in memory + val expected: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(url, streaming = false) + .raster + .resample((source.cols * 0.95).toInt , (source.rows * 0.95).toInt, NearestNeighbor) + // resample to 0.9 so we RasterSource picks the base layer and not an overview + + val resampledSource = + source.resample(expected.tile.cols, expected.tile.rows, NearestNeighbor) + + // resampledSource should have (dimensions (expected.tile.dimensions)) + + val actual: Raster[MultibandTile] = + resampledSource.read(GridBounds(0, 0, resampledSource.cols - 1, resampledSource.rows - 1)).get + + // calculated expected resolutions of overviews + // it's a rough approximation there as we're not calculating resolutions like GDAL + val ratio = resampledSource.cellSize.resolution / source.cellSize.resolution + resampledSource.resolutions.zip (source.resolutions.map { re => + val CellSize(cw, ch) = re.cellSize + RasterExtent(re.extent, CellSize(cw * ratio, ch * ratio)) + }).map { case (rea, ree) => rea.cellSize.resolution shouldBe ree.cellSize.resolution +- 1e-7 } + + withGeoTiffClue(actual, expected, resampledSource.crs) { + assertRastersEqual(actual, expected) + } + } +} diff --git a/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala new file mode 100644 index 0000000000..c59054c702 --- /dev/null +++ b/raster/src/test/scala/geotrellis/raster/geotiff/GeoTiffReprojectRasterSourceSpec.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.geotiff + +import java.io.File + +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.resample._ +import geotrellis.raster.reproject._ +import geotrellis.raster.testkit.RasterMatchers +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.GeoTiffTestUtils +import org.scalatest._ + +class GeoTiffReprojectRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen { + def rasterGeoTiffPath(name: String): String = { + def baseDataPath = "raster/data" + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } + + describe("Reprojecting a RasterSource") { + lazy val uri = rasterGeoTiffPath("vlm/aspect-tiled.tif") + + lazy val rasterSource = GeoTiffRasterSource(uri) + lazy val sourceTiff = GeoTiffReader.readMultiband(uri) + + lazy val expectedRasterExtent = { + val re = ReprojectRasterExtent(rasterSource.gridExtent, Transform(rasterSource.crs, LatLng)) + // stretch target raster extent slightly to avoid default case in ReprojectRasterExtent + RasterExtent(re.extent, CellSize(re.cellheight * 1.1, re.cellwidth * 1.1)) + } + + def testReprojection(method: ResampleMethod) = { + val warpRasterSource = rasterSource.reprojectToRegion(LatLng, expectedRasterExtent, method) + + warpRasterSource.resolutions.size shouldBe rasterSource.resolutions.size + + val testBounds = GridBounds(0, 0, expectedRasterExtent.cols, expectedRasterExtent.rows).toGridType[Long].split(64, 64).toSeq + + for (bound <- testBounds) yield { + withClue(s"Read window ${bound}: ") { + val targetExtent = expectedRasterExtent.extentFor(bound.toGridType[Int]) + val testRasterExtent = RasterExtent( + extent = targetExtent, + cellwidth = expectedRasterExtent.cellwidth, + cellheight = expectedRasterExtent.cellheight, + cols = bound.width.toInt, + rows = bound.height.toInt + ) + + val expected: Raster[MultibandTile] = { + val rr = implicitly[RasterRegionReproject[MultibandTile]] + rr.regionReproject(sourceTiff.raster, sourceTiff.crs, LatLng, testRasterExtent, testRasterExtent.extent.toPolygon, method) + } + + val actual = warpRasterSource.read(bound).get + + actual.extent.covers(expected.extent) should be(true) + actual.rasterExtent.extent.xmin should be(expected.rasterExtent.extent.xmin +- 0.00001) + actual.rasterExtent.extent.ymax should be(expected.rasterExtent.extent.ymax +- 0.00001) + actual.rasterExtent.cellwidth should be(expected.rasterExtent.cellwidth +- 0.00001) + actual.rasterExtent.cellheight should be(expected.rasterExtent.cellheight +- 0.00001) + + withGeoTiffClue(actual, expected, LatLng) { + assertRastersEqual(actual, expected) + } + } + } + } + + it("should reproject using NearestNeighbor") { + testReprojection(NearestNeighbor) + } + + it("should reproject using Bilinear") { + testReprojection(Bilinear) + } + } +} diff --git a/raster/src/test/scala/geotrellis/raster/io/geotiff/GeoTiffTestUtils.scala b/raster/src/test/scala/geotrellis/raster/io/geotiff/GeoTiffTestUtils.scala index b4dd975655..a13253f2e6 100644 --- a/raster/src/test/scala/geotrellis/raster/io/geotiff/GeoTiffTestUtils.scala +++ b/raster/src/test/scala/geotrellis/raster/io/geotiff/GeoTiffTestUtils.scala @@ -35,6 +35,12 @@ trait GeoTiffTestUtils extends Matchers { def baseDataPath = "raster/data" + def baseGeoTiffPath(name: String): String = { + val path = s"$baseDataPath/$name" + require(new File(path).exists, s"$path does not exist, unzip the archive?") + path + } + val Epsilon = 1e-9 var writtenFiles = Vector[String]() diff --git a/scripts/publish-local.sh b/scripts/publish-local.sh index 149f15e482..e43c92c127 100755 --- a/scripts/publish-local.sh +++ b/scripts/publish-local.sh @@ -27,4 +27,5 @@ ./sbt "project hbase" publishLocal && \ ./sbt "project hbase-spark" publishLocal && \ ./sbt "project s3" publishLocal && \ -./sbt "project s3-spark" publishLocal +./sbt "project s3-spark" publishLocal && \ +./sbt "project gdal" publishLocal diff --git a/scripts/publish-m2.sh b/scripts/publish-m2.sh index 2aa547d9b2..57ea02bb20 100755 --- a/scripts/publish-m2.sh +++ b/scripts/publish-m2.sh @@ -24,4 +24,5 @@ "project util" +publishM2 \ "project vector" +publishM2 \ "project vector-testkit" +publishM2 \ - "project vectortile" +publishM2 + "project vectortile" +publishM2 \ + "project gdal" +publishM2 diff --git a/scripts/publish-snapshot.sh b/scripts/publish-snapshot.sh index 55202d8671..e49cdcb837 100755 --- a/scripts/publish-snapshot.sh +++ b/scripts/publish-snapshot.sh @@ -23,4 +23,5 @@ "project util" publish \ "project vector" publish \ "project vector-testkit" publish \ - "project vectortile" publish + "project vectortile" publish \ + "project gdal" publish diff --git a/spark-testkit/src/main/scala/geotrellis/spark/testkit/LayoutRasterMatchers.scala b/spark-testkit/src/main/scala/geotrellis/spark/testkit/LayoutRasterMatchers.scala new file mode 100644 index 0000000000..4f889d021b --- /dev/null +++ b/spark-testkit/src/main/scala/geotrellis/spark/testkit/LayoutRasterMatchers.scala @@ -0,0 +1,45 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark.testkit + +import org.scalatest._ +import geotrellis.raster._ +import geotrellis.proj4._ +import geotrellis.layer.{SpatialKey, Bounds, LayoutDefinition} +import geotrellis.raster.testkit.RasterMatchers + +import matchers._ + +trait LayoutRasterMatchers { + self: Matchers with FunSpec with RasterMatchers => + def containKey(key: SpatialKey) = Matcher[Bounds[SpatialKey]] { bounds => + MatchResult(bounds.includes(key), + s"""$bounds does not contain $key""", + s"""$bounds contains $key""") + } + + def withGeoTiffClue[T]( + key: SpatialKey, + layout: LayoutDefinition, + actual: MultibandTile, + expect: MultibandTile, + crs: CRS + )(fun: => T): T = { + val extent = layout.mapTransform.keyToExtent(key) + withGeoTiffClue[T](Raster(actual, extent), Raster(expect, extent), crs)(fun) + } +} diff --git a/spark/src/main/scala/geotrellis/spark/RasterSourceRDD.scala b/spark/src/main/scala/geotrellis/spark/RasterSourceRDD.scala new file mode 100644 index 0000000000..f783723685 --- /dev/null +++ b/spark/src/main/scala/geotrellis/spark/RasterSourceRDD.scala @@ -0,0 +1,355 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark + +import geotrellis.spark.partition._ +import geotrellis.raster._ +import geotrellis.raster.resample.NearestNeighbor +import geotrellis.layer._ +import geotrellis.util._ + +import org.apache.spark.rdd._ +import org.apache.spark.{Partitioner, SparkContext} + +import scala.collection.mutable.ArrayBuilder +import scala.reflect.ClassTag +import java.time.ZonedDateTime + +object RasterSourceRDD { + final val DEFAULT_PARTITION_BYTES: Long = 128l * 1024 * 1024 + + def read( + readingSources: Seq[ReadingSource], + layout: LayoutDefinition + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + read(readingSources, layout, DEFAULT_PARTITION_BYTES) + + def read( + readingSources: Seq[ReadingSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[SpaceTimeKey, ZonedDateTime] + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = + read(readingSources, layout, keyExtractor, DEFAULT_PARTITION_BYTES) + + def read( + readingSources: Seq[ReadingSource], + layout: LayoutDefinition, + partitionBytes: Long + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + readPartitionBytes(readingSources, layout, KeyExtractor.spatialKeyExtractor, partitionBytes) + + def read( + readingSources: Seq[ReadingSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[SpaceTimeKey, ZonedDateTime], + partitionBytes: Long + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = + readPartitionBytes(readingSources, layout, keyExtractor, partitionBytes) + + def readPartitionBytes[K: SpatialComponent: ClassTag, M: Boundable]( + readingSources: Seq[ReadingSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[K, M], + partitionBytes: Long + )(implicit sc: SparkContext): MultibandTileLayerRDD[K] = { + val summary = RasterSummary.fromSeq(readingSources.map(_.source), keyExtractor.getMetadata) + val cellType = summary.cellType + val tileSize = layout.tileCols * layout.tileRows * cellType.bytes + val layerMetadata = summary.toTileLayerMetadata(layout, keyExtractor.getKey) + + def getNoDataTile = ArrayTile.alloc(cellType, layout.tileCols, layout.tileRows).fill(NODATA).interpretAs(cellType) + + val maxIndex = readingSources.map { _.sourceToTargetBand.values.max }.max + val targetIndexes: Seq[Int] = 0 to maxIndex + + val sourcesRDD: RDD[(K, (Int, Option[MultibandTile]))] = + sc.parallelize(readingSources, readingSources.size).flatMap { rs => + val m = keyExtractor.getMetadata(rs.source) + val tileKeyTransform: SpatialKey => K = { sk => keyExtractor.getKey(m, sk) } + val layoutSource = rs.source.tileToLayout(layout, tileKeyTransform) + val keys = layoutSource.keys + + RasterSourceRDD.partition(keys, partitionBytes)( _ => tileSize).flatMap { _.flatMap { key => + rs.sourceToTargetBand.map { case (sourceBand, targetBand) => + (key, (targetBand, layoutSource.read(key, Seq(sourceBand)))) + } + } } + } + + sourcesRDD.persist() + + val repartitioned = { + val count = sourcesRDD.count.toInt + if (count > sourcesRDD.partitions.size) sourcesRDD.repartition(count) + else sourcesRDD + } + + val groupedSourcesRDD: RDD[(K, Iterable[(Int, Option[MultibandTile])])] = + repartitioned.groupByKey() + + val result: RDD[(K, MultibandTile)] = + groupedSourcesRDD.mapPartitions ({ partition => + val noDataTile = getNoDataTile + + partition.map { case (key, iter) => + val mappedBands: Map[Int, Option[MultibandTile]] = iter.toSeq.sortBy { _._1 }.toMap + + val tiles: Seq[Tile] = + targetIndexes.map { index => + mappedBands.getOrElse(index, None) match { + case Some(multibandTile) => multibandTile.band(0) + case None => noDataTile + } + } + + key -> MultibandTile(tiles) + } + }, preservesPartitioning = true) + + sourcesRDD.unpersist() + + ContextRDD(result, layerMetadata) + } + + def read( + readingSourcesRDD: RDD[ReadingSource], + layout: LayoutDefinition, + partitioner: Option[Partitioner] = None + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + read(readingSourcesRDD, layout, KeyExtractor.spatialKeyExtractor, partitioner) + + def read[K: SpatialComponent: ClassTag, M: Boundable]( + readingSourcesRDD: RDD[ReadingSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[K, M], + partitioner: Option[Partitioner] + )(implicit sc: SparkContext): MultibandTileLayerRDD[K] = { + val rasterSourcesRDD = readingSourcesRDD.map { _.source } + val summary = RasterSummary.fromRDD(rasterSourcesRDD, keyExtractor.getMetadata) + val cellType = summary.cellType + val layerMetadata = summary.toTileLayerMetadata(layout, keyExtractor.getKey) + + def getNoDataTile = ArrayTile.alloc(cellType, layout.tileCols, layout.tileRows).fill(NODATA).interpretAs(cellType) + + val maxIndex = readingSourcesRDD.map { _.sourceToTargetBand.values.max }.reduce { _ max _ } + val targetIndexes: Seq[Int] = 0 to maxIndex + + val keyedRDD: RDD[(K, (Int, Option[MultibandTile]))] = + readingSourcesRDD.mapPartitions ({ partition => + partition.flatMap { rs => + val m = keyExtractor.getMetadata(rs.source) + val tileKeyTransform: SpatialKey => K = { sk => keyExtractor.getKey(m, sk) } + val layoutSource = rs.source.tileToLayout(layout, tileKeyTransform) + + layoutSource.keys.flatMap { key => + rs.sourceToTargetBand.map { case (sourceBand, targetBand) => + (key, (targetBand, layoutSource.read(key, Seq(sourceBand)))) + } + } + } + }) + + val groupedRDD: RDD[(K, Iterable[(Int, Option[MultibandTile])])] = { + // The number of partitions estimated by RasterSummary can sometimes be much + // lower than what the user set. Therefore, we assume that the larger value + // is the optimal number of partitions to use. + val partitionCount = + math.max(keyedRDD.getNumPartitions, summary.estimatePartitionsNumber) + + keyedRDD.groupByKey(partitioner.getOrElse(SpatialPartitioner[K](partitionCount))) + } + + val result: RDD[(K, MultibandTile)] = + groupedRDD.mapPartitions ({ partition => + val noDataTile = getNoDataTile + + partition.map { case (key, iter) => + val mappedBands: Map[Int, Option[MultibandTile]] = iter.toSeq.sortBy { _._1 }.toMap + + val tiles: Seq[Tile] = + targetIndexes.map { index => + mappedBands.getOrElse(index, None) match { + case Some(multibandTile) => multibandTile.band(0) + case None => noDataTile + } + } + + key -> MultibandTile(tiles) + } + }, preservesPartitioning = true) + + ContextRDD(result, layerMetadata) + } + + def tiledLayerRDD( + sources: RDD[RasterSource], + layout: LayoutDefinition + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + tiledLayerRDD(sources, layout, KeyExtractor.spatialKeyExtractor, NearestNeighbor, None, None) + + def tiledLayerRDD( + sources: RDD[RasterSource], + layout: LayoutDefinition, + resampleMethod: ResampleMethod + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + tiledLayerRDD(sources, layout, KeyExtractor.spatialKeyExtractor, resampleMethod, None, None) + + def tiledLayerRDD[K: SpatialComponent: Boundable: ClassTag, M: Boundable]( + sources: RDD[RasterSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[K, M], + resampleMethod: ResampleMethod = NearestNeighbor, + rasterSummary: Option[RasterSummary[M]] = None, + partitioner: Option[Partitioner] = None + )(implicit sc: SparkContext): MultibandTileLayerRDD[K] = { + val summary = rasterSummary.getOrElse(RasterSummary.fromRDD(sources, keyExtractor.getMetadata)) + val layerMetadata = summary.toTileLayerMetadata(layout, keyExtractor.getKey) + + val tiledLayoutSourceRDD = + sources.map { rs => + val m = keyExtractor.getMetadata(rs) + val tileKeyTransform: SpatialKey => K = { sk => keyExtractor.getKey(m, sk) } + rs.tileToLayout(layout, tileKeyTransform) + } + + val rasterRegionRDD: RDD[(K, RasterRegion)] = + tiledLayoutSourceRDD.flatMap { _.keyedRasterRegions() } + + // The number of partitions estimated by RasterSummary can sometimes be much + // lower than what the user set. Therefore, we assume that the larger value + // is the optimal number of partitions to use. + val partitionCount = + math.max(rasterRegionRDD.getNumPartitions, summary.estimatePartitionsNumber) + + val tiledRDD: RDD[(K, MultibandTile)] = + rasterRegionRDD + .groupByKey(partitioner.getOrElse(SpatialPartitioner[K](partitionCount))) + .mapValues { iter => + MultibandTile( + iter.flatMap { _.raster.toSeq.flatMap { _.tile.bands } } + ) + } + + ContextRDD(tiledRDD, layerMetadata) + } + + def spatial( + sources: Seq[RasterSource], + layout: LayoutDefinition, + partitionBytes: Long = DEFAULT_PARTITION_BYTES + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + apply(sources, layout, KeyExtractor.spatialKeyExtractor, partitionBytes) + + def spatial(source: RasterSource, layout: LayoutDefinition)(implicit sc: SparkContext): MultibandTileLayerRDD[SpatialKey] = + spatial(Seq(source), layout) + + def temporal( + sources: Seq[RasterSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[SpaceTimeKey, ZonedDateTime], + partitionBytes: Long = DEFAULT_PARTITION_BYTES + )(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = + apply(sources, layout, keyExtractor, partitionBytes) + + def temporal(source: RasterSource, layout: LayoutDefinition, keyExtractor: KeyExtractor.Aux[SpaceTimeKey, ZonedDateTime])(implicit sc: SparkContext): MultibandTileLayerRDD[SpaceTimeKey] = + temporal(Seq(source), layout, keyExtractor) + + def apply[K: SpatialComponent: Boundable, M: Boundable]( + sources: Seq[RasterSource], + layout: LayoutDefinition, + keyExtractor: KeyExtractor.Aux[K, M], + partitionBytes: Long = DEFAULT_PARTITION_BYTES + )(implicit sc: SparkContext): MultibandTileLayerRDD[K] = { + val summary = RasterSummary.fromSeq(sources, keyExtractor.getMetadata) + val extent = summary.extent + val cellType = summary.cellType + val tileSize = layout.tileCols * layout.tileRows * cellType.bytes + val layerMetadata = summary.toTileLayerMetadata(layout, keyExtractor.getKey) + + val sourcesRDD: RDD[(RasterSource, Array[SpatialKey])] = + sc.parallelize(sources).flatMap { source => + val keys: Traversable[SpatialKey] = + extent.intersection(source.extent) match { + case Some(intersection) => layout.mapTransform.keysForGeometry(intersection.toPolygon) + case None => Seq.empty[SpatialKey] + } + partition(keys, partitionBytes)( _ => tileSize).map { res => (source, res) } + } + + sourcesRDD.persist() + + val repartitioned = { + val count = sourcesRDD.count.toInt + if (count > sourcesRDD.partitions.size) sourcesRDD.repartition(count) + else sourcesRDD + } + + val result: RDD[(K, MultibandTile)] = + repartitioned.flatMap { case (source, keys) => + val m = keyExtractor.getMetadata(source) + val tileKeyTransform: SpatialKey => K = { sk => keyExtractor.getKey(m, sk) } + val tileSource = source.tileToLayout(layout, tileKeyTransform) + tileSource.readAll(keys.map(tileKeyTransform).toIterator) + } + + sourcesRDD.unpersist() + + ContextRDD(result, layerMetadata) + } + + /** Partition a set of chunks not to exceed certain size per partition */ + private def partition[T: ClassTag]( + chunks: Traversable[T], + maxPartitionSize: Long + )(chunkSize: T => Long = { c: T => 1l }): Array[Array[T]] = { + if (chunks.isEmpty) { + Array[Array[T]]() + } else { + val partition = ArrayBuilder.make[T] + partition.sizeHintBounded(128, chunks) + var partitionSize: Long = 0l + var partitionCount: Long = 0l + val partitions = ArrayBuilder.make[Array[T]] + + def finalizePartition() { + val res = partition.result + if (res.nonEmpty) partitions += res + partition.clear() + partitionSize = 0l + partitionCount = 0l + } + + def addToPartition(chunk: T) { + partition += chunk + partitionSize += chunkSize(chunk) + partitionCount += 1 + } + + for (chunk <- chunks) { + if ((partitionCount == 0) || (partitionSize + chunkSize(chunk)) < maxPartitionSize) + addToPartition(chunk) + else { + finalizePartition() + addToPartition(chunk) + } + } + + finalizePartition() + partitions.result + } + } +} diff --git a/spark/src/main/scala/geotrellis/spark/RasterSummary.scala b/spark/src/main/scala/geotrellis/spark/RasterSummary.scala new file mode 100644 index 0000000000..31f67a0448 --- /dev/null +++ b/spark/src/main/scala/geotrellis/spark/RasterSummary.scala @@ -0,0 +1,153 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark + +import geotrellis.proj4.CRS +import geotrellis.raster._ +import geotrellis.layer._ +import geotrellis.vector.Extent +import geotrellis.util._ + +import org.apache.spark.rdd.RDD +import com.typesafe.scalalogging.LazyLogging + +case class RasterSummary[M]( + crs: CRS, + cellType: CellType, + cellSize: CellSize, + extent: Extent, + cells: Long, + count: Long, + // for the internal usage only, required to collect non spatial bounds + bounds: Bounds[M] +) extends LazyLogging with Serializable { + + def estimatePartitionsNumber: Int = { + import squants.information._ + val bytes = Bytes(cellType.bytes * cells) + val numPartitions: Int = math.max((bytes / Megabytes(64)).toInt, 1) + logger.info(s"Using $numPartitions partitions for ${bytes.toString(Gigabytes)}") + numPartitions + } + + def levelFor(layoutScheme: LayoutScheme): LayoutLevel = + layoutScheme.levelFor(extent, cellSize) + + def toGridExtent: GridExtent[Long] = + new GridExtent[Long](extent, cellSize) + + def layoutDefinition(scheme: LayoutScheme): LayoutDefinition = + scheme.levelFor(extent, cellSize).layout + + def combine(other: RasterSummary[M])(implicit ev: Boundable[M]): RasterSummary[M] = { + require(other.crs == crs, s"Can't combine LayerExtent for different CRS: $crs, ${other.crs}") + val smallestCellSize = if (cellSize.resolution < other.cellSize.resolution) cellSize else other.cellSize + RasterSummary( + crs, + cellType.union(other.cellType), + smallestCellSize, + extent.combine(other.extent), + cells + other.cells, + count + other.count, + bounds.combine(other.bounds) + ) + } + + def toTileLayerMetadata[K: SpatialComponent](layoutDefinition: LayoutDefinition, keyTransform: (M, SpatialKey) => K): TileLayerMetadata[K] = { + // We want to align the data extent to pixel layout of layoutDefinition + val layerGridExtent = layoutDefinition.createAlignedGridExtent(extent) + val keyBounds = bounds match { + case KeyBounds(minDim, maxDim) => + val KeyBounds(minSpatialKey, maxSpatialKey) = KeyBounds(layoutDefinition.mapTransform.extentToBounds(layerGridExtent.extent)) + KeyBounds(keyTransform(minDim, minSpatialKey), keyTransform(maxDim, maxSpatialKey)) + case EmptyBounds => EmptyBounds + } + TileLayerMetadata(cellType, layoutDefinition, layerGridExtent.extent, crs, keyBounds) + } + + def toTileLayerMetadata(layoutDefinition: LayoutDefinition): TileLayerMetadata[SpatialKey] = + toTileLayerMetadata(layoutDefinition, (_, sk) => sk) + + def toTileLayerMetadata(layoutType: LayoutType): TileLayerMetadata[SpatialKey] = + toTileLayerMetadata(layoutType.layoutDefinitionWithZoom(crs, extent, cellSize)._1, (_, sk) => sk) + + // TODO: probably this function should be removed in the future + def resample(resampleGrid: ResampleGrid[Long]): RasterSummary[M] = { + val re = resampleGrid(toGridExtent) + RasterSummary( + crs = crs, + cellType = cellType, + cellSize = re.cellSize, + extent = re.extent, + cells = re.size, + count = count, + bounds = bounds // do nothing with it, since it contains non spatial information + ) + } +} + +object RasterSummary { + def collect[M: Boundable](rdd: RDD[RasterSource], getDimension: RasterMetadata => M): Seq[RasterSummary[M]] = { + rdd + .map { rs => + val extent = rs.extent + val crs = rs.crs + val dim = getDimension(rs) + (crs, RasterSummary(crs, rs.cellType, rs.cellSize, extent, rs.size, 1, KeyBounds(dim, dim))) + } + .reduceByKey { _ combine _ } + .values + .collect + .toSeq + } + + // TODO: should be refactored, we need to reproject all metadata into a common CRS. This code is for the current code simplification + def fromRDD[M: Boundable](rdd: RDD[RasterSource], getDimension: RasterMetadata => M): RasterSummary[M] = { + /* Notes on why this is awful: + - scalac can't infer both GetComponent and type V as CellGrid[N] + - very ad-hoc, why type constraint and lense? + - might be as simple as HasRasterSummary[V] in the first place + */ + val all = collect[M](rdd, getDimension) + require(all.size == 1, "multiple CRSs detected") // what to do in this case? + all.head + } + + def fromRDD(rdd: RDD[RasterSource]): RasterSummary[Unit] = { + val all = collect(rdd, _ => ()) + require(all.size == 1, "multiple CRSs detected") + all.head + } + + def fromSeq[M: Boundable](seq: Seq[RasterSource], getDimension: RasterMetadata => M): RasterSummary[M] = { + val all = + seq + .map { rs => + val extent = rs.extent + val crs = rs.crs + val dim = getDimension(rs) + (crs, RasterSummary(crs, rs.cellType, rs.cellSize, extent, rs.size, 1, KeyBounds(dim, dim))) + } + .groupBy(_._1) + .map { case (_, v) => v.map(_._2).reduce(_ combine _) } + + require(all.size == 1, "multiple CRSs detected") // what to do in this case? + all.head + } + + def fromSeq(seq: Seq[RasterSource]): RasterSummary[Unit] = fromSeq(seq, _ => ()) +} diff --git a/spark/src/main/scala/geotrellis/spark/partition/SpatialPartitioner.scala b/spark/src/main/scala/geotrellis/spark/partition/SpatialPartitioner.scala new file mode 100644 index 0000000000..222b256921 --- /dev/null +++ b/spark/src/main/scala/geotrellis/spark/partition/SpatialPartitioner.scala @@ -0,0 +1,43 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark.partition + +import geotrellis.layer.{SpatialComponent, SpatialKey} +import geotrellis.store.index.zcurve._ +import geotrellis.util._ + +import org.apache.spark._ + +class SpatialPartitioner[K: SpatialComponent](partitions: Int, bits: Int) extends Partitioner { + def numPartitions: Int = partitions + + def getBits: Int = bits + + def getPartition(key: Any): Int = { + val k = key.asInstanceOf[K] + val SpatialKey(col, row) = k.getComponent[SpatialKey] + ((Z2(col, row).z >> bits) % partitions).toInt + } +} + +object SpatialPartitioner { + def apply[K: SpatialComponent](partitions: Int, bits: Int): SpatialPartitioner[K] = + new SpatialPartitioner[K](partitions, bits) + + def apply[K: SpatialComponent](partitions: Int): SpatialPartitioner[K] = + new SpatialPartitioner[K](partitions, 8) +} diff --git a/spark/src/test/resources/vlm/aspect-tiled-0-1-2.tif b/spark/src/test/resources/vlm/aspect-tiled-0-1-2.tif new file mode 100644 index 0000000000..34b02c3cc4 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-0-1-2.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-0.tif b/spark/src/test/resources/vlm/aspect-tiled-0.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-0.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-1.tif b/spark/src/test/resources/vlm/aspect-tiled-1.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-1.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-2.tif b/spark/src/test/resources/vlm/aspect-tiled-2.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-2.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-2018-01-01.tif b/spark/src/test/resources/vlm/aspect-tiled-2018-01-01.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-2018-01-01.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-2018-02-01.tif b/spark/src/test/resources/vlm/aspect-tiled-2018-02-01.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-2018-02-01.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-2018-03-01.tif b/spark/src/test/resources/vlm/aspect-tiled-2018-03-01.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-2018-03-01.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-3.tif b/spark/src/test/resources/vlm/aspect-tiled-3.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-3.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-4.tif b/spark/src/test/resources/vlm/aspect-tiled-4.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-4.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled-5.tif b/spark/src/test/resources/vlm/aspect-tiled-5.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled-5.tif differ diff --git a/spark/src/test/resources/vlm/aspect-tiled.tif b/spark/src/test/resources/vlm/aspect-tiled.tif new file mode 100644 index 0000000000..e75fe90d70 Binary files /dev/null and b/spark/src/test/resources/vlm/aspect-tiled.tif differ diff --git a/spark/src/test/resources/vlm/diagonal.tif b/spark/src/test/resources/vlm/diagonal.tif new file mode 100644 index 0000000000..efb6458076 Binary files /dev/null and b/spark/src/test/resources/vlm/diagonal.tif differ diff --git a/spark/src/test/resources/vlm/left-to-right.tif b/spark/src/test/resources/vlm/left-to-right.tif new file mode 100644 index 0000000000..2e6c083af6 Binary files /dev/null and b/spark/src/test/resources/vlm/left-to-right.tif differ diff --git a/spark/src/test/resources/vlm/top-to-bottom.tif b/spark/src/test/resources/vlm/top-to-bottom.tif new file mode 100644 index 0000000000..3cb4d805d4 Binary files /dev/null and b/spark/src/test/resources/vlm/top-to-bottom.tif differ diff --git a/spark/src/test/scala/geotrellis/spark/RasterRegionSpec.scala b/spark/src/test/scala/geotrellis/spark/RasterRegionSpec.scala new file mode 100644 index 0000000000..1151fd0144 --- /dev/null +++ b/spark/src/test/scala/geotrellis/spark/RasterRegionSpec.scala @@ -0,0 +1,105 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark + +import geotrellis.raster._ +import geotrellis.raster.geotiff._ +import geotrellis.raster.io.geotiff._ +import geotrellis.proj4._ +import geotrellis.layer._ + +import geotrellis.spark.testkit._ +import geotrellis.raster.testkit.{RasterMatchers, Resource} + +import org.apache.spark.rdd._ +import org.scalatest._ +import Inspectors._ + +class RasterRegionSpec extends FunSpec with TestEnvironment with RasterMatchers with LayoutRasterMatchers with GivenWhenThen { + it("reads RDD of raster refs") { + // we're going to read these and re-build gradient.tif + + Given("some files and a LayoutDefinition") + val paths = List( + Resource.path("vlm/left-to-right.tif"), + Resource.path("vlm/top-to-bottom.tif"), + Resource.path("vlm/diagonal.tif")) + + // I might need to discover the layout at a later stage, for now we'll know + + val layout: LayoutDefinition = { + val scheme = FloatingLayoutScheme(32, 32) + val tiff = GeoTiff.readSingleband(paths.head) + scheme.levelFor(tiff.extent, tiff.cellSize).layout + } + val crs = LatLng + + When("Generating RDD of RasterRegions") + val rdd: RDD[(SpatialKey, RasterRegion)] with Metadata[TileLayerMetadata[SpatialKey]] = { + val srcRdd = sc.parallelize(paths, paths.size).map { uri => GeoTiffRasterSource(uri) } + srcRdd.cache() + + val (combinedExtent, commonCellType) = + srcRdd.map { src => (src.extent, src.cellType) } + .reduce { case ((e1, ct1), (e2, ct2)) => (e1.combine(e2), ct1.union(ct2)) } + + // we know the metadata from the layout, we just need the raster extent + val tlm = TileLayerMetadata[SpatialKey]( + cellType = commonCellType, + layout = layout, + extent = combinedExtent, + crs = crs, + bounds = KeyBounds(layout.mapTransform(combinedExtent))) + + val refRdd = srcRdd.flatMap { src => + // too easy? whats missing + val tileSource = LayoutTileSource.spatial(src, layout) + tileSource.keys.toIterator.map { key => (key, tileSource.rasterRegionForKey(key).get) } + } + + // TADA! Jobs done. + ContextRDD(refRdd, tlm) + // NEXT: + // - combine the bands to complete the test + // - make an app for this and run it againt WSEL rasters + // - run aup on EMR, figure out how to set AWS secret keys? + } + + Then("get a RasterRegion for each region of each file") + rdd.count shouldBe (8*8*3) // three 256x256 files split into 32x32 windows + + Then("convert each RasterRegion to a tile") + val realRdd: MultibandTileLayerRDD[SpatialKey] = + rdd.withContext(_.flatMap{ case (key, ref) => + for { + raster <- ref.raster + } yield (key, raster.tile) + }) + + realRdd.count shouldBe (8*8*3) // we shouldn't have lost anything + + Then("Each row matches the layout") + val rows = realRdd.collect + forAll(rows) { case (key, tile) => + realRdd.metadata.bounds should containKey(key) + tile should have ( + // dimensions (layout.tileCols, layout.tileRows), + cellType (realRdd.metadata.cellType) + ) + } + } +} diff --git a/spark/src/test/scala/geotrellis/spark/RasterSourceRDDSpec.scala b/spark/src/test/scala/geotrellis/spark/RasterSourceRDDSpec.scala new file mode 100644 index 0000000000..2f268bf9f0 --- /dev/null +++ b/spark/src/test/scala/geotrellis/spark/RasterSourceRDDSpec.scala @@ -0,0 +1,283 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark + +import geotrellis.proj4._ +import geotrellis.layer._ +import geotrellis.raster._ +import geotrellis.raster.geotiff._ +import geotrellis.raster.io.geotiff._ +import geotrellis.spark.store.hadoop._ +import geotrellis.store.hadoop._ + +import spire.syntax.cfor._ +import cats.implicits._ +import org.apache.spark.rdd.RDD + +import geotrellis.spark.testkit._ +import geotrellis.raster.testkit._ + +import org.scalatest.Inspectors._ +import org.scalatest._ + +class RasterSourceRDDSpec extends FunSpec with TestEnvironment with RasterMatchers with BeforeAndAfterAll { + val uri = Resource.path("vlm/aspect-tiled.tif") + def filePathByIndex(i: Int): String = Resource.path(s"vlm/aspect-tiled-$i.tif") + lazy val rasterSource = GeoTiffRasterSource(uri) + lazy val targetCRS = CRS.fromEpsgCode(3857) + lazy val scheme = ZoomedLayoutScheme(targetCRS) + lazy val layout = scheme.levelForZoom(13).layout + + lazy val reprojectedSource = rasterSource.reprojectToGrid(targetCRS, layout) + + describe("reading in GeoTiffs as RDDs") { + it("should have the right number of tiles") { + val expectedKeys = + layout + .mapTransform + .keysForGeometry(reprojectedSource.extent.toPolygon) + .toSeq + .sortBy { key => (key.col, key.row) } + + info(s"RasterSource CRS: ${reprojectedSource.crs}") + + val rdd = RasterSourceRDD.spatial(reprojectedSource, layout) + + val actualKeys = rdd.keys.collect().sortBy { key => (key.col, key.row) } + + for ((actual, expected) <- actualKeys.zip(expectedKeys)) { + actual should be(expected) + } + } + + it("should read in the tiles as squares") { + val reprojectedRasterSource = rasterSource.reprojectToGrid(targetCRS, layout) + val rdd = RasterSourceRDD.spatial(reprojectedRasterSource, layout) + val rows = rdd.collect() + + forAll(rows) { case (key, tile) => + withClue(s"$key") { + tile should have( + // dimensions(256, 256), + cellType(rasterSource.cellType), + bandCount(rasterSource.bandCount) + ) + } + } + } + } + + describe("Match reprojection from HadoopGeoTiffRDD") { + val floatingLayout = FloatingLayoutScheme(256) + val geoTiffRDD = HadoopGeoTiffRDD.spatialMultiband(uri) + val md = geoTiffRDD.collectMetadata[SpatialKey](floatingLayout)._2 + + val reprojectedExpectedRDD: MultibandTileLayerRDD[SpatialKey] = { + geoTiffRDD + .tileToLayout(md) + .reproject( + targetCRS, + layout + )._2.persist() + } + + def assertRDDLayersEqual( + expected: MultibandTileLayerRDD[SpatialKey], + actual: MultibandTileLayerRDD[SpatialKey], + matchRasters: Boolean = false + ): Unit = { + val joinedRDD = expected.filter { case (_, t) => !t.band(0).isNoDataTile }.leftOuterJoin(actual) + + joinedRDD.collect().foreach { case (key, (expected, actualTile)) => + actualTile match { + case Some(actual) => + /*writePngOutputTile( + actual, + name = "actual", + discriminator = s"-${key}" + ) + + writePngOutputTile( + expected, + name = "expected", + discriminator = s"-${key}" + )*/ + + // withGeoTiffClue(key, layout, actual, expected, targetCRS) { + withClue(s"$key:") { + expected.dimensions should be(actual.dimensions) + if (matchRasters) assertTilesEqual(expected, actual) + } + // } + + case None => + throw new Exception(s"$key does not exist in the rasterSourceRDD") + } + } + } + + it("should reproduce tileToLayout") { + // This should be the same as result of .tileToLayout(md.layout) + val rasterSourceRDD: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.spatial(rasterSource, md.layout) + + // Complete the reprojection + val reprojectedSource = rasterSourceRDD.reproject(targetCRS, layout)._2 + + assertRDDLayersEqual(reprojectedExpectedRDD, reprojectedSource) + } + + it("should reproduce tileToLayout followed by reproject") { + val reprojectedSourceRDD: MultibandTileLayerRDD[SpatialKey] = + RasterSourceRDD.spatial(rasterSource.reprojectToGrid(targetCRS, layout), layout) + + // geotrellis.raster.io.geotiff.GeoTiff(reprojectedExpectedRDD.stitch, targetCRS).write("/tmp/expected.tif") + // geotrellis.raster.io.geotiff.GeoTiff(reprojectedSourceRDD.stitch, targetCRS).write("/tmp/actual.tif") + + val actual = reprojectedSourceRDD.stitch.tile.band(0) + val expected = reprojectedExpectedRDD.stitch.tile.band(0) + + var (diff, pixels, mismatched) = (0d, 0d, 0) + cfor(0)(_ < math.min(actual.cols, expected.cols), _ + 1) { c => + cfor(0)(_ < math.min(actual.rows, expected.rows), _ + 1) { r => + pixels += 1d + if (math.abs(actual.get(c, r) - expected.get(c, r)) > 1e-6) + diff += 1d + if (isNoData(actual.get(c, r)) != isNoData(expected.get(c, r))) + mismatched += 1 + } + } + + assert(diff / pixels < 0.005) // half percent of pixels or less are not equal + assert(mismatched < 3) + } + + it("should reproduce tileToLayout when given an RDD[RasterSource]") { + val rasterSourceRDD: RDD[RasterSource] = sc.parallelize(Seq(rasterSource)) + + // Need to define these here or else a serialization error will occur + val targetLayout = layout + val crs = targetCRS + + val reprojectedRasterSourceRDD: RDD[RasterSource] = rasterSourceRDD.map { _.reprojectToGrid(crs, targetLayout) } + + val tiledSource: MultibandTileLayerRDD[SpatialKey] = RasterSourceRDD.tiledLayerRDD(reprojectedRasterSourceRDD, targetLayout) + + assertRDDLayersEqual(reprojectedExpectedRDD, tiledSource) + } + } + + describe("RasterSourceRDD.read") { + val floatingScheme = FloatingLayoutScheme(500, 270) + val floatingLayout = floatingScheme.levelFor(rasterSource.extent, rasterSource.cellSize).layout + + val cellType = rasterSource.cellType + + val multibandTilePath = GeoTiffPath(Resource.path("vlm/aspect-tiled-0-1-2.tif")) + + val noDataTile = ArrayTile.alloc(cellType, rasterSource.cols.toInt, rasterSource.rows.toInt).fill(NODATA).interpretAs(cellType) + + val paths: Seq[GeoTiffPath] = 0 to 5 map { index => GeoTiffPath(filePathByIndex(index)) } + + lazy val expectedMultibandTile = { + val tiles = paths.map { path => MultibandGeoTiff(path.toString, streaming = true).tile.band(0) } + MultibandTile(tiles) + } + + it("should read in singleband tiles") { + val readingSources: Seq[ReadingSource] = paths.zipWithIndex.map { case (path, index) => ReadingSource(GeoTiffRasterSource(path), 0, index) } + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + val expected = expectedMultibandTile + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + expected.dimensions shouldBe actual.dimensions + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband tiles with missing bands") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(paths(0)), 0, 0), + ReadingSource(GeoTiffRasterSource(paths(2)), 0, 1), + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 3) + ) + + val expected = MultibandTile( + expectedMultibandTile.band(0), + expectedMultibandTile.band(2), + noDataTile, + expectedMultibandTile.band(4) + ) + + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband and multiband tiles") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(multibandTilePath), 0, 0), + ReadingSource(GeoTiffRasterSource(paths(1)), 0, 1), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 2, 2), + ReadingSource(GeoTiffRasterSource(paths(3)), 0, 3), + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 4), + ReadingSource(GeoTiffRasterSource(paths(5)), 0, 5) + ) + + val expected = expectedMultibandTile + + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + + it("should read in singleband and multiband tiles with missing bands") { + val readingSources: Seq[ReadingSource] = + Seq( + ReadingSource(GeoTiffRasterSource(paths(4)), 0, 5), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 1, 0), + ReadingSource(GeoTiffRasterSource(multibandTilePath), 2, 1) + ) + + val expected = + MultibandTile( + expectedMultibandTile.band(1), + expectedMultibandTile.band(2), + noDataTile, + noDataTile, + noDataTile, + expectedMultibandTile.band(4) + ) + + val actual = RasterSourceRDD.read(readingSources, floatingLayout).stitch() + + // random chip to test agains, to speed up tests + val gridBounds = RasterExtent(randomExtentWithin(actual.extent), actual.cellSize).gridBounds + + assertEqual(expected.crop(gridBounds), actual.tile.crop(gridBounds)) + } + } +} diff --git a/spark/src/test/scala/geotrellis/spark/RasterSummarySpec.scala b/spark/src/test/scala/geotrellis/spark/RasterSummarySpec.scala new file mode 100644 index 0000000000..0cad58bb47 --- /dev/null +++ b/spark/src/test/scala/geotrellis/spark/RasterSummarySpec.scala @@ -0,0 +1,183 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.spark + +import geotrellis.spark.partition._ +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.geotiff._ +import geotrellis.raster.resample.Bilinear +import geotrellis.layer._ + +import org.apache.spark.rdd._ +import geotrellis.spark.testkit._ +import geotrellis.raster.testkit._ + +import org.scalatest._ +import java.time.{ZoneOffset, ZonedDateTime} + +class RasterSummarySpec extends FunSpec with TestEnvironment with RasterMatchers with GivenWhenThen { + + describe("Should collect GeoTiffRasterSource RasterSummary correct") { + it("should collect summary for a raw source") { + val inputPath = Resource.path("vlm/aspect-tiled.tif") + val files = inputPath :: Nil + + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GeoTiffRasterSource(uri): RasterSource) + .cache() + + val metadata = RasterSummary.fromRDD(sourceRDD) + val rasterSource = GeoTiffRasterSource(inputPath) + + rasterSource.crs shouldBe metadata.crs + rasterSource.extent shouldBe metadata.extent + rasterSource.cellSize shouldBe metadata.cellSize + rasterSource.cellType shouldBe metadata.cellType + rasterSource.size shouldBe metadata.cells + files.length shouldBe metadata.count + } + + it("should collect summary for a tiled to layout source") { + val inputPath = GeoTiffPath(Resource.path("vlm/aspect-tiled.tif")) + val files = inputPath :: Nil + val targetCRS = WebMercator + val method = Bilinear + val layoutScheme = ZoomedLayoutScheme(targetCRS, tileSize = 256) + + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GeoTiffRasterSource(uri).reproject(targetCRS, method = method): RasterSource) + .cache() + + val summary = RasterSummary.fromRDD(sourceRDD) + val LayoutLevel(zoom, layout) = summary.levelFor(layoutScheme) + val tiledLayoutSource = sourceRDD.map(_.tileToLayout(layout, method)) + + val summaryCollected = RasterSummary.fromRDD(tiledLayoutSource.map(_.source)) + val summaryResampled = summary.resample(TargetGrid(layout)) + + val metadata = summary.toTileLayerMetadata(layout) + val metadataResampled = summaryResampled.toTileLayerMetadata(GlobalLayout(256, zoom, 0.1)) + + metadata shouldBe metadataResampled + + summaryCollected.crs shouldBe summaryResampled.crs + summaryCollected.cellType shouldBe summaryResampled.cellType + + val CellSize(widthCollected, heightCollected) = summaryCollected.cellSize + val CellSize(widthResampled, heightResampled) = summaryResampled.cellSize + + // the only weird place where cellSize is a bit different + widthCollected shouldBe (widthResampled +- 1e-7) + heightCollected shouldBe (heightResampled +- 1e-7) + + summaryCollected.extent shouldBe summaryResampled.extent + summaryCollected.cells shouldBe summaryResampled.cells + summaryCollected.count shouldBe summaryResampled.count + } + } + + it("should create ContextRDD from RDD of GeoTiffRasterSources") { + val inputPath = GeoTiffPath(Resource.path("vlm/aspect-tiled.tif")) + val files = inputPath :: Nil + val targetCRS = WebMercator + val method = Bilinear + val layoutScheme = ZoomedLayoutScheme(targetCRS, tileSize = 256) + + // read sources + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GeoTiffRasterSource(uri).reproject(targetCRS, method = method): RasterSource) + .cache() + + // collect raster summary + val summary = RasterSummary.fromRDD(sourceRDD) + val LayoutLevel(_, layout) = summary.levelFor(layoutScheme) + val tiledLayoutSource = sourceRDD.map(_.tileToLayout(layout, method)) + + // Create RDD of references, references contain information how to read rasters + val rasterRefRdd: RDD[(SpatialKey, RasterRegion)] = tiledLayoutSource.flatMap(_.keyedRasterRegions()) + val tileRDD: RDD[(SpatialKey, MultibandTile)] = + rasterRefRdd // group by keys and distribute raster references using SpatialPartitioner + .groupByKey(SpatialPartitioner(summary.estimatePartitionsNumber)) + .mapValues { iter => MultibandTile(iter.flatMap(_.raster.toSeq.flatMap(_.tile.bands))) } // read rasters + + val metadata = summary.toTileLayerMetadata(layout) + val contextRDD: MultibandTileLayerRDD[SpatialKey] = ContextRDD(tileRDD, metadata) + + contextRDD.count() shouldBe rasterRefRdd.count() + contextRDD.count() shouldBe 72 + + contextRDD.stitch.tile.band(0).renderPng().write("/tmp/raster-source-contextrdd.png") + } + + it("should collect temporal contextRDD") { + val dates = "2018-01-01" :: "2018-02-01" :: "2018-03-01" :: Nil + val expectedDates = dates.map { str => + val Array(y, m, d) = str.split("-").map(_.toInt) + ZonedDateTime.of(y, m, d,0,0,0,0, ZoneOffset.UTC).toInstant.toEpochMilli + } + val files = dates.map { str => GeoTiffPath(Resource.path(s"vlm/aspect-tiled-$str.tif")) } + val targetCRS = WebMercator + val method = Bilinear + val layoutScheme = ZoomedLayoutScheme(targetCRS, tileSize = 256) + + // read sources + val sourceRDD: RDD[RasterSource] = + sc.parallelize(files, files.length) + .map(uri => GeoTiffRasterSource(uri).reproject(targetCRS, method = method): RasterSource) + .cache() + + // Im mr. happy face now (with a knife and i bite people) + val temporalKeyExtractor = TemporalKeyExtractor.fromPath { path => + val date = raw"(\d{4})-(\d{2})-(\d{2})".r.findFirstMatchIn(path.toString) + val Some((y, m, d)) = date.map { d => (d.group(1).toInt, d.group(2).toInt, d.group(3).toInt) } + + ZonedDateTime.of(y, m, d, 0, 0, 0, 0, ZoneOffset.UTC) + } + + // collect raster summary + val summary = RasterSummary.fromRDD(sourceRDD, temporalKeyExtractor.getMetadata) + // lets add layoutScheme overload + val LayoutLevel(_, layout) = summary.levelFor(layoutScheme) + + val contextRDD = + RasterSourceRDD.tiledLayerRDD(sourceRDD, layout, temporalKeyExtractor, rasterSummary = Some(summary)) + + val (minDate, maxDate) = expectedDates.head -> expectedDates.last + + contextRDD.metadata.bounds match { + case KeyBounds(minKey, maxKey) => + minKey.instant shouldBe minDate + maxKey.instant shouldBe maxDate + + case EmptyBounds => throw new Exception("EmptyBounds are not allowed here") + } + + contextRDD.count() shouldBe 72 * dates.length + + contextRDD + .toSpatial(minDate) + .stitch + .tile + .band(0) + .renderPng() + .write(s"/tmp/raster-source-contextrdd-${minDate}.png") + } +} diff --git a/spark/src/test/scala/geotrellis/store/CatalogTestEnvironment.scala b/spark/src/test/scala/geotrellis/store/CatalogTestEnvironment.scala new file mode 100644 index 0000000000..c579739b14 --- /dev/null +++ b/spark/src/test/scala/geotrellis/store/CatalogTestEnvironment.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.spark.testkit.TestEnvironment + +import org.scalatest.Suite + +import java.io.File + +trait CatalogTestEnvironment extends TestEnvironment { self: Suite => + override def beforeAll() = { + val multibandDir = new File(TestCatalog.multibandOutputPath) + // length >= 2 means that in the directory there are at least two folders - attributes and at least one layer folder + if (!(multibandDir.exists && multibandDir.list().length >= 2)) TestCatalog.createMultiband + else println(s"Test multi-band catalog exists at: $multibandDir") + val singlebandDir = new File(TestCatalog.singlebandOutputPath) + if (!(singlebandDir.exists && singlebandDir.list().length >= 2)) TestCatalog.createSingleband + else println(s"Test single-band catalog exists at: $singlebandDir") + } +} diff --git a/spark/src/test/scala/geotrellis/store/GeoTrellisConvertedRasterSourceSpec.scala b/spark/src/test/scala/geotrellis/store/GeoTrellisConvertedRasterSourceSpec.scala new file mode 100644 index 0000000000..b79f46c2fb --- /dev/null +++ b/spark/src/test/scala/geotrellis/store/GeoTrellisConvertedRasterSourceSpec.scala @@ -0,0 +1,235 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.testkit._ + +import org.scalatest.{FunSpec, GivenWhenThen} + +class GeoTrellisConvertedRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen with CatalogTestEnvironment { + val layerId = LayerId("landsat", 0) + val uriMultiband = s"file://${TestCatalog.multibandOutputPath}?layer=${layerId.name}&zoom=${layerId.zoom}" + + lazy val source = new GeoTrellisRasterSource(uriMultiband) + + lazy val expectedRaster: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(TestCatalog.filePath, streaming = false) + .raster + + describe("Converting to a different CellType") { + lazy val targetExtent = expectedRaster.extent + + lazy val expectedTile: MultibandTile = expectedRaster.tile + + describe("Bit CellType") { + it("should convert to: ByteCellType") { + val actual = source.convert(BitCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(BitCellType) } + + assertEqual(actual, expected) + } + } + + describe("Byte CellType") { + it("should convert to: ByteConstantNoDataCellType") { + val actual = source.convert(ByteConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: ByteUserDefinedNoDataCellType(10)") { + val actual = source.convert(ByteUserDefinedNoDataCellType(10)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteUserDefinedNoDataCellType(10)) } + + assertEqual(actual, expected) + } + + it("should convert to: ByteCellType") { + val actual = source.convert(ByteCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ByteCellType) } + + assertEqual(actual, expected) + } + } + + describe("UByte CellType") { + it("should convert to: UByteConstantNoDataCellType") { + val actual = source.convert(UByteConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: UByteUserDefinedNoDataCellType(10)") { + val actual = source.convert(UByteUserDefinedNoDataCellType(10)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteUserDefinedNoDataCellType(10)) } + + assertEqual(actual, expected) + } + + it("should convert to: UByteCellType") { + val actual = source.convert(UByteCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UByteCellType) } + + assertEqual(actual, expected) + } + } + + describe("Short CellType") { + it("should convert to: ShortConstantNoDataCellType") { + val actual = source.convert(ShortConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: ShortUserDefinedNoDataCellType(-1)") { + val actual = source.convert(ShortUserDefinedNoDataCellType(-1)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortUserDefinedNoDataCellType(-1)) } + + assertEqual(actual, expected) + } + + it("should convert to: ShortCellType") { + val actual = source.convert(ShortCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(ShortCellType) } + + assertEqual(actual, expected) + } + } + + describe("UShort CellType") { + it("should convert to: UShortConstantNoDataCellType") { + val actual = source.convert(UShortConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: UShortUserDefinedNoDataCellType(-1)") { + val actual = source.convert(UShortUserDefinedNoDataCellType(-1)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortUserDefinedNoDataCellType(-1)) } + + assertEqual(actual, expected) + } + + it("should convert to: UShortCellType") { + val actual = source.convert(UShortCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(UShortCellType) } + + assertEqual(actual, expected) + } + } + + describe("Int CellType") { + it("should convert to: IntConstantNoDataCellType") { + val actual = source.convert(IntConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: IntUserDefinedNoDataCellType(-100)") { + val actual = source.convert(IntUserDefinedNoDataCellType(-100)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntUserDefinedNoDataCellType(-100)) } + + assertEqual(actual, expected) + } + + it("should convert to: IntCellType") { + val actual = source.convert(IntCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(IntCellType) } + + assertEqual(actual, expected) + } + } + + describe("Float CellType") { + it("should convert to: FloatConstantNoDataCellType") { + val actual = source.convert(FloatConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: FloatUserDefinedNoDataCellType(0)") { + val actual = source.convert(FloatUserDefinedNoDataCellType(0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatUserDefinedNoDataCellType(0)) } + + assertEqual(actual, expected) + } + + it("should convert to: FloatCellType") { + val actual = source.convert(FloatCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(FloatCellType) } + + assertEqual(actual, expected) + } + } + + describe("Double CellType") { + it("should convert to: DoubleConstantNoDataCellType") { + val actual = source.convert(DoubleConstantNoDataCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleConstantNoDataCellType) } + + assertEqual(actual, expected) + } + + it("should convert to: DoubleUserDefinedNoDataCellType(1.0)") { + val actual = source.convert(DoubleUserDefinedNoDataCellType(1.0)).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleUserDefinedNoDataCellType(1.0)) } + + assertEqual(actual, expected) + } + + it("should convert to: DoubleCellType") { + val actual = source.convert(DoubleCellType).read(targetExtent).get + val expected = source.read(targetExtent).get.mapTile { _.convert(DoubleCellType) } + + assertEqual(actual, expected) + } + } + } + + describe("Chaining together operations") { + lazy val targetCellType = DoubleConstantNoDataCellType + lazy val targetExtent = expectedRaster.extent.reproject(source.crs, WebMercator) + lazy val expectedTile: MultibandTile = expectedRaster.tile + + it("should have the correct CellType after reproject") { + val actual = source.convert(targetCellType).reproject(WebMercator).read(targetExtent).get.cellType + + actual should be (targetCellType) + } + + it("should have the correct CellType after multiple conversions") { + val actual = + source + .convert(FloatUserDefinedNoDataCellType(0)) + .reproject(WebMercator) + .convert(targetCellType) + .read(targetExtent).get.cellType + + actual should be (targetCellType) + } + } +} diff --git a/spark/src/test/scala/geotrellis/store/GeoTrellisRasterSourceSpec.scala b/spark/src/test/scala/geotrellis/store/GeoTrellisRasterSourceSpec.scala new file mode 100644 index 0000000000..30590d3389 --- /dev/null +++ b/spark/src/test/scala/geotrellis/store/GeoTrellisRasterSourceSpec.scala @@ -0,0 +1,201 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.io.geotiff.{Auto, AutoHigherResolution, Base} +import geotrellis.raster.io.geotiff.reader.GeoTiffReader +import geotrellis.raster.testkit._ +import geotrellis.raster.MultibandTile +import geotrellis.raster.resample.NearestNeighbor +import geotrellis.store._ +import geotrellis.vector.Extent + +import org.scalatest._ + +class GeoTrellisRasterSourceSpec extends FunSpec with RasterMatchers with GivenWhenThen with CatalogTestEnvironment { + val layerId = LayerId("landsat", 0) + val uriMultibandNoParams = s"file://${TestCatalog.multibandOutputPath}" + val uriMultiband = s"file://${TestCatalog.multibandOutputPath}?layer=${layerId.name}&zoom=${layerId.zoom}" + val uriSingleband = s"file://${TestCatalog.singlebandOutputPath}?layer=${layerId.name}&zoom=${layerId.zoom}" + lazy val sourceMultiband = new GeoTrellisRasterSource(uriMultiband) + lazy val sourceSingleband = new GeoTrellisRasterSource(uriSingleband) + + describe("geotrellis raster source") { + + it("should read singleband tile") { + val bounds = GridBounds(0, 0, 2, 2).toGridType[Long] + // NOTE: All tiles are converted to multiband + val chip: Raster[MultibandTile] = sourceSingleband.read(bounds).get + chip should have ( + // dimensions (bounds.width, bounds.height), + cellType (sourceSingleband.cellType) + ) + } + + it("should read multiband tile") { + val bounds = GridBounds(0, 0, 2, 2).toGridType[Long] + val chip: Raster[MultibandTile] = sourceMultiband.read(bounds).get + chip should have ( + // dimensions (bounds.width, bounds.height), + cellType (sourceMultiband.cellType) + ) + } + + it("should read offset tile") { + val bounds = GridBounds(2, 2, 4, 4).toGridType[Long] + val chip: Raster[MultibandTile] = sourceMultiband.read(bounds).get + chip should have ( + // dimensions (bounds.width, bounds.height), + cellType (sourceMultiband.cellType) + ) + } + + it("should read entire file") { + val bounds = GridBounds(0, 0, sourceMultiband.cols - 1, sourceMultiband.rows - 1) + val chip: Raster[MultibandTile] = sourceMultiband.read(bounds).get + chip should have ( + // dimensions (sourceMultiband.dimensions), + cellType (sourceMultiband.cellType) + ) + } + + it("should not read past file edges") { + Given("bounds larger than raster") + val bounds = GridBounds(0, 0, sourceMultiband.cols + 100, sourceMultiband.rows + 100) + When("reading by pixel bounds") + val chip = sourceMultiband.read(bounds).get + Then("return only pixels that exist") + // chip.tile should have (dimensions (sourceMultiband.dimensions)) + } + + it("should be able to read empty layer") { + val bounds = GridBounds(9999, 9999, 10000, 10000).toGridType[Long] + assert(sourceMultiband.read(bounds) == None) + } + + it("should be able to resample") { + // read in the whole file and resample the pixels in memory + val expected: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(TestCatalog.filePath, streaming = false) + .raster + .resample((sourceMultiband.cols * 0.95).toInt, (sourceMultiband.rows * 0.95).toInt, NearestNeighbor) + // resample to 0.9 so RasterSource picks the base layer and not an overview + + val resampledSource = + sourceMultiband.resample(expected.tile.cols, expected.tile.rows, NearestNeighbor) + + // resampledSource should have (dimensions (expected.tile.dimensions)) + + val actual: Raster[MultibandTile] = + resampledSource + .resampleToGrid(expected.rasterExtent.toGridType[Long]) + .read(expected.extent) + .get + + withGeoTiffClue(actual, expected, resampledSource.crs) { + assertRastersEqual(actual, expected) + } + } + + it("should have resolutions only for given layer name") { + assert( + sourceMultiband.resolutions.length === + CollectionLayerReader(uriMultibandNoParams).attributeStore.layerIds.filter(_.name == layerId.name).length + ) + assert( + new GeoTrellisRasterSource(s"$uriMultibandNoParams?layer=bogusLayer&zoom=0").resolutions.length === 0 + ) + } + + it("should get the closest resolution") { + val extent = Extent(0.0, 0.0, 10.0, 10.0) + val rasterExtent1 = new GridExtent[Long](extent, CellSize(1.0, 1.0)) + val rasterExtent2 = new GridExtent[Long](extent, CellSize(2.0, 2.0)) + val rasterExtent3 = new GridExtent[Long](extent, CellSize(4.0, 4.0)) + + val resolutions = List(rasterExtent1, rasterExtent2, rasterExtent3) + val cellSize1 = CellSize(1.0, 1.0) + val cellSize2 = CellSize(2.0, 2.0) + + implicit def getoce(ge: GridExtent[Long]): CellSize = ge.cellSize + + assert(GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, AutoHigherResolution).get == rasterExtent1) + assert(GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize2, AutoHigherResolution).get == rasterExtent2) + + assert(GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, Auto(0)).get == rasterExtent1) + assert(GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, Auto(1)).get == rasterExtent2) + assert(GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, Auto(2)).get == rasterExtent3) + // do the best we can, we can't get index 3, so we get the closest: + val res = GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, Auto(3)) + info (s"Auto(3): ${res.map(_.cellSize)}") + assert(res == Some(rasterExtent3)) + + val resBase = GeoTrellisRasterSource.getClosestResolution(resolutions, cellSize1, Base) + info(s"Base: ${resBase.map(_.cellSize)}") + assert(resBase == Some(rasterExtent1)) + } + + // it("should get the closest layer") { + // val extent = Extent(0.0, 0.0, 10.0, 10.0) + // val rasterExtent1 = new GridExtent[Long](extent, 1.0, 1.0, 10, 10) + // val rasterExtent2 = new GridExtent[Long](extent, 2.0, 2.0, 10, 10) + // val rasterExtent3 = new GridExtent[Long](extent, 4.0, 4.0, 10, 10) + + // val resolutions = List(rasterExtent1, rasterExtent2, rasterExtent3) + + // val layerId1 = LayerId("foo", 0) + // val layerId2 = LayerId("foo", 1) + // val layerId3 = LayerId("foo", 2) + // val layerIds = List(layerId1, layerId2, layerId3) + + // val cellSize = CellSize(1.0, 1.0) + + // implicit def getoce(ge: GridExtent[Long]): CellSize = ge.cellSize + // assert(GeoTrellisRasterSource.getClosestLayer(resolutions, layerIds, layerId3, cellSize) == layerId1) + // assert(GeoTrellisRasterSource.getClosestLayer(List(), List(), layerId3, cellSize) == layerId3) + // assert(GeoTrellisRasterSource.getClosestLayer(resolutions, List(), layerId3, cellSize) == layerId3) + // } + + it("should reproject") { + val targetCRS = WebMercator + val bounds = GridBounds[Int](0, 0, sourceMultiband.cols.toInt - 1, sourceMultiband.rows.toInt - 1) + + val expected: Raster[MultibandTile] = + GeoTiffReader + .readMultiband(TestCatalog.filePath, streaming = false) + .raster + .reproject(bounds, sourceMultiband.crs, targetCRS) + + val reprojectedSource = sourceMultiband.reprojectToRegion(targetCRS, expected.rasterExtent) + + // reprojectedSource should have (dimensions (expected.tile.dimensions)) + + val actual: Raster[MultibandTile] = + reprojectedSource + .reprojectToRegion(targetCRS, expected.rasterExtent) + .read(expected.extent) + .get + + withGeoTiffClue(actual, expected, reprojectedSource.crs) { + assertRastersEqual(actual, expected) + } + } + } +} diff --git a/spark/src/test/scala/geotrellis/store/TestCatalog.scala b/spark/src/test/scala/geotrellis/store/TestCatalog.scala new file mode 100644 index 0000000000..f7a9270dfd --- /dev/null +++ b/spark/src/test/scala/geotrellis/store/TestCatalog.scala @@ -0,0 +1,86 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.layer.{LayoutDefinition, SpatialKey} +import geotrellis.raster.geotiff.GeoTiffRasterSource +import geotrellis.raster.{ArrayMultibandTile, MultibandTile} +import geotrellis.spark.store.file.FileLayerWriter +import geotrellis.spark._ +import geotrellis.store.file.FileAttributeStore +import geotrellis.store.index.ZCurveKeyIndexMethod + +import org.apache.spark.SparkContext + +import java.io.File + +object TestCatalog { + def resourcesPath(path: String): String = s"${new File("").getAbsolutePath}/spark/src/test/resources/$path" + + val filePath = resourcesPath("vlm/aspect-tiled.tif") + val multibandOutputPath = resourcesPath("vlm/catalog") + val singlebandOutputPath = resourcesPath("vlm/single_band_catalog") + + def fullPath(path: String) = new java.io.File(path).getAbsolutePath + + def createMultiband(implicit sc: SparkContext): Unit = { + // Create the attributes store that will tell us information about our catalog. + val attributeStore = FileAttributeStore(multibandOutputPath) + + // Create the writer that we will use to store the tiles in the local catalog. + val writer = FileLayerWriter(attributeStore) + + val rs = GeoTiffRasterSource(TestCatalog.filePath) + rs.resolutions.sortBy(_.cellSize.resolution).zipWithIndex.foreach { case (rasterExtent, index) => + val layout = LayoutDefinition(rasterExtent, tileSize = 256) + + val rdd: MultibandTileLayerRDD[SpatialKey] = + RasterSourceRDD.spatial(List(rs.resampleToGrid(layout)), layout) + .withContext( tiledd => + // the tiles are actually `PaddedTile`, this forces them to be ArrayTile + tiledd.mapValues { mb: MultibandTile => ArrayMultibandTile(mb.bands.map(_.toArrayTile))} + ) + + val id = LayerId("landsat", index) + writer.write(id, rdd, ZCurveKeyIndexMethod) + } + } + + def createSingleband(implicit sc: SparkContext): Unit = { + // Create the attributes store that will tell us information about our catalog. + val attributeStore = FileAttributeStore(singlebandOutputPath) + + // Create the writer that we will use to store the tiles in the local catalog. + val writer = FileLayerWriter(attributeStore) + + val rs = GeoTiffRasterSource(TestCatalog.filePath) + rs.resolutions.sortBy(_.cellSize.resolution).zipWithIndex.foreach { case (rasterExtent, index) => + val layout = LayoutDefinition(rasterExtent, tileSize = 256) + + val rdd: TileLayerRDD[SpatialKey] = + RasterSourceRDD.spatial(List(rs.resampleToGrid(layout)), layout) + .withContext( tiledd => + tiledd.mapValues { mb: MultibandTile => + ArrayMultibandTile(mb.bands.map(_.toArrayTile)).band(0) // Get only first band + } + ) + + val id = LayerId("landsat", index) + writer.write(id, rdd, ZCurveKeyIndexMethod) + } + } +} diff --git a/spark/src/test/scala/geotrellis/store/TestCatalogSpec.scala b/spark/src/test/scala/geotrellis/store/TestCatalogSpec.scala new file mode 100644 index 0000000000..2a14a9c80b --- /dev/null +++ b/spark/src/test/scala/geotrellis/store/TestCatalogSpec.scala @@ -0,0 +1,96 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.layer._ +import geotrellis.raster.geotiff._ +import geotrellis.raster.{MultibandTile, Tile} + +import org.scalatest.FunSpec + +import java.io.File + +class TestCatalogSpec extends FunSpec with CatalogTestEnvironment { + val absMultibandOutputPath = s"file://${TestCatalog.multibandOutputPath}" + val absSinglebandOutputPath = s"file://${TestCatalog.singlebandOutputPath}" + + describe("catalog test environment") { + it("should create multiband catalog before test is run") { + assert(new File(TestCatalog.multibandOutputPath).exists) + } + it("should create singleband catalog before test is run") { + assert(new File(TestCatalog.singlebandOutputPath).exists) + } + } + describe("value reader") { + it("should be able to read multiband test catalog") { + ValueReader(absMultibandOutputPath).reader[SpatialKey, Tile](LayerId("landsat", 0)) + } + it("should be able to read single test catalog") { + ValueReader(absSinglebandOutputPath).reader[SpatialKey, Tile](LayerId("landsat", 0)) + } + it("should be unable to read non-existent test catalog") { + assertThrows[AttributeNotFoundError] { + ValueReader(absMultibandOutputPath).reader[SpatialKey, Tile](LayerId("INVALID", 0)) + } + } + } + describe("collection layer reader") { + it("should be able to read multiband test catalog") { + CollectionLayerReader(absMultibandOutputPath).read[SpatialKey, MultibandTile, TileLayerMetadata[SpatialKey]](LayerId("landsat", 0)) + } + it("should be able to read singleband test catalog") { + CollectionLayerReader(absSinglebandOutputPath).read[SpatialKey, Tile, TileLayerMetadata[SpatialKey]](LayerId("landsat", 0)) + } + it("should be unable to read non-existent test catalog") { + assertThrows[LayerNotFoundError] { + CollectionLayerReader(absMultibandOutputPath).read[SpatialKey, MultibandTile, TileLayerMetadata[SpatialKey]](LayerId("INVALID", 0)) + } + } + } + + describe("test catalog") { + lazy val reader = ValueReader(absMultibandOutputPath) + lazy val rs = GeoTiffRasterSource(TestCatalog.filePath) + + it("preserves geotiff overviews") { + info(reader.attributeStore.layerIds.toString) + info(rs.resolutions.toString) + assert(reader.attributeStore.layerIds.length == rs.resolutions.length) + } + it("preserves cell size") { + info(reader.attributeStore.readMetadata[TileLayerMetadata[SpatialKey]](LayerId("landsat", 0)).cellSize.toString) + // TODO: Make geotrellis.raster.CellSize sortable + val expectedCellSizes = rs.resolutions.map(_.cellSize).sortBy(_.resolution) + info(expectedCellSizes.toString) + val actualCellSizes = reader.attributeStore.layerIds.map(layerId => reader.attributeStore.readMetadata[TileLayerMetadata[SpatialKey]](layerId).cellSize).sortBy(_.resolution) + info(actualCellSizes.toString) + assert(expectedCellSizes.length == actualCellSizes.length) + expectedCellSizes.zip(actualCellSizes).foreach { case(x, y) => + x.height shouldBe y.height +- 1e-10 + x.width shouldBe y.width +- 1e-10 + } + } + it("preserves projection") { + val expectedProjections = Set(rs.crs) + val actualProjections = reader.attributeStore.layerIds.map(layerId => reader.attributeStore.readMetadata[TileLayerMetadata[SpatialKey]](layerId).crs).toSet + expectedProjections.shouldBe(actualProjections) + } + } + +} + diff --git a/store/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider b/store/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider new file mode 100644 index 0000000000..85f4b3b452 --- /dev/null +++ b/store/src/main/resources/META-INF/services/geotrellis.raster.RasterSourceProvider @@ -0,0 +1 @@ +geotrellis.store.GeoTrellisRasterSourceProvider diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisMetadata.scala b/store/src/main/scala/geotrellis/store/GeoTrellisMetadata.scala new file mode 100644 index 0000000000..3077380afb --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisMetadata.scala @@ -0,0 +1,35 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.raster.{RasterMetadata, SourceName} +import geotrellis.proj4.CRS +import geotrellis.raster.{CellType, GridExtent} + +case class GeoTrellisMetadata( + name: SourceName, + crs: CRS, + bandCount: Int, + cellType: CellType, + gridExtent: GridExtent[Long], + resolutions: List[GridExtent[Long]], + attributes: Map[String, String] +) extends RasterMetadata { + /** GeoTrellis metadata doesn't allow to query a per band metadata by default. */ + def attributesForBand(band: Int): Map[String, String] = Map.empty +} + diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisPath.scala b/store/src/main/scala/geotrellis/store/GeoTrellisPath.scala new file mode 100644 index 0000000000..fe4ae416eb --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisPath.scala @@ -0,0 +1,83 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.raster.SourcePath + +import cats.syntax.option._ +import io.lemonlabs.uri.{Url, UrlPath, UrlWithAuthority} + +import java.net.MalformedURLException + +/** Represents a path that points to a GeoTrellis layer saved in a catalog. + * + * @param value Path to the layer. This can be either an Avro or COG layer. + * The given path needs to be in a `URI` format that include the following query + * parameters: + * - '''layer''': The name of the layer. + * - '''zoom''': The zoom level to be read. + * - '''band_count''': The number of bands of each Tile in the layer. + * Of the above three parameters, `layer` and `zoom` are required. In addition, + * this path can be prefixed with, '''gt+''' to signify that the target path + * is to be read in only by [[GeotrellisRasterSource]]. + * @example "s3://bucket/catalog?layer=layer_name&zoom=10" + * @example "hdfs://data-folder/catalog?layer=name&zoom-12&band_count=5" + * @example "gt+file:///tmp/catalog?layer=name&zoom=5" + * @note The order of the query parameters does not matter. + */ +case class GeoTrellisPath(value: String, layerName: String, zoomLevel: Option[Int], bandCount: Option[Int]) extends SourcePath { + def layerId: LayerId = LayerId(layerName, zoomLevel.get) +} + +object GeoTrellisPath { + val PREFIX = "gt+" + + implicit def toGeoTrellisDataPath(path: String): GeoTrellisPath = parse(path) + + def parseOption(path: String): Option[GeoTrellisPath] = { + val layerNameParam: String = "layer" + val zoomLevelParam: String = "zoom" + val bandCountParam: String = "band_count" + + // try to parse it, otherwise it is a path + val uri = UrlWithAuthority.parseOption(path).fold(Url().withPath(UrlPath.fromRaw(path)): Url)(identity) + val queryString = uri.query + + val catalogPath: Option[String] = { + uri.schemeOption.fold(uri.toStringRaw.some) { scheme => + val authority = + uri match { + case url: UrlWithAuthority => url.authority.userInfo.user.getOrElse("") + case _ => "" + } + + s"${scheme.split("\\+").last}://$authority${uri.path}".some + } + } + + catalogPath.fold(Option.empty[GeoTrellisPath]) { catalogPath => + val layerName: Option[String] = queryString.param(layerNameParam) + val zoomLevel: Option[Int] = queryString.param(zoomLevelParam).map(_.toInt) + val bandCount: Option[Int] = queryString.param(bandCountParam).map(_.toInt) + + layerName.map(GeoTrellisPath(catalogPath, _, zoomLevel, bandCount)) + } + } + + def parse(path: String): GeoTrellisPath = + parseOption(path).getOrElse(throw new MalformedURLException(s"Unable to parse GeoTrellisDataPath: $path")) +} diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisRasterSource.scala b/store/src/main/scala/geotrellis/store/GeoTrellisRasterSource.scala new file mode 100644 index 0000000000..731323352d --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisRasterSource.scala @@ -0,0 +1,237 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.{Auto, AutoHigherResolution, Base, OverviewStrategy} +import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} +import geotrellis.raster._ +import geotrellis.layer._ +import geotrellis.vector._ + +case class Layer(id: LayerId, metadata: TileLayerMetadata[SpatialKey], bandCount: Int) { + /** GridExtent of the data pixels in the layer */ + def gridExtent: GridExtent[Long] = metadata.layout.createAlignedGridExtent(metadata.extent) +} + +/** + * Note: GeoTrellis AttributeStore does not store the band count for the layers by default, + * thus they need to be provided from application configuration. + * + * @param dataPath geotrellis catalog DataPath + */ +class GeoTrellisRasterSource( + val attributeStore: AttributeStore, + val dataPath: GeoTrellisPath, + val sourceLayers: Stream[Layer], + val targetCellType: Option[TargetCellType] +) extends RasterSource { + def name: GeoTrellisPath = dataPath + + def this(attributeStore: AttributeStore, dataPath: GeoTrellisPath) = + this( + attributeStore, + dataPath, + GeoTrellisRasterSource.getSourceLayersByName(attributeStore, dataPath.layerName, dataPath.bandCount.getOrElse(1)), + None + ) + + def this(dataPath: GeoTrellisPath) = this(AttributeStore(dataPath.value), dataPath) + + def layerId: LayerId = dataPath.layerId + + lazy val reader = CollectionLayerReader(attributeStore, dataPath.value) + + // read metadata directly instead of searching sourceLayers to avoid unneeded reads + lazy val layerMetadata = reader.attributeStore.readMetadata[TileLayerMetadata[SpatialKey]](layerId) + + lazy val gridExtent: GridExtent[Long] = layerMetadata.layout.createAlignedGridExtent(layerMetadata.extent) + + val bandCount: Int = dataPath.bandCount.getOrElse(1) + + def crs: CRS = layerMetadata.crs + + def cellType: CellType = dstCellType.getOrElse(layerMetadata.cellType) + + def attributes: Map[String, String] = Map( + "catalogURI" -> dataPath.value, + "layerName" -> layerId.name, + "zoomLevel" -> layerId.zoom.toString, + "bandCount" -> bandCount.toString + ) + /** GeoTrellis metadata doesn't allow to query a per band metadata by default. */ + def attributesForBand(band: Int): Map[String, String] = Map.empty + + def metadata: GeoTrellisMetadata = GeoTrellisMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, attributes) + + // reference to this will fully initilze the sourceLayers stream + lazy val resolutions: List[GridExtent[Long]] = sourceLayers.map(_.gridExtent).toList + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + GeoTrellisRasterSource.read(reader, layerId, layerMetadata, extent, bands).map(convertRaster) + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = + bounds + .intersection(this.gridBounds) + .map(gridExtent.extentFor(_).buffer(- cellSize.width / 2, - cellSize.height / 2)) + .flatMap(read(_, bands)) + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + extents.toIterator.flatMap(read(_, bands)) + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + bounds.toIterator.flatMap(_.intersection(this.gridBounds).flatMap(read(_, bands))) + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = { + if (targetCRS != this.crs) { + val reprojectOptions = ResampleGrid.toReprojectOptions[Long](this.gridExtent, resampleGrid, method) + val (closestLayerId, targetGridExtent) = GeoTrellisReprojectRasterSource.getClosestSourceLayer(targetCRS, sourceLayers, reprojectOptions, strategy) + new GeoTrellisReprojectRasterSource(attributeStore, dataPath, closestLayerId, sourceLayers, targetGridExtent, targetCRS, resampleGrid, targetCellType = targetCellType) + } else { + // TODO: add unit tests for this in particular, the behavior feels murky + resampleGrid match { + case IdentityResampleGrid => + // I think I was asked to do nothing + this + case resampleGrid => + val resampledGridExtent = resampleGrid(this.gridExtent) + val closestLayerId = GeoTrellisRasterSource.getClosestResolution(sourceLayers.toList, resampledGridExtent.cellSize, strategy)(_.metadata.layout.cellSize).get.id + new GeoTrellisResampleRasterSource(attributeStore, dataPath, closestLayerId, sourceLayers, resampledGridExtent, method, targetCellType) + } + } + } + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = { + val resampledGridExtent = resampleGrid(this.gridExtent) + val closestLayerId = GeoTrellisRasterSource.getClosestResolution(sourceLayers.toList, resampledGridExtent.cellSize, strategy)(_.metadata.layout.cellSize).get.id + new GeoTrellisResampleRasterSource(attributeStore, dataPath, closestLayerId, sourceLayers, resampledGridExtent, method, targetCellType) + } + + def convert(targetCellType: TargetCellType): RasterSource = + new GeoTrellisRasterSource(attributeStore, dataPath, sourceLayers, Some(targetCellType)) + + override def toString: String = + s"GeoTrellisRasterSource($dataPath, $layerId)" +} + + +object GeoTrellisRasterSource { + // stable identifiers to match in a readTiles function + private val SpatialKeyClass = classOf[SpatialKey] + private val TileClass = classOf[Tile] + private val MultibandTileClass = classOf[MultibandTile] + + def getClosestResolution[T]( + grids: Seq[T], + cellSize: CellSize, + strategy: OverviewStrategy = AutoHigherResolution + )(implicit f: T => CellSize): Option[T] = { + val maxResultion = Some(grids.minBy(g => f(g).resolution)) + + strategy match { + case AutoHigherResolution => + grids // overviews can have erased extent information + .map { v => (cellSize.resolution - f(v).resolution) -> v } + .filter(_._1 >= 0) + .sortBy(_._1) + .map(_._2) + .headOption + .orElse(maxResultion) + case Auto(n) => + val sorted = grids.sortBy(v => math.abs(cellSize.resolution - f(v).resolution)) + sorted.lift(n).orElse(sorted.lastOption) // n can be out of bounds, + // makes only overview lookup as overview position is important + case Base => maxResultion + } + } + + /** Read metadata for all layers that share a name and sort them by their resolution */ + def getSourceLayersByName(attributeStore: AttributeStore, layerName: String, bandCount: Int): Stream[Layer] = { + attributeStore. + layerIds. + filter(_.name == layerName). + sortWith(_.zoom > _.zoom). + toStream. // We will be lazy about fetching higher zoom levels + map { id => + val metadata = attributeStore.readMetadata[TileLayerMetadata[SpatialKey]](id) + Layer(id, metadata, bandCount) + } + } + + def readTiles(reader: CollectionLayerReader[LayerId], layerId: LayerId, extent: Extent, bands: Seq[Int]): Seq[(SpatialKey, MultibandTile)] with Metadata[TileLayerMetadata[SpatialKey]] = { + val header = reader.attributeStore.readHeader[LayerHeader](layerId) + (Class.forName(header.keyClass), Class.forName(header.valueClass)) match { + case (SpatialKeyClass, TileClass) => + reader.query[SpatialKey, Tile, TileLayerMetadata[SpatialKey]](layerId) + .where(Intersects(extent)) + .result + .withContext(tiles => + // Convert single band tiles to multiband + tiles.map{ case(key, tile) => (key, MultibandTile(tile)) } + ) + case (SpatialKeyClass, MultibandTileClass) => + reader.query[SpatialKey, MultibandTile, TileLayerMetadata[SpatialKey]](layerId) + .where(Intersects(extent)) + .result + .withContext(tiles => + tiles.map{ case(key, tile) => (key, tile.subsetBands(bands)) } + ) + case _ => + throw new Exception(s"Unable to read single or multiband tiles from file: ${(header.keyClass, header.valueClass)}") + } + } + + def readIntersecting(reader: CollectionLayerReader[LayerId], layerId: LayerId, metadata: TileLayerMetadata[SpatialKey], extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val tiles = readTiles(reader, layerId, extent, bands) + sparseStitch(tiles, extent) + } + + def read(reader: CollectionLayerReader[LayerId], layerId: LayerId, metadata: TileLayerMetadata[SpatialKey], extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val tiles = readTiles(reader, layerId, extent, bands) + metadata.extent.intersection(extent) flatMap { intersectionExtent => + sparseStitch(tiles, intersectionExtent).map(_.crop(intersectionExtent)) + } + } + + /** + * The stitch method in gtcore is unable to handle missing spatialkeys correctly. + * This method works around that problem by attempting to infer any missing tiles + **/ + def sparseStitch( + tiles: Seq[(SpatialKey, MultibandTile)] with Metadata[TileLayerMetadata[SpatialKey]], + extent: Extent + ): Option[Raster[MultibandTile]] = { + val md = tiles.metadata + val expectedKeys = md + .mapTransform(extent) + .coordsIter + .map { case (x, y) => SpatialKey(x, y) } + .toList + val actualKeys = tiles.map(_._1) + val missingKeys = expectedKeys diff actualKeys + + val missingTiles = missingKeys.map { key => + (key, MultibandTile(ArrayTile.empty(md.cellType, md.tileLayout.tileCols, md.tileLayout.tileRows))) + } + val allTiles = tiles.withContext { collection => + collection.toList ::: missingTiles + } + if (allTiles.isEmpty) None + else Some(allTiles.stitch()) + } +} diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisRasterSourceProvider.scala b/store/src/main/scala/geotrellis/store/GeoTrellisRasterSourceProvider.scala new file mode 100644 index 0000000000..bb5571dfd2 --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisRasterSourceProvider.scala @@ -0,0 +1,27 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.raster.RasterSourceProvider +import geotrellis.raster.geotiff.GeoTiffPath + +class GeoTrellisRasterSourceProvider extends RasterSourceProvider { + def canProcess(path: String): Boolean = + (!path.startsWith(GeoTiffPath.PREFIX) && !path.startsWith("gdal+")) && path.nonEmpty && GeoTrellisPath.parseOption(path).nonEmpty + + def rasterSource(path: String): GeoTrellisRasterSource = new GeoTrellisRasterSource(path) +} diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisReprojectRasterSource.scala b/store/src/main/scala/geotrellis/store/GeoTrellisReprojectRasterSource.scala new file mode 100644 index 0000000000..45ecbe6559 --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisReprojectRasterSource.scala @@ -0,0 +1,221 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.vector._ +import geotrellis.raster._ +import geotrellis.raster.reproject._ +import geotrellis.raster.resample._ +import geotrellis.proj4._ +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} + +import com.typesafe.scalalogging.LazyLogging +import scala.io.AnsiColor._ + +class GeoTrellisReprojectRasterSource( + val attributeStore: AttributeStore, + val dataPath: GeoTrellisPath, + val layerId: LayerId, + val sourceLayers: Stream[Layer], + val gridExtent: GridExtent[Long], + val crs: CRS, + val targetResampleGrid: ResampleGrid[Long] = IdentityResampleGrid, + val resampleMethod: ResampleMethod = NearestNeighbor, + val strategy: OverviewStrategy = AutoHigherResolution, + val errorThreshold: Double = 0.125, + val targetCellType: Option[TargetCellType] +) extends RasterSource with LazyLogging { + def name: GeoTrellisPath = dataPath + + lazy val reader = CollectionLayerReader(attributeStore, dataPath.value) + + lazy val resolutions: List[GridExtent[Long]] = { + sourceLayers.map { layer => + ReprojectRasterExtent(layer.gridExtent, layer.metadata.crs, crs) + } + }.toList + + lazy val sourceLayer: Layer = sourceLayers.find(_.id == layerId).get + + def bandCount: Int = sourceLayer.bandCount + + def cellType: CellType = dstCellType.getOrElse(sourceLayer.metadata.cellType) + + def attributes: Map[String, String] = Map( + "catalogURI" -> dataPath.value, + "layerName" -> layerId.name, + "zoomLevel" -> layerId.zoom.toString, + "bandCount" -> bandCount.toString + ) + /** GeoTrellis metadata doesn't allow to query a per band metadata by default. */ + def attributesForBand(band: Int): Map[String, String] = Map.empty + + def metadata: GeoTrellisMetadata = GeoTrellisMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, attributes) + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val transform = Transform(sourceLayer.metadata.crs, crs) + val backTransform = Transform(crs, sourceLayer.metadata.crs) + for { + subExtent <- this.extent.intersection(extent) + targetRasterExtent = this.gridExtent.createAlignedRasterExtent(subExtent) + sourceExtent = targetRasterExtent.extent.reprojectAsPolygon(backTransform, 0.001).getEnvelopeInternal + sourceRegion = sourceLayer.metadata.layout.createAlignedGridExtent(sourceExtent) + _ = { + lazy val tileBounds = sourceLayer.metadata.mapTransform.extentToBounds(sourceExtent) + lazy val pixelsRead = (tileBounds.size * sourceLayer.metadata.layout.tileCols * sourceLayer.metadata.layout.tileRows).toDouble + lazy val pixelsQueried = targetRasterExtent.cols.toDouble * targetRasterExtent.rows.toDouble + def msg = s""" + |${GREEN}Read($extent)${RESET} = + |\t${BOLD}FROM${RESET} ${dataPath.toString} ${sourceLayer.id} + |\t${BOLD}SOURCE${RESET} $sourceExtent ${sourceLayer.metadata.cellSize} @ ${sourceLayer.metadata.crs} + |\t${BOLD}TARGET${RESET} ${targetRasterExtent.extent} ${targetRasterExtent.cellSize} @ ${crs} + |\t${BOLD}READ${RESET} ${pixelsRead/pixelsQueried} read/query ratio for ${tileBounds.size} tiles + """.stripMargin + if (tileBounds.size < 1024) // Assuming 256x256 tiles this would be a very large request + logger.debug(msg) + else + logger.warn(msg + " (large read)") + } + raster <- GeoTrellisRasterSource.readIntersecting(reader, layerId, sourceLayer.metadata, sourceExtent, bands) + } yield { + val reprojected = raster.reproject( + targetRasterExtent, + transform, + backTransform, + ResampleGrid.toReprojectOptions[Long](targetRasterExtent.toGridType[Long], targetResampleGrid, resampleMethod) + ) + convertRaster(reprojected) + } + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = + bounds + .intersection(this.gridBounds) + .map(gridExtent.extentFor(_).buffer(- cellSize.width / 2, - cellSize.height / 2)) + .flatMap(read(_, bands)) + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + extents.toIterator.flatMap(read(_, bands)) + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + bounds.toIterator.flatMap(_.intersection(this.gridBounds).flatMap(read(_, bands))) + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): RasterSource = { + if (targetCRS == sourceLayer.metadata.crs) { + val resampledGridExtent = resampleGrid(this.sourceLayer.gridExtent) + val closestLayer = GeoTrellisRasterSource.getClosestResolution(sourceLayers, resampledGridExtent.cellSize, strategy)(_.metadata.layout.cellSize).get + // TODO: if closestLayer is w/in some marging of desired CellSize, return GeoTrellisRasterSource instead + new GeoTrellisResampleRasterSource(attributeStore, dataPath, closestLayer.id, sourceLayers, resampledGridExtent, resampleMethod, targetCellType) + } else { + // Pick new layer ID + val (closestLayerId, gridExtent) = + GeoTrellisReprojectRasterSource + .getClosestSourceLayer( + targetCRS, + sourceLayers, + ResampleGrid.toReprojectOptions[Long](this.gridExtent, targetResampleGrid, resampleMethod), + strategy + ) + new GeoTrellisReprojectRasterSource( + attributeStore, + dataPath, + layerId, + sourceLayers, + gridExtent, + targetCRS, + resampleGrid, + resampleMethod, + targetCellType = targetCellType + ) + } + } + + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): RasterSource = { + val newReprojectOptions = ResampleGrid.toReprojectOptions(this.gridExtent, resampleGrid, method) + val (closestLayerId, newGridExtent) = GeoTrellisReprojectRasterSource.getClosestSourceLayer(crs, sourceLayers, newReprojectOptions, strategy) + new GeoTrellisReprojectRasterSource(attributeStore, dataPath, closestLayerId, sourceLayers, newGridExtent, crs, resampleGrid, targetCellType = targetCellType) + } + + def convert(targetCellType: TargetCellType): RasterSource = { + new GeoTrellisReprojectRasterSource(attributeStore, dataPath, layerId, sourceLayers, gridExtent, crs, targetResampleGrid, targetCellType = Some(targetCellType)) + } + + override def toString: String = + s"GeoTrellisReprojectRasterSource(${dataPath.value},$layerId,$crs,$gridExtent,${resampleMethod})" +} + +object GeoTrellisReprojectRasterSource { + /** Pick the closest source layer and decide what its GridExtent should be based on Reproject.Options + * Assumes that all layers in source Pyramid share the same CRS and the highest resolution layer is at the head. + */ + private[store] def getClosestSourceLayer( + targetCRS: CRS, + sourcePyramid: Stream[Layer], + options: Reproject.Options, + strategy: OverviewStrategy + ): (LayerId, GridExtent[Long]) = { + // most resolute layer + val baseLayer: Layer = sourcePyramid.minBy(_.metadata.cellSize.resolution) + val sourceCRS: CRS = baseLayer.metadata.crs + + if (options.targetRasterExtent.isDefined) { + val targetGrid: GridExtent[Long] = options.targetRasterExtent.get.toGridType[Long] + val sourceGrid: GridExtent[Long] = ReprojectRasterExtent(targetGrid, targetCRS, sourceCRS) + val sourceLayer = GeoTrellisRasterSource.getClosestResolution(sourcePyramid, sourceGrid.cellSize, strategy)(_.metadata.layout.cellSize).get + (sourceLayer.id, targetGrid) + + } else if (options.parentGridExtent.isDefined) { + val targetGridAlignment: GridExtent[Long] = options.parentGridExtent.get + val sourceGridInTargetCrs: GridExtent[Long] = ReprojectRasterExtent(baseLayer.gridExtent, sourceCRS, targetCRS) + val sourceLayer: Layer = { + // we know the target pixel grid but we don't know which is the closest source resolution to it + // we're going to use the same heuristic we use when reprojecting without target CellSize backwards + val provisional: GridExtent[Long] = targetGridAlignment.createAlignedGridExtent(sourceGridInTargetCrs.extent) + val aproximateSourceCellSize: CellSize = { + val newExtent = baseLayer.metadata.extent + val distance = newExtent.northWest.distance(newExtent.southEast) + val cols: Double = provisional.extent.width / provisional.cellwidth + val rows: Double = provisional.extent.height / provisional.cellheight + val pixelSize = distance / math.sqrt(cols * cols + rows * rows) + CellSize(pixelSize, pixelSize) + } + GeoTrellisRasterSource.getClosestResolution(sourcePyramid, aproximateSourceCellSize, strategy)(_.metadata.layout.cellSize).get + } + val gridExtent: GridExtent[Long] = ReprojectRasterExtent(sourceLayer.gridExtent, sourceLayer.metadata.crs, targetCRS, options) + (sourceLayer.id, gridExtent) + + } else if (options.targetCellSize.isDefined) { + val targetCellSize = options.targetCellSize.get + val sourceGridInTargetCrs: GridExtent[Long] = ReprojectRasterExtent(baseLayer.gridExtent, sourceCRS, targetCRS) + val aproximateSourceCellSize: CellSize = { + val newExtent = baseLayer.metadata.extent + val distance = newExtent.northWest.distance(newExtent.southEast) + val cols: Double = sourceGridInTargetCrs.extent.width / targetCellSize.width + val rows: Double = sourceGridInTargetCrs.extent.height / targetCellSize.height + val pixelSize = distance / math.sqrt(cols * cols + rows * rows) + CellSize(pixelSize, pixelSize) + } + val sourceLayer = GeoTrellisRasterSource.getClosestResolution(sourcePyramid, aproximateSourceCellSize, strategy)(_.metadata.layout.cellSize).get + val gridExtent: GridExtent[Long] = ReprojectRasterExtent(sourceLayer.gridExtent, sourceLayer.metadata.crs, targetCRS, options) + (sourceLayer.id, gridExtent) + + } else { // do your worst ... or best ! + val targetGrid: GridExtent[Long] = ReprojectRasterExtent(baseLayer.gridExtent, sourceCRS, targetCRS) + (sourcePyramid.head.id, targetGrid) + } + } +} diff --git a/store/src/main/scala/geotrellis/store/GeoTrellisResampleRasterSource.scala b/store/src/main/scala/geotrellis/store/GeoTrellisResampleRasterSource.scala new file mode 100644 index 0000000000..600c634f93 --- /dev/null +++ b/store/src/main/scala/geotrellis/store/GeoTrellisResampleRasterSource.scala @@ -0,0 +1,130 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.vector._ +import geotrellis.proj4._ +import geotrellis.raster._ +import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod} +import geotrellis.raster.io.geotiff.{AutoHigherResolution, OverviewStrategy} + +import com.typesafe.scalalogging.LazyLogging + +/** RasterSource that resamples on read from underlying GeoTrellis layer. + * + * Note: + * The constructor is unfriendly. + * This class is not intended to constructed directly by the users. + * Refer to [[GeoTrellisRasterSource]] for example of correct setup. + * It is expected that the caller has significant pre-computed information about the layers. + * + * @param attributeStore the source of metadata for the layers, used for reading + * @param dataPath dataPath of the GeoTrellis catalog that can format a given path to be read in by a AttributeStore + * @param layerId The specific layer we're sampling from + * @param sourceLayers list of layers we can can sample from for futher resample + * @param gridExtent the desired pixel grid for the layer + * @param resampleMethod Resampling method used when fitting data to target grid + */ +class GeoTrellisResampleRasterSource( + val attributeStore: AttributeStore, + val dataPath: GeoTrellisPath, + val layerId: LayerId, + val sourceLayers: Stream[Layer], + val gridExtent: GridExtent[Long], + val resampleMethod: ResampleMethod = NearestNeighbor, + val targetCellType: Option[TargetCellType] = None +) extends RasterSource with LazyLogging { + def name: GeoTrellisPath = dataPath + + lazy val reader = CollectionLayerReader(attributeStore, dataPath.value) + + /** Source layer metadata that needs to be resampled */ + lazy val sourceLayer: Layer = sourceLayers.find(_.id == layerId).get + + /** GridExtent of source pixels that needs to be resampled */ + lazy val sourceGridExtent: GridExtent[Long] = sourceLayer.gridExtent + + def crs: CRS = sourceLayer.metadata.crs + + def cellType: CellType = dstCellType.getOrElse(sourceLayer.metadata.cellType) + + def bandCount: Int = sourceLayer.bandCount + + def attributes: Map[String, String] = Map( + "catalogURI" -> dataPath.value, + "layerName" -> layerId.name, + "zoomLevel" -> layerId.zoom.toString, + "bandCount" -> bandCount.toString + ) + /** GeoTrellis metadata doesn't allow to query a per band metadata by default. */ + def attributesForBand(band: Int): Map[String, String] = Map.empty + + def metadata: GeoTrellisMetadata = GeoTrellisMetadata(name, crs, bandCount, cellType, gridExtent, resolutions, attributes) + + lazy val resolutions: List[GridExtent[Long]] = sourceLayers.map(_.gridExtent).toList + + def read(extent: Extent, bands: Seq[Int]): Option[Raster[MultibandTile]] = { + val tileBounds = sourceLayer.metadata.mapTransform.extentToBounds(extent) + def msg = s"\u001b[32mread($extent)\u001b[0m = ${dataPath.toString} ${sourceLayer.id} ${sourceLayer.metadata.cellSize} @ ${sourceLayer.metadata.crs} TO $cellSize -- reading ${tileBounds.size} tiles" + if (tileBounds.size < 1024) // Assuming 256x256 tiles this would be a very large request + logger.debug(msg) + else + logger.warn(msg + " (large read)") + + GeoTrellisRasterSource.readIntersecting(reader, layerId, sourceLayer.metadata, extent, bands) + .map { raster => + val targetRasterExtent = gridExtent.createAlignedRasterExtent(extent) + logger.trace(s"\u001b[31mTargetRasterExtent\u001b[0m: ${targetRasterExtent} ${targetRasterExtent.dimensions}") + raster.resample(targetRasterExtent, resampleMethod) + } + } + + def read(bounds: GridBounds[Long], bands: Seq[Int]): Option[Raster[MultibandTile]] = { + bounds + .intersection(this.gridBounds) + .map(gridExtent.extentFor(_).buffer(- cellSize.width / 2, - cellSize.height / 2)) + .flatMap(read(_, bands)) + } + + def reprojection(targetCRS: CRS, resampleGrid: ResampleGrid[Long] = IdentityResampleGrid, method: ResampleMethod = NearestNeighbor, strategy: OverviewStrategy = AutoHigherResolution): GeoTrellisReprojectRasterSource = { + val reprojectOptions = ResampleGrid.toReprojectOptions[Long](this.gridExtent, resampleGrid, method) + val (closestLayerId, gridExtent) = GeoTrellisReprojectRasterSource.getClosestSourceLayer(targetCRS, sourceLayers, reprojectOptions, strategy) + new GeoTrellisReprojectRasterSource(attributeStore, dataPath, layerId, sourceLayers, gridExtent, targetCRS, resampleGrid, targetCellType = targetCellType) + } + /** Resample underlying RasterSource to new grid extent + * Note: ResampleGrid will be applied to GridExtent of the source layer, not the GridExtent of this RasterSource + */ + def resample(resampleGrid: ResampleGrid[Long], method: ResampleMethod, strategy: OverviewStrategy): GeoTrellisResampleRasterSource = { + val resampledGridExtent = resampleGrid(this.gridExtent) + val closestLayer = GeoTrellisRasterSource.getClosestResolution(sourceLayers, resampledGridExtent.cellSize, strategy)(_.metadata.layout.cellSize).get + // TODO: if closestLayer is w/in some marging of desired CellSize, return GeoTrellisRasterSource instead + new GeoTrellisResampleRasterSource(attributeStore, dataPath, closestLayer.id, sourceLayers, resampledGridExtent, method, targetCellType) + } + + def convert(targetCellType: TargetCellType): GeoTrellisResampleRasterSource = { + new GeoTrellisResampleRasterSource(attributeStore, dataPath, layerId, sourceLayers, gridExtent, resampleMethod, Some(targetCellType)) + } + + override def readExtents(extents: Traversable[Extent], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + extents.toIterator.flatMap(read(_, bands)) + + override def readBounds(bounds: Traversable[GridBounds[Long]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = + bounds.toIterator.flatMap(_.intersection(this.gridBounds).flatMap(read(_, bands))) + + override def toString: String = + s"GeoTrellisResampleRasterSource(${dataPath.toString},$layerId,$gridExtent,$resampleMethod)" +} diff --git a/store/src/main/scala/geotrellis/store/avro/codecs/Implicits.scala b/store/src/main/scala/geotrellis/store/avro/codecs/Implicits.scala index 76c1c82ca7..17a2cdbc43 100644 --- a/store/src/main/scala/geotrellis/store/avro/codecs/Implicits.scala +++ b/store/src/main/scala/geotrellis/store/avro/codecs/Implicits.scala @@ -41,7 +41,7 @@ trait Implicits uShortConstantTileCodec ) - implicit def tileUnionCodec = new AvroUnionCodec[Tile]( + def simpleTileUnionCodec = new AvroUnionCodec[Tile]( byteArrayTileCodec, floatArrayTileCodec, doubleArrayTileCodec, @@ -60,5 +60,25 @@ trait Implicits uShortConstantTileCodec ) + implicit def tileUnionCodec = new AvroUnionCodec[Tile]( + byteArrayTileCodec, + floatArrayTileCodec, + doubleArrayTileCodec, + shortArrayTileCodec, + intArrayTileCodec, + bitArrayTileCodec, + uByteArrayTileCodec, + uShortArrayTileCodec, + byteConstantTileCodec, + floatConstantTileCodec, + doubleConstantTileCodec, + shortConstantTileCodec, + intConstantTileCodec, + bitConstantTileCodec, + uByteConstantTileCodec, + uShortConstantTileCodec, + paddedTileCodec + ) + implicit def tupleCodec[A: AvroRecordCodec, B: AvroRecordCodec]: TupleCodec[A, B] = TupleCodec[A, B] } diff --git a/store/src/main/scala/geotrellis/store/avro/codecs/TileCodecs.scala b/store/src/main/scala/geotrellis/store/avro/codecs/TileCodecs.scala index 4aa00405ab..b4edcf22b3 100644 --- a/store/src/main/scala/geotrellis/store/avro/codecs/TileCodecs.scala +++ b/store/src/main/scala/geotrellis/store/avro/codecs/TileCodecs.scala @@ -359,6 +359,36 @@ trait TileCodecs { ArrayMultibandTile(bands) } } + + implicit def paddedTileCodec: AvroRecordCodec[PaddedTile] = new AvroRecordCodec[PaddedTile] { + def schema = SchemaBuilder + .record("PaddedTile").namespace("geotrellis.raster") + .fields() + .name("chunk").`type`(simpleTileUnionCodec.schema).noDefault() + .name("colOffset").`type`().intType().noDefault() + .name("rowOffset").`type`().intType().noDefault() + .name("cols").`type`().intType().noDefault() + .name("rows").`type`().intType().noDefault() + .endRecord() + + def encode(tile: PaddedTile, rec: GenericRecord) = { + rec.put("chunk", simpleTileUnionCodec.encode(tile.chunk)) + rec.put("colOffset", tile.colOffset) + rec.put("rowOffset", tile.rowOffset) + rec.put("cols", tile.cols) + rec.put("rows", tile.rows) + } + + def decode(rec: GenericRecord) = { + val chunk = simpleTileUnionCodec.decode(rec[GenericRecord]("chunk")) + val colOffset = rec[Int]("colOffset") + val rowOffset = rec[Int]("rowOffset") + val cols = rec[Int]("cols") + val rows = rec[Int]("rows") + + PaddedTile(chunk, colOffset, rowOffset, cols, rows) + } + } } object TileCodecs extends TileCodecs diff --git a/store/src/test/scala/geotrellis/store/GeoTrellisRasterSourceProviderSpec.scala b/store/src/test/scala/geotrellis/store/GeoTrellisRasterSourceProviderSpec.scala new file mode 100644 index 0000000000..7953362ab3 --- /dev/null +++ b/store/src/test/scala/geotrellis/store/GeoTrellisRasterSourceProviderSpec.scala @@ -0,0 +1,53 @@ +/* + * Copyright 2019 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.store + +import geotrellis.raster.RasterSource + +import org.scalatest._ + +class GeoTrellisRasterSourceProviderSpec extends FunSpec { + describe("GeoTrellisRasterSourceProvider") { + val provider = new GeoTrellisRasterSourceProvider() + + it("should process a non-prefixed string") { + assert(provider.canProcess("hdfs://storage/big-catalog?layer=big&zoom=30")) + } + + it("should process a prefixed string") { + assert(provider.canProcess("gt+s3://catalog/path/blah?layer=blah!&zoom=0&band_count=4")) + } + + it("should not be able to process a path that doesn't contain a layer name") { + assert(!provider.canProcess("file://this/path/leads/to/a/bad/catalog")) + } + + it("should not be able to process a path that doesn't point to a catalog") { + assert(!provider.canProcess("s3://path/to/my/fav/files/cool-image-3.jp2")) + } + + it("should not be able to process a GDAL prefixed path") { + assert(!provider.canProcess("gdal+file:///sketch-pad/files/temp-file.tif")) + } + + it("should produce a GeoTrellisRasterSource from a string") { + val params = s"?layer=landsat&zoom=0" + val uriMultiband = s"file:///tmp/catalog$params" + assert(RasterSource(uriMultiband).isInstanceOf[GeoTrellisRasterSource]) + } + } +} diff --git a/store/src/test/scala/geotrellis/store/avro/TileCodecsSpec.scala b/store/src/test/scala/geotrellis/store/avro/TileCodecsSpec.scala index 3bcb385996..aa1bb8bd24 100644 --- a/store/src/test/scala/geotrellis/store/avro/TileCodecsSpec.scala +++ b/store/src/test/scala/geotrellis/store/avro/TileCodecsSpec.scala @@ -113,4 +113,21 @@ class TileCodecsSpec extends FunSpec with Matchers with AvroTools { roundTripWithNoDataCheck(thing) } } + + describe("PaddedTileCodecs") { + it("encode PaddedTile") { + roundTrip(PaddedTile(ByteArrayTile.fill(127,10,15), 0, 0, 10, 15)) + } + + it("encode multiband PaddedTile"){ + val tile = MultibandTile( + ByteArrayTile.fill(127,10,15), + ByteArrayTile.fill(100,10,15), + ByteArrayTile.fill(50,10,15) + ) + val ptile = tile.mapBands((_, tile) => PaddedTile(tile, 0, 0, 10, 15)) + + roundTrip(ptile) + } + } } diff --git a/util/src/main/scala/geotrellis/util/RangeReader.scala b/util/src/main/scala/geotrellis/util/RangeReader.scala index 6b08a0c79d..7aca73c03a 100644 --- a/util/src/main/scala/geotrellis/util/RangeReader.scala +++ b/util/src/main/scala/geotrellis/util/RangeReader.scala @@ -17,10 +17,11 @@ package geotrellis.util import scala.collection.JavaConverters._ - import java.net.URI import java.util.ServiceLoader +import scala.util.Try + /** * This trait defines methods for breaking up a source of bytes into * Map[Long, Array[Byte]] called a, "chunk". Where the Long is where within @@ -51,6 +52,9 @@ object RangeReader { implicit def rangeReaderToStreamingByteReader(rangeReader: RangeReader): StreamingByteReader = StreamingByteReader(rangeReader) + implicit def rangeReaderToStreamingByteReaderOpt(rangeReader: Option[RangeReader]): Option[StreamingByteReader] = + rangeReader.map(rangeReaderToStreamingByteReader) + def apply(uri: URI): RangeReader = ServiceLoader.load(classOf[RangeReaderProvider]) .iterator().asScala @@ -59,4 +63,9 @@ object RangeReader { .rangeReader(uri) def apply(uri: String): RangeReader = apply(new URI(uri)) + + /** This function checks if the source is valid, by trying to read the first byte of the data. */ + def validated(uri: URI): Option[RangeReader] = Try { apply(uri).readRange(0, 1); apply(uri) }.toOption + + def validated(uri: String): Option[RangeReader] = validated(new URI(uri)) } diff --git a/vector/src/main/scala/geotrellis/vector/Dimension.scala b/vector/src/main/scala/geotrellis/vector/Dimension.scala index f576b039d4..0e6b544eb6 100644 --- a/vector/src/main/scala/geotrellis/vector/Dimension.scala +++ b/vector/src/main/scala/geotrellis/vector/Dimension.scala @@ -16,8 +16,6 @@ package geotrellis.vector -import simulacrum._ - trait Dimension[G <: Geometry] trait AtLeastOneDimension[G <: Geometry] extends Dimension[G] diff --git a/vector/src/main/scala/geotrellis/vector/Extent.scala b/vector/src/main/scala/geotrellis/vector/Extent.scala index bc1c2d7817..337ef9e857 100644 --- a/vector/src/main/scala/geotrellis/vector/Extent.scala +++ b/vector/src/main/scala/geotrellis/vector/Extent.scala @@ -238,8 +238,10 @@ case class Extent( intersection(other) /** Create a new extent using a buffer around this extent */ - def buffer(d: Double): Extent = - Extent(xmin - d, ymin - d, xmax + d, ymax + d) + def buffer(d: Double): Extent = buffer(d, d) + + def buffer(width: Double, height: Double): Extent = + Extent(xmin - width, ymin - height, xmax + width, ymax + height) /** Orders two bounding boxes by their (geographically) lower-left corner. The bounding box * that is further south (or west in the case of a tie) comes first.