diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala index 6647f4258..098841362 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala @@ -100,9 +100,7 @@ object TileRasterizerAggregate { def apply(tlm: TileLayerMetadata[_], sampler: ResampleMethod): ProjectedRasterDefinition = { // Try to determine the actual dimensions of our data coverage - val actualSize = tlm.layout.toRasterExtent().gridBoundsFor(tlm.extent) // <--- Do we have the math right here? - val cols = actualSize.width - val rows = actualSize.height + val TileDimensions(cols, rows) = tlm.totalDimensions new ProjectedRasterDefinition(cols, rows, tlm.cellType, tlm.crs, tlm.extent, sampler) } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala index 352f78ac9..4ba658baa 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.transformers import geotrellis.raster.Tile -import org.apache.spark.sql.{Column, TypedColumn} +import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala index 563e03e87..46be52a4e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala @@ -31,6 +31,7 @@ import org.apache.spark.SparkConf import org.apache.spark.rdd.RDD import org.apache.spark.sql._ import org.apache.spark.sql.types.{MetadataBuilder, Metadata => SMetadata} +import org.locationtech.rasterframes.model.TileDimensions import spray.json.JsonFormat import scala.reflect.runtime.universe._ @@ -79,6 +80,15 @@ trait Implicits { private[rasterframes] implicit class WithMetadataBuilderMethods(val self: MetadataBuilder) extends MetadataBuilderMethods + + private[rasterframes] + implicit class TLMHasTotalCells(tlm: TileLayerMetadata[_]) { + // TODO: With upgrade to GT 3.1, replace this with the more general `Dimensions[Long]` + def totalDimensions: TileDimensions = { + val gb = tlm.layout.toRasterExtent().gridBoundsFor(tlm.extent) + TileDimensions(gb.width, gb.height) + } + } } object Implicits extends Implicits diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala similarity index 96% rename from core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala rename to core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala index af79c1c05..e78d07017 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/extensions/RFSpatialColumnMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.encoders.serialized_literal * * @since 12/15/17 */ -trait RFSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with StandardColumns { +trait LayerSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with StandardColumns { import Implicits.{WithDataFrameMethods, WithRasterFrameLayerMethods} import org.locationtech.geomesa.spark.jts._ @@ -112,7 +112,7 @@ trait RFSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with Sta */ def withCenterLatLng(colName: String = "center"): RasterFrameLayer = { val key2Center = sparkUdf(keyCol2LatLng) - self.withColumn(colName, key2Center(self.spatialKeyColumn).cast(RFSpatialColumnMethods.LngLatStructType)).certify + self.withColumn(colName, key2Center(self.spatialKeyColumn).cast(LayerSpatialColumnMethods.LngLatStructType)).certify } /** @@ -130,6 +130,6 @@ trait RFSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with Sta } } -object RFSpatialColumnMethods { +object LayerSpatialColumnMethods { private[rasterframes] val LngLatStructType = StructType(Seq(StructField("longitude", DoubleType), StructField("latitude", DoubleType))) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala index e9d375f12..49a1cfdee 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala @@ -52,7 +52,7 @@ import scala.reflect.runtime.universe._ * @since 7/18/17 */ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer] - with RFSpatialColumnMethods with MetadataKeys { + with LayerSpatialColumnMethods with MetadataKeys { import Implicits.{WithDataFrameMethods, WithRasterFrameLayerMethods} @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala index d7e71dbe8..6bd66bab4 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala @@ -23,6 +23,7 @@ package org.locationtech.rasterframes.extensions import org.apache.spark.sql._ import org.apache.spark.sql.functions._ import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.serialized_literal import org.locationtech.rasterframes.expressions.SpatialRelation import org.locationtech.rasterframes.expressions.accessors.ExtractTile import org.locationtech.rasterframes.functions.reproject_and_merge @@ -89,9 +90,12 @@ object RasterJoin { // After the aggregation we take all the tiles we've collected and resample + merge // into LHS extent/CRS. // Use a representative tile from the left for the tile dimensions - val leftTile = left.tileColumns.headOption.getOrElse(throw new IllegalArgumentException("Need at least one target tile on LHS")) + val destDims = left.tileColumns.headOption + .map(t => rf_dimensions(unresolved(t))) + .getOrElse(serialized_literal(NOMINAL_TILE_DIMS)) + val reprojCols = rightAggTiles.map(t => reproject_and_merge( - col(leftExtent2), col(leftCRS2), col(t.columnName), col(rightExtent2), col(rightCRS2), rf_dimensions(unresolved(leftTile)) + col(leftExtent2), col(leftCRS2), col(t.columnName), col(rightExtent2), col(rightCRS2), destDims ) as t.columnName) val finalCols = leftAggCols.map(unresolved) ++ reprojCols ++ rightAggOther.map(unresolved) diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala index d5e6f5e31..6c80e9d0d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala @@ -26,6 +26,8 @@ import org.apache.spark.sql._ import org.apache.spark.sql.functions.broadcast import org.locationtech.rasterframes._ import org.locationtech.rasterframes.util._ + +/** Algorithm for projecting an arbitrary RasterFrame into a layer with consistent CRS and gridding. */ object ReprojectToLayer { def apply(df: DataFrame, tlm: TileLayerMetadata[SpatialKey]): RasterFrameLayer = { // create a destination dataframe with crs and extend columns @@ -42,8 +44,9 @@ object ReprojectToLayer { e = tlm.mapTransform(sk) } yield (sk, e, crs) + // Create effectively a target RasterFrame, but with no tiles. val dest = gridItems.toSeq.toDF(SPATIAL_KEY_COLUMN.columnName, EXTENT_COLUMN.columnName, CRS_COLUMN.columnName) - dest.show(false) + val joined = RasterJoin(broadcast(dest), df) joined.asLayer(SPATIAL_KEY_COLUMN, tlm) diff --git a/core/src/test/scala/examples/CreatingRasterFrames.scala b/core/src/test/scala/examples/CreatingRasterFrames.scala deleted file mode 100644 index 8b5c00c72..000000000 --- a/core/src/test/scala/examples/CreatingRasterFrames.scala +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -/** - * - * @author sfitch - * @since 11/6/17 - */ -object CreatingRasterFrames extends App { -// # Creating RasterFrames -// -// There are a number of ways to create a `RasterFrameLayer`, as enumerated in the sections below. -// -// ## Initialization -// -// First, some standard `import`s: - - import org.locationtech.rasterframes._ - import geotrellis.raster._ - import geotrellis.raster.io.geotiff.SinglebandGeoTiff - import geotrellis.spark.io._ - import org.apache.spark.sql._ - -// Next, initialize the `SparkSession`, and call the `withRasterFrames` method on it: - - implicit val spark = SparkSession.builder(). - master("local[*]").appName("RasterFrames"). - getOrCreate(). - withRasterFrames - spark.sparkContext.setLogLevel("ERROR") - -// ## From `ProjectedExtent` -// -// The simplest mechanism for getting a RasterFrameLayer is to use the `toLayer(tileCols, tileRows)` extension method on `ProjectedRaster`. - - val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - val rf = scene.projectedRaster.toLayer(128, 128) - rf.show(5, false) - - -// ## From `TileLayerRDD` -// -// Another option is to use a GeoTrellis [`LayerReader`](https://docs.geotrellis.io/en/latest/guide/tile-backends.html), to get a `TileLayerRDD` for which there's also a `toLayer` extension method. - - -// ## Inspecting Structure -// -// `RasterFrameLayer` has a number of methods providing access to metadata about the contents of the RasterFrameLayer. -// -// ### Tile Column Names - - rf.tileColumns.map(_.toString) - -// ### Spatial Key Column Name - - rf.spatialKeyColumn.toString - -// ### Temporal Key Column -// -// Returns an `Option[Column]` since not all RasterFrames have an explicit temporal dimension. - - rf.temporalKeyColumn.map(_.toString) - -// ### Tile Layer Metadata -// -// The Tile Layer Metadata defines how the spatial/spatiotemporal domain is discretized into tiles, -// and what the key bounds are. - - import spray.json._ - // The `fold` is required because an `Either` is retured, depending on the key type. - rf.tileLayerMetadata.fold(_.toJson, _.toJson).prettyPrint - - spark.stop() -} diff --git a/core/src/test/scala/examples/MeanValue.scala b/core/src/test/scala/examples/MeanValue.scala deleted file mode 100644 index 2ee264469..000000000 --- a/core/src/test/scala/examples/MeanValue.scala +++ /dev/null @@ -1,50 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2017 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - */ - -package examples - -import org.locationtech.rasterframes._ -import geotrellis.raster.io.geotiff.SinglebandGeoTiff -import org.apache.spark.sql.SparkSession - -/** - * Compute the cell mean value of an image. - * - * @since 10/23/17 - */ -object MeanValue extends App { - - implicit val spark = SparkSession.builder() - .master("local[*]") - .appName(getClass.getName) - .getOrCreate() - .withRasterFrames - - - val scene = SinglebandGeoTiff("src/test/resources/L8-B8-Robinson-IL.tiff") - - val rf = scene.projectedRaster.toLayer(128, 128) // <-- tile size - - rf.printSchema - - val tileCol = rf("tile") - rf.agg(rf_agg_no_data_cells(tileCol), rf_agg_data_cells(tileCol), rf_agg_mean(tileCol)).show(false) - - spark.stop() -} diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala similarity index 87% rename from core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala rename to core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala index f37c5150a..931ca409d 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala @@ -23,19 +23,21 @@ package org.locationtech.rasterframes +import java.net.URI import java.sql.Timestamp import java.time.ZonedDateTime -import org.locationtech.rasterframes.util._ -import geotrellis.proj4.LatLng -import geotrellis.raster.render.{ColorMap, ColorRamp} -import geotrellis.raster.{ProjectedRaster, Tile, TileFeature, TileLayout, UByteCellType} +import geotrellis.proj4.{CRS, LatLng} +import geotrellis.raster.{MultibandTile, ProjectedRaster, Raster, Tile, TileFeature, TileLayout, UByteCellType, UByteConstantNoDataCellType} import geotrellis.spark._ import geotrellis.spark.tiling._ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.functions._ -import org.apache.spark.sql.{SQLContext, SparkSession} +import org.apache.spark.sql.{Encoders, SQLContext, SparkSession} import org.locationtech.rasterframes.model.TileDimensions +import org.locationtech.rasterframes.ref.RasterSource +import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import org.locationtech.rasterframes.util._ import scala.util.control.NonFatal @@ -44,7 +46,7 @@ import scala.util.control.NonFatal * * @since 7/10/17 */ -class RasterFrameSpec extends TestEnvironment with MetadataKeys +class RasterLayerSpec extends TestEnvironment with MetadataKeys with TestData { import TestData.randomTile import spark.implicits._ @@ -232,17 +234,40 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys assert(bounds._2 === SpaceTimeKey(3, 1, now)) } - def basicallySame(expected: Extent, computed: Extent): Unit = { - val components = Seq( - (expected.xmin, computed.xmin), - (expected.ymin, computed.ymin), - (expected.xmax, computed.xmax), - (expected.ymax, computed.ymax) - ) - forEvery(components)(c ⇒ - assert(c._1 === c._2 +- 0.000001) + it("should create layer from arbitrary RasterFrame") { + val src = RasterSource(URI.create("https://raw.githubusercontent.com/locationtech/rasterframes/develop/core/src/test/resources/LC08_RGB_Norfolk_COG.tiff")) + val srcCrs = src.crs + + def project(r: Raster[MultibandTile]): Seq[ProjectedRasterTile] = + r.tile.bands.map(b => ProjectedRasterTile(b, r.extent, srcCrs)) + + val prtEnc = ProjectedRasterTile.prtEncoder + implicit val enc = Encoders.tuple(prtEnc, prtEnc, prtEnc) + + val rasters = src.readAll(bands = Seq(0, 1, 2)).map(project).map(p => (p(0), p(1), p(2))) + + val df = rasters.toDF("red", "green", "blue") + + val crs = CRS.fromString("+proj=utm +zone=18 +datum=WGS84 +units=m +no_defs") + + val extent = Extent(364455.0, 4080315.0, 395295.0, 4109985.0) + val layout = LayoutDefinition(extent, TileLayout(2, 2, 32, 32)) + + val tlm = new TileLayerMetadata[SpatialKey]( + UByteConstantNoDataCellType, + layout, + extent, + crs, + KeyBounds(SpatialKey(0, 0), SpatialKey(1, 1)) ) - } + val layer = df.toLayer(tlm) + + val TileDimensions(cols, rows) = tlm.totalDimensions + val prt = layer.toMultibandRaster(Seq($"red", $"green", $"blue"), cols, rows) + prt.tile.dimensions should be((cols, rows)) + prt.crs should be(crs) + prt.extent should be(extent) + } it("shouldn't clip already clipped extents") { val rf = TestData.randomSpatialTileLayerRDD(1024, 1024, 8, 8).toLayer @@ -258,27 +283,8 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys basicallySame(expected2, computed2) } - def Greyscale(stops: Int): ColorRamp = { - val colors = (0 to stops) - .map(i ⇒ { - val c = java.awt.Color.HSBtoRGB(0f, 0f, i / stops.toFloat) - (c << 8) | 0xFF // Add alpha channel. - }) - ColorRamp(colors) - } - - def render(tile: Tile, tag: String): Unit = { - if(false && !isCI) { - val colors = ColorMap.fromQuantileBreaks(tile.histogram, Greyscale(128)) - val path = s"target/${getClass.getSimpleName}_$tag.png" - logger.info(s"Writing '$path'") - tile.color(colors).renderPng().write(path) - } - } - it("should rasterize with a spatiotemporal key") { val rf = TestData.randomSpatioTemporalTileLayerRDD(20, 20, 2, 2).toLayer - noException shouldBe thrownBy { rf.toRaster($"tile", 128, 128) } @@ -291,7 +297,6 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys val joinTypes = Seq("inner", "outer", "fullouter", "left_outer", "right_outer", "leftsemi") forEvery(joinTypes) { jt ⇒ val joined = rf1.spatialJoin(rf2, jt) - //println(joined.schema.json) assert(joined.tileLayerMetadata.isRight) } } diff --git a/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala index 01fbffcd0..e9af4f382 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala @@ -23,7 +23,10 @@ package org.locationtech.rasterframes import java.nio.file.{Files, Path} import com.typesafe.scalalogging.Logger +import geotrellis.raster.Tile +import geotrellis.raster.render.{ColorMap, ColorRamps} import geotrellis.raster.testkit.RasterMatchers +import geotrellis.vector.Extent import org.apache.spark.sql._ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.StructType @@ -78,6 +81,13 @@ trait TestEnvironment extends FunSpec rows.length == inRows } + def render(tile: Tile, tag: String): Unit = { + val colors = ColorMap.fromQuantileBreaks(tile.histogram, ColorRamps.greyscale(128)) + val path = s"target/${getClass.getSimpleName}_$tag.png" + logger.info(s"Writing '$path'") + tile.color(colors).renderPng().write(path) + } + /** * Constructor for creating a DataFrame with a single row and no columns. * Useful for testing the invocation of data constructing UDFs. @@ -99,6 +109,18 @@ trait TestEnvironment extends FunSpec def matchGeom(g: Geometry, tolerance: Double) = new GeometryMatcher(g, tolerance) + def basicallySame(expected: Extent, computed: Extent): Unit = { + val components = Seq( + (expected.xmin, computed.xmin), + (expected.ymin, computed.ymin), + (expected.xmax, computed.xmax), + (expected.ymax, computed.ymax) + ) + forEvery(components)(c ⇒ + assert(c._1 === c._2 +- 0.000001) + ) + } + def checkDocs(name: String): Unit = { import spark.implicits._ val docs = sql(s"DESCRIBE FUNCTION EXTENDED $name").as[String].collect().mkString("\n") @@ -107,8 +129,4 @@ trait TestEnvironment extends FunSpec docs shouldNot include("null") docs shouldNot include("N/A") } -} - -object TestEnvironment { - } \ No newline at end of file