From e70668ceffacdc83fb5b3ec460a331d270a119d6 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 21 Sep 2021 20:21:44 -0400 Subject: [PATCH 01/25] Add focal operations --- .../scala/geotrellis/raster/BufferTile.scala | 411 ++++++++++++++++++ .../focal/BufferedFocalMethods.scala | 90 ++++ .../org/apache/spark/sql/rf/TileUDT.scala | 7 +- .../rasterframes/RasterFunctions.scala | 2 +- .../expressions/focalops/Aspect.scala | 50 +++ .../expressions/focalops/Convolve.scala | 52 +++ .../expressions/focalops/FocalMax.scala | 51 +++ .../expressions/focalops/FocalMean.scala | 51 +++ .../expressions/focalops/FocalMedian.scala | 51 +++ .../expressions/focalops/FocalMin.scala | 50 +++ .../expressions/focalops/FocalMode.scala | 51 +++ .../expressions/focalops/FocalMoransI.scala | 51 +++ .../focalops/FocalNeighborhoodOp.scala | 53 +++ .../expressions/focalops/FocalOp.scala | 49 +++ .../expressions/focalops/FocalStdDev.scala | 51 +++ .../expressions/focalops/Hillshade.scala | 61 +++ .../expressions/focalops/Slope.scala | 51 +++ .../expressions/focalops/SurfaceOp.scala | 82 ++++ .../generators/RasterSourceToRasterRefs.scala | 6 +- .../generators/RasterSourceToTiles.scala | 6 +- .../rasterframes/expressions/package.scala | 14 + .../functions/FocalFunctions.scala | 62 +++ .../rasterframes/ref/RasterRef.scala | 36 +- .../tiles/ProjectedRasterTile.scala | 8 +- .../geotrellis/raster/BufferTileSpec.scala | 103 +++++ .../functions/FocalFunctionsSpec.scala | 208 +++++++++ .../functions/LocalFunctionsSpec.scala | 1 - .../rasterframes/ref/RasterRefSpec.scala | 38 +- .../raster/RasterSourceDataSource.scala | 54 ++- .../raster/RasterSourceRelation.scala | 5 +- .../src/test/scala/examples/BufferTiles.scala | 68 +++ 31 files changed, 1823 insertions(+), 50 deletions(-) create mode 100644 core/src/main/scala/geotrellis/raster/BufferTile.scala create mode 100644 core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala create mode 100644 core/src/test/scala/geotrellis/raster/BufferTileSpec.scala create mode 100644 core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala create mode 100644 datasource/src/test/scala/examples/BufferTiles.scala diff --git a/core/src/main/scala/geotrellis/raster/BufferTile.scala b/core/src/main/scala/geotrellis/raster/BufferTile.scala new file mode 100644 index 000000000..a6a473a22 --- /dev/null +++ b/core/src/main/scala/geotrellis/raster/BufferTile.scala @@ -0,0 +1,411 @@ +/* + * Copyright 2021 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.mapalgebra.focal.BufferedFocalMethods +import spire.syntax.cfor._ + +/** + * When combined with another BufferTile the two tiles will be aligned on (0, 0) pixel of tile center. + * The operation will be carried over all overlapping pixels. + * For instance: combining a tile padded with 5 pixels on all sides with tile padded with 3 pixels on all sides will + * result in buffer tile with 3 pixel padding on all sides. + * + * When combined with another BufferTile the operation will be executed over the maximum shared in + * + * TODO: + * - What should .map do? Map the buffer pixels or not? + * - toString method is friendly + * - mutable version makes sense + * - toBytes needs to encode padding size? + */ +case class BufferTile( + sourceTile: Tile, + gridBounds: GridBounds[Int] +) extends Tile { + require( + gridBounds.colMin >=0 && gridBounds.rowMin >= 0 && gridBounds.colMax < sourceTile.cols && gridBounds.rowMax < sourceTile.rows, + s"Tile center bounds $gridBounds exceed underlying tile dimensions ${sourceTile.dimensions}" + ) + + val cols: Int = gridBounds.width + val rows: Int = gridBounds.height + + val cellType: CellType = sourceTile.cellType + + private def colMin: Int = gridBounds.colMin + private def rowMin: Int = gridBounds.rowMin + private def sourceCols: Int = sourceTile.cols + private def sourceRows: Int = sourceTile.rows + + def bufferTop: Int = gridBounds.rowMin + def bufferLeft: Int = gridBounds.colMin + def bufferRight: Int = sourceTile.cols - gridBounds.colMin - gridBounds.colMax + def bufferBottom: Int = sourceTile.rows - gridBounds.rowMin - gridBounds.rowMax + + /** + * Returns a [[Tile]] equivalent to this tile, except with cells of + * the given type. + * + * @param targetCellType The type of cells that the result should have + * @return The new Tile + */ + def convert(targetCellType: CellType): Tile = + mutable(targetCellType) + + def withNoData(noDataValue: Option[Double]): BufferTile = + BufferTile(sourceTile.withNoData(noDataValue), gridBounds) + + def interpretAs(newCellType: CellType): BufferTile = + BufferTile(sourceTile.interpretAs(newCellType), gridBounds) + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Int datum found at the given location + */ + def get(col: Int, row: Int): Int = { + val c = col + colMin + val r = row + rowMin + if(c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) { + throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") + } else { + sourceTile.get(c, r) + } + } + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Double datum found at the given location + */ + def getDouble(col: Int, row: Int): Double = { + val c = col + colMin + val r = row + rowMin + + if(c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) { + throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") + } else { + sourceTile.getDouble(col + gridBounds.colMin, row + gridBounds.rowMin) + } + } + + /** + * Another name for the 'mutable' method on this class. + * + * @return An [[ArrayTile]] + */ + def toArrayTile: ArrayTile = mutable + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(): MutableArrayTile = + mutable(cellType) + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(targetCellType: CellType): MutableArrayTile = { + val tile = ArrayTile.alloc(targetCellType, cols, rows) + + if(!cellType.isFloatingPoint) { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, get(col, row)) + } + } + } else { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, getDouble(col, row)) + } + } + } + + tile + } + + /** + * Return the data behind this tile as an array of integers. + * + * @return The copy as an Array[Int] + */ + def toArray: Array[Int] = { + val arr = Array.ofDim[Int](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = get(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the data behind this tile as an array of doubles. + * + * @return The copy as an Array[Int] + */ + def toArrayDouble: Array[Double] = { + val arr = Array.ofDim[Double](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = getDouble(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the underlying data behind this tile as an array. + * + * @return An array of bytes + */ + def toBytes(): Array[Byte] = toArrayTile.toBytes + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Int to Unit + */ + def foreach(f: Int => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(get(col, row)) + } + } + } + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Double to Unit + */ + def foreachDouble(f: Double => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(getDouble(col, row)) + } + } + } + + /** + * Execute an [[IntTileVisitor]] at each cell of the present tile. + * + * @param visitor An IntTileVisitor + */ + def foreachIntVisitor(visitor: IntTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, get(col, row)) + } + } + } + + /** + * Execute an [[DoubleTileVisitor]] at each cell of the present tile. + * + * @param visitor An DoubleTileVisitor + */ + def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, getDouble(col, row)) + } + } + } + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Int to Int, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def map(f: Int => Int): BufferTile = mapTile(_.map(f)) + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Double to Double, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def mapDouble(f: Double => Double): BufferTile = mapTile(_.mapDouble(f)) + + /** + * Map an [[IntTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapIntMapper(mapper: IntTileMapper): BufferTile = mapTile(_.mapIntMapper(mapper)) + + /** + * Map an [[DoubleTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapDoubleMapper(mapper: DoubleTileMapper): Tile = mapTile(_.mapDoubleMapper(mapper)) + + private def combine(other: BufferTile)(f: (Int, Int) => Int): Tile = { + if((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) { + throw new GeoAttrsError("Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + } + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.get(col, row) + val rightV = other.get(col, row) + tile.set(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile(tile, GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1 + )) + } + + def combineDouble(other: BufferTile)(f: (Double, Double) => Double): Tile = { + if((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) { + throw new GeoAttrsError("Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + } + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.getDouble(col, row) + val rightV = other.getDouble(col, row) + tile.setDouble(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile(tile, GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1)) + } + + /** + * Combine two tiles' cells into new cells using the given integer + * function. For every (x, y) cell coordinate, get each of the + * tiles' integer values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combine(other: Tile)(f: (Int, Int) => Int): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => this.combine(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row), other.get(col, row))) + } + } + tile + } + } + + /** + * Combine two tiles' cells into new cells using the given double + * function. For every (x, y) cell coordinate, get each of the + * tiles' double values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combineDouble(other: Tile)(f: (Double, Double) => Double): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => + this.combineDouble(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row), other.getDouble(col, row))) + } + } + tile + } + } + + def mapTile(f: Tile => Tile): BufferTile = BufferTile(f(sourceTile), gridBounds) +} + +object BufferTile { + implicit class BufferTileOps(val self: BufferTile) extends BufferedFocalMethods +} diff --git a/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala b/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala new file mode 100644 index 000000000..5863615e9 --- /dev/null +++ b/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala @@ -0,0 +1,90 @@ +/* + * Copyright 2016 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster.mapalgebra.focal + +import geotrellis.raster._ +import geotrellis.util.MethodExtensions + +trait BufferedFocalMethods extends MethodExtensions[BufferTile] { + + /** Computes the minimum value of a neighborhood */ + def focalMin(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalMin(n, bounds, target)) + + /** Computes the maximum value of a neighborhood */ + def focalMax(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalMax(n, bounds, target)) + + /** Computes the mode of a neighborhood */ + def focalMode(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalMode(n, bounds, target)) + + /** Computes the median of a neighborhood */ + def focalMedian(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalMedian(n, bounds, target)) + + /** Computes the mean of a neighborhood */ + def focalMean(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalMean(n, bounds, target)) + + /** Computes the sum of a neighborhood */ + def focalSum(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalSum(n, bounds, target)) + + /** Computes the standard deviation of a neighborhood */ + def focalStandardDeviation(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.focalStandardDeviation(n, bounds, target)) + + /** Computes the next step of Conway's Game of Life */ + def focalConway(bounds: Option[GridBounds[Int]] = None): BufferTile = + self.mapTile(_.focalConway(bounds)) + + /** Computes the convolution of the raster for the given kernl */ + def convolve(kernel: Kernel, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.convolve(kernel, bounds, target)) + + /** + * Calculates spatial autocorrelation of cells based on the + * similarity to neighboring values. + */ + def tileMoransI(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.tileMoransI(n, bounds, target)) + + /** + * Calculates global spatial autocorrelation of a raster based on + * the similarity to neighboring values. + */ + def scalarMoransI(n: Neighborhood, bounds: Option[GridBounds[Int]] = None): Double = + self.sourceTile.scalarMoransI(n, bounds) + + /** + * Calculates the slope of each cell in a raster. + * + * @param cs cellSize of the raster + * @param zFactor Number of map units to one elevation unit. + */ + def slope(cs: CellSize, zFactor: Double = 1.0, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.slope(cs, zFactor, bounds, target)) + + /** + * Calculates the aspect of each cell in a raster. + * + * @param cs cellSize of the raster + */ + def aspect(cs: CellSize, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = + self.mapTile(_.aspect(cs, bounds, target)) +} diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala index 6c4f38654..5fc2a7b5d 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala @@ -20,10 +20,11 @@ */ package org.apache.spark.sql.rf -import geotrellis.raster._ + +import geotrellis.raster.{ArrayTile, CellType, ConstantTile, Tile} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport -import org.apache.spark.sql.types.{DataType, _} +import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String import org.locationtech.rasterframes.encoders.syntax._ import org.locationtech.rasterframes.ref.RasterRef @@ -84,7 +85,7 @@ class TileUDT extends UserDefinedType[Tile] { /** TODO: a compatible encoder for the ProjectedRasterTile */ val tile: Tile = - if (! row.isNullAt(4)) { + if (!row.isNullAt(4)) { Try { val ir = row.getStruct(4, 4) val ref = ir.as[RasterRef] diff --git a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala index c8bfa3813..accca888d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala @@ -26,4 +26,4 @@ import org.locationtech.rasterframes.functions._ * Mix-in for UDFs for working with Tiles in Spark DataFrames. * @since 4/3/17 */ -trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions +trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions with FocalFunctions diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala new file mode 100644 index 000000000..311f837b8 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -0,0 +1,50 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, CellSize, Tile} +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.locationtech.rasterframes.model.TileContext + +@ExpressionDescription( + usage = "_FUNC_(tile) - Performs aspect on tile.", + arguments = """ + Arguments: + * tile - a tile to apply operation""", + examples = """ + Examples: + > SELECT _FUNC_(tile); + ..""" +) +case class Aspect(child: Expression) extends SurfaceOp { + override def nodeName: String = Aspect.name + def op(t: Tile, ctx: TileContext): Tile = t match { + case bt: BufferTile => bt.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) + case _ => t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) + } +} + +object Aspect { + def name: String = "rf_aspect" + def apply(tile: Column): Column = new Column(Aspect(tile.expr)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala new file mode 100644 index 000000000..be4d63c7b --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -0,0 +1,52 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs convolve on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * kernel - a focal operation kernel""", + examples = """ + Examples: + > SELECT _FUNC_(tile, kernel); + ..""" +) +case class Convolve(child: Expression, kernel: Kernel) extends FocalOp { + override def nodeName: String = Convolve.name + + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.convolve(kernel) + case _ => t.convolve(kernel) + } +} + +object Convolve { + def name: String = "rf_convolve" + def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala new file mode 100644 index 000000000..1af40018c --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMax on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMax.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalMax(neighborhood) + case _ => t.focalMax(neighborhood) + } +} + +object FocalMax { + def name: String = "rf_focal_max" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMax(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala new file mode 100644 index 000000000..732249d01 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMean on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMean.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalMean(neighborhood) + case _ => t.focalMean(neighborhood) + } +} + +object FocalMean { + def name:String = "rf_focal_mean" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMean(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala new file mode 100644 index 000000000..40ae8a570 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMedian on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMedian.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalMedian(neighborhood) + case _ => t.focalMedian(neighborhood) + } +} + +object FocalMedian { + def name: String = "rf_focal_median" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMedian(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala new file mode 100644 index 000000000..e6f152d5d --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -0,0 +1,50 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMin on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMin.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalMin(neighborhood) + case _ => t.focalMin(neighborhood) + } +} + +object FocalMin { + def name: String = "rf_focal_min" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMin(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala new file mode 100644 index 000000000..4bc4cf5a4 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMode on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMode.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalMode(neighborhood) + case _ => t.focalMode(neighborhood) + } +} + +object FocalMode { + def name: String = "rf_focal_mode" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMode(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala new file mode 100644 index 000000000..b0ca86379 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalMoransI on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalMoransI.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.tileMoransI(neighborhood) + case _ => t.tileMoransI(neighborhood) + } +} + +object FocalMoransI { + def name: String = "rf_focal_moransi" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMoransI(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala new file mode 100644 index 000000000..2a91f4d34 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -0,0 +1,53 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp, row} +import org.locationtech.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +trait FocalNeighborhoodOp extends UnaryLocalRasterOp with NullToValue with CodegenFallback { + def na: Any = null + def neighborhood: Neighborhood + + override protected def nullSafeEval(input: Any): Any = { + val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) + val literral = childTile match { + // if it is RasterRef, we want the BufferTile + case ref: RasterRef => ref.realizedTile + // if it is a ProjectedRasterTile, can we flatten it? + case prt: ProjectedRasterTile => prt.tile match { + // if it is RasterRef, we can get what's inside + case rr: RasterRef => rr.realizedTile + // otherwise it is some tile + case _ => + println(s"prt.getClass: ${prt.tile.getClass}") + prt.tile + } + } + val result = op(literral) + toInternalRow(result, childCtx) + } +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala new file mode 100644 index 000000000..47cee7290 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala @@ -0,0 +1,49 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp, row} +import org.locationtech.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +trait FocalOp extends UnaryLocalRasterOp with NullToValue with CodegenFallback { + def na: Any = null + + override protected def nullSafeEval(input: Any): Any = { + val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) + val literral = childTile match { + // if it is RasterRef, we want the BufferTile + case ref: RasterRef => ref.realizedTile + // if it is a ProjectedRasterTile, can we flatten it? + case prt: ProjectedRasterTile => prt.tile match { + // if it is RasterRef, we can get what's inside + case rr: RasterRef => rr.realizedTile + // otherwise it is some tile + case _ => prt + } + } + val result = op(literral) + toInternalRow(result, childCtx) + } +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala new file mode 100644 index 000000000..0098fac96 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} + +@ExpressionDescription( + usage = "_FUNC_(tile, neighborhood) - Performs focalStandardDeviation on tile in the neighborhood.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * neighborhood - a focal operation neighborhood""", + examples = """ + Examples: + > SELECT _FUNC_(tile, Square(1)); + ..""" +) +case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { + override def nodeName: String = FocalStdDev.name + protected def op(t: Tile): Tile = t match { + case bt: BufferTile => bt.focalStandardDeviation(neighborhood) + case _ => t.focalStandardDeviation(neighborhood) + } +} + +object FocalStdDev { + def name: String = "rf_focal_stddevd" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalStdDev(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala new file mode 100644 index 000000000..7b65ca25a --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -0,0 +1,61 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, CellSize, Tile} +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.locationtech.rasterframes.model.TileContext + +@ExpressionDescription( + usage = "_FUNC_(tile, azimuth, altitude, zFactor) - Performs hillshade on tile.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * azimuth + * altitude + * zFactor""", + examples = """ + Examples: + > SELECT _FUNC_(tile, azimuth, altitude, zFactor); + ..""" +) +case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOp { + override def nodeName: String = Hillshade.name + + /** + * Apply hillshade differently to the BufferedTile and a regular tile. + * In case of a BufferedTile case we'd like to apply the hillshade to the underlying tile. + * + * Check [[SurfaceOp.nullSafeEval()]] to see the details of unpacking the BufferTile. + */ + protected def op(t: Tile, ctx: TileContext): Tile = t match { + case bt: BufferTile => bt.mapTile(_.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor)) + case _ => t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) + } +} + +object Hillshade { + def name: String = "rf_hillshade" + def apply(tile: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + new Column(Hillshade(tile.expr, azimuth, altitude, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala new file mode 100644 index 000000000..97d521974 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.{BufferTile, CellSize, Tile} +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.locationtech.rasterframes.model.TileContext + +@ExpressionDescription( + usage = "_FUNC_(tile, zFactor) - Performs slope on tile.", + arguments = """ + Arguments: + * tile - a tile to apply operation + * zFactor - a slope operation zFactor""", + examples = """ + Examples: + > SELECT _FUNC_(tile, 0.2); + ..""" +) +case class Slope(child: Expression, zFactor: Double) extends SurfaceOp { + override def nodeName: String = Slope.name + protected def op(t: Tile, ctx: TileContext): Tile = t match { + case bt: BufferTile => bt.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) + case _ => t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) + } +} + +object Slope { + def name: String = "rf_slope" + def apply(tile: Column, zFactor: Double): Column = new Column(Slope(tile.expr, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala new file mode 100644 index 000000000..bc87d67ce --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala @@ -0,0 +1,82 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import org.slf4j.LoggerFactory +import com.typesafe.scalalogging.Logger +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.expressions.row +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.ref.RasterRef + +import org.locationtech.rasterframes.encoders.syntax._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.NullToValue +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +/** Operation on a tile returning a tile. */ +trait SurfaceOp extends UnaryExpression with NullToValue with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + def dataType: DataType = child.dataType + def na: Any = null + + override def checkInputDataTypes(): TypeCheckResult = { + if (!tileExtractor.isDefinedAt(child.dataType)) { + TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") + } else TypeCheckSuccess + } + + override protected def nullSafeEval(input: Any): Any = { + val (tile, ctx) = tileExtractor(child.dataType)(row(input)) + + val literral = tile match { + // if it is RasterRef, we want the BufferTile + case ref: RasterRef => ref.realizedTile + // if it is a ProjectedRasterTile, can we flatten it? + case prt: ProjectedRasterTile => prt.tile match { + // if it is RasterRef, we can get what's inside + case rr: RasterRef => rr.realizedTile + // otherwise it is some tile + case _ => prt + } + } + eval(literral, ctx) + } + + protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { + ctx match { + case Some(ctx) => + val ret = op(tile, ctx) + ctx.toProjectRasterTile(ret).toInternalRow + + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + } + + protected def op(t: Tile, ctx: TileContext): Tile +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala index e29966854..1d9b82abc 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala @@ -44,7 +44,7 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression +case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None, bufferSize: Short = 0) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { def inputTypes: Seq[DataType] = Seq.fill(children.size)(rasterSourceUDT) @@ -57,7 +57,7 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[ } yield StructField(name, RasterRef.rasterRefEncoder.schema, true)) private def band2ref(src: RFRasterSource, grid: Option[GridBounds[Int]], extent: Option[Extent])(b: Int): RasterRef = - if (b < src.bandCount) RasterRef(src, b, extent, grid.map(Subgrid.apply)) else null + if (b < src.bandCount) RasterRef(src, b, extent, grid.map(Subgrid.apply), bufferSize) else null def eval(input: InternalRow): TraversableOnce[InternalRow] = try { @@ -88,6 +88,8 @@ object RasterSourceToRasterRefs { def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*) def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile] + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = + new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims, bufferSize)).as[ProjectedRasterTile] private[rasterframes] def bandNames(basename: String, bandIndexes: Seq[Int]): Seq[String] = bandIndexes match { case Seq() => Seq.empty diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala index 85a7be8f9..8f28eb916 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala @@ -45,7 +45,7 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) +case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None, bufferSize: Short = 0) extends Expression with RasterResult with Generator with CodegenFallback with ExpectsInputTypes { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) @@ -89,7 +89,9 @@ case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], object RasterSourceToTiles { def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*) def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = - new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile] + new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims, 0.toShort)).as[ProjectedRasterTile] + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = + new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims, bufferSize)).as[ProjectedRasterTile] } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala index 1fd99725e..9fa191ae4 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala @@ -34,6 +34,7 @@ import org.locationtech.rasterframes.expressions.aggregates.CellCountAggregate.D import org.locationtech.rasterframes.expressions.aggregates._ import org.locationtech.rasterframes.expressions.generators._ import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.focalops._ import org.locationtech.rasterframes.expressions.tilestats._ import org.locationtech.rasterframes.expressions.transformers._ @@ -137,6 +138,19 @@ package object expressions { registry.registerExpression[LocalCountAggregate.LocalNoDataCellsUDAF]("rf_agg_local_no_data_cells") registry.registerExpression[LocalMeanAggregate]("rf_agg_local_mean") + registry.registerExpression[FocalMax](FocalMax.name) + registry.registerExpression[FocalMin](FocalMin.name) + registry.registerExpression[FocalMean](FocalMean.name) + registry.registerExpression[FocalMode](FocalMode.name) + registry.registerExpression[FocalMedian](FocalMedian.name) + registry.registerExpression[FocalMoransI](FocalMoransI.name) + registry.registerExpression[FocalStdDev](FocalStdDev.name) + registry.registerExpression[Convolve](Convolve.name) + + registry.registerExpression[Slope](Slope.name) + registry.registerExpression[Aspect](Aspect.name) + registry.registerExpression[Hillshade](Hillshade.name) + registry.registerExpression[Mask.MaskByDefined]("rf_mask") registry.registerExpression[Mask.InverseMaskByDefined]("rf_inverse_mask") registry.registerExpression[Mask.MaskByValue]("rf_mask_by_value") diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala new file mode 100644 index 000000000..bc58db471 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -0,0 +1,62 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions + +import geotrellis.raster.Neighborhood +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.locationtech.rasterframes.expressions.focalops._ + +trait FocalFunctions { + def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMean(tileCol, neighborhood) + + def rf_focal_median(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMedian(tileCol, neighborhood) + + def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMode(tileCol, neighborhood) + + def rf_focal_max(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMax(tileCol, neighborhood) + + def rf_focal_min(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMin(tileCol, neighborhood) + + def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood): Column = + FocalStdDev(tileCol, neighborhood) + + def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMoransI(tileCol, neighborhood) + + def rf_convolve(tileCol: Column, kernel: Kernel): Column = + Convolve(tileCol, kernel) + + def rf_slope(tileCol: Column, zFactor: Double): Column = + Slope(tileCol, zFactor) + + def rf_aspect(tileCol: Column): Column = + Aspect(tileCol) + + def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + Hillshade(tileCol, azimuth, altitude, zFactor) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index 04497f489..fbc567e9d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -24,7 +24,7 @@ package org.locationtech.rasterframes.ref import com.typesafe.scalalogging.LazyLogging import frameless.TypedExpressionEncoder import geotrellis.proj4.CRS -import geotrellis.raster.{CellType, GridBounds, Tile} +import geotrellis.raster.{BufferTile, CellType, GridBounds, Tile} import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.locationtech.rasterframes.tiles.ProjectedRasterTile @@ -34,35 +34,49 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile * * @since 8/21/18 */ -case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[Subgrid]) extends ProjectedRasterTile { +case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[Subgrid], bufferSize: Short) extends ProjectedRasterTile { def tile: Tile = this def extent: Extent = subextent.getOrElse(source.extent) def crs: CRS = source.crs - def delegate = realizedTile + def delegate: BufferTile = realizedTile override def cols: Int = grid.width override def rows: Int = grid.height override def cellType: CellType = source.cellType - protected lazy val grid: GridBounds[Int] = - subgrid.map(_.toGridBounds).getOrElse(source.rasterExtent.gridBoundsFor(extent, true)) + protected lazy val grid: GridBounds[Int] = subgrid.map(_.toGridBounds).getOrElse(source.rasterExtent.gridBoundsFor(extent, true)) - lazy val realizedTile: Tile = { - RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source") - source.read(grid, Seq(bandIndex)).tile.band(0) + lazy val realizedTile: BufferTile = { + RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source with bufferSize: $bufferSize") + // Pixel bounds we would like to read, including buffer + val bufferedGrid = grid.buffer(bufferSize) + + // Pixel bounds we can read, including buffer + val possibleGrid = bufferedGrid.intersection(source.gridBounds).get + // Pixel bounds of center/non-buffer pixels in read tile + val tileCenterBounds = grid.offset( + colOffset = - possibleGrid.colMin, + rowOffset = - possibleGrid.rowMin + ) + + val raster = source.read(possibleGrid, Seq(bandIndex)).mapTile(_.band(0)) + BufferTile(raster.tile, tileCenterBounds) } - override def toString: String = s"RasterRef($source,$bandIndex,$cellType)" + override def toString: String = s"RasterRef($source, $bandIndex, $cellType, $subextent, $subgrid, $bufferSize)" } object RasterRef extends LazyLogging { private val log = logger def apply(source: RFRasterSource, bandIndex: Int): RasterRef = - RasterRef(source, bandIndex, None, None) + RasterRef(source, bandIndex, None, None, 0) def apply(source: RFRasterSource, bandIndex: Int, subextent: Extent, subgrid: GridBounds[Int]): RasterRef = - RasterRef(source, bandIndex, Some(subextent), Some(Subgrid(subgrid))) + RasterRef(source, bandIndex, Some(subextent), Some(Subgrid(subgrid)), 0) + + def apply(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[Subgrid]): RasterRef = + new RasterRef(source, bandIndex, subextent, subgrid, 0) implicit val rasterRefEncoder: ExpressionEncoder[RasterRef] = TypedExpressionEncoder[RasterRef].asInstanceOf[ExpressionEncoder[RasterRef]] diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala index 564664211..4bee10992 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala @@ -37,9 +37,10 @@ trait ProjectedRasterTile extends DelegatingTile with ProjectedRasterLike with D def tile: Tile def extent: Extent def crs: CRS + def delegate: Tile def projectedExtent: ProjectedExtent = ProjectedExtent(extent, crs) - def projectedRaster: ProjectedRaster[Tile] = ProjectedRaster[Tile](this, extent, crs) - def mapTile(f: Tile => Tile): ProjectedRasterTile = ProjectedRasterTile(f(this), extent, crs) + def projectedRaster: ProjectedRaster[Tile] = ProjectedRaster[Tile](delegate, extent, crs) + def mapTile(f: Tile => Tile): ProjectedRasterTile = ProjectedRasterTile(f(delegate), extent, crs) } object ProjectedRasterTile { @@ -55,8 +56,7 @@ object ProjectedRasterTile { } } - def unapply(prt: ProjectedRasterTile): Option[(Tile, Extent, CRS)] = - Some((prt.tile, prt.extent, prt.crs)) + def unapply(prt: ProjectedRasterTile): Option[(Tile, Extent, CRS)] = Some((prt.tile, prt.extent, prt.crs)) implicit lazy val projectedRasterTileEncoder: ExpressionEncoder[ProjectedRasterTile] = ExpressionEncoder() } diff --git a/core/src/test/scala/geotrellis/raster/BufferTileSpec.scala b/core/src/test/scala/geotrellis/raster/BufferTileSpec.scala new file mode 100644 index 000000000..af8c048a3 --- /dev/null +++ b/core/src/test/scala/geotrellis/raster/BufferTileSpec.scala @@ -0,0 +1,103 @@ +/* + * Copyright 2021 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import geotrellis.raster.testkit._ + +import org.scalatest.matchers.should.Matchers +import org.scalatest.funspec.AnyFunSpec + +class BufferTileSpec extends AnyFunSpec with Matchers with RasterMatchers with TileBuilders { + + val tile = IntArrayTile( + Array( + 2,2, + 2,2 + ), + 2, 2 + ) + // center is all 1s, buffer is all 0 + val padded: BufferTile = BufferTile( + ArrayTile( + Array( + 1,2,3,4, + 1,1,1,4, + 1,1,1,4, + 1,2,3,4 + ), + 4, 4 + ), // GridBounds(0, 0, 4, 4) + GridBounds(1, 1, 2, 2) // for zeroes? + ) + + it("padded + tile => tile") { + val ans = padded + tile + ans.get(0,0) shouldBe 3 + ans.dimensions shouldBe Dimensions(2, 2) + // info("\n" + ans.asciiDraw()) + } + + it("padded + padded => padded") { + val ans = padded.combine(padded)(_ + _).asInstanceOf[BufferTile] + + ans.bufferTop shouldBe 1 + ans.bufferLeft shouldBe 1 + ans.bufferRight shouldBe 1 + ans.bufferBottom shouldBe 1 + ans.dimensions shouldBe Dimensions(2, 2) + + // info("\n" + ans.sourceTile.asciiDraw()) + val ansDouble = padded.combineDouble(padded)(_ + _).asInstanceOf[BufferTile] + + ansDouble.bufferTop shouldBe 1 + ansDouble.bufferLeft shouldBe 1 + ansDouble.bufferRight shouldBe 1 + ansDouble.bufferBottom shouldBe 1 + ansDouble.dimensions shouldBe Dimensions(2, 2) + } + + it("padded + padded => padded (as Tile)") { + val tile: Tile = padded + val ans = (tile.combine(tile)(_ + _)).asInstanceOf[BufferTile] + + ans.bufferTop shouldBe 1 + ans.bufferLeft shouldBe 1 + ans.bufferRight shouldBe 1 + ans.bufferBottom shouldBe 1 + ans.dimensions shouldBe Dimensions(2, 2) + // info("\n" + ans.sourceTile.asciiDraw()) + + val ansDouble = (tile.combineDouble(tile)(_ + _)).asInstanceOf[BufferTile] + + ansDouble.bufferTop shouldBe 1 + ansDouble.bufferLeft shouldBe 1 + ansDouble.bufferRight shouldBe 1 + ansDouble.bufferBottom shouldBe 1 + ansDouble.dimensions shouldBe Dimensions(2, 2) + } + + it("tile center bounds must be contained by underlying tile") { + BufferTile(tile, GridBounds[Int](0,0,1,1)) + + assertThrows[IllegalArgumentException] { + BufferTile(tile, GridBounds[Int](0,0,2,2)) + } + assertThrows[IllegalArgumentException] { + BufferTile(tile, GridBounds[Int](-1,0,1,1)) + } + } +} diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala new file mode 100644 index 000000000..1a0b9356f --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -0,0 +1,208 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions + +import geotrellis.raster.mapalgebra.focal.{Circle, Kernel, Square} +import geotrellis.raster.{BufferTile, CellSize} +import geotrellis.raster.testkit.RasterMatchers +import org.locationtech.rasterframes.ref.{RFRasterSource, RasterRef, Subgrid} +import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import org.locationtech.rasterframes._ + +import java.nio.file.Paths + +class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { + + import spark.implicits._ + + describe("focal operations") { + lazy val path = + if(Paths.get("").toUri.toString.endsWith("core/")) Paths.get("src/test/resources/L8-B7-Elkton-VA.tiff").toUri + else Paths.get("core/src/test/resources/L8-B7-Elkton-VA.tiff").toUri + + lazy val src = RFRasterSource(path) + lazy val fullTile = src.read(src.extent).tile.band(0) + + // read a smaller region to read + lazy val subGridBounds = src.gridBounds.buffer(-10) + // read the region above, but buffered + lazy val bufferedRaster = new RasterRef(src, 0, None, Some(Subgrid(subGridBounds)), 10) + + lazy val bt = BufferTile(fullTile, subGridBounds) + lazy val btCellSize = CellSize(src.extent, bt.cols, bt.rows) + + it("should provide focal mean") { + checkDocs("rf_focal_mean") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_mean($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalMean(Square(1)), actual) + assertEqual(fullTile.focalMean(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal median") { + checkDocs("rf_focal_median") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_median($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalMedian(Square(1)), actual) + assertEqual(fullTile.focalMedian(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal mode") { + checkDocs("rf_focal_mode") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_mode($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalMode(Square(1)), actual) + assertEqual(fullTile.focalMode(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal max") { + checkDocs("rf_focal_max") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_max($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalMax(Square(1)), actual) + assertEqual(fullTile.focalMax(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal min") { + checkDocs("rf_focal_min") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_min($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalMin(Square(1)), actual) + assertEqual(fullTile.focalMin(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal stddev") { + checkDocs("rf_focal_moransi") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_stddev($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.focalStandardDeviation(Square(1)), actual) + assertEqual(fullTile.focalStandardDeviation(Square(1)).crop(subGridBounds), actual) + } + it("should provide focal Moran's I") { + checkDocs("rf_focal_moransi") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_focal_moransi($"proj_raster", Square(1))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.tileMoransI(Square(1)), actual) + assertEqual(fullTile.tileMoransI(Square(1)).crop(subGridBounds), actual) + } + it("should provide convolve") { + checkDocs("rf_convolve") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_convolve($"proj_raster", Kernel(Circle(2d)))) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.convolve(Kernel(Circle(2d))), actual) + assertEqual(fullTile.convolve(Kernel(Circle(2d))).crop(subGridBounds), actual) + } + it("should provide slope") { + checkDocs("rf_slope") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_slope($"proj_raster", 2d)) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.slope(btCellSize, 2d), actual) + assertEqual(fullTile.slope(btCellSize, 2d).crop(subGridBounds), actual) + } + it("should provide aspect") { + checkDocs("rf_aspect") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_aspect($"proj_raster")) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.aspect(btCellSize), actual) + assertEqual(fullTile.aspect(btCellSize).crop(subGridBounds), actual) + } + it("should provide hillshade") { + checkDocs("rf_hillshade") + val actual = + Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) + .toDF("proj_raster") + .select(rf_hillshade($"proj_raster", 315, 45, 1)) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(bt.mapTile(_.hillshade(btCellSize, 315, 45, 1)), actual) + assertEqual(fullTile.hillshade(btCellSize, 315, 45, 1).crop(subGridBounds), actual) + } + } +} diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/LocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/LocalFunctionsSpec.scala index 3a7d13321..ee8940b61 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/LocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/LocalFunctionsSpec.scala @@ -21,7 +21,6 @@ package org.locationtech.rasterframes.functions -import geotrellis.raster.testkit.RasterMatchers import org.locationtech.rasterframes.TestEnvironment import geotrellis.raster._ import geotrellis.raster.testkit.RasterMatchers diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala index aea2d13ae..7dba36e84 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala @@ -180,6 +180,40 @@ class RasterRefSpec extends TestEnvironment with TestData { } } + describe("buffering") { + val src = RFRasterSource(remoteMODIS) + val refs = src + .layoutExtents(NOMINAL_TILE_DIMS) + .map(e => RasterRef(src, 0, Some(e), None, bufferSize = 3)) + val refTiles = refs.map(r => r: Tile) + + it("should maintain reported tile size with buffering") { + val dims = refTiles + .map(_.dimensions) + .distinct + + forEvery(dims) { d => + // println(s"NOMINAL_TILE_SIZE: ${NOMINAL_TILE_SIZE}") + // println(s"d: $d") + d._1 should be <= NOMINAL_TILE_SIZE + d._2 should be <= NOMINAL_TILE_SIZE + } + } + + it("should read a buffered ref") { + val ref = refs.head + + val tile = ref: Tile + // RasterRefTile is lazy on tile content + // val v = tile.get(0, 0) + + // println(s"tile.getClass: ${ref.delegate.getClass}") + // I can't inspect the BufferTile because its hidden behind RasterRefTile.delegate + info(s"tile.get(max + 1, max + 1): ${tile.get(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE)}") + } + + } + describe("RasterSourceToRasterRefs") { it("should convert and expand RasterSource") { val src = RFRasterSource(remoteMODIS) @@ -234,7 +268,7 @@ class RasterRefSpec extends TestEnvironment with TestData { import RasterRef.rasterRefEncoder // This shouldn't be required, but product encoder gets choosen. val r: RasterRef = subRaster val df = Seq(r).toDF() - val result = df.select(rf_tile(struct($"source", $"bandIndex", $"subextent", $"subgrid"))).first() + val result = df.select(rf_tile(struct($"source", $"bandIndex", $"subextent", $"subgrid", $"bufferSize"))).first() result.isInstanceOf[RasterRef] should be(false) assertEqual(r.tile.toArrayTile(), result) } @@ -242,7 +276,7 @@ class RasterRefSpec extends TestEnvironment with TestData { it("should resolve a RasterRefTile") { new Fixture { - val result = Seq(subRaster).toDF().select(rf_tile(struct($"source", $"bandIndex", $"subextent", $"subgrid"))).first() + val result = Seq(subRaster).toDF().select(rf_tile(struct($"source", $"bandIndex", $"subextent", $"subgrid", $"bufferSize"))).first() result.isInstanceOf[RasterRef] should be(false) assertEqual(subRaster.toArrayTile(), result) } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala index 5515b7513..5ed034f71 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala @@ -36,15 +36,16 @@ import scala.util.Try class RasterSourceDataSource extends DataSourceRegister with RelationProvider { import RasterSourceDataSource._ - override def shortName(): String = SHORT_NAME - override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { + def shortName(): String = SHORT_NAME + def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { val bands = parameters.bandIndexes val tiling = parameters.tileDims.orElse(Some(NOMINAL_TILE_DIMS)) + val bufferSize = parameters.bufferSize val lazyTiles = parameters.lazyTiles val spatialIndex = parameters.spatialIndex val spec = parameters.pathSpec val catRef = spec.fold(_.registerAsTable(sqlContext), identity) - RasterSourceRelation(sqlContext, catRef, bands, tiling, lazyTiles, spatialIndex) + RasterSourceRelation(sqlContext, catRef, bands, tiling, bufferSize, lazyTiles, spatialIndex) } } @@ -54,6 +55,7 @@ object RasterSourceDataSource { final val PATHS_PARAM = "paths" final val BAND_INDEXES_PARAM = "band_indexes" final val TILE_DIMS_PARAM = "tile_dimensions" + final val BUFFER_SIZE_PARAM = "buffer_size" final val CATALOG_TABLE_PARAM = "catalog_table" final val CATALOG_TABLE_COLS_PARAM = "catalog_col_names" final val CATALOG_CSV_PARAM = "catalog_csv" @@ -110,20 +112,22 @@ object RasterSourceDataSource { def tokenize(csv: String): Seq[String] = csv.split(',').map(_.trim) def tileDims: Option[Dimensions[Int]] = - parameters.get(TILE_DIMS_PARAM) + parameters + .get(TILE_DIMS_PARAM) .map(tokenize(_).map(_.toInt)) .map { case Seq(cols, rows) => Dimensions(cols, rows)} - def bandIndexes: Seq[Int] = parameters - .get(BAND_INDEXES_PARAM) - .map(tokenize(_).map(_.toInt)) - .getOrElse(Seq(0)) + def bandIndexes: Seq[Int] = + parameters + .get(BAND_INDEXES_PARAM) + .map(tokenize(_).map(_.toInt)) + .getOrElse(Seq(0)) + + def lazyTiles: Boolean = parameters.get(LAZY_TILES_PARAM).forall(_.toBoolean) - def lazyTiles: Boolean = parameters - .get(LAZY_TILES_PARAM).forall(_.toBoolean) + def bufferSize: Short = parameters.get(BUFFER_SIZE_PARAM).map(_.toShort).getOrElse(0.toShort) // .getOrElse(-1.toShort) - def spatialIndex: Option[Int] = parameters - .get(SPATIAL_INDEX_PARTITIONS_PARAM).flatMap(p => Try(p.toInt).toOption) + def spatialIndex: Option[Int] = parameters.get(SPATIAL_INDEX_PARTITIONS_PARAM).flatMap(p => Try(p.toInt).toOption) def catalog: Option[RasterSourceCatalog] = { val paths = ( @@ -143,16 +147,18 @@ object RasterSourceDataSource { ) } - def catalogTableCols: Seq[String] = parameters - .get(CATALOG_TABLE_COLS_PARAM) - .map(tokenize(_).filter(_.nonEmpty).toSeq) - .getOrElse(Seq.empty) + def catalogTableCols: Seq[String] = + parameters + .get(CATALOG_TABLE_COLS_PARAM) + .map(tokenize(_).filter(_.nonEmpty).toSeq) + .getOrElse(Seq.empty) - def catalogTable: Option[RasterSourceCatalogRef] = parameters - .get(CATALOG_TABLE_PARAM) - .map(p => RasterSourceCatalogRef(p, catalogTableCols: _*)) + def catalogTable: Option[RasterSourceCatalogRef] = + parameters + .get(CATALOG_TABLE_PARAM) + .map(p => RasterSourceCatalogRef(p, catalogTableCols: _*)) - def pathSpec: Either[RasterSourceCatalog, RasterSourceCatalogRef] = { + def pathSpec: Either[RasterSourceCatalog, RasterSourceCatalogRef] = (catalog, catalogTable) match { case (Some(f), None) => Left(f) case (None, Some(p)) => Right(p) @@ -161,7 +167,6 @@ object RasterSourceDataSource { case _ => throw new IllegalArgumentException( "Only one of a set of file paths OR a paths table column may be provided.") } - } } /** Mixin for adding extension methods on DataFrameReader for RasterSourceDataSource-like readers. */ @@ -179,7 +184,7 @@ object RasterSourceDataSource { type TaggedReader = DataFrameReader @@ ReaderTag val reader: TaggedReader - protected def tmpTableName() = UUID.randomUUID().toString.replace("-", "") + protected def tmpTableName(): String = UUID.randomUUID().toString.replace("-", "") /** Set the zero-based band indexes to read. Defaults to Seq(0). */ def withBandIndexes(bandIndexes: Int*): TaggedReader = @@ -192,6 +197,11 @@ object RasterSourceDataSource { reader.option(RasterSourceDataSource.TILE_DIMS_PARAM, s"$cols,$rows") ) + def withBufferSize(bufferSize: Short): TaggedReader = + tag[ReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.BUFFER_SIZE_PARAM, bufferSize) + ) + /** Indicate if tile reading should be delayed until cells are fetched. Defaults to `true`. */ def withLazyTiles(state: Boolean): TaggedReader = tag[ReaderTag][DataFrameReader]( diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala index 3b729df53..658f862f4 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala @@ -51,6 +51,7 @@ case class RasterSourceRelation( catalogTable: RasterSourceCatalogRef, bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]], + bufferSize: Short, lazyTiles: Boolean, spatialIndexPartitions: Option[Int] ) extends BaseRelation with TableScan { @@ -127,7 +128,7 @@ case class RasterSourceRelation( // Expand RasterSource into multiple columns per band, and multiple rows per tile // There's some unintentional fragility here in that the structure of the expression // is expected to line up with our column structure here. - val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, srcs: _*) as refColNames + val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, bufferSize, srcs: _*) as refColNames // RasterSourceToRasterRef is a generator, which means you have to do the Tile conversion // in a separate select statement (Query planner doesn't know how many columns ahead of time). @@ -139,7 +140,7 @@ case class RasterSourceRelation( .select(extras ++ paths :+ refs: _*) .select(paths ++ refsToTiles ++ extras: _*) } else { - val tiles = RasterSourceToTiles(subtileDims, bandIndexes, srcs: _*) as tileColNames + val tiles = RasterSourceToTiles(subtileDims, bandIndexes, bufferSize, srcs: _*) as tileColNames withPaths.select((paths :+ tiles) ++ extras: _*) } diff --git a/datasource/src/test/scala/examples/BufferTiles.scala b/datasource/src/test/scala/examples/BufferTiles.scala new file mode 100644 index 000000000..66be3e979 --- /dev/null +++ b/datasource/src/test/scala/examples/BufferTiles.scala @@ -0,0 +1,68 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package examples + +import geotrellis.raster.mapalgebra.focal.Square +import org.apache.spark.sql._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +object BufferTiles extends App { + + implicit val spark = + SparkSession + .builder() + .master("local[*]") + .appName("RasterFrames") + .withKryoSerialization + .getOrCreate() + .withRasterFrames + + spark.sparkContext.setLogLevel("ERROR") + + import spark.implicits._ + + val example = "https://raw.githubusercontent.com/locationtech/rasterframes/develop/core/src/test/resources/LC08_B7_Memphis_COG.tiff" + + val tile = + spark + .read + .raster + .from(example) + .withBufferSize(1) + .withTileDimensions(100, 100) + .load() + .limit(1) + .select($"proj_raster") + .select(rf_focal_max($"proj_raster", Square(1))) + // .select(rf_aspect($"proj_raster")) + // .select(rf_hillshade($"proj_raster", 315, 45, 1)) + .as[Option[ProjectedRasterTile]] + // .show(false) + .first() + + // tile.get.renderPng().write("/tmp/hillshade-buffered.png") + // tile.get.renderPng().write("/tmp/hillshade-nobuffered.png") + + // spark.stop() +} From 38fba10702d5084d1515410f9dff76fedebb0ffd Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 21 Sep 2021 20:24:30 -0400 Subject: [PATCH 02/25] Finish Focal ops BufferTile falttening --- .../focal/BufferedFocalMethods.scala | 2 +- .../encoders/StandardEncoders.scala | 9 +++- .../focalops/FocalNeighborhoodOp.scala | 4 +- .../expressions/focalops/FocalOp.scala | 2 +- .../expressions/focalops/SurfaceOp.scala | 2 +- .../rasterframes/util/package.scala | 51 ++++++++++++++++++- .../functions/FocalFunctionsSpec.scala | 46 +++++++++-------- 7 files changed, 85 insertions(+), 31 deletions(-) diff --git a/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala b/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala index 5863615e9..bf1987d66 100644 --- a/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala +++ b/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala @@ -1,5 +1,5 @@ /* - * Copyright 2016 Azavea + * Copyright 2021 Azavea * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala index 25cadd0d2..3b3cdc29b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala @@ -24,7 +24,7 @@ package org.locationtech.rasterframes.encoders import org.locationtech.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics} import org.locationtech.jts.geom.Envelope import geotrellis.proj4.CRS -import geotrellis.raster.{CellSize, CellType, Dimensions, Raster, Tile, TileLayout, GridBounds, CellGrid} +import geotrellis.raster.{CellGrid, CellSize, CellType, Dimensions, GridBounds, Raster, Tile, TileLayout} import geotrellis.layer._ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder @@ -32,10 +32,10 @@ import org.apache.spark.sql.catalyst.util.QuantileSummaries import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders import org.locationtech.rasterframes.model.{CellContext, LongExtent, TileContext, TileDataContext} import frameless.TypedEncoder +import geotrellis.raster.mapalgebra.focal.{Square, Circle, Nesw, Wedge, Annulus} import java.net.URI import java.sql.Timestamp - import scala.reflect.ClassTag import scala.reflect.runtime.universe._ @@ -54,6 +54,11 @@ trait StandardEncoders extends SpatialEncoders with TypedEncoders { implicit lazy val localCellStatsEncoder: ExpressionEncoder[LocalCellStatistics] = ExpressionEncoder() implicit lazy val uriEncoder: ExpressionEncoder[URI] = typedExpressionEncoder[URI] + implicit lazy val squareNeighborhoodEncoder: ExpressionEncoder[Square] = typedExpressionEncoder[Square] + implicit lazy val circleNeighborhoodEncoder: ExpressionEncoder[Circle] = typedExpressionEncoder[Circle] + implicit lazy val neswNeighborhoodEncoder: ExpressionEncoder[Nesw] = typedExpressionEncoder[Nesw] + implicit lazy val wedgeNeighborhoodEncoder: ExpressionEncoder[Wedge] = typedExpressionEncoder[Wedge] + implicit lazy val annulusNeighborhoodEncoder: ExpressionEncoder[Annulus] = typedExpressionEncoder[Annulus] implicit lazy val quantileSummariesEncoder: ExpressionEncoder[QuantileSummaries] = typedExpressionEncoder[QuantileSummaries] implicit lazy val envelopeEncoder: ExpressionEncoder[Envelope] = typedExpressionEncoder implicit lazy val longExtentEncoder: ExpressionEncoder[LongExtent] = typedExpressionEncoder diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala index 2a91f4d34..4b5a31234 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -42,9 +42,7 @@ trait FocalNeighborhoodOp extends UnaryLocalRasterOp with NullToValue with Codeg // if it is RasterRef, we can get what's inside case rr: RasterRef => rr.realizedTile // otherwise it is some tile - case _ => - println(s"prt.getClass: ${prt.tile.getClass}") - prt.tile + case _ => prt.tile } } val result = op(literral) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala index 47cee7290..86f2b9888 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala @@ -40,7 +40,7 @@ trait FocalOp extends UnaryLocalRasterOp with NullToValue with CodegenFallback { // if it is RasterRef, we can get what's inside case rr: RasterRef => rr.realizedTile // otherwise it is some tile - case _ => prt + case _ => prt.tile } } val result = op(literral) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala index bc87d67ce..3dc0f9494 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala @@ -62,7 +62,7 @@ trait SurfaceOp extends UnaryExpression with NullToValue with CodegenFallback { // if it is RasterRef, we can get what's inside case rr: RasterRef => rr.realizedTile // otherwise it is some tile - case _ => prt + case _ => prt.tile } } eval(literral, ctx) diff --git a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala index 4f91873d7..0169d3473 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala @@ -185,6 +185,55 @@ package object util extends DataFrameRenderers { def apply() = mapping.keys.toSeq } + object FocalNeighborhood { + import scala.util.Try + import geotrellis.raster.Neighborhood + import geotrellis.raster.mapalgebra.focal._ + + // pattern matching and string interpolation work only since Scala 2.13 + def unapply(name: String): Option[Neighborhood] = + name.toLowerCase().trim() match { + case s if s.startsWith("square-") => Try(Square(Integer.parseInt(s.split("square-").last))).toOption + case s if s.startsWith("circle-") => Try(Circle(java.lang.Double.parseDouble(s.split("circle-").last))).toOption + case s if s.startsWith("nesw-") => Try(Nesw(Integer.parseInt(s.split("nesw-").last))).toOption + case s if s.startsWith("wedge-") => Try { + val List(radius: Double, startAngle: Double, endAngle: Double) = + s + .split("wedge-") + .last + .split("-") + .toList + .map(java.lang.Double.parseDouble) + + Wedge(radius, startAngle, endAngle) + }.toOption + + case s if s.startsWith("annulus-") => Try { + val List(innerRadius: Double, outerRadius: Double) = + s + .split("annulus-") + .last + .split("-") + .toList + .map(java.lang.Double.parseDouble) + + Annulus(innerRadius, outerRadius) + }.toOption + case _ => None + } + + def apply(neighborhood: Neighborhood): String = { + neighborhood match { + case Square(e) => s"square-$e" + case Circle(e) => s"circle-$e" + case Nesw(e) => s"nesw-$e" + case Wedge(radius, startAngle, endAngle) => s"nesw-$radius-$startAngle-$endAngle" + case Annulus(innerRadius, outerRadius) => s"annulus-$innerRadius-$outerRadius" + case _ => throw new IllegalArgumentException(s"Unrecognized Neighborhood ${neighborhood.toString}") + } + } + } + object ResampleMethod { import geotrellis.raster.resample.{ResampleMethod => GTResampleMethod, _} def unapply(name: String): Option[GTResampleMethod] = { @@ -217,7 +266,7 @@ package object util extends DataFrameRenderers { case Max => "max" case Min => "min" case Sum => "sum" - case _ => throw new IllegalArgumentException(s"Unrecogized ResampleMethod ${gtr.toString()}") + case _ => throw new IllegalArgumentException(s"Unrecognized ResampleMethod ${gtr.toString()}") } } } diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala index 1a0b9356f..f6811a626 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -24,6 +24,7 @@ package org.locationtech.rasterframes.functions import geotrellis.raster.mapalgebra.focal.{Circle, Kernel, Square} import geotrellis.raster.{BufferTile, CellSize} import geotrellis.raster.testkit.RasterMatchers +import org.apache.spark.sql.functions.typedLit import org.locationtech.rasterframes.ref.{RFRasterSource, RasterRef, Subgrid} import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes._ @@ -50,25 +51,35 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { lazy val bt = BufferTile(fullTile, subGridBounds) lazy val btCellSize = CellSize(src.extent, bt.cols, bt.rows) + lazy val df = Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))).toDF("proj_raster") + it("should provide focal mean") { checkDocs("rf_focal_mean") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_mean($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() .get .tile + /*val actualExpr = + df + .selectExpr("rf_focal_mean(proj_raster, \"square-1\")") + .first()*/ + // .as[Option[ProjectedRasterTile]] + // .first() + // .get + // .tile + + // assertEqual(actualExpr, actual) assertEqual(bt.focalMean(Square(1)), actual) assertEqual(fullTile.focalMean(Square(1)).crop(subGridBounds), actual) } it("should provide focal median") { checkDocs("rf_focal_median") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_median($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -81,8 +92,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide focal mode") { checkDocs("rf_focal_mode") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_mode($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -95,8 +105,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide focal max") { checkDocs("rf_focal_max") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_max($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -109,8 +118,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide focal min") { checkDocs("rf_focal_min") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_min($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -123,8 +131,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide focal stddev") { checkDocs("rf_focal_moransi") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_stddev($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -137,8 +144,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide focal Moran's I") { checkDocs("rf_focal_moransi") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_focal_moransi($"proj_raster", Square(1))) .as[Option[ProjectedRasterTile]] .first() @@ -151,8 +157,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide convolve") { checkDocs("rf_convolve") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_convolve($"proj_raster", Kernel(Circle(2d)))) .as[Option[ProjectedRasterTile]] .first() @@ -165,8 +170,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide slope") { checkDocs("rf_slope") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_slope($"proj_raster", 2d)) .as[Option[ProjectedRasterTile]] .first() @@ -179,8 +183,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide aspect") { checkDocs("rf_aspect") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_aspect($"proj_raster")) .as[Option[ProjectedRasterTile]] .first() @@ -193,8 +196,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { it("should provide hillshade") { checkDocs("rf_hillshade") val actual = - Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))) - .toDF("proj_raster") + df .select(rf_hillshade($"proj_raster", 315, 45, 1)) .as[Option[ProjectedRasterTile]] .first() From a99f02bc29d453a6648bbf9fe96230e8c1fefaf8 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 21 Sep 2021 21:41:50 -0400 Subject: [PATCH 03/25] Rename and reorganize FocalOp expressions --- build.sbt | 2 +- ...terOp.scala => BinaryRasterFunction.scala} | 2 +- ...sterOp.scala => UnaryRasterFunction.scala} | 25 ++--- .../expressions/UnaryRasterOp.scala | 28 ++--- .../expressions/accessors/ExtractTile.scala | 4 +- .../accessors/GetTileContext.scala | 4 +- .../expressions/focalops/Aspect.scala | 2 +- .../expressions/focalops/Convolve.scala | 2 +- .../expressions/focalops/FocalMax.scala | 2 +- .../expressions/focalops/FocalMean.scala | 2 +- .../expressions/focalops/FocalMedian.scala | 2 +- .../expressions/focalops/FocalMin.scala | 2 +- .../expressions/focalops/FocalMode.scala | 2 +- .../expressions/focalops/FocalMoransI.scala | 2 +- .../focalops/FocalNeighborhoodOp.scala | 25 +---- .../expressions/focalops/FocalOp.scala | 21 +--- .../expressions/focalops/FocalStdDev.scala | 2 +- .../expressions/focalops/Hillshade.scala | 2 +- .../expressions/focalops/Slope.scala | 2 +- .../expressions/focalops/SurfaceOp.scala | 54 ++------- .../expressions/focalops/package.scala | 19 ++++ .../expressions/localops/Abs.scala | 4 +- .../expressions/localops/Add.scala | 4 +- .../expressions/localops/BiasedAdd.scala | 4 +- .../expressions/localops/Defined.scala | 4 +- .../expressions/localops/Divide.scala | 4 +- .../expressions/localops/Equal.scala | 4 +- .../expressions/localops/Exp.scala | 10 +- .../expressions/localops/Greater.scala | 4 +- .../expressions/localops/GreaterEqual.scala | 4 +- .../expressions/localops/Identity.scala | 4 +- .../expressions/localops/Less.scala | 4 +- .../expressions/localops/LessEqual.scala | 4 +- .../expressions/localops/Log.scala | 10 +- .../expressions/localops/Max.scala | 4 +- .../expressions/localops/Min.scala | 4 +- .../expressions/localops/Multiply.scala | 4 +- .../expressions/localops/Round.scala | 4 +- .../expressions/localops/Sqrt.scala | 4 +- .../expressions/localops/Subtract.scala | 4 +- .../expressions/localops/Undefined.scala | 4 +- .../expressions/localops/Unequal.scala | 4 +- .../expressions/tilestats/DataCells.scala | 4 +- .../expressions/tilestats/Exists.scala | 4 +- .../expressions/tilestats/ForAll.scala | 4 +- .../expressions/tilestats/IsNoDataTile.scala | 4 +- .../expressions/tilestats/NoDataCells.scala | 4 +- .../expressions/tilestats/Sum.scala | 4 +- .../expressions/tilestats/TileHistogram.scala | 4 +- .../expressions/tilestats/TileMax.scala | 4 +- .../expressions/tilestats/TileMean.scala | 4 +- .../expressions/tilestats/TileMin.scala | 4 +- .../expressions/tilestats/TileStats.scala | 4 +- .../transformers/DebugRender.scala | 4 +- .../expressions/transformers/RenderPNG.scala | 4 +- .../transformers/TileToArrayDouble.scala | 4 +- .../transformers/TileToArrayInt.scala | 4 +- .../test/resources/L8-B7-Elkton-VA-small.tiff | Bin 0 -> 22800 bytes .../geotrellis/raster/BufferTileSpec.scala | 103 ------------------ .../functions/FocalFunctionsSpec.scala | 19 +--- .../rasterframes/ref/RasterRefSpec.scala | 34 ------ 61 files changed, 155 insertions(+), 363 deletions(-) rename core/src/main/scala/org/locationtech/rasterframes/expressions/{BinaryLocalRasterOp.scala => BinaryRasterFunction.scala} (97%) rename core/src/main/scala/org/locationtech/rasterframes/expressions/{UnaryLocalRasterOp.scala => UnaryRasterFunction.scala} (70%) create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala create mode 100644 core/src/test/resources/L8-B7-Elkton-VA-small.tiff delete mode 100644 core/src/test/scala/geotrellis/raster/BufferTileSpec.scala diff --git a/build.sbt b/build.sbt index a04cadd66..51991d480 100644 --- a/build.sbt +++ b/build.sbt @@ -59,7 +59,7 @@ lazy val core = project circe("generic").value, circe("parser").value, circe("generic-extras").value, - frameless excludeAll ExclusionRule("com.github.mpilquist", "simulacrum"), + frameless excludeAll ExclusionRule(organization = "com.github.mpilquist"), `jts-core`, `spray-json`, geomesa("z3").value, diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala similarity index 97% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala index 18d337bdc..094c157d7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala @@ -31,7 +31,7 @@ import org.locationtech.rasterframes.expressions.DynamicExtractors._ import org.slf4j.LoggerFactory /** Operation combining two tiles or a tile and a scalar into a new tile. */ -trait BinaryLocalRasterOp extends BinaryExpression with RasterResult { +trait BinaryRasterFunction extends BinaryExpression with RasterResult { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala similarity index 70% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala index 2904fe57d..6eb4e7a69 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala @@ -21,34 +21,27 @@ package org.locationtech.rasterframes.expressions -import com.typesafe.scalalogging.Logger +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.UnaryExpression -import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.DynamicExtractors._ -import org.slf4j.LoggerFactory - -/** Operation on a tile returning a tile. */ -trait UnaryLocalRasterOp extends UnaryExpression with RasterResult { - @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - - def dataType: DataType = child.dataType +import org.locationtech.rasterframes.model.TileContext +/** Boilerplate for expressions operating on a single Tile-like . */ +trait UnaryRasterFunction extends UnaryExpression { override def checkInputDataTypes(): TypeCheckResult = { if (!tileExtractor.isDefinedAt(child.dataType)) { TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") - } - else TypeCheckSuccess + } else TypeCheckSuccess } override protected def nullSafeEval(input: Any): Any = { - val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - val result = op(childTile) - toInternalRow(result, childCtx) + // TODO: Ensure InternalRowTile is preserved + val (tile, ctx) = tileExtractor(child.dataType)(row(input)) + eval(tile, ctx) } - protected def op(child: Tile): Tile + protected def eval(tile: Tile, ctx: Option[TileContext]): Any } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala index 8d2b532c8..dcb4871c8 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala @@ -21,27 +21,21 @@ package org.locationtech.rasterframes.expressions -import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import com.typesafe.scalalogging.Logger import geotrellis.raster.Tile -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes.model.TileContext +import org.slf4j.LoggerFactory -/** Boilerplate for expressions operating on a single Tile-like . */ -trait UnaryRasterOp extends UnaryExpression { - override def checkInputDataTypes(): TypeCheckResult = { - if (!tileExtractor.isDefinedAt(child.dataType)) { - TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") - } else TypeCheckSuccess - } +/** Operation on a tile returning a tile. */ +trait UnaryRasterOp extends UnaryRasterFunction with RasterResult { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - override protected def nullSafeEval(input: Any): Any = { - // TODO: Ensure InternalRowTile is preserved - val (tile, ctx) = tileExtractor(child.dataType)(row(input)) - eval(tile, ctx) - } + def dataType: DataType = child.dataType - protected def eval(tile: Tile, ctx: Option[TileContext]): Any + protected def eval(tile: Tile, ctx: Option[TileContext]): Any = + toInternalRow(op(tile), ctx) + + protected def op(child: Tile): Tile } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala index 529c88996..ea615843a 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.accessors -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -32,7 +32,7 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes._ /** Expression to extract at tile from several types that contain tiles.*/ -case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ExtractTile(child: Expression) extends UnaryRasterFunction with CodegenFallback { def dataType: DataType = tileUDT override def nodeName: String = "rf_extract_tile" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala index 52bc4074e..eb1fb9675 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala @@ -28,10 +28,10 @@ import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes._ import org.locationtech.rasterframes.encoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext -case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class GetTileContext(child: Expression) extends UnaryRasterFunction with CodegenFallback { def dataType: DataType = tileContextEncoder.schema override def nodeName: String = "get_tile_context" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala index 311f837b8..d906403e9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -34,7 +34,7 @@ import org.locationtech.rasterframes.model.TileContext examples = """ Examples: > SELECT _FUNC_(tile); - ..""" + ...""" ) case class Aspect(child: Expression) extends SurfaceOp { override def nodeName: String = Aspect.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala index be4d63c7b..2559706c1 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, kernel); - ..""" + ...""" ) case class Convolve(child: Expression, kernel: Kernel) extends FocalOp { override def nodeName: String = Convolve.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala index 1af40018c..50f8c3e7c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMax.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala index 732249d01..6ae045146 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMean.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala index 40ae8a570..d6aaae2bc 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMedian.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala index e6f152d5d..071a2e2ee 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMin.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala index 4bc4cf5a4..9ec10ee5e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMode.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala index b0ca86379..c52a71cd7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalMoransI.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala index 4b5a31234..24e299bab 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -22,30 +22,7 @@ package org.locationtech.rasterframes.expressions.focalops import geotrellis.raster.mapalgebra.focal.Neighborhood -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp, row} -import org.locationtech.rasterframes.ref.RasterRef -import org.locationtech.rasterframes.tiles.ProjectedRasterTile -trait FocalNeighborhoodOp extends UnaryLocalRasterOp with NullToValue with CodegenFallback { - def na: Any = null +trait FocalNeighborhoodOp extends FocalOp { def neighborhood: Neighborhood - - override protected def nullSafeEval(input: Any): Any = { - val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - val literral = childTile match { - // if it is RasterRef, we want the BufferTile - case ref: RasterRef => ref.realizedTile - // if it is a ProjectedRasterTile, can we flatten it? - case prt: ProjectedRasterTile => prt.tile match { - // if it is RasterRef, we can get what's inside - case rr: RasterRef => rr.realizedTile - // otherwise it is some tile - case _ => prt.tile - } - } - val result = op(literral) - toInternalRow(result, childCtx) - } } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala index 86f2b9888..6169cf88c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala @@ -21,29 +21,16 @@ package org.locationtech.rasterframes.expressions.focalops -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp, row} -import org.locationtech.rasterframes.ref.RasterRef -import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp, row} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -trait FocalOp extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +trait FocalOp extends UnaryRasterOp with NullToValue with CodegenFallback { def na: Any = null override protected def nullSafeEval(input: Any): Any = { val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - val literral = childTile match { - // if it is RasterRef, we want the BufferTile - case ref: RasterRef => ref.realizedTile - // if it is a ProjectedRasterTile, can we flatten it? - case prt: ProjectedRasterTile => prt.tile match { - // if it is RasterRef, we can get what's inside - case rr: RasterRef => rr.realizedTile - // otherwise it is some tile - case _ => prt.tile - } - } - val result = op(literral) + val result = op(extractBufferTile(childTile)) toInternalRow(result, childCtx) } } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala index 0098fac96..9927f7874 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript examples = """ Examples: > SELECT _FUNC_(tile, Square(1)); - ..""" + ...""" ) case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { override def nodeName: String = FocalStdDev.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala index 7b65ca25a..e3c693ec9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.model.TileContext examples = """ Examples: > SELECT _FUNC_(tile, azimuth, altitude, zFactor); - ..""" + ...""" ) case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOp { override def nodeName: String = Hillshade.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala index 97d521974..80558daa6 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -35,7 +35,7 @@ import org.locationtech.rasterframes.model.TileContext examples = """ Examples: > SELECT _FUNC_(tile, 0.2); - ..""" + ...""" ) case class Slope(child: Expression, zFactor: Double) extends SurfaceOp { override def nodeName: String = Slope.name diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala index 3dc0f9494..a0e8fc1c7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala @@ -21,61 +21,31 @@ package org.locationtech.rasterframes.expressions.focalops -import org.slf4j.LoggerFactory -import com.typesafe.scalalogging.Logger -import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.row -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.locationtech.rasterframes.ref.RasterRef - +import org.locationtech.rasterframes.expressions.{NullToValue, RasterResult, UnaryRasterFunction, row} import org.locationtech.rasterframes.encoders.syntax._ import org.locationtech.rasterframes.expressions.DynamicExtractors._ -import geotrellis.raster.Tile -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.UnaryExpression import org.locationtech.rasterframes.model.TileContext -import org.locationtech.rasterframes.expressions.NullToValue -import org.locationtech.rasterframes.tiles.ProjectedRasterTile +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.DataType +import org.slf4j.LoggerFactory +import com.typesafe.scalalogging.Logger /** Operation on a tile returning a tile. */ -trait SurfaceOp extends UnaryExpression with NullToValue with CodegenFallback { +trait SurfaceOp extends UnaryRasterFunction with RasterResult with NullToValue with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - def dataType: DataType = child.dataType def na: Any = null - - override def checkInputDataTypes(): TypeCheckResult = { - if (!tileExtractor.isDefinedAt(child.dataType)) { - TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") - } else TypeCheckSuccess - } + def dataType: DataType = child.dataType override protected def nullSafeEval(input: Any): Any = { val (tile, ctx) = tileExtractor(child.dataType)(row(input)) - - val literral = tile match { - // if it is RasterRef, we want the BufferTile - case ref: RasterRef => ref.realizedTile - // if it is a ProjectedRasterTile, can we flatten it? - case prt: ProjectedRasterTile => prt.tile match { - // if it is RasterRef, we can get what's inside - case rr: RasterRef => rr.realizedTile - // otherwise it is some tile - case _ => prt.tile - } - } - eval(literral, ctx) + eval(extractBufferTile(tile), ctx) } - protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { - ctx match { - case Some(ctx) => - val ret = op(tile, ctx) - ctx.toProjectRasterTile(ret).toInternalRow - - case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") - } + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") } protected def op(t: Tile, ctx: TileContext): Tile diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala new file mode 100644 index 000000000..a1e47eaba --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala @@ -0,0 +1,19 @@ +package org.locationtech.rasterframes.expressions + +import geotrellis.raster.Tile +import org.locationtech.rasterframes.ref.RasterRef +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +package object focalops extends Serializable { + private [focalops] def extractBufferTile(tile: Tile): Tile = tile match { + // if it is RasterRef, we want the BufferTile + case ref: RasterRef => ref.realizedTile + // if it is a ProjectedRasterTile, can we flatten it? + case prt: ProjectedRasterTile => prt.tile match { + // if it is RasterRef, we can get what's inside + case rr: RasterRef => rr.realizedTile + // otherwise it is some tile + case _ => prt.tile + } + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala index 153eeb5fa..19cbe3090 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} @ExpressionDescription( usage = "_FUNC_(tile) - Compute the absolute value of each cell.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Abs(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Abs(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { override def nodeName: String = "rf_abs" def na: Any = null protected def op(t: Tile): Tile = t.localAbs() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala index ff23eb646..7f231797b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction import org.locationtech.rasterframes.expressions.DynamicExtractors @ExpressionDescription( @@ -43,7 +43,7 @@ import org.locationtech.rasterframes.expressions.DynamicExtractors > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Add(left: Expression, right: Expression) extends BinaryLocalRasterOp +case class Add(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_add" protected def op(left: Tile, right: Tile): Tile = left.localAdd(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala index e31dd17eb..300103154 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor import org.locationtech.rasterframes.util.DataBiasedOp @@ -45,7 +45,7 @@ import org.locationtech.rasterframes.util.DataBiasedOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class BiasedAdd(left: Expression, right: Expression) extends BinaryLocalRasterOp +case class BiasedAdd(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_biased_add" protected def op(left: Tile, right: Tile): Tile = DataBiasedOp.BiasedAdd(left, right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala index 1a7af9b25..035a5ad84 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Defined(child: Expression) extends UnaryLocalRasterOp +case class Defined(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_data" def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala index f90fb4225..ce0d0be1c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise division between two tiles or a tile and a scalar.", @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Divide(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Divide(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_divide" protected def op(left: Tile, right: Tile): Tile = left.localDivide(right) protected def op(left: Tile, right: Double): Tile = left.localDivide(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala index c1804708f..b83fcee7e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise equality test between two tiles.", @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Equal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Equal(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_equal" protected def op(left: Tile, right: Tile): Tile = left.localEqual(right) protected def op(left: Tile, right: Double): Tile = left.localEqual(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala index 01d45e19d..21f57d1f6 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOp, fpTile} @ExpressionDescription( usage = "_FUNC_(tile) - Performs cell-wise exponential.", @@ -38,7 +38,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Exp(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_exp" protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E) @@ -59,7 +59,7 @@ object Exp { > SELECT _FUNC_(tile); ...""" ) -case class Exp10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp10(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(10.0) @@ -80,7 +80,7 @@ object Exp10 { > SELECT _FUNC_(tile); ...""" ) -case class Exp2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp2(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_exp2" protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0) @@ -101,7 +101,7 @@ object Exp2 { > SELECT _FUNC_(tile); ...""" ) -case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class ExpM1(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_expm1" protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala index b318329fc..e820f94f5 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise greater-than (>) test between two tiles.", @@ -38,7 +38,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Greater(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Greater(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_greater" protected def op(left: Tile, right: Tile): Tile = left.localGreater(right) protected def op(left: Tile, right: Double): Tile = left.localGreater(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala index e4d1dcfc1..dd33e3415 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise greater-than-or-equal (>=) test between two tiles.", @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class GreaterEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class GreaterEqual(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_greater_equal" protected def op(left: Tile, right: Tile): Tile = left.localGreaterOrEqual(right) protected def op(left: Tile, right: Double): Tile = left.localGreaterOrEqual(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala index 001688a1c..418ddf780 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} @ExpressionDescription( usage = "_FUNC_(tile) - Return the given tile or projected raster unchanged. Useful in debugging round-trip serialization across various language and memory boundaries.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Identity(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Identity(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { override def nodeName: String = "rf_identity" def na: Any = null protected def op(t: Tile): Tile = t diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala index 76543e34e..8f5ac719f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise less-than (<) test between two tiles.", @@ -38,7 +38,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Less(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Less(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_less" protected def op(left: Tile, right: Tile): Tile = left.localLess(right) protected def op(left: Tile, right: Double): Tile = left.localLess(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala index 116b3c712..ae51ab2f1 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise less-than-or-equal (<=) test between two tiles.", @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class LessEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class LessEqual(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_less_equal" protected def op(left: Tile, right: Tile): Tile = left.localLessOrEqual(right) protected def op(left: Tile, right: Double): Tile = left.localLessOrEqual(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala index c428cc922..2ebd84412 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOp, fpTile} @ExpressionDescription( usage = "_FUNC_(tile) - Performs cell-wise natural logarithm.", @@ -38,7 +38,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Log(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "log" protected def op(tile: Tile): Tile = fpTile(tile).localLog() @@ -59,7 +59,7 @@ object Log { > SELECT _FUNC_(tile); ...""" ) -case class Log10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log10(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_log10" protected def op(tile: Tile): Tile = fpTile(tile).localLog10() @@ -80,7 +80,7 @@ object Log10 { > SELECT _FUNC_(tile); ...""" ) -case class Log2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log2(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_log2" protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0) @@ -101,7 +101,7 @@ object Log2 { > SELECT _FUNC_(tile); ...""" ) -case class Log1p(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log1p(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_log1p" protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala index b68e49955..01019543f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise maximum two tiles or a tile and a scalar.", @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Max(left: Expression, right:Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Max(left: Expression, right:Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName = "rf_local_max" protected def op(left: Tile, right: Tile): Tile = left.localMax(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala index 0af8b3117..171812929 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise minimum two tiles or a tile and a scalar.", @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Min(left: Expression, right:Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Min(left: Expression, right:Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName = "rf_local_min" protected def op(left: Tile, right: Tile): Tile = left.localMin(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala index 4dc7e8548..7bf3367d4 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise multiplication between two tiles or a tile and a scalar.", @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Multiply(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Multiply(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_multiply" protected def op(left: Tile, right: Tile): Tile = left.localMultiply(right) protected def op(left: Tile, right: Double): Tile = left.localMultiply(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala index 90bf4b508..d4238c27f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} @ExpressionDescription( usage = "_FUNC_(tile) - Round cell values to the nearest integer without changing the cell type.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Round(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Round(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { override def nodeName: String = "rf_round" def na: Any = null protected def op(child: Tile): Tile = child.localRound() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala index d8e86fb34..ad3ed376d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOp, fpTile} @ExpressionDescription( usage = "_FUNC_(tile) - Perform cell-wise square root", @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile) ... """ ) -case class Sqrt(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Sqrt(child: Expression) extends UnaryRasterOp with CodegenFallback { override val nodeName: String = "rf_sqrt" protected def op(tile: Tile): Tile = fpTile(tile).localPow(0.5) override def dataType: DataType = child.dataType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala index 645049ce2..708e7e207 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(tile, rhs) - Performs cell-wise subtraction between two tiles or a tile and a scalar.", @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Subtract(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Subtract(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_subtract" protected def op(left: Tile, right: Tile): Tile = left.localSubtract(right) protected def op(left: Tile, right: Double): Tile = left.localSubtract(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala index fb146451f..bd533f4b7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Undefined(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Undefined(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_no_data" def na: Any = null protected def op(child: Tile): Tile = child.localUndefined() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala index 2cdc30292..9bab9b86b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.functions.lit -import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp +import org.locationtech.rasterframes.expressions.BinaryRasterFunction @ExpressionDescription( usage = "_FUNC_(lhs, rhs) - Performs cell-wise inequality test between two tiles.", @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp > SELECT _FUNC_(tile1, tile2); ...""" ) -case class Unequal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback { +case class Unequal(left: Expression, right: Expression) extends BinaryRasterFunction with CodegenFallback { override val nodeName: String = "rf_local_unequal" protected def op(left: Tile, right: Tile): Tile = left.localUnequal(right) protected def op(left: Tile, right: Double): Tile = left.localUnequal(right) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala index a27b78328..52dc8c1ed 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 357""" ) -case class DataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue { +case class DataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_data_cells" def dataType: DataType = LongType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = DataCells.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala index 1fa187409..ebb2156d7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ import org.locationtech.rasterframes.isCellTrue -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -24,7 +24,7 @@ import spire.syntax.cfor.cfor true """ ) -case class Exists(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Exists(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "exists" def dataType: DataType = BooleanType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala index a49888845..f553de047 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes._ import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -24,7 +24,7 @@ import spire.syntax.cfor.cfor true """ ) -case class ForAll(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ForAll(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "for_all" def dataType: DataType = BooleanType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala index f796e6019..e03b96194 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); false""" ) -case class IsNoDataTile(child: Expression) extends UnaryRasterOp +case class IsNoDataTile(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_is_no_data_tile" def na: Any = true diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala index 2601bc4ae..556abd715 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 12""" ) -case class NoDataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue { +case class NoDataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_no_data_cells" def dataType: DataType = LongType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = NoDataCells.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala index 9e1861cda..9e3ff1f8c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster._ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile5); 2135.34""" ) -case class Sum(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Sum(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_sum" def dataType: DataType = DoubleType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala index 567216ac5..a4a5fffa3 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileHistogram(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileHistogram(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_histogram" protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileHistogram.converter(TileHistogram.op(tile)) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala index 8d3cd285a..cbbe1a52c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 1""" ) -case class TileMax(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { +case class TileMax(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_max" protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile) def dataType: DataType = DoubleType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala index 5fb7b1805..2f0bdedb5 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMean(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { +case class TileMean(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_mean" protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile) def dataType: DataType = DoubleType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala index 66698824e..c3d26fb4a 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.tilestats import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMin(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback { +case class TileMin(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_min" protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile) def dataType: DataType = DoubleType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala index 2ef501faa..ebf6bf67c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -41,7 +41,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileStats(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileStats(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_stats" protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileStats.converter(TileStats.op(tile).orNull) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala index 5f54506df..76be3ba16 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala @@ -29,11 +29,11 @@ import org.apache.spark.sql.types.{DataType, StringType} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor -abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp with CodegenFallback with Serializable { +abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterFunction with CodegenFallback with Serializable { import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix def dataType: DataType = StringType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala index a896a4342..9d3639910 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types.{BinaryType, DataType} import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext /** @@ -36,7 +36,7 @@ import org.locationtech.rasterframes.model.TileContext * @param child tile column * @param ramp color ramp to use for non-composite tiles. */ -abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterOp with CodegenFallback with Serializable { +abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterFunction with CodegenFallback with Serializable { def dataType: DataType = BinaryType protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { val png = ramp.map(tile.renderPng).getOrElse(tile.renderPng()) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala index 6e52ed9ca..a856b917b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.transformers import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayDouble(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayDouble(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_double" def dataType: DataType = DataTypes.createArrayType(DoubleType, false) protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala index 07b5dc58b..e6bbbd4a7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType} import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.encoders.SparkBasicEncoders._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayInt(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayInt(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_int" def dataType: DataType = DataTypes.createArrayType(IntegerType, false) protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/test/resources/L8-B7-Elkton-VA-small.tiff b/core/src/test/resources/L8-B7-Elkton-VA-small.tiff new file mode 100644 index 0000000000000000000000000000000000000000..4ed7a5e48947374c645a98af3a86776468bad0fd GIT binary patch literal 22800 zcmZU*1#lZlv^MH6gArAd#i5M zTbCn6tH{&S-RFGNE>p%AkTxJ7AWJ|%vJ?Ty;8{obPWC^~o5Si_dRFj>lg6!6+n_zwO*uZR54a}BSD!t;OMvmd_k|9MZ<|MNWakc|Jm=OVmk@X(9_ zsp0V@e6JavB7nmqh9A6cc=7-W&+Gkr5%6CgmWSD%KJd5oJRsnCq5oclKgtyTf6xB+ z@qb_c|38ir@ySB_nIZub2IRh(w{nsHeMhqY?njz{fRu1Y{{8yzF?m4plUKuADpxPi$6{!#%M|9xzonDW0zuc}$9D$f(gqj>dd6?mVH9h&o)+!g!%zgPPO z6vMB@2ia8pKuu5<^w^5!Y58$>TkZ2MiQX1FBYJW~b$5b)kJk>r;!pJFGmCU4^N$I2 zuz9B=bQ#)&HbI?Hev*;oAv^Fzd=CeZI69txlbh%fwwLrn3#`l5A30y9kQ+sS43;ao zmnY(us3IDT8lXz(EUM|Iiz?(zqjXdwRaCE0JyndoKs7Vx%m^uTZiPiHyEGjFB@nQmuZxsJazLLx2UAY<`#6#q9 z;qhjQs2O=GdU<>=Gfj1r#dQ{QM4d6Eodc$njxY_J+~%_yC1(r2*rc}L)9kw)LkJno z|IYhBB;f0`8UCs>=(Wn$P2l^5C zxDctL3P(AS$0B{jTeKW~`l*UwJF59Nx3~F*&ctCCl^s<$!ylia!G_eS1`OdE4J$Nk8R zQSbB{UB^V2P_y04F^5zi-IJwNA9Y(?8&md}EWxW_bkFf>ybCuX4S5IISjX!?iSh5y zMd+y1vZfj%MvL2gIo~R{dMoCNP4+1{SPggD=`8MV_k~$!^7xVY<_t8%HzV%_wqH%x zFPz!-74txMf$<&oruVLLzS|h@LpgClmV^h=wTzjUrj4828LN8n`sS9K-@8F4y35>9 zxkT0!okblw4!^Qqqx&L=8*yK4ax(d}2+w;v!mC+mMo#Te;Xj4z!b3Xb~`LfubO->O>R^yL$AyJL{_zre~ z_mjzGGP|dpXh(?7=7{saTOs4GkIS4O$sqa z3>IFITm*_1-YW2}2I_@u=G>IkOfxMahM6a#uo`R<%rkjIowiG>yW%QuCUfeg&IVh< z&n)E>@=Z32+&ylPW6KR*>7S{;>8T=G%)<<~v?F*mo}0(;D49Y9%R6>A^UL{S3b}pV za&C-wsy&S@CqCP_UA=|uyUtI&S|8F4bzQkhPu9EfPt=ZPW@}Y`Q_XaQ9Wv1_tG?qE zw2LUFN9bZ^6doSd9Ceqr+AA~4`=T;G$`VAR+N|HmwsJTh%X{(-Q2Q)i)>lX-LG7p` z7s!WBYE#ja^`3)WK29yuUMERDQ-2#RSF^4Y(Xty4XAPLUmL;6oF1gXRrOouI(>k zBYj)P=xRDiTv5IBc~wu|w<_Z9v>$D!a@lw6^{TSj>MU0bkEI#p5%CJv?h(8oY&+hh zKFZ_ruzaeD5lMdG-J~2ZB{G`sDp(&BMfgcRPj7cRikB*#cm)+=g*qr}t2S;K6C#uj zktt!;N;t3eay4H>!s?yP>pQER=k^qPl|9;h;3oGO_wVr3-b!{?naF>s>iV4wVOz11 z6HPABhpSY5ETFUT zCCE>j@Sk#$vrPBVshq#nd*`8?hg*`}^0&OGhRb*~)#?=5MGsZ;R3z-xEBF_Ff#=}8 zcqmD0XSIFKS$m8+rjwb;YB(FNI@&4Befu=;BQLv!RVrtSNw3z*j&@}lBKN3xompqo zy`A>X2*+dJvO7C%oeNGgH>1-!daiR_twiU^XT3}QP;JFZJH4R^MyXu%VBe|`$)t%%qr!)$5K zP;<4%_;dl&N%m%gWqtZu98^BljP>Tn;VsGs<>28KtJ7$!yP) z1JP=HFKD33sJ=suEUD&^m-s$T#C1q9xlDC3d(Gc|kGs+x=I&S9MY_luaid}zi%i}m z+vm@(f9uEkm6@Ze%JI%DomV`P#Z96;7ZAw+)5CPPcbi#GBmZrunh>-QEkkGHRJ|1f^_vgoWnd;Fcyd|JJ55)I8C=0r@~rktzLf+;N9!|e ziuq*HnQQ92S||3a_wp6*$Xruh{w3<@ooca3q0>8^)dLx-eC9cK$Rn~QSja&2M~+pA zx;Q(5r{ZbY z)aIyOY47l^&{}m<0jiBYthbo~x`NEk8T9Z;C~2DOb*hU@YY!6#NjT{lvRHCeM=emdb$79y&WF{hWE1&bKUUk+ zMg2%cs8MbUVN^r^Bi}sT$#>m%G`z1jKK!t^yS>w`WIq+f?0V{*n51T^g*@0fY==Sx zN>FdrDW0HutCM&$eu58?Iye$<#DjPdK7(AxNq7VwCt_qr86z7=uG=_Da(Ty;^j>#v zi8VY8zfXP>n{A^fOh!`6B!`-A+3R_7l~+`=pP6vc)avznG`DNMRzKsiK>5)lGF=E>zjwD&d9IAXQUM^fflMbaxY~ zo~y08iqqBi*!x?S^38TqI6FltwV&tYcSS?@5FnjSvZBtXZ>T*g(vFt(RTJ?XS0D}{ zcm(m1FsR@!RT26d=OrnfIK5P1wMEvI)#RMmh&(NQmGx-74(7ZqxkysLDs9YrlGUV$ z?wlhoW_(0`XR`f~mlhjE5xHJHV$*q`IwZ2O&Ww_UYEk5Mol0+)tG!;>iCxS>w2p4Y zF=8%TY6~?@1qsG|CZjH>v+Gl81mLq0s-*5IGkd!@WnyZ_hP#v8^ZtIiw%+Ov6&aP$ zYef=&C>QEcs-He1v)Dq^kY!D3Q&3OiZ&fQ{tI}#{fW%&t8wbFd@3r+BPKk@8{Y;M;3RiL8%;aW|9SJ!eU-zn@9M|)gs6PpMXCq+h!3JC_%6AFAE0#j5n72KlHpL7$B<~i zeC6b66=J8+jl~)~8Q9`Y@&Qx)%o>K%`?240GY=Ls{ik^yM!m|E)W5wi)DiQ{PG{P&tK>M_;Z5lTi$eBm6D&803B0`S z;CSr|g25*}vHIv4u$JuTMZR-v86Ay#@y+&QMb#(uf?u--dv_=t=`m^T^3DVOo@I1% zngqMDTx7@S@w5rAD_fACxB;m|o{|eV75;{9qATcc^Z-3W{qQanN&-kA8^BMKKNyj= zBt5-?BScHUC=2Y~G{3kc7Rgk=DF?}4D!VDeda+NW8S55x&39d-7g=O2NqC~uAjd`M zZz;WV?3{pqUfI`7lzAz7;NDo#a68oLs@gywEUj;(a-qWH-7u##e)XeG@^kjBU`FGd>mVk%j3Oh2AV=%vUcJZ@c3G^ zx4bIuic|Tr#Se=so%2VofZShW|3rM12T%{Tnp|WT=x36snwkRML!un7K(puUXilj{ z?nRYSU6a=V_qs|!caIfg>G9C$Ak5cG!X}~tcmNxYzM}lN0`lQQCY zJyuNO2ZgX_$nD}34_2*k7$B(JZfZ4;Uw|IepN|4=TpGrek=#dqQ~;Gmr>qs$bL$x@ zkNTi7=pcHI&f^wjDn1YOB|R+x73Kn31h1|kecY7Z8>%y^k3H%vuPkObJEHG~_sLZt zylC`~=mqiuRcw;%LU-b=@ULS;erLARQ(81yeDXDMD){=EwCWn)%~JrM$t9k#-Rux+ z!it8b3*O1k(7V_|l}Id4j&~pfcl0!RgQg-I7l1C6h1?~|R3j-!LsE$zr$y=6$OGXs z$#b`(z82lqnWh)=fh0Hiif$3*N@C>vQjYOd|yGTN3pr;OpidhQP!~^kql7!>& z9GVx_N^;Z@6o*#m1KNZ$(1u8$Pv{<-%4W0Vpsh@FYIq0vg5!#KpX!zJr2Z!I0l$2r zhuNPUN2t>I?5Lvgp*%CpOx%{O?V;4DbiV)zC-uV^EXYxcS^&x+Q$_by?4h)m^<>j37_x z5iy??6`9p%6{fH1NNMZxG#Kp(X%SM|3PiD(vGIVFMziCzFK;AD$O9}?e&!zm6W8`u zG2?VEy+oC9OSn1RyE;nl)ql)n?Q!R8U@x&t*^P3B>#GvmSD@-HGiTgJ zPHQv5+^0#XFG-6pqRK2Cok5G?p5!JTgo>k))L|WHb(%oW^MZQ2*`a!Pmx9)D-rZnZ zs)xKT3!AfUZtn*h^!FA&%v_IdWuBu9K_i8AQ z{hW3N<)L!$$B^5h9Z`KUjYNBfiwwLQ_3?BnOhm|!JcV}TRF%^7(99mBSKHN0Ywr$S z40d0CxPRQfVRqOnG}jf}b@paEr8CR)GhI|+-O6ohFEV8#D@D5AfuE=%Q*uNv|m@t2hd_26R z_YNqUxpV>DLRSX-(AxV&&z0NMBQ*_u#RJhASlJqJ+J$^TlT~AO(MpMjvNmD@-H0lM zbquW+`aZaO@Zzv{xQ@zZXEPa14HKaz@)r3uZ z_lc^j%b3@oplWl`UF;5YOM&iC+R5m>7d1V0nR`@cp%-x~GKZw0`SBKgTx`G%aU=YS zSJ#Ib7Yp1+hB+&p1a)5KRmsI^>GOWck<9xC7~&D+Q_uI)4r5=BgO3ewh_clBw!W zbI#ePoziZC`?r0=uHk0XAIv_Ifu`rFKt*WFT7f$L5UO%foMeuvJTi@THH+|s-?HD? zwH*%&x0Cyp$0m5ox^b$P)5q6cA2W-bY0iB+i|*l&@Y~+UJVc}v=lNQw1sk}<&!Gf7 z561V%>V%e|I;0wmWIw)1y!4ek>sBLGMG$F1a*@4c3TcPm1s@81W0fVDcwRPvtpi;; z!Fy9@wFjx;s+}q-AAn(XQkj$%QyCXUaqBrrsR0D6+l!Xr-m^ z?RU;Por?}2%b`jXbRO|5BqM%gbwTe?Wwn7{SJ#1ee9@P&GV?mF0qE z;(wS|=NLm7`JdS6Oc)HG$Ik;cw5xz|9;j>v|(Uf&^j>~fPBok?htGy~( zMvKe#8#&hu25;b}JY^d_AJ*$Nanl^u(n(?Gi#=lMkg|9mUd48bQEpLlo43)`K?9ts zsXR>+@h8AB{eAgWVebZ$*VVwqZzRebda>r|p_{3OB9&+ogFk*-O-TdCd@*(Jx}7 zYN4Bn3#vD;%qr?H-GOX_dnHJ5+yZ-WB6}wXx!3JyD$KY}X6>-fWDRzl3UZL|jDKJC zBmSn&95c-A3CiFKsM1+i6(Q~*l-i>H*d#e$t zrV7y4tft;5hp1k__wLC#y1xF$^mTH0zu1%YT+knTIz3EY^V`nlJ!N;&OYK0Ycs+16 zoDR3f%gAROihZ~n?tnR)FaL29&2-&NzU3=qT5lmYpEt?dKmL{T#`$+;M9X#3qi4!I zP9HtV9&QGMs+Nb%pcm;+I*iwFuvh|>Bo%jgJU`1kd?tI&V(~19li{Dl!ujxVv=1|pQNxJXpeKoL+8!Sf74X# z7_5oQfbE;AfvSdnB}<6cERz^ZiW3FQV=tPE8lhzLI4MJVlF6hixvM6bf9N_;SRUYi z043H3J!~g#Lr&s+WD%N=mWCBXTj*)Ngw?mRnDGk9Wi*h~GqdfZph#D9HvpoE)@^mH zF|eOT(?D6urhuT{@rAe#4#W<=4fSz9>VaqC=jb?!C!29c@=bhqdawwd+6{Al=^%Tw z`D(9?{9vz_>Fq+M8)-`alHJV-vk(|`oT=@b2c0y)kt~)?u#<&1g?n;9d1Q52Mi!CH zWnp&p4R<1}s{q56gj`<`<=^c8AI^j!c54XK`C$H<26uV?5 z+1H`INw#C!8jn-XKirok{4wyOQ_k$@H_<(!cF7qsUM>*Lkv`Bxd!egH0-%4gu}=k`j37u zH-L+~Tr9UMI<4#){xWuMJFPiwX6o&#xii=88WA14IcMAA9l~eYT5sj?;sYNnF7XrK zTK5rWge&8~3%?{Mk_V_0x(K``6KM^5unQ@~TZ&>7%e8=g=F3!OxObLQNJr|?g2`c^ z9o66icmdE(@AGRkm9-b0B`-)eb4CqS*_?ynTVp%&dh#Hkl(SAj_cQeCs`dxB6FGA;0c|QwM~2bqv_#2 z;goQ;gBsUGJvV`Flyf+Ia=sEpi^e}UjyFvI(L$_(E_Z_`K&|^M1Sp}s)kM4n zEkIX+Db6JQ0n7d*7eq21PHW2hptT$X?YFURAWMNWe3u{OTX}czwQKUaye)hUqswU) zPduiisi~^=iSt@TTLNl{HU^Bsfg*ulTN zM2k{A;>tyCHAlfGbwn!R7o~tbM~eYsuaL5k+$am!N0Gz>0Q)o~<>(qvKL?Y(=(v>$ zozdNRJ`tk^i>h+8*e{L=(0auVSm8Z+F20K|;X`?AHkM2eSp!N=Gd@k1&~LU=KZ{c06(}4!T^KF`D>Xfy04jbue1VmA z<~ohdQdNVrptIx-=edcq$2f;Ty}N5K2ZghQX%V|GZb*(r?#P^TqB};$gQmGy`9uZL zL<|C?vP5K)onh5nbXWU|N7PbTX=6Hz@|Lw9d0_`8SG8GyNsKt^Pf!V> zifAL+$*eL^e4&Y;=?ufO=t??*WI!3iTvSeVlOxqRv0OA%P4pW@z%AS@|45<}4YM2C zugr3l*~IXrs-vo^a?v^bw1^Vt$X@ymPb1lI08Rn;i_+nmX~4$r2sbM!Pw(+gb{%`R z7^1e?;r39wjNRUR(<38#=boOwzyCn^R9{o}k4*AmT?g_Z5#qF1DzEFbzKfurx`0s2 zfEN9?90WM+z3ilK@CrfGMiLE zIY~cS4>{<)XSpYX>J2-hkNii@lS@^8ol+f^Gena5W!F}zR5kF3!g(@Wj~s>w);m2>F5DgHn@O$H_?P1CM2_uZAea>afhJuZjP8R zUuK&QH4S6l%dBiST}G$UL*xzFLWYrr^euP}6=*4NnLp5cEP!4l$ps-NXc((cKazAb zD;-D6lH0f!9*<*jJ5Zil;6?Z@3c@kK%zn{Y^kYy(E6Dz$SEwrDi|im)s()w;{*HBl z`tuum#UVAq)P`ikF#Qf6MR9zEexMw+69vHDbMYwTzE;i_5&x%xu?rXCA?;O_yM!ozlx>?m7)hN2G?x< zRVB%C@&$al7r>2&lkC)GZ`j=MFX9F(Mdy(nq$a#8ie8~j$#*;-pTzBPQ&7d`;kx7n zIuB~(bvly1quqjt$79z}DIurRS?7@-z`^<@`l`m`TR^|F%b5CrGqaN2PQj>`v^}hYRlqIN(G7s7zo8OX;wmt@b*QcS zhc6^U!P86TthZg;Z-;^Q@XCJaEQ)#J`vz&D#k>oy&o9__y?uCHx{S3V-AH3{1=d?E z=&BvWS$dD&q9y1g`kM}-&FCFkQD)>-+261(N0U|HY@CJBr6tEm6gdV;cr38k(%=Q$ z$Ms>?tw*=%76zFZd?2W2SUkA>71a-3UrI4mmj}(Kn_Y-u%;-^k3f+)vB0fcY;roEW ze2|w>Ydnlsafb4V=pM>~@1RoT4EYGEdtrPY>dFmJdbo;_N8oC!lJ;VbUTZw=2Vc+F zcDcvKt<7^V_gQzocnNg?dpGa}tQ)TdSHF$^fnD|jcGx>$sN3LcB-u@x(X5c4o4|h3 zCA1Rziv>cx4x|Uj4f2@GAd6vT-UVK86?{Vu^Oy*Fl_4dtL4VQZz~UFNWvE#2K`Y7L zrUt9};-Z|RhB*`PSqrNQXfRy+KFwJH+*oP5Vfi8)@ zq8BR+vp1Ohq@!c6%PET{uIJ!ji7em=x`Wa@isfwm^NMW*$+$KfHED}gG*@f3Y4(B}1Nf4}u z%QzSI^LOGC8^!KmC$yB6TIMldoK@~@I|m@kUhW#`&A(7r{0iKTXVxydiu6X4NhVWRG36R!d*i9W!RnTzy!(MaoYqS@Xhg+yQI*Cf5%78VWp_jm*GXN_Zg#-C? zzFp?l1LafasxupMD=S(=%xPzht8|3Z6>44$oI}RLb(V3u;vvvS;_+IP3BN=K0O@sy zf13@TK*_<;I|Qt`9P&Z+3BWf|8hFP&)Dv>aqxc!eiY(@fr>AjlI*DCV>(vi-1}b>4VDs2mCfpcg8bniY;|1X0&I~sh} z)95E~^K!TJK4;lKG7 z`B{xnY2`Fpmlb2L`A(;cD9anOXz1aK%{}mKim4RztJp~UkecAqOF&{TX*^p8pBxE0 z<_N~PAncMMD2J6dWLQ{q*zwRVLEnOYvh5@p`_2QmE}57O9H=lofZnloFa~+~EaDb8 z={3Bm@KCfE)xx(>5!?yy24(a;IIeBcbl7caAU{STxtBX!k& zaTa!YEcllc=ftB(c^Dn0A@m$s3R=ir6bS5j8*NWwXf)p`qV)#73YhmJz+d@9Uw)9~ z;-&agb_PIO^@;e85wb3ikw@R|+^*DdJ^CfR!8vm2ofZ_}P%&P7f<85Uwi~ ztxraidY}}}_52FX1YhHVw}qVXWB{eGAm|1L+@ISbMik?zWfF^LuVJ^&BHhU?7A+6( z$~1-yr$tF3+^38pp7sVMa}3<`g}61E6~e;KdQ#xwC;~n9M8oRc?VO_dQ86~I{k7c- z)mODozO_aJ4*h`IpmMZ;neRQ|n`;W-!H{$)3N`mQ?5q1wQ(QRHumLpM%b|{g!}*K-o@&7>f^vi`MRoBZ zHV21WA43)gjSgxZl8m$cv^&}u`t}{JitUd;TpI&Bq6`3OLQ#o`$_DRZmUL$`M{6*K|Q@9dy}P- z+qHe+KJ9x%E~3KZ9V!USI20IU4m=m?$0bw-jf36z%qj&M%MX;D)}!knzdMdPq#ynY z70f32!ByA*mGK$w1jr&IxrV1(e}x?j8WuLzDy}$WCBNuuz_s?Eww})+U4vEz?+Qx^ zK4fROD>0-gKMn5fQovlR0G*_zbtz;;XcX+|6TAT(qra(__D9`L^+j)jL&FApszU!N z52Gs!vw-zEmRSr%f8iOlJ)H~sFL<=D{sN%JrVhW&&Z_`OWjFFx_g;{F@GJ5Qg~6=; zMui{?lM!dZ9%SIt9$N@;So2>{3aEL2`cMMdg!Tjf=|%=&SW~nBbmq;#S}x;RsAOoq z;Fz$JY!+`Ly7Fy$kNF+7T`l9eWC8hAHelC7?gp(58z9TUI+=*pTM0Nl4+42G z(b?oMc}S+vA{?1=vaL!Zh3c;fFBLrAGX#*uLa5+(pz~o;nG4_%->k*#KHE*6t6I7z zA7drLZkm8%@KNwq3X&HxQSG-s+voH$wjDH$zA$H(K$V?{-&%vMo>mZ=0~qBHb~P*upy?!}pbuS2Q@xIlS8Z9hm(x>slsAohur zGOt)3Iuel60{j(R%s{*eXQt!W7^tP?$P}o{nN)johNefEPzwAvD9m;75p)8u%6a66 zYi&p;K($|muc0Zh{xSj*TMXZ!XpP0eRo;V(K|LM^Ik-y1B57cTLP-KnB1^!{s0mk= zn!W`l(+>aTSre2uq^b1L0<4fyku17QIJJxD`=X?s%^n**4?NP#dQ;?&$hEQ{Xr))+ zCpUt0(arpb6qSpT%eJc_I%_6V?z_z)fK9_k#r1ejFuN z;rZmJb;w#rQqXtcJ|(Hz=#RAz_GMAfV5b948AfhHO&kYFm%GsC;&ExxjHfcl0)5fG6nin=E^ey?t51C}n z5#MnErxSc{BYxPMyno^wd5Q17?{3uN$QQl^zG(kde@1^9(%6#+P~Rs=1bjuQQ3Wd{ z`i=XLTBJAmK4ua7_uw!skIl+y)&x4l$^9nOHXM!P7lI ztX3#JV6NKGyXN5yP)A-tSNH_#$ui*E=b&rnM`ZyU)+VL!2dIy^p>wCiB~T004P2#K z(8+@VNlbyRRt8R#rK8!XrtiQ#X@!mi_4JezA3>R(?FQP>Ziv0h^ok(91b44f)|ccR z?%wtu^oDw8`=(QeN@Y>~$F6+zqqd~J(XbP*Yc`&m~hCxM`= z=A~hx1DsoXL_1g)J*~(R(@Isb-daNdubqHBNnkX$(JWYd51_}b#Tpct(N;F=23i5# zJP_A`bA1P_GWZsbggy}pC;3Le9l3!X!cGsSrAb%lt&xCun}MqJH@bt6XLMKV?axZn}nmW7*L;7pS$rC>p{a*ePczp}(k9G#{GT-d*TM=u*EBRN)oQ=Bb$>&++ znP$DWvU+NSJ_-IOq`y_#ibe?ta(!^_urA<;9y#dLdD!>3X10Sdi`gv)Xg`Uvis#vLDM7yC2^g{!Y2ko>npuE6_ z_u(|C8RRV}Dh-UkB@KmLe;m7zaL>-(l45YD*P)yt{;Zn{PgCQB#(;gsu zi$rraXV*eWQL7@_+u7X+zw0gNTkBmF-rwi(-t#4?W4sS;Z}qimS!Jwuo|#q=^xIk< z+Ap|F(2}5EA!|afTfZz9g@eyJDDi^{pc6)nwsX@+WU70t9-F?HlZFR6lTYapvaEB+NN>-+@TEX3eW(Fk- z9u=H3Bu&Wjpg}=}J+&2+kE|-{nySxk0QcBNM|fs?=36zbF|dJCW~1ne)0yaK$u82rzyz)Pz` ztyNHcK3PRVCh<=6DG`*?Z*008Z00&ake14n(B1y7>x75J-OO_}SA2~WMbm{}$x%G! zsz27r;ePTLjhkUc@^N^u^~CCiw$g>XI%Q-Ps)0VjN?nBl;p#Gwy`avGf@Ewao>O1J zoJ8TTvV+9*31m1uC=N}A6k-Z^Z#mFM(gBwk3A#%dWS4E=-)msi4MRsEmH830lbR^e znq*Z#6@Yi&K$ZGnAJwJS3)t*ta;u_{Zr~i)Z;awFm*BR>n=}YpD^ndaD!qd4g{B6xqeiHgj zE4af`fa(3hFUeVpvHI36D;waFDAEe>do5r99+_G`!g)wh(j4~%O=cha#z&AQz^>QB zE?NrSUo@<;T`(IA6c`UE@@wI>Nl-&G!|3M1jE+ZrtZ?fpR33p=f#RDNZ3Jvt7fk~_ zB@O;gR=|0ZRxqki@*41EN-~w_pg|$|tOKCi6WN>w()zR)4GDYllN$%Mbp{VX}sXLkivtoU5)UQ1-T4`x~k}ym7=1 z8N|BKjO-5mW43$WMs<(r?!OYTPHR8$Ux`m2E8?Ge@8lXBbra4X6>+~f7tJ=`bjW=F z)P+q^V0Be^2S8nk`~vv-UDzY`Qid5OkMPB;GyMg+VkerLKj3*udPs9B5(6h%ib3V* z1H0-KIH^qmXZT#YPAMdz64fn7mNXq z-$#=mZ|72rR-gmHi%9|mS{OR!vCuuF08gzt_@_cXCY77lP_de(1*I%CIFLcEjp9fFJGy|9C89JrdPZ zWlSB_7*1dogtOSGc^)|!(%+lmsvmmF0eAj_TCuWXI`EUNcsum$Kk@+lie&e7j;uJ9!}MLliwh(u$q3fQ|Q|=8JrWn1$#t70(-Z$ z*Xl+dLLROpxd;B-ZN3SxNClLLlS57E2>$nR$O(EO=^jVVgD&43RKnq)h)~c|qEQO# zk!LR;*&C?8wcq-H?tn@+0(6nr~T^wR3!<^+fe{0K0cqL6QIjhoRl zobZ=qB&5E#u#z0x&FmVcn;s`O$*D3BRD9d)2gUKJUE2HrSNn)AECWpqHJ)t+#v01L zgZkc{edPh-J!4{=xQydeU;908$bR4@RvSRMO{{%Z4pdhO5)P`yb&^chCj}tIwgLEB z0%*yZ;QabTNFxP+p6>(Sf13qI6Nnu}be zdqGkCOd1gkC-A<)T)qbVe;!eATKOnsYjcPTWG9KGMOcW4f;k>TV_9{+PJOZ)x|QIB z-!-{a5fcmPv>T8?cT@*)gAasGJd+I(Yt<}p3=&8K`ix|wLs(Nj2$D7J#6P?N{FFp} zP>#hV0f)S>3ZM&++~|y(nsmVF&yZfQ5W&NdeGSP4)omOO z2hHmcRLLl)p}C+p26~dha(I4&Z~YHC29DEw*lDG}2fhoL^#+jnzl&RfXSf~E^F+e< zIhKy!VA>MfN)KT|&3_Hvbvsh|@MJ3<@LvV6H5{YPswa&UoC(tC&-p8pJ(Uj3o-`o?qKjbTj8BNKi$rolA?edXM!`5 z40hll=;JNmo-p9yiO{QCz!(ccHSL9)((_RN)&b{TMoQC6kOQtwuj3bF3vER^g8nrO zFwRB15OmZ%q%2Df%<=*|%cirdtOrZae$aEEH&g=d@Eg{~C$hjf>nss9Sx34RYV9<< z8;!!@G)NJ)i(Y3@a3?2$%3O!8C5>?=(U#YNw9P4=hrt;Jx-~Kew2gtF+MTih4vP$| zvhqV-r-_|`?gNexK!=dQxDI4IE3jvzIPm+PPyyrUa$t0oFoW817qE+7*2|i6OeifWcTzX&{B5*e}0NL0OKx$ zZlbd^GjxVixHG(-0KcQ*DNYUD!AJf@%=JJgx($j6rxi#6aGnUQ&lZU(Xen*Ru7E%F z0^cD)EI%k207wQ$V zn`$r0vC5zae#IUB!y>EjJCWLc0Jq7JUw2-qEn8 z>H>N?g9gF9>_ro47@X2LPZgk}P695l7+(dHe*r4gM>_Z4 zTnjjTqrq)i2tMEx@I2en+%%rP0-t9eWRn-dn(~7OcY&UQ4v(soD+HmWSQEkj%<--VaS59ouGM0SUdS~ z9cBUYnF1bIFOrG{nzh;^68Lji>3z&6=-N%mEIiaYW7PodrZk*YZVM>(3~5L=!YQY| zs2C^(=RxTy3u^ZS;M;XUZ5fD9knwOP=_j5+j?px%4~O&dkOe7CvXIF%g8ISNC_oiW zBnpxMKgbtal1>Kfnt_$1#Ysy*m0fWc&>ylu((n+u2CKM-JxE?+S@~bjZr1aQD!sn)zYsP|v2+jK2t2QL9DPKA@ZrD+m+N{flNT({#Z z!Ko@K#_90?de}~aVb^Vfdq0?EhAVjij_qhbU3r0#T>(#OEcn6MKwZc{dcY3N591r~ zpKrAspCnPB3wTvF76V%olHBEpD^#IU_NevdOi*G z^CZCBHmH23Az!kc{-6&)>mw|X2l8z!nD>(BconhPE1U$Up&O~5$zr^gzQ(KK0MKf_ zvRH7;?m+778JtIM1I)#X(*kF{k2lebG?+i2Dfj{2oYV$&r~?`eNqB;T-HbdX-Ay;p zyK(}gsb0YE3%ZPIuyd#ZB&c5_4RiKyiu*U{L7m}q2cp){-*SSg&ULO)wSt^~MN6X9f;ZbN+_abU?nDW_s{vg{A|Bh_%bJl>~h;Mk#JBCclye7bDH^? zPRGx!!fFar@1yDQapw14n!2n8sbTg|-CZ=n;TT3jrmi{H8f$i<_BA)Fgw{J#E>PPBOYwpZJ^*X>h+l{LIoY4$_vmF(tC;IFmHBY^% zrs)KA3=RGYZ*aZXMry>81He0-QPgXxm+&>e)YZiV;=g=Ot$Tx2xEAF8v^pI6+RAX+Gk+Q9)Ug)A!)ubsX(p3!{_l(7Y8xo3tGa{o zP?x-BBPvdzDl`8Q`D7V-f=>FPn|CMGw;8u8O8p`|y^+bwx_S%K7IW~5+r#D>s%WlK zS?$E{{eq|V47lL}__va|Rc~fay~r8mudxS*KJ-WUef$T@K#i zl#LCzrz2obX>z(**FNejw)WUz=Z4)9cX_Efz**pS@fMO$cuijsQ(&+i)Kasj-fqTO z59?Cr0~Ehru(l+)YGe9Iw76od7Aa&0HqqzGnC>j58$TxM$$DxXXExkK`Q0WG{#I^N z=!Kh^v^ybA%dx5l3Q>l-hnBI7x7}4HtK;xPOYNf5Md@k{q9A#D4ZRd5?&KNfv2?3zZ0wqrkT5B z3u{QwIMOi4kZJMx#yA~EGOK`{lMS0H#r3PAdRvp=O^M8RPIPa=ML!T_su}O@nEcfI zR8?WFWrZjKEe}FTDMc}fMZa4hDvP$L=z~D@AF5B(m++s;^n^0}@Q$#+M0WOlxc5>~ zM6W+0`|79kEx81=^aEGkWGoe|!*Y$as{M?$q?rVt{#8|Dqo-nFq95 z1l7!MIe{w;-}J6k%g+txhdcXqB7-6`Lf<-Z;f2AaV0)+}etO+s>s`bLxe5}0L0;4k z`c+9!-P9R+o1JgoGsl_#aQAs#$!bjkAzqSYMnkeqi>&v-C5@;BY4SE|LJ8WzuOR*w zFc~GM;cI@a9->ZVqsIl z#q2VAbu7Fh3Oq9iv@r1RXP?I>r1cxw&mz5w#C}5^0+q&saBGsF$_jT3xA8kf=0TcAu3fw ztcTPo*vK{VOAo-R8lW$op4nGl>!c+1}*K0tAp8>Hd>Tn$s}; zAAfgtn(M4kMn~}x z_^>x@tv7D%R{Hw$a3eUN_!nB`d8$)Qx_ya=QTK6;|A4iA&A$6mey44??;&v9Kycf7 zod?5cjmEc@Zuc(vr84|IU%x=-tLGoJm%%DxD!vOmu&b5hbx8QG;jM(C$e3WPuCGpq zI(m=#Lxb8uRM0+><4q%3w^j~RIdsnyt(j%&Xjfu-{8{Io%+|`D7fDs|oU)gyn}cjC zQ(v<5aT2gc`K@b65$z(Qkb>Xv8|rymS;E}aEIpOFW2;`~Nz}4De*Q3QZ#$Ubh|yWR z%uf3gj@OOM7i_DObVe~huf%C2z3qdyB)IL0;VJOshsMKxc0nU>79-sRI!yDe%dSO3iWo&o}2PNwD?Q3aH_mbI%n z-yoS4w2ORP4RHOBdWPB~-Z4tij*7)UdB+=!HaJ7Q=n?5;`yKH2n`Hf-M#YT*F+Cwp z8o6RV&c$~)xI56&eh|-sZ*pMYAAu^?}Qcbtuqd(c?YH|rvhMn-3 zp5mOb952dyHnh@D2(%ZCO3_4(prXZ+hK!agtu1b9D21~U`>Cna$O!xPTUhQKoW%@$ z?DteT{b&rt-lP&H8b6+E>^Hyhy+GI$lGJTvwSKEqLaz_jXW8W54Isal9Ydw{Y*3GsUgp=DE}DXzK`P#Z<)s%Of>P&dbVjc8Clw?BBOX zIpxmwaO?UvlZHn&k%0|!j#>Y3o;#UcoviA+k9dtj3qu$2Vr)_Z=jCjb?iu6v%(V2R998Y8d z3ERiGms0AKE8jOB$1R$}=MOOPSI7>V0oQ9u&00xqKFn1o({F0B=8~KOPQ2>oWxBri zjF)VW_Oh(=DqCMxn?-xhcagG#SLl_xE!;EiFL8OW&+6c&tC~Snc(!w0r-xIW zV@{d7$PLW9?7{ctdvYZe=Vv;yhd=PKI-m{s=?pMSff}kSa_8q@bVB=ipVU@G^%O4Cd;g?l>7-DUPV z`&96JG$JdsNA02)J~YDcZUA@B0<1*#k0yE1;yaQPB^v tile") { - val ans = padded + tile - ans.get(0,0) shouldBe 3 - ans.dimensions shouldBe Dimensions(2, 2) - // info("\n" + ans.asciiDraw()) - } - - it("padded + padded => padded") { - val ans = padded.combine(padded)(_ + _).asInstanceOf[BufferTile] - - ans.bufferTop shouldBe 1 - ans.bufferLeft shouldBe 1 - ans.bufferRight shouldBe 1 - ans.bufferBottom shouldBe 1 - ans.dimensions shouldBe Dimensions(2, 2) - - // info("\n" + ans.sourceTile.asciiDraw()) - val ansDouble = padded.combineDouble(padded)(_ + _).asInstanceOf[BufferTile] - - ansDouble.bufferTop shouldBe 1 - ansDouble.bufferLeft shouldBe 1 - ansDouble.bufferRight shouldBe 1 - ansDouble.bufferBottom shouldBe 1 - ansDouble.dimensions shouldBe Dimensions(2, 2) - } - - it("padded + padded => padded (as Tile)") { - val tile: Tile = padded - val ans = (tile.combine(tile)(_ + _)).asInstanceOf[BufferTile] - - ans.bufferTop shouldBe 1 - ans.bufferLeft shouldBe 1 - ans.bufferRight shouldBe 1 - ans.bufferBottom shouldBe 1 - ans.dimensions shouldBe Dimensions(2, 2) - // info("\n" + ans.sourceTile.asciiDraw()) - - val ansDouble = (tile.combineDouble(tile)(_ + _)).asInstanceOf[BufferTile] - - ansDouble.bufferTop shouldBe 1 - ansDouble.bufferLeft shouldBe 1 - ansDouble.bufferRight shouldBe 1 - ansDouble.bufferBottom shouldBe 1 - ansDouble.dimensions shouldBe Dimensions(2, 2) - } - - it("tile center bounds must be contained by underlying tile") { - BufferTile(tile, GridBounds[Int](0,0,1,1)) - - assertThrows[IllegalArgumentException] { - BufferTile(tile, GridBounds[Int](0,0,2,2)) - } - assertThrows[IllegalArgumentException] { - BufferTile(tile, GridBounds[Int](-1,0,1,1)) - } - } -} diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala index f6811a626..cb45208bd 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -24,7 +24,6 @@ package org.locationtech.rasterframes.functions import geotrellis.raster.mapalgebra.focal.{Circle, Kernel, Square} import geotrellis.raster.{BufferTile, CellSize} import geotrellis.raster.testkit.RasterMatchers -import org.apache.spark.sql.functions.typedLit import org.locationtech.rasterframes.ref.{RFRasterSource, RasterRef, Subgrid} import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes._ @@ -51,7 +50,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { lazy val bt = BufferTile(fullTile, subGridBounds) lazy val btCellSize = CellSize(src.extent, bt.cols, bt.rows) - lazy val df = Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))).toDF("proj_raster") + lazy val df = Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))).toDF("proj_raster").cache() it("should provide focal mean") { checkDocs("rf_focal_mean") @@ -63,16 +62,6 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile - /*val actualExpr = - df - .selectExpr("rf_focal_mean(proj_raster, \"square-1\")") - .first()*/ - // .as[Option[ProjectedRasterTile]] - // .first() - // .get - // .tile - - // assertEqual(actualExpr, actual) assertEqual(bt.focalMean(Square(1)), actual) assertEqual(fullTile.focalMean(Square(1)).crop(subGridBounds), actual) } @@ -171,14 +160,14 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { checkDocs("rf_slope") val actual = df - .select(rf_slope($"proj_raster", 2d)) + .select(rf_slope($"proj_raster", 1d)) .as[Option[ProjectedRasterTile]] .first() .get .tile - assertEqual(bt.slope(btCellSize, 2d), actual) - assertEqual(fullTile.slope(btCellSize, 2d).crop(subGridBounds), actual) + assertEqual(bt.slope(btCellSize, 1d), actual) + assertEqual(fullTile.slope(btCellSize, 1d).crop(subGridBounds), actual) } it("should provide aspect") { checkDocs("rf_aspect") diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala index 7dba36e84..f63cbc9fc 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala @@ -180,40 +180,6 @@ class RasterRefSpec extends TestEnvironment with TestData { } } - describe("buffering") { - val src = RFRasterSource(remoteMODIS) - val refs = src - .layoutExtents(NOMINAL_TILE_DIMS) - .map(e => RasterRef(src, 0, Some(e), None, bufferSize = 3)) - val refTiles = refs.map(r => r: Tile) - - it("should maintain reported tile size with buffering") { - val dims = refTiles - .map(_.dimensions) - .distinct - - forEvery(dims) { d => - // println(s"NOMINAL_TILE_SIZE: ${NOMINAL_TILE_SIZE}") - // println(s"d: $d") - d._1 should be <= NOMINAL_TILE_SIZE - d._2 should be <= NOMINAL_TILE_SIZE - } - } - - it("should read a buffered ref") { - val ref = refs.head - - val tile = ref: Tile - // RasterRefTile is lazy on tile content - // val v = tile.get(0, 0) - - // println(s"tile.getClass: ${ref.delegate.getClass}") - // I can't inspect the BufferTile because its hidden behind RasterRefTile.delegate - info(s"tile.get(max + 1, max + 1): ${tile.get(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE)}") - } - - } - describe("RasterSourceToRasterRefs") { it("should convert and expand RasterSource") { val src = RFRasterSource(remoteMODIS) From 020a07f69916c51f128cc1abb066f1193035405c Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 22 Sep 2021 19:36:56 -0400 Subject: [PATCH 04/25] Add nullable gridBounds fields into the TileUDT to support BufferTile --- .../org/apache/spark/sql/rf/TileUDT.scala | 31 +++++++++++++------ .../rasterframes/encoders/EncodingSpec.scala | 31 +++++++++++++++++-- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala index 5fc2a7b5d..4c8fa341e 100644 --- a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala +++ b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala @@ -21,11 +21,12 @@ package org.apache.spark.sql.rf -import geotrellis.raster.{ArrayTile, CellType, ConstantTile, Tile} +import geotrellis.raster.{ArrayTile, BufferTile, CellType, ConstantTile, GridBounds, Tile} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String +import org.locationtech.rasterframes._ import org.locationtech.rasterframes.encoders.syntax._ import org.locationtech.rasterframes.ref.RasterRef import org.locationtech.rasterframes.tiles.{ProjectedRasterTile, ShowableTile} @@ -50,6 +51,7 @@ class TileUDT extends UserDefinedType[Tile] { StructField("cols", IntegerType, false), StructField("rows", IntegerType, false), StructField("cells", BinaryType, true), + StructField("gridBounds", gridBoundsEncoder[Int].schema, true), // make it parquet compliant, only expanded UDTs can be in a UDT schema StructField("ref", ParquetReadSupport.expandUDT(RasterRef.rasterRefEncoder.schema), true) )) @@ -60,22 +62,26 @@ class TileUDT extends UserDefinedType[Tile] { // TODO: review matches there case ref: RasterRef => val ct = UTF8String.fromString(ref.cellType.toString()) - InternalRow(ct, ref.cols, ref.rows, null, ref.toInternalRow) + InternalRow(ct, ref.cols, ref.rows, null, null, ref.toInternalRow) case ProjectedRasterTile(ref: RasterRef, _, _) => val ct = UTF8String.fromString(ref.cellType.toString()) - InternalRow(ct, ref.cols, ref.rows, null, ref.toInternalRow) + InternalRow(ct, ref.cols, ref.rows, null, null, ref.toInternalRow) case prt: ProjectedRasterTile => val tile = prt.tile val ct = UTF8String.fromString(tile.cellType.toString()) - InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null) + InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null, null) + case bt: BufferTile => + val tile = bt.sourceTile.toArrayTile() + val ct = UTF8String.fromString(tile.cellType.toString()) + InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), bt.gridBounds.toInternalRow, null) case const: ConstantTile => // Must expand constant tiles so they can be interpreted properly in catalyst and Python. val tile = const.toArrayTile() val ct = UTF8String.fromString(tile.cellType.toString()) - InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null) + InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null, null) case tile => val ct = UTF8String.fromString(tile.cellType.toString()) - InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null) + InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null, null) } } @@ -83,11 +89,11 @@ class TileUDT extends UserDefinedType[Tile] { if (datum == null) return null val row = datum.asInstanceOf[InternalRow] - /** TODO: a compatible encoder for the ProjectedRasterTile */ + /** TODO: a compatible encoder for the ProjectedRasterTile? */ val tile: Tile = - if (!row.isNullAt(4)) { + if (!row.isNullAt(5)) { Try { - val ir = row.getStruct(4, 4) + val ir = row.getStruct(5, 5) val ref = ir.as[RasterRef] ref }/*.orElse { @@ -99,6 +105,13 @@ class TileUDT extends UserDefinedType[Tile] { .tile ) }*/.get + } else if(!row.isNullAt(4)) { + val ct = CellType.fromName(row.getString(0)) + val cols = row.getInt(1) + val rows = row.getInt(2) + val bytes = row.getBinary(3) + val gridBounds = row.getStruct(4, 5).as[GridBounds[Int]] + BufferTile(ArrayTile.fromBytes(bytes, ct, cols, rows), gridBounds) } else { val ct = CellType.fromName(row.getString(0)) val cols = row.getInt(1) diff --git a/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala index 1b2b931e1..95fc4fb41 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala @@ -23,10 +23,9 @@ package org.locationtech.rasterframes.encoders import java.io.File import java.net.URI - import geotrellis.layer._ import geotrellis.proj4._ -import geotrellis.raster.{ArrayTile, CellType, Raster, Tile} +import geotrellis.raster.{ArrayTile, BufferTile, CellType, GridBounds, Raster, Tile} import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.SparkConf import org.apache.spark.sql.Row @@ -43,7 +42,6 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile */ class EncodingSpec extends TestEnvironment with TestData { - import spark.implicits._ describe("Spark encoding on standard types") { @@ -58,6 +56,19 @@ class EncodingSpec extends TestEnvironment with TestData { } } + it("should serialize BufferTile") { + val tileUDT = new TileUDT() + val tile = one.tile + val expected = BufferTile(tile, GridBounds(tile.dimensions)) + val actual = tileUDT.deserialize(tileUDT.serialize(expected)) + + assert(actual.isInstanceOf[BufferTile] === true) + val actualBufferTile = actual.asInstanceOf[BufferTile] + + actualBufferTile.gridBounds shouldBe expected.gridBounds + assertEqual(actualBufferTile.sourceTile, expected.sourceTile) + } + it("should code RDD[Tile]") { val rdd = sc.makeRDD(Seq(byteArrayTile: Tile, null)) val ds = rdd.toDF("tile") @@ -65,6 +76,20 @@ class EncodingSpec extends TestEnvironment with TestData { assert(ds.toDF.as[Tile].collect().head === byteArrayTile) } + it("should code RDD[BufferTile]") { + val tile = one.tile + val expected = BufferTile(tile, GridBounds(tile.dimensions)) + val ds = Seq(expected: Tile).toDS() + write(ds) + val actual = ds.toDF.as[Tile].first() + + assert(actual.isInstanceOf[BufferTile] === true) + val actualBufferTile = actual.asInstanceOf[BufferTile] + + actualBufferTile.gridBounds shouldBe expected.gridBounds + assertEqual(actualBufferTile.sourceTile, expected.sourceTile) + } + it("should code RDD[(Int, Tile)]") { val ds = Seq((1, byteArrayTile: Tile), (2, null)).toDS write(ds) From 2882e6d70365fc547ba409cefaca8a3b2d9be6ef Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 22 Sep 2021 20:16:45 -0400 Subject: [PATCH 05/25] Switch to JDK11 --- .../test/resources/L8-B7-Elkton-VA-small.tiff | Bin 22800 -> 0 bytes .../rasterframes/RasterLayerSpec.scala | 3 ++ .../rasterframes/TestEnvironment.scala | 27 ++++++++++-------- project/RFProjectPlugin.scala | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) delete mode 100644 core/src/test/resources/L8-B7-Elkton-VA-small.tiff diff --git a/core/src/test/resources/L8-B7-Elkton-VA-small.tiff b/core/src/test/resources/L8-B7-Elkton-VA-small.tiff deleted file mode 100644 index 4ed7a5e48947374c645a98af3a86776468bad0fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22800 zcmZU*1#lZlv^MH6gArAd#i5M zTbCn6tH{&S-RFGNE>p%AkTxJ7AWJ|%vJ?Ty;8{obPWC^~o5Si_dRFj>lg6!6+n_zwO*uZR54a}BSD!t;OMvmd_k|9MZ<|MNWakc|Jm=OVmk@X(9_ zsp0V@e6JavB7nmqh9A6cc=7-W&+Gkr5%6CgmWSD%KJd5oJRsnCq5oclKgtyTf6xB+ z@qb_c|38ir@ySB_nIZub2IRh(w{nsHeMhqY?njz{fRu1Y{{8yzF?m4plUKuADpxPi$6{!#%M|9xzonDW0zuc}$9D$f(gqj>dd6?mVH9h&o)+!g!%zgPPO z6vMB@2ia8pKuu5<^w^5!Y58$>TkZ2MiQX1FBYJW~b$5b)kJk>r;!pJFGmCU4^N$I2 zuz9B=bQ#)&HbI?Hev*;oAv^Fzd=CeZI69txlbh%fwwLrn3#`l5A30y9kQ+sS43;ao zmnY(us3IDT8lXz(EUM|Iiz?(zqjXdwRaCE0JyndoKs7Vx%m^uTZiPiHyEGjFB@nQmuZxsJazLLx2UAY<`#6#q9 z;qhjQs2O=GdU<>=Gfj1r#dQ{QM4d6Eodc$njxY_J+~%_yC1(r2*rc}L)9kw)LkJno z|IYhBB;f0`8UCs>=(Wn$P2l^5C zxDctL3P(AS$0B{jTeKW~`l*UwJF59Nx3~F*&ctCCl^s<$!ylia!G_eS1`OdE4J$Nk8R zQSbB{UB^V2P_y04F^5zi-IJwNA9Y(?8&md}EWxW_bkFf>ybCuX4S5IISjX!?iSh5y zMd+y1vZfj%MvL2gIo~R{dMoCNP4+1{SPggD=`8MV_k~$!^7xVY<_t8%HzV%_wqH%x zFPz!-74txMf$<&oruVLLzS|h@LpgClmV^h=wTzjUrj4828LN8n`sS9K-@8F4y35>9 zxkT0!okblw4!^Qqqx&L=8*yK4ax(d}2+w;v!mC+mMo#Te;Xj4z!b3Xb~`LfubO->O>R^yL$AyJL{_zre~ z_mjzGGP|dpXh(?7=7{saTOs4GkIS4O$sqa z3>IFITm*_1-YW2}2I_@u=G>IkOfxMahM6a#uo`R<%rkjIowiG>yW%QuCUfeg&IVh< z&n)E>@=Z32+&ylPW6KR*>7S{;>8T=G%)<<~v?F*mo}0(;D49Y9%R6>A^UL{S3b}pV za&C-wsy&S@CqCP_UA=|uyUtI&S|8F4bzQkhPu9EfPt=ZPW@}Y`Q_XaQ9Wv1_tG?qE zw2LUFN9bZ^6doSd9Ceqr+AA~4`=T;G$`VAR+N|HmwsJTh%X{(-Q2Q)i)>lX-LG7p` z7s!WBYE#ja^`3)WK29yuUMERDQ-2#RSF^4Y(Xty4XAPLUmL;6oF1gXRrOouI(>k zBYj)P=xRDiTv5IBc~wu|w<_Z9v>$D!a@lw6^{TSj>MU0bkEI#p5%CJv?h(8oY&+hh zKFZ_ruzaeD5lMdG-J~2ZB{G`sDp(&BMfgcRPj7cRikB*#cm)+=g*qr}t2S;K6C#uj zktt!;N;t3eay4H>!s?yP>pQER=k^qPl|9;h;3oGO_wVr3-b!{?naF>s>iV4wVOz11 z6HPABhpSY5ETFUT zCCE>j@Sk#$vrPBVshq#nd*`8?hg*`}^0&OGhRb*~)#?=5MGsZ;R3z-xEBF_Ff#=}8 zcqmD0XSIFKS$m8+rjwb;YB(FNI@&4Befu=;BQLv!RVrtSNw3z*j&@}lBKN3xompqo zy`A>X2*+dJvO7C%oeNGgH>1-!daiR_twiU^XT3}QP;JFZJH4R^MyXu%VBe|`$)t%%qr!)$5K zP;<4%_;dl&N%m%gWqtZu98^BljP>Tn;VsGs<>28KtJ7$!yP) z1JP=HFKD33sJ=suEUD&^m-s$T#C1q9xlDC3d(Gc|kGs+x=I&S9MY_luaid}zi%i}m z+vm@(f9uEkm6@Ze%JI%DomV`P#Z96;7ZAw+)5CPPcbi#GBmZrunh>-QEkkGHRJ|1f^_vgoWnd;Fcyd|JJ55)I8C=0r@~rktzLf+;N9!|e ziuq*HnQQ92S||3a_wp6*$Xruh{w3<@ooca3q0>8^)dLx-eC9cK$Rn~QSja&2M~+pA zx;Q(5r{ZbY z)aIyOY47l^&{}m<0jiBYthbo~x`NEk8T9Z;C~2DOb*hU@YY!6#NjT{lvRHCeM=emdb$79y&WF{hWE1&bKUUk+ zMg2%cs8MbUVN^r^Bi}sT$#>m%G`z1jKK!t^yS>w`WIq+f?0V{*n51T^g*@0fY==Sx zN>FdrDW0HutCM&$eu58?Iye$<#DjPdK7(AxNq7VwCt_qr86z7=uG=_Da(Ty;^j>#v zi8VY8zfXP>n{A^fOh!`6B!`-A+3R_7l~+`=pP6vc)avznG`DNMRzKsiK>5)lGF=E>zjwD&d9IAXQUM^fflMbaxY~ zo~y08iqqBi*!x?S^38TqI6FltwV&tYcSS?@5FnjSvZBtXZ>T*g(vFt(RTJ?XS0D}{ zcm(m1FsR@!RT26d=OrnfIK5P1wMEvI)#RMmh&(NQmGx-74(7ZqxkysLDs9YrlGUV$ z?wlhoW_(0`XR`f~mlhjE5xHJHV$*q`IwZ2O&Ww_UYEk5Mol0+)tG!;>iCxS>w2p4Y zF=8%TY6~?@1qsG|CZjH>v+Gl81mLq0s-*5IGkd!@WnyZ_hP#v8^ZtIiw%+Ov6&aP$ zYef=&C>QEcs-He1v)Dq^kY!D3Q&3OiZ&fQ{tI}#{fW%&t8wbFd@3r+BPKk@8{Y;M;3RiL8%;aW|9SJ!eU-zn@9M|)gs6PpMXCq+h!3JC_%6AFAE0#j5n72KlHpL7$B<~i zeC6b66=J8+jl~)~8Q9`Y@&Qx)%o>K%`?240GY=Ls{ik^yM!m|E)W5wi)DiQ{PG{P&tK>M_;Z5lTi$eBm6D&803B0`S z;CSr|g25*}vHIv4u$JuTMZR-v86Ay#@y+&QMb#(uf?u--dv_=t=`m^T^3DVOo@I1% zngqMDTx7@S@w5rAD_fACxB;m|o{|eV75;{9qATcc^Z-3W{qQanN&-kA8^BMKKNyj= zBt5-?BScHUC=2Y~G{3kc7Rgk=DF?}4D!VDeda+NW8S55x&39d-7g=O2NqC~uAjd`M zZz;WV?3{pqUfI`7lzAz7;NDo#a68oLs@gywEUj;(a-qWH-7u##e)XeG@^kjBU`FGd>mVk%j3Oh2AV=%vUcJZ@c3G^ zx4bIuic|Tr#Se=so%2VofZShW|3rM12T%{Tnp|WT=x36snwkRML!un7K(puUXilj{ z?nRYSU6a=V_qs|!caIfg>G9C$Ak5cG!X}~tcmNxYzM}lN0`lQQCY zJyuNO2ZgX_$nD}34_2*k7$B(JZfZ4;Uw|IepN|4=TpGrek=#dqQ~;Gmr>qs$bL$x@ zkNTi7=pcHI&f^wjDn1YOB|R+x73Kn31h1|kecY7Z8>%y^k3H%vuPkObJEHG~_sLZt zylC`~=mqiuRcw;%LU-b=@ULS;erLARQ(81yeDXDMD){=EwCWn)%~JrM$t9k#-Rux+ z!it8b3*O1k(7V_|l}Id4j&~pfcl0!RgQg-I7l1C6h1?~|R3j-!LsE$zr$y=6$OGXs z$#b`(z82lqnWh)=fh0Hiif$3*N@C>vQjYOd|yGTN3pr;OpidhQP!~^kql7!>& z9GVx_N^;Z@6o*#m1KNZ$(1u8$Pv{<-%4W0Vpsh@FYIq0vg5!#KpX!zJr2Z!I0l$2r zhuNPUN2t>I?5Lvgp*%CpOx%{O?V;4DbiV)zC-uV^EXYxcS^&x+Q$_by?4h)m^<>j37_x z5iy??6`9p%6{fH1NNMZxG#Kp(X%SM|3PiD(vGIVFMziCzFK;AD$O9}?e&!zm6W8`u zG2?VEy+oC9OSn1RyE;nl)ql)n?Q!R8U@x&t*^P3B>#GvmSD@-HGiTgJ zPHQv5+^0#XFG-6pqRK2Cok5G?p5!JTgo>k))L|WHb(%oW^MZQ2*`a!Pmx9)D-rZnZ zs)xKT3!AfUZtn*h^!FA&%v_IdWuBu9K_i8AQ z{hW3N<)L!$$B^5h9Z`KUjYNBfiwwLQ_3?BnOhm|!JcV}TRF%^7(99mBSKHN0Ywr$S z40d0CxPRQfVRqOnG}jf}b@paEr8CR)GhI|+-O6ohFEV8#D@D5AfuE=%Q*uNv|m@t2hd_26R z_YNqUxpV>DLRSX-(AxV&&z0NMBQ*_u#RJhASlJqJ+J$^TlT~AO(MpMjvNmD@-H0lM zbquW+`aZaO@Zzv{xQ@zZXEPa14HKaz@)r3uZ z_lc^j%b3@oplWl`UF;5YOM&iC+R5m>7d1V0nR`@cp%-x~GKZw0`SBKgTx`G%aU=YS zSJ#Ib7Yp1+hB+&p1a)5KRmsI^>GOWck<9xC7~&D+Q_uI)4r5=BgO3ewh_clBw!W zbI#ePoziZC`?r0=uHk0XAIv_Ifu`rFKt*WFT7f$L5UO%foMeuvJTi@THH+|s-?HD? zwH*%&x0Cyp$0m5ox^b$P)5q6cA2W-bY0iB+i|*l&@Y~+UJVc}v=lNQw1sk}<&!Gf7 z561V%>V%e|I;0wmWIw)1y!4ek>sBLGMG$F1a*@4c3TcPm1s@81W0fVDcwRPvtpi;; z!Fy9@wFjx;s+}q-AAn(XQkj$%QyCXUaqBrrsR0D6+l!Xr-m^ z?RU;Por?}2%b`jXbRO|5BqM%gbwTe?Wwn7{SJ#1ee9@P&GV?mF0qE z;(wS|=NLm7`JdS6Oc)HG$Ik;cw5xz|9;j>v|(Uf&^j>~fPBok?htGy~( zMvKe#8#&hu25;b}JY^d_AJ*$Nanl^u(n(?Gi#=lMkg|9mUd48bQEpLlo43)`K?9ts zsXR>+@h8AB{eAgWVebZ$*VVwqZzRebda>r|p_{3OB9&+ogFk*-O-TdCd@*(Jx}7 zYN4Bn3#vD;%qr?H-GOX_dnHJ5+yZ-WB6}wXx!3JyD$KY}X6>-fWDRzl3UZL|jDKJC zBmSn&95c-A3CiFKsM1+i6(Q~*l-i>H*d#e$t zrV7y4tft;5hp1k__wLC#y1xF$^mTH0zu1%YT+knTIz3EY^V`nlJ!N;&OYK0Ycs+16 zoDR3f%gAROihZ~n?tnR)FaL29&2-&NzU3=qT5lmYpEt?dKmL{T#`$+;M9X#3qi4!I zP9HtV9&QGMs+Nb%pcm;+I*iwFuvh|>Bo%jgJU`1kd?tI&V(~19li{Dl!ujxVv=1|pQNxJXpeKoL+8!Sf74X# z7_5oQfbE;AfvSdnB}<6cERz^ZiW3FQV=tPE8lhzLI4MJVlF6hixvM6bf9N_;SRUYi z043H3J!~g#Lr&s+WD%N=mWCBXTj*)Ngw?mRnDGk9Wi*h~GqdfZph#D9HvpoE)@^mH zF|eOT(?D6urhuT{@rAe#4#W<=4fSz9>VaqC=jb?!C!29c@=bhqdawwd+6{Al=^%Tw z`D(9?{9vz_>Fq+M8)-`alHJV-vk(|`oT=@b2c0y)kt~)?u#<&1g?n;9d1Q52Mi!CH zWnp&p4R<1}s{q56gj`<`<=^c8AI^j!c54XK`C$H<26uV?5 z+1H`INw#C!8jn-XKirok{4wyOQ_k$@H_<(!cF7qsUM>*Lkv`Bxd!egH0-%4gu}=k`j37u zH-L+~Tr9UMI<4#){xWuMJFPiwX6o&#xii=88WA14IcMAA9l~eYT5sj?;sYNnF7XrK zTK5rWge&8~3%?{Mk_V_0x(K``6KM^5unQ@~TZ&>7%e8=g=F3!OxObLQNJr|?g2`c^ z9o66icmdE(@AGRkm9-b0B`-)eb4CqS*_?ynTVp%&dh#Hkl(SAj_cQeCs`dxB6FGA;0c|QwM~2bqv_#2 z;goQ;gBsUGJvV`Flyf+Ia=sEpi^e}UjyFvI(L$_(E_Z_`K&|^M1Sp}s)kM4n zEkIX+Db6JQ0n7d*7eq21PHW2hptT$X?YFURAWMNWe3u{OTX}czwQKUaye)hUqswU) zPduiisi~^=iSt@TTLNl{HU^Bsfg*ulTN zM2k{A;>tyCHAlfGbwn!R7o~tbM~eYsuaL5k+$am!N0Gz>0Q)o~<>(qvKL?Y(=(v>$ zozdNRJ`tk^i>h+8*e{L=(0auVSm8Z+F20K|;X`?AHkM2eSp!N=Gd@k1&~LU=KZ{c06(}4!T^KF`D>Xfy04jbue1VmA z<~ohdQdNVrptIx-=edcq$2f;Ty}N5K2ZghQX%V|GZb*(r?#P^TqB};$gQmGy`9uZL zL<|C?vP5K)onh5nbXWU|N7PbTX=6Hz@|Lw9d0_`8SG8GyNsKt^Pf!V> zifAL+$*eL^e4&Y;=?ufO=t??*WI!3iTvSeVlOxqRv0OA%P4pW@z%AS@|45<}4YM2C zugr3l*~IXrs-vo^a?v^bw1^Vt$X@ymPb1lI08Rn;i_+nmX~4$r2sbM!Pw(+gb{%`R z7^1e?;r39wjNRUR(<38#=boOwzyCn^R9{o}k4*AmT?g_Z5#qF1DzEFbzKfurx`0s2 zfEN9?90WM+z3ilK@CrfGMiLE zIY~cS4>{<)XSpYX>J2-hkNii@lS@^8ol+f^Gena5W!F}zR5kF3!g(@Wj~s>w);m2>F5DgHn@O$H_?P1CM2_uZAea>afhJuZjP8R zUuK&QH4S6l%dBiST}G$UL*xzFLWYrr^euP}6=*4NnLp5cEP!4l$ps-NXc((cKazAb zD;-D6lH0f!9*<*jJ5Zil;6?Z@3c@kK%zn{Y^kYy(E6Dz$SEwrDi|im)s()w;{*HBl z`tuum#UVAq)P`ikF#Qf6MR9zEexMw+69vHDbMYwTzE;i_5&x%xu?rXCA?;O_yM!ozlx>?m7)hN2G?x< zRVB%C@&$al7r>2&lkC)GZ`j=MFX9F(Mdy(nq$a#8ie8~j$#*;-pTzBPQ&7d`;kx7n zIuB~(bvly1quqjt$79z}DIurRS?7@-z`^<@`l`m`TR^|F%b5CrGqaN2PQj>`v^}hYRlqIN(G7s7zo8OX;wmt@b*QcS zhc6^U!P86TthZg;Z-;^Q@XCJaEQ)#J`vz&D#k>oy&o9__y?uCHx{S3V-AH3{1=d?E z=&BvWS$dD&q9y1g`kM}-&FCFkQD)>-+261(N0U|HY@CJBr6tEm6gdV;cr38k(%=Q$ z$Ms>?tw*=%76zFZd?2W2SUkA>71a-3UrI4mmj}(Kn_Y-u%;-^k3f+)vB0fcY;roEW ze2|w>Ydnlsafb4V=pM>~@1RoT4EYGEdtrPY>dFmJdbo;_N8oC!lJ;VbUTZw=2Vc+F zcDcvKt<7^V_gQzocnNg?dpGa}tQ)TdSHF$^fnD|jcGx>$sN3LcB-u@x(X5c4o4|h3 zCA1Rziv>cx4x|Uj4f2@GAd6vT-UVK86?{Vu^Oy*Fl_4dtL4VQZz~UFNWvE#2K`Y7L zrUt9};-Z|RhB*`PSqrNQXfRy+KFwJH+*oP5Vfi8)@ zq8BR+vp1Ohq@!c6%PET{uIJ!ji7em=x`Wa@isfwm^NMW*$+$KfHED}gG*@f3Y4(B}1Nf4}u z%QzSI^LOGC8^!KmC$yB6TIMldoK@~@I|m@kUhW#`&A(7r{0iKTXVxydiu6X4NhVWRG36R!d*i9W!RnTzy!(MaoYqS@Xhg+yQI*Cf5%78VWp_jm*GXN_Zg#-C? zzFp?l1LafasxupMD=S(=%xPzht8|3Z6>44$oI}RLb(V3u;vvvS;_+IP3BN=K0O@sy zf13@TK*_<;I|Qt`9P&Z+3BWf|8hFP&)Dv>aqxc!eiY(@fr>AjlI*DCV>(vi-1}b>4VDs2mCfpcg8bniY;|1X0&I~sh} z)95E~^K!TJK4;lKG7 z`B{xnY2`Fpmlb2L`A(;cD9anOXz1aK%{}mKim4RztJp~UkecAqOF&{TX*^p8pBxE0 z<_N~PAncMMD2J6dWLQ{q*zwRVLEnOYvh5@p`_2QmE}57O9H=lofZnloFa~+~EaDb8 z={3Bm@KCfE)xx(>5!?yy24(a;IIeBcbl7caAU{STxtBX!k& zaTa!YEcllc=ftB(c^Dn0A@m$s3R=ir6bS5j8*NWwXf)p`qV)#73YhmJz+d@9Uw)9~ z;-&agb_PIO^@;e85wb3ikw@R|+^*DdJ^CfR!8vm2ofZ_}P%&P7f<85Uwi~ ztxraidY}}}_52FX1YhHVw}qVXWB{eGAm|1L+@ISbMik?zWfF^LuVJ^&BHhU?7A+6( z$~1-yr$tF3+^38pp7sVMa}3<`g}61E6~e;KdQ#xwC;~n9M8oRc?VO_dQ86~I{k7c- z)mODozO_aJ4*h`IpmMZ;neRQ|n`;W-!H{$)3N`mQ?5q1wQ(QRHumLpM%b|{g!}*K-o@&7>f^vi`MRoBZ zHV21WA43)gjSgxZl8m$cv^&}u`t}{JitUd;TpI&Bq6`3OLQ#o`$_DRZmUL$`M{6*K|Q@9dy}P- z+qHe+KJ9x%E~3KZ9V!USI20IU4m=m?$0bw-jf36z%qj&M%MX;D)}!knzdMdPq#ynY z70f32!ByA*mGK$w1jr&IxrV1(e}x?j8WuLzDy}$WCBNuuz_s?Eww})+U4vEz?+Qx^ zK4fROD>0-gKMn5fQovlR0G*_zbtz;;XcX+|6TAT(qra(__D9`L^+j)jL&FApszU!N z52Gs!vw-zEmRSr%f8iOlJ)H~sFL<=D{sN%JrVhW&&Z_`OWjFFx_g;{F@GJ5Qg~6=; zMui{?lM!dZ9%SIt9$N@;So2>{3aEL2`cMMdg!Tjf=|%=&SW~nBbmq;#S}x;RsAOoq z;Fz$JY!+`Ly7Fy$kNF+7T`l9eWC8hAHelC7?gp(58z9TUI+=*pTM0Nl4+42G z(b?oMc}S+vA{?1=vaL!Zh3c;fFBLrAGX#*uLa5+(pz~o;nG4_%->k*#KHE*6t6I7z zA7drLZkm8%@KNwq3X&HxQSG-s+voH$wjDH$zA$H(K$V?{-&%vMo>mZ=0~qBHb~P*upy?!}pbuS2Q@xIlS8Z9hm(x>slsAohur zGOt)3Iuel60{j(R%s{*eXQt!W7^tP?$P}o{nN)johNefEPzwAvD9m;75p)8u%6a66 zYi&p;K($|muc0Zh{xSj*TMXZ!XpP0eRo;V(K|LM^Ik-y1B57cTLP-KnB1^!{s0mk= zn!W`l(+>aTSre2uq^b1L0<4fyku17QIJJxD`=X?s%^n**4?NP#dQ;?&$hEQ{Xr))+ zCpUt0(arpb6qSpT%eJc_I%_6V?z_z)fK9_k#r1ejFuN z;rZmJb;w#rQqXtcJ|(Hz=#RAz_GMAfV5b948AfhHO&kYFm%GsC;&ExxjHfcl0)5fG6nin=E^ey?t51C}n z5#MnErxSc{BYxPMyno^wd5Q17?{3uN$QQl^zG(kde@1^9(%6#+P~Rs=1bjuQQ3Wd{ z`i=XLTBJAmK4ua7_uw!skIl+y)&x4l$^9nOHXM!P7lI ztX3#JV6NKGyXN5yP)A-tSNH_#$ui*E=b&rnM`ZyU)+VL!2dIy^p>wCiB~T004P2#K z(8+@VNlbyRRt8R#rK8!XrtiQ#X@!mi_4JezA3>R(?FQP>Ziv0h^ok(91b44f)|ccR z?%wtu^oDw8`=(QeN@Y>~$F6+zqqd~J(XbP*Yc`&m~hCxM`= z=A~hx1DsoXL_1g)J*~(R(@Isb-daNdubqHBNnkX$(JWYd51_}b#Tpct(N;F=23i5# zJP_A`bA1P_GWZsbggy}pC;3Le9l3!X!cGsSrAb%lt&xCun}MqJH@bt6XLMKV?axZn}nmW7*L;7pS$rC>p{a*ePczp}(k9G#{GT-d*TM=u*EBRN)oQ=Bb$>&++ znP$DWvU+NSJ_-IOq`y_#ibe?ta(!^_urA<;9y#dLdD!>3X10Sdi`gv)Xg`Uvis#vLDM7yC2^g{!Y2ko>npuE6_ z_u(|C8RRV}Dh-UkB@KmLe;m7zaL>-(l45YD*P)yt{;Zn{PgCQB#(;gsu zi$rraXV*eWQL7@_+u7X+zw0gNTkBmF-rwi(-t#4?W4sS;Z}qimS!Jwuo|#q=^xIk< z+Ap|F(2}5EA!|afTfZz9g@eyJDDi^{pc6)nwsX@+WU70t9-F?HlZFR6lTYapvaEB+NN>-+@TEX3eW(Fk- z9u=H3Bu&Wjpg}=}J+&2+kE|-{nySxk0QcBNM|fs?=36zbF|dJCW~1ne)0yaK$u82rzyz)Pz` ztyNHcK3PRVCh<=6DG`*?Z*008Z00&ake14n(B1y7>x75J-OO_}SA2~WMbm{}$x%G! zsz27r;ePTLjhkUc@^N^u^~CCiw$g>XI%Q-Ps)0VjN?nBl;p#Gwy`avGf@Ewao>O1J zoJ8TTvV+9*31m1uC=N}A6k-Z^Z#mFM(gBwk3A#%dWS4E=-)msi4MRsEmH830lbR^e znq*Z#6@Yi&K$ZGnAJwJS3)t*ta;u_{Zr~i)Z;awFm*BR>n=}YpD^ndaD!qd4g{B6xqeiHgj zE4af`fa(3hFUeVpvHI36D;waFDAEe>do5r99+_G`!g)wh(j4~%O=cha#z&AQz^>QB zE?NrSUo@<;T`(IA6c`UE@@wI>Nl-&G!|3M1jE+ZrtZ?fpR33p=f#RDNZ3Jvt7fk~_ zB@O;gR=|0ZRxqki@*41EN-~w_pg|$|tOKCi6WN>w()zR)4GDYllN$%Mbp{VX}sXLkivtoU5)UQ1-T4`x~k}ym7=1 z8N|BKjO-5mW43$WMs<(r?!OYTPHR8$Ux`m2E8?Ge@8lXBbra4X6>+~f7tJ=`bjW=F z)P+q^V0Be^2S8nk`~vv-UDzY`Qid5OkMPB;GyMg+VkerLKj3*udPs9B5(6h%ib3V* z1H0-KIH^qmXZT#YPAMdz64fn7mNXq z-$#=mZ|72rR-gmHi%9|mS{OR!vCuuF08gzt_@_cXCY77lP_de(1*I%CIFLcEjp9fFJGy|9C89JrdPZ zWlSB_7*1dogtOSGc^)|!(%+lmsvmmF0eAj_TCuWXI`EUNcsum$Kk@+lie&e7j;uJ9!}MLliwh(u$q3fQ|Q|=8JrWn1$#t70(-Z$ z*Xl+dLLROpxd;B-ZN3SxNClLLlS57E2>$nR$O(EO=^jVVgD&43RKnq)h)~c|qEQO# zk!LR;*&C?8wcq-H?tn@+0(6nr~T^wR3!<^+fe{0K0cqL6QIjhoRl zobZ=qB&5E#u#z0x&FmVcn;s`O$*D3BRD9d)2gUKJUE2HrSNn)AECWpqHJ)t+#v01L zgZkc{edPh-J!4{=xQydeU;908$bR4@RvSRMO{{%Z4pdhO5)P`yb&^chCj}tIwgLEB z0%*yZ;QabTNFxP+p6>(Sf13qI6Nnu}be zdqGkCOd1gkC-A<)T)qbVe;!eATKOnsYjcPTWG9KGMOcW4f;k>TV_9{+PJOZ)x|QIB z-!-{a5fcmPv>T8?cT@*)gAasGJd+I(Yt<}p3=&8K`ix|wLs(Nj2$D7J#6P?N{FFp} zP>#hV0f)S>3ZM&++~|y(nsmVF&yZfQ5W&NdeGSP4)omOO z2hHmcRLLl)p}C+p26~dha(I4&Z~YHC29DEw*lDG}2fhoL^#+jnzl&RfXSf~E^F+e< zIhKy!VA>MfN)KT|&3_Hvbvsh|@MJ3<@LvV6H5{YPswa&UoC(tC&-p8pJ(Uj3o-`o?qKjbTj8BNKi$rolA?edXM!`5 z40hll=;JNmo-p9yiO{QCz!(ccHSL9)((_RN)&b{TMoQC6kOQtwuj3bF3vER^g8nrO zFwRB15OmZ%q%2Df%<=*|%cirdtOrZae$aEEH&g=d@Eg{~C$hjf>nss9Sx34RYV9<< z8;!!@G)NJ)i(Y3@a3?2$%3O!8C5>?=(U#YNw9P4=hrt;Jx-~Kew2gtF+MTih4vP$| zvhqV-r-_|`?gNexK!=dQxDI4IE3jvzIPm+PPyyrUa$t0oFoW817qE+7*2|i6OeifWcTzX&{B5*e}0NL0OKx$ zZlbd^GjxVixHG(-0KcQ*DNYUD!AJf@%=JJgx($j6rxi#6aGnUQ&lZU(Xen*Ru7E%F z0^cD)EI%k207wQ$V zn`$r0vC5zae#IUB!y>EjJCWLc0Jq7JUw2-qEn8 z>H>N?g9gF9>_ro47@X2LPZgk}P695l7+(dHe*r4gM>_Z4 zTnjjTqrq)i2tMEx@I2en+%%rP0-t9eWRn-dn(~7OcY&UQ4v(soD+HmWSQEkj%<--VaS59ouGM0SUdS~ z9cBUYnF1bIFOrG{nzh;^68Lji>3z&6=-N%mEIiaYW7PodrZk*YZVM>(3~5L=!YQY| zs2C^(=RxTy3u^ZS;M;XUZ5fD9knwOP=_j5+j?px%4~O&dkOe7CvXIF%g8ISNC_oiW zBnpxMKgbtal1>Kfnt_$1#Ysy*m0fWc&>ylu((n+u2CKM-JxE?+S@~bjZr1aQD!sn)zYsP|v2+jK2t2QL9DPKA@ZrD+m+N{flNT({#Z z!Ko@K#_90?de}~aVb^Vfdq0?EhAVjij_qhbU3r0#T>(#OEcn6MKwZc{dcY3N591r~ zpKrAspCnPB3wTvF76V%olHBEpD^#IU_NevdOi*G z^CZCBHmH23Az!kc{-6&)>mw|X2l8z!nD>(BconhPE1U$Up&O~5$zr^gzQ(KK0MKf_ zvRH7;?m+778JtIM1I)#X(*kF{k2lebG?+i2Dfj{2oYV$&r~?`eNqB;T-HbdX-Ay;p zyK(}gsb0YE3%ZPIuyd#ZB&c5_4RiKyiu*U{L7m}q2cp){-*SSg&ULO)wSt^~MN6X9f;ZbN+_abU?nDW_s{vg{A|Bh_%bJl>~h;Mk#JBCclye7bDH^? zPRGx!!fFar@1yDQapw14n!2n8sbTg|-CZ=n;TT3jrmi{H8f$i<_BA)Fgw{J#E>PPBOYwpZJ^*X>h+l{LIoY4$_vmF(tC;IFmHBY^% zrs)KA3=RGYZ*aZXMry>81He0-QPgXxm+&>e)YZiV;=g=Ot$Tx2xEAF8v^pI6+RAX+Gk+Q9)Ug)A!)ubsX(p3!{_l(7Y8xo3tGa{o zP?x-BBPvdzDl`8Q`D7V-f=>FPn|CMGw;8u8O8p`|y^+bwx_S%K7IW~5+r#D>s%WlK zS?$E{{eq|V47lL}__va|Rc~fay~r8mudxS*KJ-WUef$T@K#i zl#LCzrz2obX>z(**FNejw)WUz=Z4)9cX_Efz**pS@fMO$cuijsQ(&+i)Kasj-fqTO z59?Cr0~Ehru(l+)YGe9Iw76od7Aa&0HqqzGnC>j58$TxM$$DxXXExkK`Q0WG{#I^N z=!Kh^v^ybA%dx5l3Q>l-hnBI7x7}4HtK;xPOYNf5Md@k{q9A#D4ZRd5?&KNfv2?3zZ0wqrkT5B z3u{QwIMOi4kZJMx#yA~EGOK`{lMS0H#r3PAdRvp=O^M8RPIPa=ML!T_su}O@nEcfI zR8?WFWrZjKEe}FTDMc}fMZa4hDvP$L=z~D@AF5B(m++s;^n^0}@Q$#+M0WOlxc5>~ zM6W+0`|79kEx81=^aEGkWGoe|!*Y$as{M?$q?rVt{#8|Dqo-nFq95 z1l7!MIe{w;-}J6k%g+txhdcXqB7-6`Lf<-Z;f2AaV0)+}etO+s>s`bLxe5}0L0;4k z`c+9!-P9R+o1JgoGsl_#aQAs#$!bjkAzqSYMnkeqi>&v-C5@;BY4SE|LJ8WzuOR*w zFc~GM;cI@a9->ZVqsIl z#q2VAbu7Fh3Oq9iv@r1RXP?I>r1cxw&mz5w#C}5^0+q&saBGsF$_jT3xA8kf=0TcAu3fw ztcTPo*vK{VOAo-R8lW$op4nGl>!c+1}*K0tAp8>Hd>Tn$s}; zAAfgtn(M4kMn~}x z_^>x@tv7D%R{Hw$a3eUN_!nB`d8$)Qx_ya=QTK6;|A4iA&A$6mey44??;&v9Kycf7 zod?5cjmEc@Zuc(vr84|IU%x=-tLGoJm%%DxD!vOmu&b5hbx8QG;jM(C$e3WPuCGpq zI(m=#Lxb8uRM0+><4q%3w^j~RIdsnyt(j%&Xjfu-{8{Io%+|`D7fDs|oU)gyn}cjC zQ(v<5aT2gc`K@b65$z(Qkb>Xv8|rymS;E}aEIpOFW2;`~Nz}4De*Q3QZ#$Ubh|yWR z%uf3gj@OOM7i_DObVe~huf%C2z3qdyB)IL0;VJOshsMKxc0nU>79-sRI!yDe%dSO3iWo&o}2PNwD?Q3aH_mbI%n z-yoS4w2ORP4RHOBdWPB~-Z4tij*7)UdB+=!HaJ7Q=n?5;`yKH2n`Hf-M#YT*F+Cwp z8o6RV&c$~)xI56&eh|-sZ*pMYAAu^?}Qcbtuqd(c?YH|rvhMn-3 zp5mOb952dyHnh@D2(%ZCO3_4(prXZ+hK!agtu1b9D21~U`>Cna$O!xPTUhQKoW%@$ z?DteT{b&rt-lP&H8b6+E>^Hyhy+GI$lGJTvwSKEqLaz_jXW8W54Isal9Ydw{Y*3GsUgp=DE}DXzK`P#Z<)s%Of>P&dbVjc8Clw?BBOX zIpxmwaO?UvlZHn&k%0|!j#>Y3o;#UcoviA+k9dtj3qu$2Vr)_Z=jCjb?iu6v%(V2R998Y8d z3ERiGms0AKE8jOB$1R$}=MOOPSI7>V0oQ9u&00xqKFn1o({F0B=8~KOPQ2>oWxBri zjF)VW_Oh(=DqCMxn?-xhcagG#SLl_xE!;EiFL8OW&+6c&tC~Snc(!w0r-xIW zV@{d7$PLW9?7{ctdvYZe=Vv;yhd=PKI-m{s=?pMSff}kSa_8q@bVB=ipVU@G^%O4Cd;g?l>7-DUPV z`&96JG$JdsNA02)J~YDcZUA@B0<1*#k0yE1;yaQPB^v Date: Thu, 23 Sep 2021 15:37:06 -0400 Subject: [PATCH 06/25] Remove GeometryUDT --- .../org/apache/spark/sql/stac/GeometryUDT.scala | 14 -------------- .../stac/api/encoders/StacSerializers.scala | 16 ++++++++-------- 2 files changed, 8 insertions(+), 22 deletions(-) delete mode 100644 datasource/src/main/scala/org/apache/spark/sql/stac/GeometryUDT.scala diff --git a/datasource/src/main/scala/org/apache/spark/sql/stac/GeometryUDT.scala b/datasource/src/main/scala/org/apache/spark/sql/stac/GeometryUDT.scala deleted file mode 100644 index 6421fe4b6..000000000 --- a/datasource/src/main/scala/org/apache/spark/sql/stac/GeometryUDT.scala +++ /dev/null @@ -1,14 +0,0 @@ -package org.apache.spark.sql.stac - -import org.locationtech.jts.geom._ -import org.apache.spark.sql.jts.AbstractGeometryUDT -import org.locationtech.jts.geom.Geometry - -class PointUDT extends AbstractGeometryUDT[Point]("point") -class MultiPointUDT extends AbstractGeometryUDT[MultiPoint]("multipoint") -class LineStringUDT extends AbstractGeometryUDT[LineString]("linestring") -class MultiLineStringUDT extends AbstractGeometryUDT[MultiLineString]("multilinestring") -class PolygonUDT extends AbstractGeometryUDT[Polygon]("polygon") -class MultiPolygonUDT extends AbstractGeometryUDT[MultiPolygon]("multipolygon") -class GeometryUDT extends AbstractGeometryUDT[Geometry]("geometry") -class GeometryCollectionUDT extends AbstractGeometryUDT[GeometryCollection]("geometrycollection") \ No newline at end of file diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala index c5a8e2fd3..3029ace2e 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala @@ -8,20 +8,20 @@ import com.azavea.stac4s._ import eu.timepit.refined.api.{RefType, Validate} import frameless.{Injection, SQLTimestamp, TypedEncoder, TypedExpressionEncoder} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.stac._ +import org.apache.spark.sql.jts.JTSTypes import java.time.Instant /** STAC API Dataframe relies on the Frameless Expressions derivation. */ trait StacSerializers { /** GeoMesa UDTs, should be defined as implicits so frameless would pick them up */ - implicit val pointUDT: PointUDT = new PointUDT - implicit val multiPointUDT: MultiPointUDT = new MultiPointUDT - implicit val multiLineStringUDT: MultiLineStringUDT = new MultiLineStringUDT - implicit val polygonUDT: PolygonUDT = new PolygonUDT - implicit val multiPolygonUDT: MultiPolygonUDT = new MultiPolygonUDT - implicit val geometryUDT: GeometryUDT = new GeometryUDT - implicit val geometryCollectionUDT: GeometryCollectionUDT = new GeometryCollectionUDT + implicit val pointUDT = JTSTypes.PointTypeInstance + implicit val multiPointUDT = JTSTypes.MultiPointTypeInstance + implicit val multiLineStringUDT = JTSTypes.MultiLineStringTypeInstance + implicit val polygonUDT = JTSTypes.PolygonTypeInstance + implicit val multiPolygonUDT = JTSTypes.MultipolygonTypeInstance + implicit val geometryUDT = JTSTypes.GeometryTypeInstance + implicit val geometryCollectionUDT = JTSTypes.GeometryCollectionTypeInstance /** Injections to Encode stac4s objects */ implicit val stacLinkTypeInjection: Injection[StacLinkType, String] = Injection(_.repr, _.asJson.asUnsafe[StacLinkType]) From b571fb49b1276a32d5f7e8005c84f916bce1a798 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Fri, 24 Sep 2021 14:52:45 -0400 Subject: [PATCH 07/25] Bump stac4s client version up --- .../stac4s/api/client/search/package.scala | 10 ------- .../api/encoders/ItemDatetimeCatalyst.scala | 29 ++++++++++++------- .../encoders/ItemDatetimeCatalystType.scala | 1 + .../stac/api/encoders/StacSerializers.scala | 1 + .../geotrellis/TileFeatureSupportSpec.scala | 7 +---- project/RFDependenciesPlugin.scala | 2 +- 6 files changed, 22 insertions(+), 28 deletions(-) delete mode 100644 datasource/src/main/scala/com/azavea/stac4s/api/client/search/package.scala diff --git a/datasource/src/main/scala/com/azavea/stac4s/api/client/search/package.scala b/datasource/src/main/scala/com/azavea/stac4s/api/client/search/package.scala deleted file mode 100644 index a383ff7b8..000000000 --- a/datasource/src/main/scala/com/azavea/stac4s/api/client/search/package.scala +++ /dev/null @@ -1,10 +0,0 @@ -package com.azavea.stac4s.api.client - -import com.azavea.stac4s.StacItem -import fs2.Stream - -package object search { - implicit class Stac4sClientOps[F[_]](val self: SttpStacClient[F]) extends AnyVal { - def search(filter: Option[SearchFilters]): Stream[F, StacItem] = filter.fold(self.search)(self.search) - } -} diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalyst.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalyst.scala index 0d6970200..d8692e96e 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalyst.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalyst.scala @@ -1,27 +1,34 @@ package org.locationtech.rasterframes.datasource.stac.api.encoders -import com.azavea.stac4s.ItemDatetime +import cats.data.Ior import frameless.SQLTimestamp import cats.syntax.option._ +import com.azavea.stac4s.{PointInTime, TimeRange} +import com.azavea.stac4s.types.ItemDatetime import java.time.Instant -case class ItemDatetimeCatalyst(start: SQLTimestamp, end: Option[SQLTimestamp], _type: ItemDatetimeCatalystType) +case class ItemDatetimeCatalyst(datetime: Option[SQLTimestamp], start: Option[SQLTimestamp], end: Option[SQLTimestamp], _type: ItemDatetimeCatalystType) object ItemDatetimeCatalyst { def toDatetime(dt: ItemDatetimeCatalyst): ItemDatetime = { - val ItemDatetimeCatalyst(start, endo, _type) = dt - (_type, endo) match { - case (ItemDatetimeCatalystType.PointInTime, _) => ItemDatetime.PointInTime(Instant.ofEpochMilli(start.us)) - case (ItemDatetimeCatalystType.TimeRange, Some(end)) => ItemDatetime.TimeRange(Instant.ofEpochMilli(start.us), Instant.ofEpochMilli(end.us)) - case err => throw new Exception(s"ItemDatetimeCatalyst decoding is not possible, $err") + dt match { + case ItemDatetimeCatalyst(Some(datetime), Some(start), Some(end), ItemDatetimeCatalystType.PointInTimeAndTimeRange) => + Ior.Both(PointInTime(Instant.ofEpochMilli(datetime.us)), TimeRange(Instant.ofEpochMilli(start.us), Instant.ofEpochMilli(end.us))) + case ItemDatetimeCatalyst(Some(datetime), _, _, ItemDatetimeCatalystType.PointInTime) => + Ior.Left(PointInTime(Instant.ofEpochMilli(datetime.us))) + case ItemDatetimeCatalyst(_, Some(start), Some(end), ItemDatetimeCatalystType.PointInTime) => + Ior.Right(TimeRange(Instant.ofEpochMilli(start.us), Instant.ofEpochMilli(end.us))) + case e => throw new Exception(s"ItemDatetimeCatalyst decoding is not possible, $e") } } def fromItemDatetime(dt: ItemDatetime): ItemDatetimeCatalyst = dt match { - case ItemDatetime.PointInTime(when) => - ItemDatetimeCatalyst(SQLTimestamp(when.toEpochMilli), None, ItemDatetimeCatalystType.PointInTime) - case ItemDatetime.TimeRange(start, end) => - ItemDatetimeCatalyst(SQLTimestamp(start.toEpochMilli), SQLTimestamp(end.toEpochMilli).some, ItemDatetimeCatalystType.PointInTime) + case Ior.Left(PointInTime(datetime)) => + ItemDatetimeCatalyst(SQLTimestamp(datetime.toEpochMilli).some, None, None, ItemDatetimeCatalystType.PointInTime) + case Ior.Right(TimeRange(start, end)) => + ItemDatetimeCatalyst(None, SQLTimestamp(start.toEpochMilli).some, SQLTimestamp(end.toEpochMilli).some, ItemDatetimeCatalystType.PointInTime) + case Ior.Both(PointInTime(datetime), TimeRange(start, end)) => + ItemDatetimeCatalyst(SQLTimestamp(datetime.toEpochMilli).some, SQLTimestamp(start.toEpochMilli).some, SQLTimestamp(end.toEpochMilli).some, ItemDatetimeCatalystType.PointInTime) } } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalystType.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalystType.scala index ab2da1117..31f88c2c8 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalystType.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/ItemDatetimeCatalystType.scala @@ -4,6 +4,7 @@ sealed trait ItemDatetimeCatalystType { lazy val repr: String = this.getClass.ge object ItemDatetimeCatalystType { case object PointInTime extends ItemDatetimeCatalystType case object TimeRange extends ItemDatetimeCatalystType + case object PointInTimeAndTimeRange extends ItemDatetimeCatalystType def fromString(str: String): ItemDatetimeCatalystType = str match { case PointInTime.repr => PointInTime diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala index 3029ace2e..9f085a8c0 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/encoders/StacSerializers.scala @@ -5,6 +5,7 @@ import io.circe.{Json, JsonObject} import io.circe.syntax._ import cats.syntax.either._ import com.azavea.stac4s._ +import com.azavea.stac4s.types.ItemDatetime import eu.timepit.refined.api.{RefType, Validate} import frameless.{Injection, SQLTimestamp, TypedEncoder, TypedExpressionEncoder} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder diff --git a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala index fd3069e24..79c82b3ab 100644 --- a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotrellis/TileFeatureSupportSpec.scala @@ -40,10 +40,7 @@ import org.scalatest.BeforeAndAfter import scala.reflect.ClassTag - -class TileFeatureSupportSpec extends TestEnvironment - with TestData - with BeforeAndAfter { +class TileFeatureSupportSpec extends TestEnvironment with TestData with BeforeAndAfter { val strTF1 = TileFeature(squareIncrementingTile(3), List("data1")) val strTF2 = TileFeature(squareIncrementingTile(3), List("data2")) @@ -54,10 +51,8 @@ class TileFeatureSupportSpec extends TestEnvironment val geoms = Seq(ext2.toPolygon()) val maskOpts: Rasterizer.Options = Rasterizer.Options.DEFAULT - describe("TileFeatureSupport") { it("should support merge, prototype operations") { - val merged = strTF1.merge(strTF2) assert(merged.tile == strTF1.tile.merge(strTF2.tile)) assert(merged.data == List("data1", "data2")) diff --git a/project/RFDependenciesPlugin.scala b/project/RFDependenciesPlugin.scala index c3f830930..766934a06 100644 --- a/project/RFDependenciesPlugin.scala +++ b/project/RFDependenciesPlugin.scala @@ -52,7 +52,7 @@ object RFDependenciesPlugin extends AutoPlugin { val scaffeine = "com.github.blemale" %% "scaffeine" % "4.0.2" val `spray-json` = "io.spray" %% "spray-json" % "1.3.4" val `scala-logging` = "com.typesafe.scala-logging" %% "scala-logging" % "3.8.0" - val stac4s = "com.azavea.stac4s" %% "client" % "0.6.2" + val stac4s = "com.azavea.stac4s" %% "client" % "0.7.1" val sttpCatsCe2 = "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats-ce2" % "3.3.6" val frameless = "org.typelevel" %% "frameless-dataset" % "0.10.1" } From 05ae5c6c106d35521779887b2156222aaa7a16a3 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 27 Sep 2021 14:15:37 -0400 Subject: [PATCH 08/25] Adjust tests --- .../scala/org/locationtech/rasterframes/RasterLayerSpec.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala index e7b2a4a9f..1dce2a6ca 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala @@ -82,9 +82,6 @@ class RasterLayerSpec extends TestEnvironment with MetadataKeys } describe("RasterFrameLayer") { - // Try to GC to avoid OOM on low memory instances. - // TODO: remove once we have a larger CI - System.gc() it("should implicitly convert from spatial layer type") { val tileLayerRDD = TestData.randomSpatialTileLayerRDD(20, 20, 2, 2) From 286f370c6b17e8a01b3cae104dc909060c4859dc Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 27 Sep 2021 14:34:47 -0400 Subject: [PATCH 09/25] Add py focal functions --- .../main/python/pyrasterframes/__init__.py | 5 +- .../python/pyrasterframes/rasterfunctions.py | 57 +++++++++++++++++++ .../main/python/pyrasterframes/rf_types.py | 6 +- .../rasterframes/py/PyRFContext.scala | 17 +++--- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index add1c42da..0be01bbf2 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -117,6 +117,7 @@ def _raster_reader( source=None, catalog_col_names: Optional[List[str]] = None, band_indexes: Optional[List[int]] = None, + buffer_size: int = 0, tile_dimensions: Tuple[int] = (256, 256), lazy_tiles: bool = True, spatial_index_partitions=None, @@ -134,6 +135,7 @@ def _raster_reader( :param catalog_col_names: required if `source` is a DataFrame or CSV string. It is a list of strings giving the names of columns containing URIs to read. :param band_indexes: list of integers indicating which bands, zero-based, to read from the raster files specified; default is to read only the first band. :param tile_dimensions: tuple or list of two indicating the default tile dimension as (columns, rows). + :param buffer_size: buffer each tile read by this many cells on all sides. :param lazy_tiles: If true (default) only generate minimal references to tile contents; if false, fetch tile cell values. :param spatial_index_partitions: If true, partitions read tiles by a Z2 spatial index using the default shuffle partitioning. If a values > 0, the given number of partitions are created instead of the default. @@ -176,7 +178,8 @@ def temp_name(): options.update({ "band_indexes": to_csv(band_indexes), "tile_dimensions": to_csv(tile_dimensions), - "lazy_tiles": str(lazy_tiles) + "lazy_tiles": str(lazy_tiles), + "buffer_size": int(buffer_size) }) # Parse the `source` argument diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index 0c48e91af..fde68e855 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -65,6 +65,15 @@ def to_jvm(ct): elif isinstance(cell_type_arg, CellType): return to_jvm(cell_type_arg.cell_type_name) +def _parse_neighborhood(neighborhood_arg: str) -> JavaObject: + """ Convert the cell type representation to the expected JVM CellType object.""" + def to_jvm(n): + return _context_call('_parse_neighborhood', n) + + if isinstance(neighborhood_arg, str): + return to_jvm(neighborhood_arg) + else: + raise NotImplementedError def rf_cell_types() -> List[CellType]: """Return a list of standard cell types""" @@ -781,6 +790,54 @@ def rf_identity(tile_col: Column_type) -> Column: """Pass tile through unchanged""" return _apply_column_function('rf_identity', tile_col) +def rf_focal_max(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mean(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mean value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mean') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_median(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_median') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_min(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the min value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_min') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mode(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mode value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mode') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_std_dev(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the standard deviation value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_std_dev') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_moransI(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_aspect(tile_col: Column_type) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + return _apply_column_function('rf_aspect', tile_col) + +def rf_slope(tile_col: Column_type, z_factor: float) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_slope') + return Column(jfcn(_to_java_column(tile_col), float(z_factor))) + +def rf_hillshade(tile_col: Column_type, azimuth: float, altitude: float, z_factor: float) -> Column: + """Calculates the hillshade of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_hillshade') + return Column(jfcn(_to_java_column(tile_col), float(azimuth), float(altitude), float(z_factor))) def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_types.py b/pyrasterframes/src/main/python/pyrasterframes/rf_types.py index 516a0eb2c..a0f682f69 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rf_types.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rf_types.py @@ -461,7 +461,7 @@ def sqlType(cls): StructField("xmax",DoubleType(), True), StructField("ymax",DoubleType(), True) ]) - subgrid = StructType([ + grid = StructType([ StructField("colMin", IntegerType(), True), StructField("rowMin", IntegerType(), True), StructField("colMax", IntegerType(), True), @@ -474,7 +474,7 @@ def sqlType(cls): ]),True), StructField("bandIndex", IntegerType(), True), StructField("subextent", extent ,True), - StructField("subgrid", subgrid, True), + StructField("subgrid", grid, True), ]) return StructType([ @@ -482,6 +482,7 @@ def sqlType(cls): StructField("cols", IntegerType(), False), StructField("rows", IntegerType(), False), StructField("cells", BinaryType(), True), + StructField("gridBounds", grid, True), StructField("ref", ref, True) ]) @@ -501,6 +502,7 @@ def serialize(self, tile): dims[0], dims[1], cells, + None, None ] diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala index 9c9ca9c4b..71ab8ca16 100644 --- a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -21,22 +21,21 @@ package org.locationtech.rasterframes.py import java.nio.ByteBuffer - import geotrellis.proj4.CRS -import geotrellis.raster.{CellType, MultibandTile} +import geotrellis.raster.{CellType, MultibandTile, Neighborhood} import geotrellis.spark._ import geotrellis.layer._ import geotrellis.vector.Extent import org.apache.spark.sql._ import org.locationtech.rasterframes -import org.locationtech.rasterframes.util.ResampleMethod +import org.locationtech.rasterframes.util.{FocalNeighborhood, KryoSupport, ResampleMethod} import org.locationtech.rasterframes.extensions.RasterJoin import org.locationtech.rasterframes.model.LazyCRS -import org.locationtech.rasterframes.ref.{GDALRasterSource, RasterRef, RFRasterSource} -import org.locationtech.rasterframes.util.KryoSupport +import org.locationtech.rasterframes.ref.{GDALRasterSource, RFRasterSource, RasterRef} import org.locationtech.rasterframes.{RasterFunctions, _} import spray.json._ import org.locationtech.rasterframes.util.JsonCodecs._ + import scala.collection.JavaConverters._ /** @@ -134,8 +133,6 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions * Left spatial join managing reprojection and merging of `other`; uses joinExprs to conduct initial join then extent and CRS columns to determine if rows intersect */ def rasterJoin(df: DataFrame, other: DataFrame, joinExprs: Column, leftExtent: Column, leftCRS: Column, rightExtent: Column, rightCRS: Column, resamplingMethod: String): DataFrame = { - - val m = resamplingMethod match { case ResampleMethod(mm) => mm case _ => throw new IllegalArgumentException(s"Incorrect resampling method passed: ${resamplingMethod}") @@ -143,12 +140,16 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions RasterJoin(df, other, joinExprs, leftExtent, leftCRS, rightExtent, rightCRS, m, None) } - /** * Convenience functions for use in Python */ def _parse_cell_type(name: String): CellType = CellType.fromName(name) + def _parse_neighborhood(name: String): Neighborhood = name match { + case FocalNeighborhood(n) => n + case _ => throw new Exception(s"$name is an unsupported neighborhood") + } + /** * Convenience list of valid cell type strings * @return Java List of String, which py4j can interpret as a python `list` From 48d6dcf07253a4a470af41bfcd781af0d319bd98 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Mon, 27 Sep 2021 16:43:52 -0400 Subject: [PATCH 10/25] Make the correct tile compatible BufferTile python support --- .../src/main/python/pyrasterframes/rf_types.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_types.py b/pyrasterframes/src/main/python/pyrasterframes/rf_types.py index a0f682f69..9366fe07e 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rf_types.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rf_types.py @@ -371,7 +371,7 @@ def __repr__(self): class Tile(object): - def __init__(self, cells, cell_type=None): + def __init__(self, cells, cell_type=None, grid_bounds=None): if cell_type is None: # infer cell type from the cells dtype and whether or not it is masked ct = CellType.from_numpy_dtype(cells.dtype) @@ -390,6 +390,11 @@ def __init__(self, cells, cell_type=None): # if the value in the array is `nd_value`, it is masked as nodata self.cells = np.ma.masked_equal(self.cells, nd_value) + # is it a buffer tile? crop it on extraction to preserve the tile behavior + if grid_bounds is not None: + colmin, rowmin, colmax, rowmax = grid_bounds + self.cells = self.cells[rowmin:(rowmax+1), colmin:(colmax+1)] + def __eq__(self, other): if type(other) is type(self): return self.cell_type == other.cell_type and \ @@ -535,7 +540,7 @@ def deserialize(self, datum): try: as_numpy = np.frombuffer(cell_data_bytes, dtype=cell_type.to_numpy_dtype()) reshaped = as_numpy.reshape((rows, cols)) - t = Tile(reshaped, cell_type) + t = Tile(reshaped, cell_type, datum.gridBounds) except ValueError as e: raise ValueError({ "cell_type": cell_type, @@ -543,7 +548,8 @@ def deserialize(self, datum): "rows": rows, "cell_data.length": len(cell_data_bytes), "cell_data.type": type(cell_data_bytes), - "cell_data.values": repr(cell_data_bytes) + "cell_data.values": repr(cell_data_bytes), + "grid_bounds": datum.gridBounds }, e) return t From 2b2b23564b967f12ee39fa816c0de33ac1e9fcc5 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 28 Sep 2021 12:31:50 -0400 Subject: [PATCH 11/25] Add notebooks --- rf-notebook/src/main/docker/Dockerfile | 16 +- .../src/main/docker/requirements-nb.txt | 2 +- .../src/main/notebooks/FocalOperations.ipynb | 172 ++++++++++++++++++ 3 files changed, 178 insertions(+), 12 deletions(-) create mode 100644 rf-notebook/src/main/notebooks/FocalOperations.ipynb diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile index be0c95a8b..ae38eb494 100644 --- a/rf-notebook/src/main/docker/Dockerfile +++ b/rf-notebook/src/main/docker/Dockerfile @@ -1,21 +1,15 @@ # jupyter/scipy-notebook isn't semantically versioned. # We pick this arbitrary one from Sept 2019 because it's what latest was on Oct 17 2019. -FROM jupyter/scipy-notebook:7a0c7325e470 +FROM jupyter/all-spark-notebook:python-3.8.8 LABEL maintainer="Astraea, Inc. " USER root -RUN \ - apt-get -y update && \ - apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV APACHE_SPARK_VERSION 2.4.7 -ENV HADOOP_VERSION 2.7 +ENV APACHE_SPARK_VERSION 3.1.2 +ENV HADOOP_VERSION 3.2 # On MacOS compute this with `shasum -a 512` -ARG APACHE_SPARK_CHECKSUM="0f5455672045f6110b030ce343c049855b7ba86c0ecb5e39a075ff9d093c7f648da55ded12e72ffe65d84c32dcd5418a6d764f2d6295a3f894a4286cc80ef478" +ARG APACHE_SPARK_CHECKSUM="2385cb772f21b014ce2abd6b8f5e815721580d6e8bc42a26d70bbcdda8d303d886a6f12b36d40f6971b5547b70fae62b5a96146f0421cb93d4e51491308ef5d5" ARG APACHE_SPARK_FILENAME="spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" ARG APACHE_SPARK_REMOTE_PATH="spark-${APACHE_SPARK_VERSION}/${APACHE_SPARK_FILENAME}" @@ -33,7 +27,7 @@ RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERS # Spark config ENV SPARK_HOME /usr/local/spark -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip +ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info ENV RF_LIB_LOC=/usr/local/rasterframes diff --git a/rf-notebook/src/main/docker/requirements-nb.txt b/rf-notebook/src/main/docker/requirements-nb.txt index cff89d025..720a38f60 100644 --- a/rf-notebook/src/main/docker/requirements-nb.txt +++ b/rf-notebook/src/main/docker/requirements-nb.txt @@ -1,5 +1,5 @@ pyspark>=3.1 -gdal==3.1.2 +gdal==3.1.4 numpy pandas shapely diff --git a/rf-notebook/src/main/notebooks/FocalOperations.ipynb b/rf-notebook/src/main/notebooks/FocalOperations.ipynb new file mode 100644 index 000000000..464fe45c4 --- /dev/null +++ b/rf-notebook/src/main/notebooks/FocalOperations.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Focal Operations with RastrFrames Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Spark Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "from pyrasterframes.utils import create_rf_spark_session\n", + "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", + "from pyrasterframes.rasterfunctions import *\n", + "import pyspark.sql.functions as F" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "spark = create_rf_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get a PySpark DataFrame from elevation raster\n", + "\n", + "Read a single scene of elevation into DataFrame or raster tiles.\n", + "Each tile overlaps its neighbor by \"buffer_size\" of pixels, providing focal operations neighbor information around tile edges.\n", + "You can configure the default size of these tiles, by passing a tuple of desired columns and rows as: `raster(uri, tile_dimensions=(96, 96))`. The default is `(256, 256)`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "uri = 'https://geotrellis-demo.s3.us-east-1.amazonaws.com/cogs/harrisburg-pa/elevation.tif'\n", + "df = spark.read.raster(uri, tile_dimensions=(512, 512), buffer_size=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "root\n |-- proj_raster_path: string (nullable = false)\n |-- proj_raster: struct (nullable = true)\n | |-- tile_context: struct (nullable = true)\n | | |-- extent: struct (nullable = false)\n | | | |-- xmin: double (nullable = false)\n | | | |-- ymin: double (nullable = false)\n | | | |-- xmax: double (nullable = false)\n | | | |-- ymax: double (nullable = false)\n | | |-- crs: struct (nullable = false)\n | | | |-- crsProj4: string (nullable = false)\n | |-- tile: tile (nullable = false)\n\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The extent struct tells us where in the [CRS](https://spatialreference.org/ref/sr-org/6842/) the tile data covers. The granule is split into arbitrary sized chunks. Each row is a different chunk. Let's see how many." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "81" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], + "source": [ + "df.count()" + ] + }, + { + "source": [ + "## Focal Operations\n", + "Additional transformations are complished through use of column functions.\n", + "The functions used here are mapped to their Scala implementation and applied per row.\n", + "For each row the source elevation data is fetched only once before it's used as input." + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[rf_crs(proj_raster): struct, rf_extent(proj_raster): struct, rf_aspect(proj_raster): struct,crs:struct>,tile:udt>, rf_slope(proj_raster): struct,crs:struct>,tile:udt>, rf_hillshade(proj_raster): struct,crs:struct>,tile:udt>]" + ], + "text/html": "\n\n\n\n\n\n\n\n\n\n\n\n
Showing only top 5 rows
rf_crs(proj_raster)rf_extent(proj_raster)rf_aspect(proj_raster)rf_slope(proj_raster)rf_hillshade(proj_raster)
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][302369.2154, 4478399.0319, 317729.2154, 4493759.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4398599.0319, 287009.2154, 4401599.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][225569.2154, 4416959.0319, 240929.2154, 4432319.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4463039.0319, 287009.2154, 4478399.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][317729.2154, 4416959.0319, 333089.2154, 4432319.0319]
", + "text/markdown": "\n_Showing only top 5 rows_.\n\n| rf_crs(proj_raster) | rf_extent(proj_raster) | rf_aspect(proj_raster) | rf_slope(proj_raster) | rf_hillshade(proj_raster) |\n|---|---|---|---|---|\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[302369.2154, 4478399.0319, 317729.2154, 4493759.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4398599.0319, 287009.2154, 4401599.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[225569.2154, 4416959.0319, 240929.2154, 4432319.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4463039.0319, 287009.2154, 4478399.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[317729.2154, 4416959.0319, 333089.2154, 4432319.0319] | | | |" + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "df.select(\n", + " rf_crs(df.proj_raster), \n", + " rf_extent(df.proj_raster), \n", + " rf_aspect(df.proj_raster), \n", + " rf_slope(df.proj_raster, z_factor=1), \n", + " rf_hillshade(df.proj_raster, azimuth=315, altitude=45, z_factor=1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file From 9cb9c7fce933890c7b8766dfade5e89e7b747d0f Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 28 Sep 2021 13:13:20 -0400 Subject: [PATCH 12/25] Move to the scipy notebook version --- rf-notebook/src/main/docker/Dockerfile | 8 +++++++- rf-notebook/src/main/docker/requirements-nb.txt | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile index ae38eb494..e207613ca 100644 --- a/rf-notebook/src/main/docker/Dockerfile +++ b/rf-notebook/src/main/docker/Dockerfile @@ -1,11 +1,17 @@ # jupyter/scipy-notebook isn't semantically versioned. # We pick this arbitrary one from Sept 2019 because it's what latest was on Oct 17 2019. -FROM jupyter/all-spark-notebook:python-3.8.8 +FROM jupyter/scipy-notebook:python-3.8.8 LABEL maintainer="Astraea, Inc. " USER root +RUN \ + apt-get -y update && \ + apt-get install --no-install-recommends -y openjdk-11-jdk ca-certificates-java && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + ENV APACHE_SPARK_VERSION 3.1.2 ENV HADOOP_VERSION 3.2 # On MacOS compute this with `shasum -a 512` diff --git a/rf-notebook/src/main/docker/requirements-nb.txt b/rf-notebook/src/main/docker/requirements-nb.txt index 720a38f60..cff89d025 100644 --- a/rf-notebook/src/main/docker/requirements-nb.txt +++ b/rf-notebook/src/main/docker/requirements-nb.txt @@ -1,5 +1,5 @@ pyspark>=3.1 -gdal==3.1.4 +gdal==3.1.2 numpy pandas shapely From 005492fee35f6731725e165cd1ee6eae90b05ada Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 28 Sep 2021 17:20:50 -0400 Subject: [PATCH 13/25] Add FocalFunctions specs --- .../functions/FocalFunctionsSpec.scala | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala index cb45208bd..c58448033 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -27,6 +27,8 @@ import geotrellis.raster.testkit.RasterMatchers import org.locationtech.rasterframes.ref.{RFRasterSource, RasterRef, Subgrid} import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes._ +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.local.Implicits._ import java.nio.file.Paths @@ -195,5 +197,49 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { assertEqual(bt.mapTile(_.hillshade(btCellSize, 315, 45, 1)), actual) assertEqual(fullTile.hillshade(btCellSize, 315, 45, 1).crop(subGridBounds), actual) } + // that is the original use case + // to read a buffered source, perform a focal operation + // the followup functions would work with the buffered tile as + // with a regular tile without a buffer (all ops will work within the window) + it("should perform a focal operation and a valid local operation after that") { + val actual = + df + .select(rf_aspect($"proj_raster").as("aspect")) + .select(rf_local_add($"aspect", $"aspect")) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + val a: Tile = bt.aspect(btCellSize) + assertEqual(a.localAdd(a), actual) + } + + // if we read a buffered tile the local buffer would preserve the buffer information + // however rf_local_* functions don't preserve that type information + // and the Buffer Tile is upcasted into the Tile and stored as a regular tile (within the buffer, with the buffer lost) + // the follow up focal operation would be non buffered + it("should perform a local operation and a valid focal operation after that with the buffer lost") { + val actual = + df + .select(rf_local_add($"proj_raster", $"proj_raster") as "added") + .select(rf_aspect($"added")) + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + // that's what we would like eventually + // val expected = bt.localAdd(bt) match { + // case b: BufferTile => b.aspect(btCellSize) + // case _ => throw new Exception("Not a Buffer Tile") + // } + + // that's what we have actually + // even though local ops can preserve the output tile + // we don't handle that + val expected = bt.localAdd(bt).aspect(btCellSize) + assertEqual(expected, actual) + } } } From de57a3bce90bfc9a8eef9bc9cb04eceeff39a271 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Tue, 28 Sep 2021 18:37:48 -0400 Subject: [PATCH 14/25] Add a comment about how to achieve BufferTile preservation --- .../rasterframes/expressions/BinaryRasterFunction.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala index 094c157d7..425e6c4e7 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterFunction.scala @@ -59,6 +59,7 @@ trait BinaryRasterFunction extends BinaryExpression with RasterResult { if(leftCtx.isDefined && rightCtx.isDefined && leftCtx != rightCtx) logger.warn(s"Both '${left}' and '${right}' provided an extent and CRS, but they are different. Left-hand side will be used.") + // TODO: extract BufferTile here to preserve the buffer op(leftTile, rightTile) case DoubleArg(d) => op(fpTile(leftTile), d) case IntegerArg(i) => op(leftTile, i) From d569195d1bf2ecee614dd5e3362ac5b80c6ef0b8 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 29 Sep 2021 17:02:49 -0400 Subject: [PATCH 15/25] Make FocalOps valid expressions --- .../encoders/StandardEncoders.scala | 9 +- .../rasterframes/encoders/TypedEncoders.scala | 10 +- .../expressions/DynamicExtractors.scala | 8 +- .../expressions/focalops/Aspect.scala | 29 ++++- .../expressions/focalops/Convolve.scala | 38 +++++- .../expressions/focalops/FocalMax.scala | 8 +- .../expressions/focalops/FocalMean.scala | 8 +- .../expressions/focalops/FocalMedian.scala | 8 +- .../expressions/focalops/FocalMin.scala | 8 +- .../expressions/focalops/FocalMode.scala | 8 +- .../expressions/focalops/FocalMoransI.scala | 8 +- .../focalops/FocalNeighborhoodOp.scala | 61 +++++---- .../expressions/focalops/FocalOp.scala | 36 ----- .../expressions/focalops/FocalStdDev.scala | 10 +- .../expressions/focalops/Hillshade.scala | 55 ++++++-- .../expressions/focalops/Slope.scala | 41 +++++- .../expressions/focalops/SurfaceOp.scala | 52 -------- .../functions/FocalFunctions.scala | 55 ++++++-- .../functions/FocalFunctionsSpec.scala | 123 ++++++++++++++++-- 19 files changed, 380 insertions(+), 195 deletions(-) delete mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala delete mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala index 3b3cdc29b..3301bca70 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.util.QuantileSummaries import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders import org.locationtech.rasterframes.model.{CellContext, LongExtent, TileContext, TileDataContext} import frameless.TypedEncoder -import geotrellis.raster.mapalgebra.focal.{Square, Circle, Nesw, Wedge, Annulus} +import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood} import java.net.URI import java.sql.Timestamp @@ -54,11 +54,8 @@ trait StandardEncoders extends SpatialEncoders with TypedEncoders { implicit lazy val localCellStatsEncoder: ExpressionEncoder[LocalCellStatistics] = ExpressionEncoder() implicit lazy val uriEncoder: ExpressionEncoder[URI] = typedExpressionEncoder[URI] - implicit lazy val squareNeighborhoodEncoder: ExpressionEncoder[Square] = typedExpressionEncoder[Square] - implicit lazy val circleNeighborhoodEncoder: ExpressionEncoder[Circle] = typedExpressionEncoder[Circle] - implicit lazy val neswNeighborhoodEncoder: ExpressionEncoder[Nesw] = typedExpressionEncoder[Nesw] - implicit lazy val wedgeNeighborhoodEncoder: ExpressionEncoder[Wedge] = typedExpressionEncoder[Wedge] - implicit lazy val annulusNeighborhoodEncoder: ExpressionEncoder[Annulus] = typedExpressionEncoder[Annulus] + implicit lazy val neighborhoodEncoder: ExpressionEncoder[Neighborhood] = typedExpressionEncoder[Neighborhood] + implicit lazy val kernelEncoder: ExpressionEncoder[Kernel] = typedExpressionEncoder[Kernel] implicit lazy val quantileSummariesEncoder: ExpressionEncoder[QuantileSummaries] = typedExpressionEncoder[QuantileSummaries] implicit lazy val envelopeEncoder: ExpressionEncoder[Envelope] = typedExpressionEncoder implicit lazy val longExtentEncoder: ExpressionEncoder[LongExtent] = typedExpressionEncoder diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala index d0dc97f1b..1fe15f15b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala @@ -3,13 +3,14 @@ package org.locationtech.rasterframes.encoders import frameless._ import geotrellis.layer.{KeyBounds, LayoutDefinition, TileLayerMetadata} import geotrellis.proj4.CRS -import geotrellis.raster.{CellType, Dimensions, GridBounds, Raster, Tile, CellGrid} +import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood} +import geotrellis.raster.{CellGrid, CellType, Dimensions, GridBounds, Raster, Tile} import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.util.QuantileSummaries import org.apache.spark.sql.rf.{CrsUDT, RasterSourceUDT, TileUDT} import org.locationtech.jts.geom.Envelope -import org.locationtech.rasterframes.util.KryoSupport +import org.locationtech.rasterframes.util.{FocalNeighborhood, KryoSupport} import java.net.URI import java.nio.ByteBuffer @@ -33,6 +34,9 @@ trait TypedEncoders { implicit val uriInjection: Injection[URI, String] = Injection(_.toString, new URI(_)) implicit val uriTypedEncoder: TypedEncoder[URI] = TypedEncoder.usingInjection + implicit val neighborhoodInjection: Injection[Neighborhood, String] = Injection(FocalNeighborhood(_), FocalNeighborhood.unapply(_).get) + implicit val neighborhoodTypedEncoder: TypedEncoder[Neighborhood] = TypedEncoder.usingInjection + implicit val envelopeTypedEncoder: TypedEncoder[Envelope] = ManualTypedEncoder.newInstance[Envelope]( fields = List( @@ -81,6 +85,8 @@ trait TypedEncoders { implicit val tileTypedEncoder: TypedEncoder[Tile] = TypedEncoder.usingUserDefinedType[Tile] implicit def rasterTileTypedEncoder[T <: CellGrid[Int]: TypedEncoder]: TypedEncoder[Raster[T]] = TypedEncoder.usingDerivation + + implicit val kernelTypedEncoder: TypedEncoder[Kernel] = TypedEncoder.usingDerivation } object TypedEncoders extends TypedEncoders diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala index e92326b03..743ee8e97 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions import geotrellis.proj4.CRS -import geotrellis.raster.{CellGrid, Raster, Tile} +import geotrellis.raster.{CellGrid, Neighborhood, Raster, Tile} import geotrellis.vector.Extent import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow @@ -38,6 +38,7 @@ import org.locationtech.rasterframes.model.{LazyCRS, LongExtent, TileContext} import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef} import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.apache.spark.sql.rf.CrsUDT +import org.locationtech.rasterframes.util.FocalNeighborhood private[rasterframes] object DynamicExtractors { @@ -224,4 +225,9 @@ object DynamicExtractors { case c: Char => IntegerArg(c.toInt) } } + + lazy val neighborhoodExtractor: PartialFunction[DataType, Any => Neighborhood] = { + case _: StringType => (v: Any) => FocalNeighborhood.unapply(v.asInstanceOf[UTF8String].toString).get + case n if n.conformsToSchema(neighborhoodEncoder.schema) => { case ir: InternalRow => ir.as[Neighborhood] } + } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala index d906403e9..10ba6727d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -21,10 +21,18 @@ package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.{BufferTile, CellSize, Tile} +import geotrellis.raster.{BufferTile, CellSize} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.locationtech.rasterframes.expressions.{NullToValue, RasterResult, UnaryRasterFunction, row} +import org.locationtech.rasterframes.encoders.syntax._ +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.DataType +import org.slf4j.LoggerFactory +import com.typesafe.scalalogging.Logger @ExpressionDescription( usage = "_FUNC_(tile) - Performs aspect on tile.", @@ -36,8 +44,25 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class Aspect(child: Expression) extends SurfaceOp { +case class Aspect(child: Expression) extends UnaryRasterFunction with RasterResult with NullToValue with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + def na: Any = null + + def dataType: DataType = child.dataType + + override protected def nullSafeEval(input: Any): Any = { + val (tile, ctx) = tileExtractor(child.dataType)(row(input)) + eval(extractBufferTile(tile), ctx) + } + + protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + override def nodeName: String = Aspect.name + def op(t: Tile, ctx: TileContext): Tile = t match { case bt: BufferTile => bt.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) case _ => t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala index 2559706c1..594c8b871 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -21,13 +21,24 @@ package org.locationtech.rasterframes.expressions.focalops +import com.typesafe.scalalogging.Logger import geotrellis.raster.{BufferTile, Tile} import geotrellis.raster.mapalgebra.focal.Kernel import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription} +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders._ +import org.locationtech.rasterframes.encoders.syntax._ +import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.expressions.{RasterResult, row} +import org.slf4j.LoggerFactory @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs convolve on tile in the neighborhood.", + usage = "_FUNC_(tile, kernel) - Performs convolve on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation @@ -37,10 +48,27 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript > SELECT _FUNC_(tile, kernel); ...""" ) -case class Convolve(child: Expression, kernel: Kernel) extends FocalOp { +case class Convolve(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + override def nodeName: String = Convolve.name - protected def op(t: Tile): Tile = t match { + def dataType: DataType = left.dataType + + override def checkInputDataTypes(): TypeCheckResult = + if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") + else if (!right.dataType.conformsToSchema(kernelEncoder.schema)) { + TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a kernel type.") + } else TypeCheckSuccess + + override protected def nullSafeEval(tileInput: Any, kernelInput: Any): Any = { + val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) + val kernel = row(kernelInput).as[Kernel] + val result = op(extractBufferTile(tile), kernel) + toInternalRow(result, ctx) + } + + protected def op(t: Tile, kernel: Kernel): Tile = t match { case bt: BufferTile => bt.convolve(kernel) case _ => t.convolve(kernel) } @@ -48,5 +76,5 @@ case class Convolve(child: Expression, kernel: Kernel) extends FocalOp { object Convolve { def name: String = "rf_convolve" - def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) + def apply(tile: Column, kernel: Column): Column = new Column(Convolve(tile.expr, kernel.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala index 50f8c3e7c..a7220f941 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -34,12 +34,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMax(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMax.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalMax(neighborhood) case _ => t.focalMax(neighborhood) } @@ -47,5 +47,5 @@ case class FocalMax(child: Expression, neighborhood: Neighborhood) extends Focal object FocalMax { def name: String = "rf_focal_max" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMax(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMax(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala index 6ae045146..b72019d2b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -34,12 +34,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMean(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMean.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalMean(neighborhood) case _ => t.focalMean(neighborhood) } @@ -47,5 +47,5 @@ case class FocalMean(child: Expression, neighborhood: Neighborhood) extends Foca object FocalMean { def name:String = "rf_focal_mean" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMean(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMean(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala index d6aaae2bc..4dc11d029 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -34,12 +34,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMedian(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMedian.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalMedian(neighborhood) case _ => t.focalMedian(neighborhood) } @@ -47,5 +47,5 @@ case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends Fo object FocalMedian { def name: String = "rf_focal_median" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMedian(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMedian(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala index 071a2e2ee..fc5cfac70 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -33,12 +33,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMin(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMin.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalMin(neighborhood) case _ => t.focalMin(neighborhood) } @@ -46,5 +46,5 @@ case class FocalMin(child: Expression, neighborhood: Neighborhood) extends Focal object FocalMin { def name: String = "rf_focal_min" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMin(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMin(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala index 9ec10ee5e..af5ff14fd 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -34,12 +34,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMode(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMode.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalMode(neighborhood) case _ => t.focalMode(neighborhood) } @@ -47,5 +47,5 @@ case class FocalMode(child: Expression, neighborhood: Neighborhood) extends Foca object FocalMode { def name: String = "rf_focal_mode" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMode(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMode(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala index c52a71cd7..e09dd5681 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -34,12 +34,12 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalMoransI(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMoransI.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.tileMoransI(neighborhood) case _ => t.tileMoransI(neighborhood) } @@ -47,5 +47,5 @@ case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends F object FocalMoransI { def name: String = "rf_focal_moransi" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMoransI(tile.expr, neighborhood)) + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMoransI(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala index 24e299bab..20db8bf6f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -1,28 +1,39 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2020 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.mapalgebra.focal.Neighborhood +import com.typesafe.scalalogging.Logger +import geotrellis.raster.{Neighborhood, Tile} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.expressions.DynamicExtractors.{neighborhoodExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.{RasterResult, row} +import org.slf4j.LoggerFactory + +trait FocalNeighborhoodOp extends BinaryExpression with RasterResult with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + // tile + def left: Expression + // neighborhood + def right: Expression + + def dataType: DataType = left.dataType + + override def checkInputDataTypes(): TypeCheckResult = + if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") + else if(!neighborhoodExtractor.isDefinedAt(right.dataType)) { + TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a string neighborhood type.") + } else TypeCheckSuccess + + override protected def nullSafeEval(tileInput: Any, neighborhoodInput: Any): Any = { + val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) + val neighborhood = neighborhoodExtractor(right.dataType)(neighborhoodInput) + val result = op(extractBufferTile(tile), neighborhood) + toInternalRow(result, ctx) + } + + protected def op(child: Tile, neighborhood: Neighborhood): Tile +} -trait FocalNeighborhoodOp extends FocalOp { - def neighborhood: Neighborhood -} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala deleted file mode 100644 index 6169cf88c..000000000 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalOp.scala +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2021 Astraea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.locationtech.rasterframes.expressions.focalops - -import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp, row} -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback - -trait FocalOp extends UnaryRasterOp with NullToValue with CodegenFallback { - def na: Any = null - - override protected def nullSafeEval(input: Any): Any = { - val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - val result = op(extractBufferTile(childTile)) - toInternalRow(result, childCtx) - } -} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala index 9927f7874..7ec881544 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -34,18 +34,18 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript * neighborhood - a focal operation neighborhood""", examples = """ Examples: - > SELECT _FUNC_(tile, Square(1)); + > SELECT _FUNC_(tile, 'square-1'); ...""" ) -case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOp { +case class FocalStdDev(left: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalStdDev.name - protected def op(t: Tile): Tile = t match { + protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { case bt: BufferTile => bt.focalStandardDeviation(neighborhood) case _ => t.focalStandardDeviation(neighborhood) } } object FocalStdDev { - def name: String = "rf_focal_stddevd" - def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalStdDev(tile.expr, neighborhood)) + def name: String = "rf_focal_stddev" + def apply(tile: Column, neighborhood: Column): Column = new Column(FocalStdDev(tile.expr, neighborhood.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala index e3c693ec9..84c9580b0 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -21,10 +21,19 @@ package org.locationtech.rasterframes.expressions.focalops +import com.typesafe.scalalogging.Logger import geotrellis.raster.{BufferTile, CellSize, Tile} import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, QuaternaryExpression} +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.encoders.syntax._ +import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.locationtech.rasterframes.model.TileContext +import org.slf4j.LoggerFactory @ExpressionDescription( usage = "_FUNC_(tile, azimuth, altitude, zFactor) - Performs hillshade on tile.", @@ -39,16 +48,40 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile, azimuth, altitude, zFactor); ...""" ) -case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOp { +case class Hillshade(first: Expression, second: Expression, third: Expression, fourth: Expression) extends QuaternaryExpression with RasterResult with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + override def nodeName: String = Hillshade.name - /** - * Apply hillshade differently to the BufferedTile and a regular tile. - * In case of a BufferedTile case we'd like to apply the hillshade to the underlying tile. - * - * Check [[SurfaceOp.nullSafeEval()]] to see the details of unpacking the BufferTile. - */ - protected def op(t: Tile, ctx: TileContext): Tile = t match { + def dataType: DataType = first.dataType + + val children: Seq[Expression] = Seq(first, second, third, fourth) + + override def checkInputDataTypes(): TypeCheckResult = + if (!tileExtractor.isDefinedAt(first.dataType)) TypeCheckFailure(s"Input type '${first.dataType}' does not conform to a raster type.") + else if (!children.tail.forall(expr => numberArgExtractor.isDefinedAt(expr.dataType))) { + TypeCheckFailure(s"Input type '${second.dataType}' does not conform to a numeric type.") + } else TypeCheckSuccess + + override protected def nullSafeEval(tileInput: Any, azimuthInput: Any, altitudeInput: Any, zFactorInput: Any): Any = { + val (tile, ctx) = tileExtractor(first.dataType)(row(tileInput)) + val List(azimuth, altitude, zFactor) = + children + .tail + .zip(List(azimuthInput, altitudeInput, zFactorInput)) + .map { case (expr, datum) => numberArgExtractor(expr.dataType)(datum) match { + case DoubleArg(value) => value + case IntegerArg(value) => value.toDouble + } } + eval(extractBufferTile(tile), ctx, azimuth, altitude, zFactor) + } + + protected def eval(tile: Tile, ctx: Option[TileContext], azimuth: Double, altitude: Double, zFactor: Double): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, azimuth, altitude, zFactor)).toInternalRow + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + + protected def op(t: Tile, ctx: TileContext, azimuth: Double, altitude: Double, zFactor: Double): Tile = t match { case bt: BufferTile => bt.mapTile(_.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor)) case _ => t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) } @@ -56,6 +89,6 @@ case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFact object Hillshade { def name: String = "rf_hillshade" - def apply(tile: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = - new Column(Hillshade(tile.expr, azimuth, altitude, zFactor)) + def apply(tile: Column, azimuth: Column, altitude: Column, zFactor: Column): Column = + new Column(Hillshade(tile.expr, azimuth.expr, altitude.expr, zFactor.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala index 80558daa6..9932b4406 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -21,10 +21,19 @@ package org.locationtech.rasterframes.expressions.focalops +import com.typesafe.scalalogging.Logger import geotrellis.raster.{BufferTile, CellSize, Tile} import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription} +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.encoders.syntax._ +import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.locationtech.rasterframes.model.TileContext +import org.slf4j.LoggerFactory @ExpressionDescription( usage = "_FUNC_(tile, zFactor) - Performs slope on tile.", @@ -37,9 +46,33 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile, 0.2); ...""" ) -case class Slope(child: Expression, zFactor: Double) extends SurfaceOp { +case class Slope(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + override def nodeName: String = Slope.name - protected def op(t: Tile, ctx: TileContext): Tile = t match { + + def dataType: DataType = left.dataType + + override def checkInputDataTypes(): TypeCheckResult = + if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") + else if (!numberArgExtractor.isDefinedAt(right.dataType)) { + TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a numeric type.") + } else TypeCheckSuccess + + override protected def nullSafeEval(tileInput: Any, zFactorInput: Any): Any = { + val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) + val zFactor = numberArgExtractor(right.dataType)(zFactorInput) match { + case DoubleArg(value) => value + case IntegerArg(value) => value.toDouble + } + eval(extractBufferTile(tile), ctx, zFactor) + } + protected def eval(tile: Tile, ctx: Option[TileContext], zFactor: Double): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, zFactor)).toInternalRow + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + + protected def op(t: Tile, ctx: TileContext, zFactor: Double): Tile = t match { case bt: BufferTile => bt.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) case _ => t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) } @@ -47,5 +80,5 @@ case class Slope(child: Expression, zFactor: Double) extends SurfaceOp { object Slope { def name: String = "rf_slope" - def apply(tile: Column, zFactor: Double): Column = new Column(Slope(tile.expr, zFactor)) + def apply(tile: Column, zFactor: Column): Column = new Column(Slope(tile.expr, zFactor.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala deleted file mode 100644 index a0e8fc1c7..000000000 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOp.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * This software is licensed under the Apache 2 license, quoted below. - * - * Copyright 2021 Azavea, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * [http://www.apache.org/licenses/LICENSE-2.0] - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - * - * SPDX-License-Identifier: Apache-2.0 - * - */ - -package org.locationtech.rasterframes.expressions.focalops - -import org.locationtech.rasterframes.expressions.{NullToValue, RasterResult, UnaryRasterFunction, row} -import org.locationtech.rasterframes.encoders.syntax._ -import org.locationtech.rasterframes.expressions.DynamicExtractors._ -import org.locationtech.rasterframes.model.TileContext -import geotrellis.raster.Tile -import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.types.DataType -import org.slf4j.LoggerFactory -import com.typesafe.scalalogging.Logger - -/** Operation on a tile returning a tile. */ -trait SurfaceOp extends UnaryRasterFunction with RasterResult with NullToValue with CodegenFallback { - @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - - def na: Any = null - def dataType: DataType = child.dataType - - override protected def nullSafeEval(input: Any): Any = { - val (tile, ctx) = tileExtractor(child.dataType)(row(input)) - eval(extractBufferTile(tile), ctx) - } - - override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ctx match { - case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow - case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") - } - - protected def op(t: Tile, ctx: TileContext): Tile -} diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala index bc58db471..cdfe8e18d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -24,39 +24,72 @@ package org.locationtech.rasterframes.functions import geotrellis.raster.Neighborhood import geotrellis.raster.mapalgebra.focal.Kernel import org.apache.spark.sql.Column +import org.apache.spark.sql.functions.lit +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.encoders.serialized_literal import org.locationtech.rasterframes.expressions.focalops._ trait FocalFunctions { def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMean(tileCol, neighborhood) + rf_focal_mean(tileCol, serialized_literal(neighborhood)) + + def rf_focal_mean(tileCol: Column, neighborhoodCol: Column): Column = + FocalMean(tileCol, neighborhoodCol) def rf_focal_median(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMedian(tileCol, neighborhood) + rf_focal_median(tileCol, serialized_literal(neighborhood)) + + def rf_focal_median(tileCol: Column, neighborhoodCol: Column): Column = + FocalMedian(tileCol, neighborhoodCol) def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMode(tileCol, neighborhood) + rf_focal_mode(tileCol, serialized_literal(neighborhood)) + + def rf_focal_mode(tileCol: Column, neighborhoodCol: Column): Column = + FocalMode(tileCol, neighborhoodCol) def rf_focal_max(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMax(tileCol, neighborhood) + rf_focal_max(tileCol, serialized_literal(neighborhood)) + + def rf_focal_max(tileCol: Column, neighborhoodCol: Column): Column = + FocalMax(tileCol, neighborhoodCol) def rf_focal_min(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMin(tileCol, neighborhood) + rf_focal_min(tileCol, serialized_literal(neighborhood)) + + def rf_focal_min(tileCol: Column, neighborhoodCol: Column): Column = + FocalMin(tileCol, neighborhoodCol) def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood): Column = - FocalStdDev(tileCol, neighborhood) + rf_focal_stddev(tileCol, serialized_literal(neighborhood)) + + def rf_focal_stddev(tileCol: Column, neighborhoodCol: Column): Column = + FocalStdDev(tileCol, neighborhoodCol) def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood): Column = - FocalMoransI(tileCol, neighborhood) + rf_focal_moransi(tileCol, serialized_literal(neighborhood)) + + def rf_focal_moransi(tileCol: Column, neighborhoodCol: Column): Column = + FocalMoransI(tileCol, neighborhoodCol) def rf_convolve(tileCol: Column, kernel: Kernel): Column = - Convolve(tileCol, kernel) + rf_convolve(tileCol, serialized_literal(kernel)) + + def rf_convolve(tileCol: Column, kernelCol: Column): Column = + Convolve(tileCol, kernelCol) - def rf_slope(tileCol: Column, zFactor: Double): Column = - Slope(tileCol, zFactor) + def rf_slope[T: Numeric](tileCol: Column, zFactor: T): Column = + rf_slope(tileCol, lit(zFactor)) + + def rf_slope(tileCol: Column, zFactorCol: Column): Column = + Slope(tileCol, zFactorCol) def rf_aspect(tileCol: Column): Column = Aspect(tileCol) - def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + def rf_hillshade[T: Numeric](tileCol: Column, azimuth: T, altitude: T, zFactor: T): Column = + rf_hillshade(tileCol, lit(azimuth), lit(altitude), lit(zFactor)) + + def rf_hillshade(tileCol: Column, azimuth: Column, altitude: Column, zFactor: Column): Column = Hillshade(tileCol, azimuth, altitude, zFactor) } diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala index c58448033..73271bb35 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -29,6 +29,7 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes._ import geotrellis.raster.Tile import geotrellis.raster.mapalgebra.local.Implicits._ +import org.locationtech.rasterframes.encoders.serialized_literal import java.nio.file.Paths @@ -54,7 +55,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { lazy val df = Seq(Option(ProjectedRasterTile(bufferedRaster, src.extent, src.crs))).toDF("proj_raster").cache() - it("should provide focal mean") { + it("should perform focal mean") { checkDocs("rf_focal_mean") val actual = df @@ -64,10 +65,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_mean(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalMean(Square(1)), actual) assertEqual(fullTile.focalMean(Square(1)).crop(subGridBounds), actual) } - it("should provide focal median") { + it("should perform focal median") { checkDocs("rf_focal_median") val actual = df @@ -77,10 +87,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_median(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalMedian(Square(1)), actual) assertEqual(fullTile.focalMedian(Square(1)).crop(subGridBounds), actual) } - it("should provide focal mode") { + it("should perform focal mode") { checkDocs("rf_focal_mode") val actual = df @@ -90,10 +109,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_mode(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalMode(Square(1)), actual) assertEqual(fullTile.focalMode(Square(1)).crop(subGridBounds), actual) } - it("should provide focal max") { + it("should perform focal max") { checkDocs("rf_focal_max") val actual = df @@ -103,10 +131,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_max(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalMax(Square(1)), actual) assertEqual(fullTile.focalMax(Square(1)).crop(subGridBounds), actual) } - it("should provide focal min") { + it("should perform focal min") { checkDocs("rf_focal_min") val actual = df @@ -116,10 +153,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_min(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalMin(Square(1)), actual) assertEqual(fullTile.focalMin(Square(1)).crop(subGridBounds), actual) } - it("should provide focal stddev") { + it("should perform focal stddev") { checkDocs("rf_focal_moransi") val actual = df @@ -129,10 +175,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_stddev(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.focalStandardDeviation(Square(1)), actual) assertEqual(fullTile.focalStandardDeviation(Square(1)).crop(subGridBounds), actual) } - it("should provide focal Moran's I") { + it("should perform focal Moran's I") { checkDocs("rf_focal_moransi") val actual = df @@ -142,10 +197,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_focal_moransi(proj_raster, 'square-1')") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.tileMoransI(Square(1)), actual) assertEqual(fullTile.tileMoransI(Square(1)).crop(subGridBounds), actual) } - it("should provide convolve") { + it("should perform convolve") { checkDocs("rf_convolve") val actual = df @@ -155,10 +219,20 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .withColumn("kernel", serialized_literal(Kernel(Circle(2d)))) + .selectExpr(s"rf_convolve(proj_raster, kernel)") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.convolve(Kernel(Circle(2d))), actual) assertEqual(fullTile.convolve(Kernel(Circle(2d))).crop(subGridBounds), actual) } - it("should provide slope") { + it("should perform slope") { checkDocs("rf_slope") val actual = df @@ -168,10 +242,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_slope(proj_raster, 1)") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.slope(btCellSize, 1d), actual) assertEqual(fullTile.slope(btCellSize, 1d).crop(subGridBounds), actual) } - it("should provide aspect") { + it("should perform aspect") { checkDocs("rf_aspect") val actual = df @@ -181,10 +264,19 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_aspect(proj_raster)") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.aspect(btCellSize), actual) assertEqual(fullTile.aspect(btCellSize).crop(subGridBounds), actual) } - it("should provide hillshade") { + it("should perform hillshade") { checkDocs("rf_hillshade") val actual = df @@ -194,6 +286,15 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { .get .tile + val actualExpr = + df + .selectExpr(s"rf_hillshade(proj_raster, 315, 45, 1)") + .as[Option[ProjectedRasterTile]] + .first() + .get + .tile + + assertEqual(actual, actualExpr) assertEqual(bt.mapTile(_.hillshade(btCellSize, 315, 45, 1)), actual) assertEqual(fullTile.hillshade(btCellSize, 315, 45, 1).crop(subGridBounds), actual) } From 07723dae8b6857bd218778cd8cb0dd731d08815c Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 29 Sep 2021 17:32:05 -0400 Subject: [PATCH 16/25] Update FocalOps Python bindings --- .../python/pyrasterframes/rasterfunctions.py | 81 ++++++++++--------- .../rasterframes/py/PyRFContext.scala | 11 +-- 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index fde68e855..dcc3ce156 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -65,16 +65,6 @@ def to_jvm(ct): elif isinstance(cell_type_arg, CellType): return to_jvm(cell_type_arg.cell_type_name) -def _parse_neighborhood(neighborhood_arg: str) -> JavaObject: - """ Convert the cell type representation to the expected JVM CellType object.""" - def to_jvm(n): - return _context_call('_parse_neighborhood', n) - - if isinstance(neighborhood_arg, str): - return to_jvm(neighborhood_arg) - else: - raise NotImplementedError - def rf_cell_types() -> List[CellType]: """Return a list of standard cell types""" return [CellType(str(ct)) for ct in _context_call('rf_cell_types')] @@ -790,54 +780,67 @@ def rf_identity(tile_col: Column_type) -> Column: """Pass tile through unchanged""" return _apply_column_function('rf_identity', tile_col) -def rf_focal_max(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_max(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the max value in its neighborhood of each cell""" - jfcn = RFContext.active().lookup('rf_focal_max') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_max', tile_col, neighborhood) -def rf_focal_mean(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_mean(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the mean value in its neighborhood of each cell""" - jfcn = RFContext.active().lookup('rf_focal_mean') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_mean', tile_col, neighborhood) -def rf_focal_median(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_median(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the max in its neighborhood value of each cell""" - jfcn = RFContext.active().lookup('rf_focal_median') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_median', tile_col, neighborhood) -def rf_focal_min(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_min(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the min value in its neighborhood of each cell""" - jfcn = RFContext.active().lookup('rf_focal_min') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_min', tile_col, neighborhood) -def rf_focal_mode(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_mode(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the mode value in its neighborhood of each cell""" - jfcn = RFContext.active().lookup('rf_focal_mode') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_mode', tile_col, neighborhood) -def rf_focal_std_dev(tile_col: Column_type, neighborhood: str) -> Column: +def rf_focal_std_dev(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: """Compute the standard deviation value in its neighborhood of each cell""" - jfcn = RFContext.active().lookup('rf_focal_std_dev') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_std_dev', tile_col, neighborhood) -def rf_moransI(tile_col: Column_type, neighborhood: str) -> Column: - """Compute the max in its neighborhood value of each cell""" - jfcn = RFContext.active().lookup('rf_focal_max') - return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) +def rf_moransI(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: + """Compute moransI in its neighborhood value of each cell""" + if isinstance(neighborhood, str): + neighborhood = lit(neighborhood) + return _apply_column_function('rf_focal_moransi', tile_col, neighborhood) def rf_aspect(tile_col: Column_type) -> Column: """Calculates the aspect of each cell in an elevation raster""" return _apply_column_function('rf_aspect', tile_col) -def rf_slope(tile_col: Column_type, z_factor: float) -> Column: - """Calculates the aspect of each cell in an elevation raster""" - jfcn = RFContext.active().lookup('rf_slope') - return Column(jfcn(_to_java_column(tile_col), float(z_factor))) +def rf_slope(tile_col: Column_type, z_factor: Union[float, Column_type]) -> Column: + """Calculates slope of each cell in an elevation raster""" + if isinstance(z_factor, float): + z_factor = lit(z_factor) + return _apply_column_function('rf_slope', tile_col, z_factor) -def rf_hillshade(tile_col: Column_type, azimuth: float, altitude: float, z_factor: float) -> Column: +def rf_hillshade(tile_col: Column_type, azimuth: Union[float, Column_type], altitude: Union[float, Column_type], z_factor: Union[float, Column_type]) -> Column: """Calculates the hillshade of each cell in an elevation raster""" - jfcn = RFContext.active().lookup('rf_hillshade') - return Column(jfcn(_to_java_column(tile_col), float(azimuth), float(altitude), float(z_factor))) + if isinstance(azimuth, float): + azimuth = lit(azimuth) + if isinstance(altitude, float): + altitude = lit(altitude) + if isinstance(z_factor, float): + z_factor = lit(z_factor) + return _apply_column_function('rf_hillshade', tile_col, azimuth, altitude, z_factor) def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala index 71ab8ca16..2e5bdd8f0 100644 --- a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -22,17 +22,17 @@ package org.locationtech.rasterframes.py import java.nio.ByteBuffer import geotrellis.proj4.CRS -import geotrellis.raster.{CellType, MultibandTile, Neighborhood} +import geotrellis.raster.{CellType, MultibandTile} import geotrellis.spark._ import geotrellis.layer._ import geotrellis.vector.Extent import org.apache.spark.sql._ import org.locationtech.rasterframes -import org.locationtech.rasterframes.util.{FocalNeighborhood, KryoSupport, ResampleMethod} +import org.locationtech.rasterframes.util.{KryoSupport, ResampleMethod} import org.locationtech.rasterframes.extensions.RasterJoin import org.locationtech.rasterframes.model.LazyCRS import org.locationtech.rasterframes.ref.{GDALRasterSource, RFRasterSource, RasterRef} -import org.locationtech.rasterframes.{RasterFunctions, _} +import org.locationtech.rasterframes._ import spray.json._ import org.locationtech.rasterframes.util.JsonCodecs._ @@ -145,11 +145,6 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions */ def _parse_cell_type(name: String): CellType = CellType.fromName(name) - def _parse_neighborhood(name: String): Neighborhood = name match { - case FocalNeighborhood(n) => n - case _ => throw new Exception(s"$name is an unsupported neighborhood") - } - /** * Convenience list of valid cell type strings * @return Java List of String, which py4j can interpret as a python `list` From b127329e54e9d385925d18551ff22b04da58b3db Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 29 Sep 2021 17:55:40 -0400 Subject: [PATCH 17/25] Add a missing copyright header --- .../focalops/FocalNeighborhoodOp.scala | 21 +++++++++++++++++++ .../expressions/focalops/Hillshade.scala | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala index 20db8bf6f..b73db0341 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -1,3 +1,24 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + package org.locationtech.rasterframes.expressions.focalops import com.typesafe.scalalogging.Logger diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala index 84c9580b0..256419435 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -60,7 +60,7 @@ case class Hillshade(first: Expression, second: Expression, third: Expression, f override def checkInputDataTypes(): TypeCheckResult = if (!tileExtractor.isDefinedAt(first.dataType)) TypeCheckFailure(s"Input type '${first.dataType}' does not conform to a raster type.") else if (!children.tail.forall(expr => numberArgExtractor.isDefinedAt(expr.dataType))) { - TypeCheckFailure(s"Input type '${second.dataType}' does not conform to a numeric type.") + TypeCheckFailure(s"Input type '${second.dataType}', '${third.dataType}' or '${fourth.dataType}' do not conform to a numeric type.") } else TypeCheckSuccess override protected def nullSafeEval(tileInput: Any, azimuthInput: Any, altitudeInput: Any, zFactorInput: Any): Any = { From a092d497f8da19001a698931b74651ee6bd8bee2 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 29 Sep 2021 22:51:41 -0400 Subject: [PATCH 18/25] Add integers support in py focal functions --- .../main/python/pyrasterframes/rasterfunctions.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index dcc3ce156..b9b67e247 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -826,19 +826,19 @@ def rf_aspect(tile_col: Column_type) -> Column: """Calculates the aspect of each cell in an elevation raster""" return _apply_column_function('rf_aspect', tile_col) -def rf_slope(tile_col: Column_type, z_factor: Union[float, Column_type]) -> Column: +def rf_slope(tile_col: Column_type, z_factor: Union[int, float, Column_type]) -> Column: """Calculates slope of each cell in an elevation raster""" - if isinstance(z_factor, float): + if isinstance(z_factor, (int, float)): z_factor = lit(z_factor) return _apply_column_function('rf_slope', tile_col, z_factor) -def rf_hillshade(tile_col: Column_type, azimuth: Union[float, Column_type], altitude: Union[float, Column_type], z_factor: Union[float, Column_type]) -> Column: +def rf_hillshade(tile_col: Column_type, azimuth: Union[int, float, Column_type], altitude: Union[int, float, Column_type], z_factor: Union[int, float, Column_type]) -> Column: """Calculates the hillshade of each cell in an elevation raster""" - if isinstance(azimuth, float): + if isinstance(azimuth, (int, float)): azimuth = lit(azimuth) - if isinstance(altitude, float): + if isinstance(altitude, (int, float)): altitude = lit(altitude) - if isinstance(z_factor, float): + if isinstance(z_factor, (int, float)): z_factor = lit(z_factor) return _apply_column_function('rf_hillshade', tile_col, azimuth, altitude, z_factor) From c406536f529cd7c099b6c96ecd114e3cefb1bc55 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Wed, 29 Sep 2021 23:27:17 -0400 Subject: [PATCH 19/25] Update FocalOperations.ipynb results --- .../src/main/notebooks/FocalOperations.ipynb | 103 +++++++++++++++--- 1 file changed, 85 insertions(+), 18 deletions(-) diff --git a/rf-notebook/src/main/notebooks/FocalOperations.ipynb b/rf-notebook/src/main/notebooks/FocalOperations.ipynb index 464fe45c4..9afffaa6c 100644 --- a/rf-notebook/src/main/notebooks/FocalOperations.ipynb +++ b/rf-notebook/src/main/notebooks/FocalOperations.ipynb @@ -33,7 +33,25 @@ "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by bash)\n", + "bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by bash)\n", + "WARNING: An illegal reflective access operation has occurred\n", + "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar) to constructor java.nio.DirectByteBuffer(long,int)\n", + "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n", + "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n", + "WARNING: All illegal access operations will be denied in a future release\n", + "21/09/30 03:19:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n" + ] + } + ], "source": [ "spark = create_rf_spark_session()" ] @@ -65,10 +83,20 @@ "metadata": {}, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ - "root\n |-- proj_raster_path: string (nullable = false)\n |-- proj_raster: struct (nullable = true)\n | |-- tile_context: struct (nullable = true)\n | | |-- extent: struct (nullable = false)\n | | | |-- xmin: double (nullable = false)\n | | | |-- ymin: double (nullable = false)\n | | | |-- xmax: double (nullable = false)\n | | | |-- ymax: double (nullable = false)\n | | |-- crs: struct (nullable = false)\n | | | |-- crsProj4: string (nullable = false)\n | |-- tile: tile (nullable = false)\n\n" + "root\n", + " |-- proj_raster_path: string (nullable = false)\n", + " |-- proj_raster: struct (nullable = true)\n", + " | |-- tile: tile (nullable = true)\n", + " | |-- extent: struct (nullable = true)\n", + " | | |-- xmin: double (nullable = false)\n", + " | | |-- ymin: double (nullable = false)\n", + " | | |-- xmax: double (nullable = false)\n", + " | | |-- ymax: double (nullable = false)\n", + " | |-- crs: crs (nullable = true)\n", + "\n" ] } ], @@ -89,14 +117,21 @@ "metadata": {}, "outputs": [ { - "output_type": "execute_result", + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { "data": { "text/plain": [ "81" ] }, + "execution_count": 5, "metadata": {}, - "execution_count": 5 + "output_type": "execute_result" } ], "source": [ @@ -104,31 +139,63 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Focal Operations\n", "Additional transformations are complished through use of column functions.\n", "The functions used here are mapped to their Scala implementation and applied per row.\n", "For each row the source elevation data is fetched only once before it's used as input." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { - "output_type": "execute_result", + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { "data": { - "text/plain": [ - "DataFrame[rf_crs(proj_raster): struct, rf_extent(proj_raster): struct, rf_aspect(proj_raster): struct,crs:struct>,tile:udt>, rf_slope(proj_raster): struct,crs:struct>,tile:udt>, rf_hillshade(proj_raster): struct,crs:struct>,tile:udt>]" + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Showing only top 5 rows
rf_crs(proj_raster)rf_extent(proj_raster)rf_aspect(proj_raster)rf_slope(proj_raster, 1)rf_hillshade(proj_raster, 315, 45, 1)
utm-CS{240929.2154, 4398599.0319, 256289.2154, 4401599.0319}
utm-CS{210209.2154, 4432319.0319, 225569.2154, 4447679.0319}
utm-CS{256289.2154, 4416959.0319, 271649.2154, 4432319.0319}
utm-CS{271649.2154, 4509119.0319, 287009.2154, 4524479.0319}
utm-CS{333089.2154, 4398599.0319, 341969.2154, 4401599.0319}
" ], - "text/html": "\n\n\n\n\n\n\n\n\n\n\n\n
Showing only top 5 rows
rf_crs(proj_raster)rf_extent(proj_raster)rf_aspect(proj_raster)rf_slope(proj_raster)rf_hillshade(proj_raster)
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][302369.2154, 4478399.0319, 317729.2154, 4493759.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4398599.0319, 287009.2154, 4401599.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][225569.2154, 4416959.0319, 240929.2154, 4432319.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4463039.0319, 287009.2154, 4478399.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][317729.2154, 4416959.0319, 333089.2154, 4432319.0319]
", - "text/markdown": "\n_Showing only top 5 rows_.\n\n| rf_crs(proj_raster) | rf_extent(proj_raster) | rf_aspect(proj_raster) | rf_slope(proj_raster) | rf_hillshade(proj_raster) |\n|---|---|---|---|---|\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[302369.2154, 4478399.0319, 317729.2154, 4493759.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4398599.0319, 287009.2154, 4401599.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[225569.2154, 4416959.0319, 240929.2154, 4432319.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4463039.0319, 287009.2154, 4478399.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[317729.2154, 4416959.0319, 333089.2154, 4432319.0319] | | | |" + "text/markdown": [ + "\n", + "_Showing only top 5 rows_.\n", + "\n", + "| rf_crs(proj_raster) | rf_extent(proj_raster) | rf_aspect(proj_raster) | rf_slope(proj_raster, 1) | rf_hillshade(proj_raster, 315, 45, 1) |\n", + "|---|---|---|---|---|\n", + "| utm-CS | {240929.2154, 4398599.0319, 256289.2154, 4401599.0319} | | | |\n", + "| utm-CS | {210209.2154, 4432319.0319, 225569.2154, 4447679.0319} | | | |\n", + "| utm-CS | {256289.2154, 4416959.0319, 271649.2154, 4432319.0319} | | | |\n", + "| utm-CS | {271649.2154, 4509119.0319, 287009.2154, 4524479.0319} | | | |\n", + "| utm-CS | {333089.2154, 4398599.0319, 341969.2154, 4401599.0319} | | | |" + ], + "text/plain": [ + "DataFrame[rf_crs(proj_raster): udt, rf_extent(proj_raster): struct, rf_aspect(proj_raster): struct,crs:udt>, rf_slope(proj_raster, 1): struct,crs:udt>, rf_hillshade(proj_raster, 315, 45, 1): struct,crs:udt>]" + ] }, + "execution_count": 6, "metadata": {}, - "execution_count": 7 + "output_type": "execute_result" } ], "source": [ @@ -150,7 +217,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -164,9 +231,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10-final" + "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} From d79a6edf22fe09a90aa1496938f060cb4197ec4e Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Thu, 30 Sep 2021 13:00:48 -0400 Subject: [PATCH 20/25] make extractBufferTile function total, FocalNeighborhood.fromString function signature change --- .../rasterframes/encoders/TypedEncoders.scala | 2 +- .../expressions/DynamicExtractors.scala | 2 +- .../expressions/focalops/package.scala | 1 + .../rasterframes/util/package.scala | 21 ++++++++++--------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala index 1fe15f15b..dff2453b2 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala @@ -34,7 +34,7 @@ trait TypedEncoders { implicit val uriInjection: Injection[URI, String] = Injection(_.toString, new URI(_)) implicit val uriTypedEncoder: TypedEncoder[URI] = TypedEncoder.usingInjection - implicit val neighborhoodInjection: Injection[Neighborhood, String] = Injection(FocalNeighborhood(_), FocalNeighborhood.unapply(_).get) + implicit val neighborhoodInjection: Injection[Neighborhood, String] = Injection(FocalNeighborhood(_), FocalNeighborhood.fromString(_).get) implicit val neighborhoodTypedEncoder: TypedEncoder[Neighborhood] = TypedEncoder.usingInjection implicit val envelopeTypedEncoder: TypedEncoder[Envelope] = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala index 743ee8e97..1dcd15ce6 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala @@ -227,7 +227,7 @@ object DynamicExtractors { } lazy val neighborhoodExtractor: PartialFunction[DataType, Any => Neighborhood] = { - case _: StringType => (v: Any) => FocalNeighborhood.unapply(v.asInstanceOf[UTF8String].toString).get + case _: StringType => (v: Any) => FocalNeighborhood.fromString(v.asInstanceOf[UTF8String].toString).get case n if n.conformsToSchema(neighborhoodEncoder.schema) => { case ir: InternalRow => ir.as[Neighborhood] } } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala index a1e47eaba..2221b4d68 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/package.scala @@ -15,5 +15,6 @@ package object focalops extends Serializable { // otherwise it is some tile case _ => prt.tile } + case _ => tile } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala index 0169d3473..2bcaa53a6 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala @@ -190,13 +190,13 @@ package object util extends DataFrameRenderers { import geotrellis.raster.Neighborhood import geotrellis.raster.mapalgebra.focal._ - // pattern matching and string interpolation work only since Scala 2.13 - def unapply(name: String): Option[Neighborhood] = + // pattern matching and string interpolation works only since Scala 2.13 + def fromString(name: String): Try[Neighborhood] = Try { name.toLowerCase().trim() match { - case s if s.startsWith("square-") => Try(Square(Integer.parseInt(s.split("square-").last))).toOption - case s if s.startsWith("circle-") => Try(Circle(java.lang.Double.parseDouble(s.split("circle-").last))).toOption - case s if s.startsWith("nesw-") => Try(Nesw(Integer.parseInt(s.split("nesw-").last))).toOption - case s if s.startsWith("wedge-") => Try { + case s if s.startsWith("square-") => Square(Integer.parseInt(s.split("square-").last)) + case s if s.startsWith("circle-") => Circle(java.lang.Double.parseDouble(s.split("circle-").last)) + case s if s.startsWith("nesw-") => Nesw(Integer.parseInt(s.split("nesw-").last)) + case s if s.startsWith("wedge-") => { val List(radius: Double, startAngle: Double, endAngle: Double) = s .split("wedge-") @@ -206,9 +206,9 @@ package object util extends DataFrameRenderers { .map(java.lang.Double.parseDouble) Wedge(radius, startAngle, endAngle) - }.toOption + } - case s if s.startsWith("annulus-") => Try { + case s if s.startsWith("annulus-") => { val List(innerRadius: Double, outerRadius: Double) = s .split("annulus-") @@ -218,9 +218,10 @@ package object util extends DataFrameRenderers { .map(java.lang.Double.parseDouble) Annulus(innerRadius, outerRadius) - }.toOption - case _ => None + } + case _ => throw new IllegalArgumentException(s"Unrecognized Neighborhood $name") } + } def apply(neighborhood: Neighborhood): String = { neighborhood match { From d24fd3a9b48e4316b5220e39c784dffef59cb101 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Thu, 30 Sep 2021 17:48:49 -0400 Subject: [PATCH 21/25] Add STAC API Python proxy --- .../main/python/pyrasterframes/__init__.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index 0be01bbf2..fc9f46b28 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -252,6 +252,24 @@ def temp_name(): .format("raster") \ .load(path, **options) +def _stac_api_reader( + df_reader: DataFrameReader, + uri: str, + filters: dict = None, + search_limit: Optional[int] = None) -> DataFrame: + """ + uri - STAC API uri + filters - a STAC API Search filters dict (bbox, datetime, intersects, collections, items, limit, query, next) + search_limit - search results convenient limit method + """ + import json + + return df_reader \ + .format("stac-api") \ + .option("uri", uri) \ + .option("search-filters", json.dumps(filters)) \ + .option("asset-limit", search_limit) \ + .load() def _geotiff_writer( df_writer: DataFrameWriter, @@ -305,3 +323,4 @@ def set_dims(parts): DataFrameReader.geotrellis = lambda df_reader, path: _layer_reader(df_reader, "geotrellis", path) DataFrameReader.geotrellis_catalog = lambda df_reader, path: _aliased_reader(df_reader, "geotrellis-catalog", path) DataFrameWriter.geotrellis = lambda df_writer, path: _aliased_writer(df_writer, "geotrellis", path) +DataFrameReader.stacapi = _stac_api_reader From 934fa0be4fcfc444c684db3c001ae99606562dcc Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Thu, 30 Sep 2021 19:11:16 -0400 Subject: [PATCH 22/25] Change the STAC API search limit handling --- .../org/locationtech/rasterframes/datasource/package.scala | 2 +- .../rasterframes/datasource/stac/api/StacApiDataSource.scala | 2 +- .../rasterframes/datasource/stac/api/StacApiTable.scala | 4 ++-- .../rasterframes/datasource/stac/api/package.scala | 2 +- pyrasterframes/src/main/python/pyrasterframes/__init__.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala index bfe4bfb3e..a671d8618 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/package.scala @@ -52,7 +52,7 @@ package object datasource { private[rasterframes] def intParam(key: String, parameters: CaseInsensitiveStringMap): Option[Int] = - if(parameters.containsKey(key)) parameters.get(key).toInt.some + if(parameters.containsKey(key)) Option(parameters.get(key)).map(_.toInt) else None private[rasterframes] diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala index bce9191be..3e4f2dc23 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala @@ -23,5 +23,5 @@ object StacApiDataSource { final val SHORT_NAME = "stac-api" final val URI_PARAM = "uri" final val SEARCH_FILTERS_PARAM = "search-filters" - final val ASSET_LIMIT_PARAM = "asset-limit" + final val SEARCH_LIMIT_PARAM = "search-limit" } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala index 0db7a34f2..aafab0eec 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala @@ -7,7 +7,7 @@ import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapabil import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.locationtech.rasterframes.datasource.stac.api.StacApiDataSource.{ASSET_LIMIT_PARAM, SEARCH_FILTERS_PARAM, URI_PARAM} +import org.locationtech.rasterframes.datasource.stac.api.StacApiDataSource.{SEARCH_LIMIT_PARAM, SEARCH_FILTERS_PARAM, URI_PARAM} import org.locationtech.rasterframes.datasource.{intParam, jsonParam, uriParam} import sttp.model.Uri @@ -36,6 +36,6 @@ object StacApiTable { .flatMap(_.as[SearchFilters].toOption) .getOrElse(SearchFilters(limit = NonNegInt.from(30).toOption)) - def searchLimit: Option[NonNegInt] = intParam(ASSET_LIMIT_PARAM, options).flatMap(NonNegInt.from(_).toOption) + def searchLimit: Option[NonNegInt] = intParam(SEARCH_LIMIT_PARAM, options).flatMap(NonNegInt.from(_).toOption) } } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala index b99515d38..e189aa46c 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala @@ -53,7 +53,7 @@ package object api { stacApi() .option(StacApiDataSource.URI_PARAM, uri) .option(StacApiDataSource.SEARCH_FILTERS_PARAM, filters.asJson.noSpaces) - .option(StacApiDataSource.ASSET_LIMIT_PARAM, searchLimit) + .option(StacApiDataSource.SEARCH_LIMIT_PARAM, searchLimit) ) } } diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index fc9f46b28..e8d2eb2af 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -268,7 +268,7 @@ def _stac_api_reader( .format("stac-api") \ .option("uri", uri) \ .option("search-filters", json.dumps(filters)) \ - .option("asset-limit", search_limit) \ + .option("search-limit", search_limit) \ .load() def _geotiff_writer( From fdd46cb178ba22e466e50ef036e7b98e1a59e5db Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Thu, 30 Sep 2021 20:37:04 -0400 Subject: [PATCH 23/25] Add STAC API Example jupyther notebook --- ...perations.ipynb => Focal Operations.ipynb} | 7 - .../src/main/notebooks/STAC API Example.ipynb | 477 ++++++++++++++++++ 2 files changed, 477 insertions(+), 7 deletions(-) rename rf-notebook/src/main/notebooks/{FocalOperations.ipynb => Focal Operations.ipynb} (99%) create mode 100644 rf-notebook/src/main/notebooks/STAC API Example.ipynb diff --git a/rf-notebook/src/main/notebooks/FocalOperations.ipynb b/rf-notebook/src/main/notebooks/Focal Operations.ipynb similarity index 99% rename from rf-notebook/src/main/notebooks/FocalOperations.ipynb rename to rf-notebook/src/main/notebooks/Focal Operations.ipynb index 9afffaa6c..262c685bf 100644 --- a/rf-notebook/src/main/notebooks/FocalOperations.ipynb +++ b/rf-notebook/src/main/notebooks/Focal Operations.ipynb @@ -206,13 +206,6 @@ " rf_slope(df.proj_raster, z_factor=1), \n", " rf_hillshade(df.proj_raster, azimuth=315, altitude=45, z_factor=1))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/rf-notebook/src/main/notebooks/STAC API Example.ipynb b/rf-notebook/src/main/notebooks/STAC API Example.ipynb new file mode 100644 index 000000000..4c54bf0aa --- /dev/null +++ b/rf-notebook/src/main/notebooks/STAC API Example.ipynb @@ -0,0 +1,477 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# STAC API with RasterFrames Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Spark Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "from pyrasterframes.utils import create_rf_spark_session\n", + "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", + "from pyrasterframes.rasterfunctions import *\n", + "import pyspark.sql.functions as F\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by bash)\n", + "bash: /opt/conda/lib/libtinfo.so.6: no version information available (required by bash)\n", + "WARNING: An illegal reflective access operation has occurred\n", + "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/spark-3.1.2-bin-hadoop3.2/jars/spark-unsafe_2.12-3.1.2.jar) to constructor java.nio.DirectByteBuffer(long,int)\n", + "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n", + "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n", + "WARNING: All illegal access operations will be denied in a future release\n", + "21/10/01 00:25:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n", + "Setting default log level to \"WARN\".\n", + "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n" + ] + } + ], + "source": [ + "spark = create_rf_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get a STAC API DataFrame\n", + "\n", + "Read a DataFrame that consists of STAC Items retrieved from the STAC API service." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# read assets from the landsat-8-l1-c1 collection\n", + "# due to the collection size and query parameters\n", + "# it makes sense to limit the amount of items retrieved from the STAC API\n", + "uri = 'https://earth-search.aws.element84.com/v0'\n", + "df = spark.read.stacapi(uri, {'collections': ['landsat-8-l1-c1']}, search_limit=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n", + " |-- id: string (nullable = true)\n", + " |-- stacVersion: string (nullable = true)\n", + " |-- stacExtensions: array (nullable = true)\n", + " | |-- element: string (containsNull = true)\n", + " |-- _type: string (nullable = true)\n", + " |-- geometry: geometry (nullable = true)\n", + " |-- bbox: struct (nullable = true)\n", + " | |-- xmin: double (nullable = true)\n", + " | |-- ymin: double (nullable = true)\n", + " | |-- xmax: double (nullable = true)\n", + " | |-- ymax: double (nullable = true)\n", + " |-- links: array (nullable = true)\n", + " | |-- element: struct (containsNull = true)\n", + " | | |-- href: string (nullable = true)\n", + " | | |-- rel: string (nullable = true)\n", + " | | |-- _type: string (nullable = true)\n", + " | | |-- title: string (nullable = true)\n", + " | | |-- extensionFields: string (nullable = true)\n", + " |-- assets: map (nullable = true)\n", + " | |-- key: string\n", + " | |-- value: struct (valueContainsNull = true)\n", + " | | |-- href: string (nullable = true)\n", + " | | |-- title: string (nullable = true)\n", + " | | |-- description: string (nullable = true)\n", + " | | |-- roles: array (nullable = true)\n", + " | | | |-- element: string (containsNull = true)\n", + " | | |-- _type: string (nullable = true)\n", + " | | |-- extensionFields: string (nullable = true)\n", + " |-- collection: string (nullable = true)\n", + " |-- properties: struct (nullable = true)\n", + " | |-- datetime: struct (nullable = true)\n", + " | | |-- datetime: timestamp (nullable = true)\n", + " | | |-- start: timestamp (nullable = true)\n", + " | | |-- end: timestamp (nullable = true)\n", + " | | |-- _type: string (nullable = true)\n", + " | |-- title: string (nullable = true)\n", + " | |-- description: string (nullable = true)\n", + " | |-- created: timestamp (nullable = true)\n", + " | |-- updated: timestamp (nullable = true)\n", + " | |-- license: string (nullable = true)\n", + " | |-- providers: struct (nullable = true)\n", + " | | |-- head: struct (nullable = true)\n", + " | | | |-- name: string (nullable = true)\n", + " | | | |-- description: string (nullable = true)\n", + " | | | |-- roles: array (nullable = true)\n", + " | | | | |-- element: string (containsNull = true)\n", + " | | | |-- url: string (nullable = true)\n", + " | | |-- tail: array (nullable = true)\n", + " | | | |-- element: struct (containsNull = true)\n", + " | | | | |-- name: string (nullable = true)\n", + " | | | | |-- description: string (nullable = true)\n", + " | | | | |-- roles: array (nullable = true)\n", + " | | | | | |-- element: string (containsNull = true)\n", + " | | | | |-- url: string (nullable = true)\n", + " | |-- platform: string (nullable = true)\n", + " | |-- instruments: struct (nullable = true)\n", + " | | |-- head: string (nullable = true)\n", + " | | |-- tail: array (nullable = true)\n", + " | | | |-- element: string (containsNull = true)\n", + " | |-- constellation: string (nullable = true)\n", + " | |-- mission: string (nullable = true)\n", + " | |-- gsd: double (nullable = true)\n", + " | |-- extensionFields: string (nullable = true)\n", + "\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Each item in the DataFrame represents the entire STAC Item." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "100" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Showing only top 5 rows
idcollectiongeometry
LC08_L1TP_232093_20210716_20210717_01_T1landsat-8-l1-c1POLYGON ((-74.64766964714028 -46.3435154...
LC08_L1TP_232092_20210716_20210717_01_T1landsat-8-l1-c1POLYGON ((-74.07682865409966 -44.9166888...
LC08_L1TP_232091_20210716_20210717_01_T1landsat-8-l1-c1POLYGON ((-73.54155930424828 -43.4885910...
LC08_L1TP_232090_20210716_20210717_01_T1landsat-8-l1-c1POLYGON ((-73.02667875381594 -42.0589406...
LC08_L1TP_232089_20210716_20210717_01_T1landsat-8-l1-c1POLYGON ((-72.67424121162182 -40.6804236...
" + ], + "text/markdown": [ + "\n", + "_Showing only top 5 rows_.\n", + "\n", + "| id | collection | geometry |\n", + "|---|---|---|\n", + "| LC08_L1TP_232093_20210716_20210717_01_T1 | landsat-8-l1-c1 | POLYGON ((-74.64766964714028 -46.3435154... |\n", + "| LC08_L1TP_232092_20210716_20210717_01_T1 | landsat-8-l1-c1 | POLYGON ((-74.07682865409966 -44.9166888... |\n", + "| LC08_L1TP_232091_20210716_20210717_01_T1 | landsat-8-l1-c1 | POLYGON ((-73.54155930424828 -43.4885910... |\n", + "| LC08_L1TP_232090_20210716_20210717_01_T1 | landsat-8-l1-c1 | POLYGON ((-73.02667875381594 -42.0589406... |\n", + "| LC08_L1TP_232089_20210716_20210717_01_T1 | landsat-8-l1-c1 | POLYGON ((-72.67424121162182 -40.6804236... |" + ], + "text/plain": [ + "DataFrame[id: string, collection: string, geometry: udt]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.select(df.id, df.collection, df.geometry)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To read rasters we don't need STAC Items, but we need STAC Item Assets.\n", + "Each STAC Item in the DataFrame can contain more than a single asset => to covert such STAC Item DataFrame into the STAC Item Assets DataFrame we need to explode the assets column. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# select the first Landsat STAC Item\n", + "# explode its assets \n", + "# select blue, red, green, and nir assets only\n", + "# name each asset link as the band column\n", + "assets = df \\\n", + " .limit(1) \\\n", + " .select(df.id, F.explode(df.assets)) \\\n", + " .filter(F.col(\"key\").isin([\"B2\", \"B3\", \"B4\", \"B5\"])) \\\n", + " .select(F.col(\"value.href\").alias(\"band\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n", + " |-- band: string (nullable = true)\n", + "\n" + ] + } + ], + "source": [ + "assets.printSchema()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "4" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "assets.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# read rasters from the exploded STAC Assets DataFrame\n", + "# select only the blue asset to speed up notebook\n", + "rs = spark.read.raster(assets.limit(1), tile_dimensions=(512, 512), buffer_size=2, catalog_col_names=[\"band\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/plain": [ + "256" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rs.count()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "root\n", + " |-- band_path: string (nullable = false)\n", + " |-- band: struct (nullable = true)\n", + " | |-- tile: tile (nullable = true)\n", + " | |-- extent: struct (nullable = true)\n", + " | | |-- xmin: double (nullable = false)\n", + " | | |-- ymin: double (nullable = false)\n", + " | | |-- xmax: double (nullable = false)\n", + " | | |-- ymax: double (nullable = false)\n", + " | |-- crs: crs (nullable = true)\n", + "\n" + ] + } + ], + "source": [ + "rs.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Focal Operations\n", + "Additional transformations are complished through use of column functions.\n", + "The functions used here are mapped to their Scala implementation and applied per row.\n", + "For each row the source elevation data is fetched only once before it's used as input." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Showing only top 5 rows
rf_crs(band)rf_extent(band)rf_aspect(band)rf_slope(band, 1)rf_hillshade(band, 315, 45, 1)
utm-CS{488445.0, -5335365.0, 503805.0, -5320005.0}
utm-CS{657405.0, -5335365.0, 672765.0, -5320005.0}
utm-CS{688125.0, -5335365.0, 703485.0, -5320005.0}
utm-CS{642045.0, -5197125.0, 657405.0, -5181765.0}
utm-CS{549885.0, -5366085.0, 565245.0, -5350725.0}
" + ], + "text/markdown": [ + "\n", + "_Showing only top 5 rows_.\n", + "\n", + "| rf_crs(band) | rf_extent(band) | rf_aspect(band) | rf_slope(band, 1) | rf_hillshade(band, 315, 45, 1) |\n", + "|---|---|---|---|---|\n", + "| utm-CS | {488445.0, -5335365.0, 503805.0, -5320005.0} | | | |\n", + "| utm-CS | {657405.0, -5335365.0, 672765.0, -5320005.0} | | | |\n", + "| utm-CS | {688125.0, -5335365.0, 703485.0, -5320005.0} | | | |\n", + "| utm-CS | {642045.0, -5197125.0, 657405.0, -5181765.0} | | | |\n", + "| utm-CS | {549885.0, -5366085.0, 565245.0, -5350725.0} | | | |" + ], + "text/plain": [ + "DataFrame[rf_crs(band): udt, rf_extent(band): struct, rf_aspect(band): struct,crs:udt>, rf_slope(band, 1): struct,crs:udt>, rf_hillshade(band, 315, 45, 1): struct,crs:udt>]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rs.select(\n", + " rf_crs(rs.band), \n", + " rf_extent(rs.band), \n", + " rf_aspect(rs.band), \n", + " rf_slope(rs.band, z_factor=1), \n", + " rf_hillshade(rs.band, azimuth=315, altitude=45, z_factor=1))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 1a743382453c4f7e0c373e47ae914e5475180007 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Fri, 1 Oct 2021 15:29:47 -0400 Subject: [PATCH 24/25] Remove STACDataFrame searchLimit parameter --- .../stac/api/StacApiDataSource.scala | 3 +- .../stac/api/StacApiPartition.scala | 26 ++++++--------- .../stac/api/StacApiScanBuilder.scala | 8 ++--- .../datasource/stac/api/StacApiTable.scala | 8 ++--- .../datasource/stac/api/package.scala | 30 ++++++++++++++--- .../stac/api/StacApiDataSourceTest.scala | 32 +++++++++++-------- .../main/python/pyrasterframes/__init__.py | 5 +-- .../src/main/notebooks/STAC API Example.ipynb | 2 +- 8 files changed, 63 insertions(+), 51 deletions(-) diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala index 3e4f2dc23..47772072a 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSource.scala @@ -16,12 +16,11 @@ class StacApiDataSource extends TableProvider with DataSourceRegister { def getTable(structType: StructType, transforms: Array[Transform], map: util.Map[String, String]): Table = new StacApiTable() - override def shortName(): String = "stac-api" + def shortName(): String = StacApiDataSource.SHORT_NAME } object StacApiDataSource { final val SHORT_NAME = "stac-api" final val URI_PARAM = "uri" final val SEARCH_FILTERS_PARAM = "search-filters" - final val SEARCH_LIMIT_PARAM = "search-limit" } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala index a11f85b8c..41842cab1 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala @@ -7,13 +7,12 @@ import com.azavea.stac4s.StacItem import geotrellis.store.util.BlockingThreadPool import sttp.client3.asynchttpclient.cats.AsyncHttpClientCatsBackend import com.azavea.stac4s.api.client._ -import eu.timepit.refined.types.numeric.NonNegInt import cats.effect.IO import sttp.model.Uri import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory} -case class StacApiPartition(uri: Uri, searchFilters: SearchFilters, searchLimit: Option[NonNegInt]) extends InputPartition +case class StacApiPartition(uri: Uri, searchFilters: SearchFilters) extends InputPartition class StacApiPartitionReaderFactory extends PartitionReaderFactory { override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { @@ -25,24 +24,17 @@ class StacApiPartitionReaderFactory extends PartitionReaderFactory { } class StacApiPartitionReader(partition: StacApiPartition) extends PartitionReader[InternalRow] { - lazy val partitionValues: Iterator[StacItem] = { - implicit val cs = IO.contextShift(BlockingThreadPool.executionContext) - AsyncHttpClientCatsBackend - .resource[IO]() - .use { backend => - SttpStacClient(backend, partition.uri) - .search(partition.searchFilters) - .take(partition.searchLimit.map(_.value)) - .compile - .toList - } - .map(_.toIterator) - .unsafeRunSync() - } + + @transient private implicit lazy val cs = IO.contextShift(BlockingThreadPool.executionContext) + @transient private lazy val backend = AsyncHttpClientCatsBackend[IO]().unsafeRunSync() + @transient private lazy val partitionValues: Iterator[StacItem] = + SttpStacClient(backend, partition.uri) + .search(partition.searchFilters) + .toIterator(_.unsafeRunSync()) def next: Boolean = partitionValues.hasNext def get: InternalRow = partitionValues.next.toInternalRow - def close(): Unit = { } + def close(): Unit = backend.close().unsafeRunSync() } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala index 30ed8c8fa..a7886f81e 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala @@ -8,12 +8,12 @@ import org.apache.spark.sql.connector.read.{Batch, InputPartition, PartitionRead import org.apache.spark.sql.types.StructType import sttp.model.Uri -class StacApiScanBuilder(uri: Uri, searchFilters: SearchFilters, searchLimit: Option[NonNegInt]) extends ScanBuilder { - override def build(): Scan = new StacApiBatchScan(uri, searchFilters, searchLimit) +class StacApiScanBuilder(uri: Uri, searchFilters: SearchFilters) extends ScanBuilder { + def build(): Scan = new StacApiBatchScan(uri, searchFilters) } /** Batch Reading Support. The schema is repeated here as it can change after column pruning, etc. */ -class StacApiBatchScan(uri: Uri, searchFilters: SearchFilters, searchLimit: Option[NonNegInt]) extends Scan with Batch { +class StacApiBatchScan(uri: Uri, searchFilters: SearchFilters) extends Scan with Batch { def readSchema(): StructType = stacItemEncoder.schema override def toBatch: Batch = this @@ -23,6 +23,6 @@ class StacApiBatchScan(uri: Uri, searchFilters: SearchFilters, searchLimit: Opti * To perform a distributed load, we'd need to know some internals about how the next page token is computed. * This can be a good idea for the STAC Spec extension. * */ - def planInputPartitions(): Array[InputPartition] = Array(StacApiPartition(uri, searchFilters, searchLimit)) + def planInputPartitions(): Array[InputPartition] = Array(StacApiPartition(uri, searchFilters)) def createReaderFactory(): PartitionReaderFactory = new StacApiPartitionReaderFactory() } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala index aafab0eec..fe6a2e5e0 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiTable.scala @@ -7,8 +7,8 @@ import org.apache.spark.sql.connector.catalog.{SupportsRead, Table, TableCapabil import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.locationtech.rasterframes.datasource.stac.api.StacApiDataSource.{SEARCH_LIMIT_PARAM, SEARCH_FILTERS_PARAM, URI_PARAM} -import org.locationtech.rasterframes.datasource.{intParam, jsonParam, uriParam} +import org.locationtech.rasterframes.datasource.stac.api.StacApiDataSource.{SEARCH_FILTERS_PARAM, URI_PARAM} +import org.locationtech.rasterframes.datasource.{jsonParam, uriParam} import sttp.model.Uri import scala.collection.JavaConverters._ @@ -24,7 +24,7 @@ class StacApiTable extends Table with SupportsRead { def capabilities(): util.Set[TableCapability] = Set(TableCapability.BATCH_READ).asJava def newScanBuilder(options: CaseInsensitiveStringMap): ScanBuilder = - new StacApiScanBuilder(options.uri, options.searchFilters, options.searchLimit) + new StacApiScanBuilder(options.uri, options.searchFilters) } object StacApiTable { @@ -35,7 +35,5 @@ object StacApiTable { jsonParam(SEARCH_FILTERS_PARAM, options) .flatMap(_.as[SearchFilters].toOption) .getOrElse(SearchFilters(limit = NonNegInt.from(30).toOption)) - - def searchLimit: Option[NonNegInt] = intParam(SEARCH_LIMIT_PARAM, options).flatMap(NonNegInt.from(_).toOption) } } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala index e189aa46c..d2834f963 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/package.scala @@ -1,9 +1,11 @@ package org.locationtech.rasterframes.datasource.stac +import cats.Monad +import cats.syntax.functor._ import com.azavea.stac4s.api.client.SearchFilters import org.apache.spark.sql.{DataFrame, DataFrameReader} import io.circe.syntax._ -import fs2.Stream +import fs2.{Pull, Stream} import shapeless.tag import shapeless.tag.@@ import org.apache.spark.sql.SparkSession @@ -17,6 +19,7 @@ package object api { implicit class StacApiDataFrameReaderOps(val reader: StacApiDataFrameReader) extends AnyVal { def loadStac: StacApiDataFrame = tag[StacApiDataFrameTag][DataFrame](reader.load) + def loadStac(limit: Int): StacApiDataFrame = tag[StacApiDataFrameTag][DataFrame](reader.load.limit(limit)) } implicit class StacApiDataFrameOps(val df: StacApiDataFrame) extends AnyVal { @@ -38,7 +41,27 @@ package object api { } implicit class Fs2StreamOps[F[_], T](val self: Stream[F, T]) { - def take(n: Option[Int]): Stream[F, T] = n.fold(self)(self.take(_)) + /** Unsafe API to interop with the Spark API. */ + def toIterator(run: F[Option[(T, fs2.Stream[F, T])]] => Option[(T, fs2.Stream[F, T])]) + (implicit monad: Monad[F], compiler: Stream.Compiler[F, F]): Iterator[T] = new Iterator[T] { + private var head = self + private def nextF: F[Option[(T, fs2.Stream[F, T])]] = + head + .pull.uncons1 + .flatMap(Pull.output1) + .stream + .compile + .last + .map(_.flatten) + + def hasNext(): Boolean = run(nextF).nonEmpty + + def next(): T = { + val (item, tail) = run(nextF).get + this.head = tail + item + } + } } implicit class DataFrameReaderOps(val self: DataFrameReader) extends AnyVal { @@ -48,12 +71,11 @@ package object api { implicit class DataFrameReaderStacApiOps(val reader: DataFrameReader) extends AnyVal { def stacApi(): StacApiDataFrameReader = tag[StacApiDataFrameTag][DataFrameReader](reader.format(StacApiDataSource.SHORT_NAME)) - def stacApi(uri: String, filters: SearchFilters = SearchFilters(), searchLimit: Option[Int] = None): StacApiDataFrameReader = + def stacApi(uri: String, filters: SearchFilters = SearchFilters()): StacApiDataFrameReader = tag[StacApiDataFrameTag][DataFrameReader]( stacApi() .option(StacApiDataSource.URI_PARAM, uri) .option(StacApiDataSource.SEARCH_FILTERS_PARAM, filters.asJson.noSpaces) - .option(StacApiDataSource.SEARCH_LIMIT_PARAM, searchLimit) ) } } diff --git a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSourceTest.scala b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSourceTest.scala index a778b5db9..93e1d0446 100644 --- a/datasource/src/test/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSourceTest.scala +++ b/datasource/src/test/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiDataSourceTest.scala @@ -25,9 +25,7 @@ import org.locationtech.rasterframes.datasource.raster._ import org.locationtech.rasterframes.datasource.stac.api.encoders._ import com.azavea.stac4s.StacItem import com.azavea.stac4s.api.client.{SearchFilters, SttpStacClient} -import cats.syntax.option._ import cats.effect.IO -import eu.timepit.refined.auto._ import geotrellis.store.util.BlockingThreadPool import org.apache.spark.sql.functions.explode import org.locationtech.rasterframes.TestEnvironment @@ -45,9 +43,10 @@ class StacApiDataSourceTest extends TestEnvironment { self => .read .stacApi( "https://franklin.nasa-hsi.azavea.com/", - filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")), - searchLimit = Some(1) - ).load + filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")) + ) + .load + .limit(1) results.rdd.partitions.length shouldBe 1 results.count() shouldBe 1L @@ -78,9 +77,10 @@ class StacApiDataSourceTest extends TestEnvironment { self => .read .stacApi( "https://franklin.nasa-hsi.azavea.com/", - filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")), - searchLimit = Some(1) - ).load + filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")) + ) + .load + .limit(1) results.rdd.partitions.length shouldBe 1 @@ -118,10 +118,9 @@ class StacApiDataSourceTest extends TestEnvironment { self => .read .stacApi( "https://franklin.nasa-hsi.azavea.com/", - filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")), - searchLimit = Some(1) + filters = SearchFilters(items = List("aviris-l1-cogs_f130329t01p00r06_sc01")) ) - .loadStac + .loadStac(limit = 1) // to preserve the STAC DataFrame type val assets = items @@ -149,7 +148,7 @@ class StacApiDataSourceTest extends TestEnvironment { self => it("should read from Astraea Earth service") { import spark.implicits._ - val results = spark.read.stacApi("https://eod-catalog-svc-prod.astraea.earth/", searchLimit = Some(1)).load + val results = spark.read.stacApi("https://eod-catalog-svc-prod.astraea.earth/").load.limit(1) // results.printSchema() @@ -178,8 +177,9 @@ class StacApiDataSourceTest extends TestEnvironment { self => val items = spark .read - .stacApi("https://eod-catalog-svc-prod.astraea.earth/", searchLimit = 1.some) + .stacApi("https://eod-catalog-svc-prod.astraea.earth/") .load + .limit(1) println(items.collect().toList.length) @@ -199,7 +199,11 @@ class StacApiDataSourceTest extends TestEnvironment { self => ignore("should fetch rasters from the Datacube STAC API service") { import spark.implicits._ - val items = spark.read.stacApi("https://datacube.services.geo.ca/api", filters = SearchFilters(collections=List("markham")), searchLimit = Some(1)).load + val items = spark + .read + .stacApi("https://datacube.services.geo.ca/api", filters = SearchFilters(collections=List("markham"))) + .load + .limit(1) println(items.collect().toList.length) diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index e8d2eb2af..fed59d28e 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -255,12 +255,10 @@ def temp_name(): def _stac_api_reader( df_reader: DataFrameReader, uri: str, - filters: dict = None, - search_limit: Optional[int] = None) -> DataFrame: + filters: dict = None) -> DataFrame: """ uri - STAC API uri filters - a STAC API Search filters dict (bbox, datetime, intersects, collections, items, limit, query, next) - search_limit - search results convenient limit method """ import json @@ -268,7 +266,6 @@ def _stac_api_reader( .format("stac-api") \ .option("uri", uri) \ .option("search-filters", json.dumps(filters)) \ - .option("search-limit", search_limit) \ .load() def _geotiff_writer( diff --git a/rf-notebook/src/main/notebooks/STAC API Example.ipynb b/rf-notebook/src/main/notebooks/STAC API Example.ipynb index 4c54bf0aa..3e5cf4e47 100644 --- a/rf-notebook/src/main/notebooks/STAC API Example.ipynb +++ b/rf-notebook/src/main/notebooks/STAC API Example.ipynb @@ -75,7 +75,7 @@ "# due to the collection size and query parameters\n", "# it makes sense to limit the amount of items retrieved from the STAC API\n", "uri = 'https://earth-search.aws.element84.com/v0'\n", - "df = spark.read.stacapi(uri, {'collections': ['landsat-8-l1-c1']}, search_limit=100)" + "df = spark.read.stacapi(uri, {'collections': ['landsat-8-l1-c1']}).limit(100)" ] }, { From c7d8f4874d2283fa5a10bbc7411a33d2c365b368 Mon Sep 17 00:00:00 2001 From: Grigory Pomadchin Date: Fri, 1 Oct 2021 15:47:33 -0400 Subject: [PATCH 25/25] Update release notes and fix pydocs --- docs/src/main/paradox/release-notes.md | 8 ++++++++ pyrasterframes/src/main/python/pyrasterframes/__init__.py | 4 ++-- rf-notebook/src/main/docker/Dockerfile | 3 +-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/src/main/paradox/release-notes.md b/docs/src/main/paradox/release-notes.md index 9d738c6ad..6feed51b6 100644 --- a/docs/src/main/paradox/release-notes.md +++ b/docs/src/main/paradox/release-notes.md @@ -1,5 +1,13 @@ # Release Notes +## 0.10.x + +### 0.10.0 + +* Upgraded to Spark 3.1.2, Scala 2.12 and GeoTrellis 3.6.0 +* Added FocalOperations support +* Added STAC API DataFrames implementation + ## 0.9.x ### 0.9.1 diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index fed59d28e..65b0eaed4 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -257,8 +257,8 @@ def _stac_api_reader( uri: str, filters: dict = None) -> DataFrame: """ - uri - STAC API uri - filters - a STAC API Search filters dict (bbox, datetime, intersects, collections, items, limit, query, next) + :param uri: STAC API uri + :param filters: STAC API Search filters dict (bbox, datetime, intersects, collections, items, limit, query, next), see the STAC API Spec for more details https://github.com/radiantearth/stac-api-spec """ import json diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile index e207613ca..f00dc5acb 100644 --- a/rf-notebook/src/main/docker/Dockerfile +++ b/rf-notebook/src/main/docker/Dockerfile @@ -1,5 +1,4 @@ -# jupyter/scipy-notebook isn't semantically versioned. -# We pick this arbitrary one from Sept 2019 because it's what latest was on Oct 17 2019. +# Python version compatible with Spark and GDAL 3.1.2 FROM jupyter/scipy-notebook:python-3.8.8 LABEL maintainer="Astraea, Inc. "