diff --git a/.circleci/config.yml b/.circleci/config.yml index aafc628f2..020f6d9f7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ orbs: - image: s22s/circleci-openjdk-conda-gdal:b8e30ee working_directory: ~/repo environment: - SBT_OPTS: "-Xms64m -Xmx1536m -Djava.awt.headless=true -Dsun.io.serialization.extendedDebugInfo=true" + SBT_OPTS: "-Xms32M -Xmx2G -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp" commands: setup: description: Setup for sbt build diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 9e205bc5f..e4406498b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -128,4 +128,4 @@ jobs: uses: actions/upload-artifact@v2 with: name: rf-site - path: docs/target/site \ No newline at end of file + path: docs/target/site diff --git a/.gitignore b/.gitignore index 838c6abec..bb5b4c3b8 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ Thumbs.db *.log # sbt specific + .bsp .cache .history @@ -19,6 +20,7 @@ project/boot/ project/plugins/project/ # Scala-IDE specific + .scala_dependencies .worksheet .idea @@ -34,3 +36,14 @@ scoverage-report* zz-* rf-notebook/src/main/notebooks/.ipython + +# VSCode files + +.vscode +.history + +# Metals + +.metals +.bloop +metals.sbt diff --git a/core/src/main/scala/geotrellis/raster/BufferTile.scala b/core/src/main/scala/geotrellis/raster/BufferTile.scala deleted file mode 100644 index a6a473a22..000000000 --- a/core/src/main/scala/geotrellis/raster/BufferTile.scala +++ /dev/null @@ -1,411 +0,0 @@ -/* - * Copyright 2021 Azavea - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package geotrellis.raster - -import geotrellis.raster.mapalgebra.focal.BufferedFocalMethods -import spire.syntax.cfor._ - -/** - * When combined with another BufferTile the two tiles will be aligned on (0, 0) pixel of tile center. - * The operation will be carried over all overlapping pixels. - * For instance: combining a tile padded with 5 pixels on all sides with tile padded with 3 pixels on all sides will - * result in buffer tile with 3 pixel padding on all sides. - * - * When combined with another BufferTile the operation will be executed over the maximum shared in - * - * TODO: - * - What should .map do? Map the buffer pixels or not? - * - toString method is friendly - * - mutable version makes sense - * - toBytes needs to encode padding size? - */ -case class BufferTile( - sourceTile: Tile, - gridBounds: GridBounds[Int] -) extends Tile { - require( - gridBounds.colMin >=0 && gridBounds.rowMin >= 0 && gridBounds.colMax < sourceTile.cols && gridBounds.rowMax < sourceTile.rows, - s"Tile center bounds $gridBounds exceed underlying tile dimensions ${sourceTile.dimensions}" - ) - - val cols: Int = gridBounds.width - val rows: Int = gridBounds.height - - val cellType: CellType = sourceTile.cellType - - private def colMin: Int = gridBounds.colMin - private def rowMin: Int = gridBounds.rowMin - private def sourceCols: Int = sourceTile.cols - private def sourceRows: Int = sourceTile.rows - - def bufferTop: Int = gridBounds.rowMin - def bufferLeft: Int = gridBounds.colMin - def bufferRight: Int = sourceTile.cols - gridBounds.colMin - gridBounds.colMax - def bufferBottom: Int = sourceTile.rows - gridBounds.rowMin - gridBounds.rowMax - - /** - * Returns a [[Tile]] equivalent to this tile, except with cells of - * the given type. - * - * @param targetCellType The type of cells that the result should have - * @return The new Tile - */ - def convert(targetCellType: CellType): Tile = - mutable(targetCellType) - - def withNoData(noDataValue: Option[Double]): BufferTile = - BufferTile(sourceTile.withNoData(noDataValue), gridBounds) - - def interpretAs(newCellType: CellType): BufferTile = - BufferTile(sourceTile.interpretAs(newCellType), gridBounds) - - /** - * Fetch the datum at the given column and row of the tile. - * - * @param col The column - * @param row The row - * @return The Int datum found at the given location - */ - def get(col: Int, row: Int): Int = { - val c = col + colMin - val r = row + rowMin - if(c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) { - throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") - } else { - sourceTile.get(c, r) - } - } - - /** - * Fetch the datum at the given column and row of the tile. - * - * @param col The column - * @param row The row - * @return The Double datum found at the given location - */ - def getDouble(col: Int, row: Int): Double = { - val c = col + colMin - val r = row + rowMin - - if(c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) { - throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") - } else { - sourceTile.getDouble(col + gridBounds.colMin, row + gridBounds.rowMin) - } - } - - /** - * Another name for the 'mutable' method on this class. - * - * @return An [[ArrayTile]] - */ - def toArrayTile: ArrayTile = mutable - - /** - * Return the [[MutableArrayTile]] equivalent of this tile. - * - * @return An MutableArrayTile - */ - def mutable(): MutableArrayTile = - mutable(cellType) - - /** - * Return the [[MutableArrayTile]] equivalent of this tile. - * - * @return An MutableArrayTile - */ - def mutable(targetCellType: CellType): MutableArrayTile = { - val tile = ArrayTile.alloc(targetCellType, cols, rows) - - if(!cellType.isFloatingPoint) { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - tile.set(col, row, get(col, row)) - } - } - } else { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - tile.setDouble(col, row, getDouble(col, row)) - } - } - } - - tile - } - - /** - * Return the data behind this tile as an array of integers. - * - * @return The copy as an Array[Int] - */ - def toArray: Array[Int] = { - val arr = Array.ofDim[Int](cols * rows) - - var i = 0 - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - arr(i) = get(col, row) - i += 1 - } - } - - arr - } - - /** - * Return the data behind this tile as an array of doubles. - * - * @return The copy as an Array[Int] - */ - def toArrayDouble: Array[Double] = { - val arr = Array.ofDim[Double](cols * rows) - - var i = 0 - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - arr(i) = getDouble(col, row) - i += 1 - } - } - - arr - } - - /** - * Return the underlying data behind this tile as an array. - * - * @return An array of bytes - */ - def toBytes(): Array[Byte] = toArrayTile.toBytes - - /** - * Execute a function on each cell of the tile. The function - * returns Unit, so it presumably produces side-effects. - * - * @param f A function from Int to Unit - */ - def foreach(f: Int => Unit): Unit = { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - f(get(col, row)) - } - } - } - - /** - * Execute a function on each cell of the tile. The function - * returns Unit, so it presumably produces side-effects. - * - * @param f A function from Double to Unit - */ - def foreachDouble(f: Double => Unit): Unit = { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - f(getDouble(col, row)) - } - } - } - - /** - * Execute an [[IntTileVisitor]] at each cell of the present tile. - * - * @param visitor An IntTileVisitor - */ - def foreachIntVisitor(visitor: IntTileVisitor): Unit = { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - visitor(col, row, get(col, row)) - } - } - } - - /** - * Execute an [[DoubleTileVisitor]] at each cell of the present tile. - * - * @param visitor An DoubleTileVisitor - */ - def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = { - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - visitor(col, row, getDouble(col, row)) - } - } - } - - /** - * Map each cell in the given tile to a new one, using the given - * function. - * - * @param f A function from Int to Int, executed at each point of the tile - * @return The result, a [[Tile]] - */ - def map(f: Int => Int): BufferTile = mapTile(_.map(f)) - - /** - * Map each cell in the given tile to a new one, using the given - * function. - * - * @param f A function from Double to Double, executed at each point of the tile - * @return The result, a [[Tile]] - */ - def mapDouble(f: Double => Double): BufferTile = mapTile(_.mapDouble(f)) - - /** - * Map an [[IntTileMapper]] over the present tile. - * - * @param mapper The mapper - * @return The result, a [[Tile]] - */ - def mapIntMapper(mapper: IntTileMapper): BufferTile = mapTile(_.mapIntMapper(mapper)) - - /** - * Map an [[DoubleTileMapper]] over the present tile. - * - * @param mapper The mapper - * @return The result, a [[Tile]] - */ - def mapDoubleMapper(mapper: DoubleTileMapper): Tile = mapTile(_.mapDoubleMapper(mapper)) - - private def combine(other: BufferTile)(f: (Int, Int) => Int): Tile = { - if((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) { - throw new GeoAttrsError("Cannot combine rasters with different dimensions: " + - s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") - } - - val bufferTop = math.min(this.bufferTop, other.bufferTop) - val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) - val bufferRight = math.min(this.bufferRight, other.bufferRight) - val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) - val cols = bufferLeft + gridBounds.width + bufferRight - val rows = bufferTop + gridBounds.height + bufferBottom - - val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) - - // index both tiles relative to (0, 0) pixel - cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => - cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => - val leftV = this.get(col, row) - val rightV = other.get(col, row) - tile.set(col + bufferLeft, row + bufferTop, f(leftV, rightV)) - } - } - - if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) - tile - else - BufferTile(tile, GridBounds[Int]( - colMin = bufferLeft, - rowMin = bufferTop, - colMax = bufferLeft + gridBounds.width - 1, - rowMax = bufferTop + gridBounds.height - 1 - )) - } - - def combineDouble(other: BufferTile)(f: (Double, Double) => Double): Tile = { - if((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) { - throw new GeoAttrsError("Cannot combine rasters with different dimensions: " + - s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") - } - - val bufferTop = math.min(this.bufferTop, other.bufferTop) - val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) - val bufferRight = math.min(this.bufferRight, other.bufferRight) - val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) - val cols = bufferLeft + gridBounds.width + bufferRight - val rows = bufferTop + gridBounds.height + bufferBottom - - val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) - - // index both tiles relative to (0, 0) pixel - cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => - cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => - val leftV = this.getDouble(col, row) - val rightV = other.getDouble(col, row) - tile.setDouble(col + bufferLeft, row + bufferTop, f(leftV, rightV)) - } - } - - if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) - tile - else - BufferTile(tile, GridBounds[Int]( - colMin = bufferLeft, - rowMin = bufferTop, - colMax = bufferLeft + gridBounds.width - 1, - rowMax = bufferTop + gridBounds.height - 1)) - } - - /** - * Combine two tiles' cells into new cells using the given integer - * function. For every (x, y) cell coordinate, get each of the - * tiles' integer values, map them to a new value, and assign it to - * the output's (x, y) cell. - * - * @param other The other Tile - * @param f A function from (Int, Int) to Int - * @return The result, an Tile - */ - def combine(other: Tile)(f: (Int, Int) => Int): Tile = { - (this, other).assertEqualDimensions - - other match { - case bt: BufferTile => this.combine(bt)(f) - case _ => - val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - tile.set(col, row, f(get(col, row), other.get(col, row))) - } - } - tile - } - } - - /** - * Combine two tiles' cells into new cells using the given double - * function. For every (x, y) cell coordinate, get each of the - * tiles' double values, map them to a new value, and assign it to - * the output's (x, y) cell. - * - * @param other The other Tile - * @param f A function from (Int, Int) to Int - * @return The result, an Tile - */ - def combineDouble(other: Tile)(f: (Double, Double) => Double): Tile = { - (this, other).assertEqualDimensions - - other match { - case bt: BufferTile => - this.combineDouble(bt)(f) - case _ => - val tile = ArrayTile.alloc(cellType, cols, rows) - cfor(0)(_ < rows, _ + 1) { row => - cfor(0)(_ < cols, _ + 1) { col => - tile.setDouble(col, row, f(getDouble(col, row), other.getDouble(col, row))) - } - } - tile - } - } - - def mapTile(f: Tile => Tile): BufferTile = BufferTile(f(sourceTile), gridBounds) -} - -object BufferTile { - implicit class BufferTileOps(val self: BufferTile) extends BufferedFocalMethods -} diff --git a/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala b/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala deleted file mode 100644 index bf1987d66..000000000 --- a/core/src/main/scala/geotrellis/raster/mapalgebra/focal/BufferedFocalMethods.scala +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2021 Azavea - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package geotrellis.raster.mapalgebra.focal - -import geotrellis.raster._ -import geotrellis.util.MethodExtensions - -trait BufferedFocalMethods extends MethodExtensions[BufferTile] { - - /** Computes the minimum value of a neighborhood */ - def focalMin(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalMin(n, bounds, target)) - - /** Computes the maximum value of a neighborhood */ - def focalMax(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalMax(n, bounds, target)) - - /** Computes the mode of a neighborhood */ - def focalMode(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalMode(n, bounds, target)) - - /** Computes the median of a neighborhood */ - def focalMedian(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalMedian(n, bounds, target)) - - /** Computes the mean of a neighborhood */ - def focalMean(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalMean(n, bounds, target)) - - /** Computes the sum of a neighborhood */ - def focalSum(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalSum(n, bounds, target)) - - /** Computes the standard deviation of a neighborhood */ - def focalStandardDeviation(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.focalStandardDeviation(n, bounds, target)) - - /** Computes the next step of Conway's Game of Life */ - def focalConway(bounds: Option[GridBounds[Int]] = None): BufferTile = - self.mapTile(_.focalConway(bounds)) - - /** Computes the convolution of the raster for the given kernl */ - def convolve(kernel: Kernel, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.convolve(kernel, bounds, target)) - - /** - * Calculates spatial autocorrelation of cells based on the - * similarity to neighboring values. - */ - def tileMoransI(n: Neighborhood, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.tileMoransI(n, bounds, target)) - - /** - * Calculates global spatial autocorrelation of a raster based on - * the similarity to neighboring values. - */ - def scalarMoransI(n: Neighborhood, bounds: Option[GridBounds[Int]] = None): Double = - self.sourceTile.scalarMoransI(n, bounds) - - /** - * Calculates the slope of each cell in a raster. - * - * @param cs cellSize of the raster - * @param zFactor Number of map units to one elevation unit. - */ - def slope(cs: CellSize, zFactor: Double = 1.0, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.slope(cs, zFactor, bounds, target)) - - /** - * Calculates the aspect of each cell in a raster. - * - * @param cs cellSize of the raster - */ - def aspect(cs: CellSize, bounds: Option[GridBounds[Int]] = None, target: TargetCell = TargetCell.All): BufferTile = - self.mapTile(_.aspect(cs, bounds, target)) -} diff --git a/core/src/main/scala/org/apache/spark/sql/rf/QuinaryExpression.scala b/core/src/main/scala/org/apache/spark/sql/rf/QuinaryExpression.scala new file mode 100644 index 000000000..2f4ce827b --- /dev/null +++ b/core/src/main/scala/org/apache/spark/sql/rf/QuinaryExpression.scala @@ -0,0 +1,111 @@ +package org.apache.spark.sql.rf + +import org.apache.spark.sql.catalyst.expressions.codegen.Block._ +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.codegen.{CodeGenerator, CodegenContext, ExprCode, FalseLiteral} + +/** + * An expression with five inputs and one output. The output is by default evaluated to null if any input is evaluated to null + */ +abstract class QuinaryExpression extends Expression { + + override def foldable: Boolean = children.forall(_.foldable) + + override def nullable: Boolean = children.exists(_.nullable) + + /** + * Default behavior of evaluation according to the default nullability of QuaternaryExpression. + * If subclass of QuaternaryExpression override nullable, probably should also override this. + */ + override def eval(input: InternalRow): Any = { + val exprs = children + val value1 = exprs(0).eval(input) + if (value1 != null) { + val value2 = exprs(1).eval(input) + if (value2 != null) { + val value3 = exprs(2).eval(input) + if (value3 != null) { + val value4 = exprs(3).eval(input) + if (value4 != null) { + val value5 = exprs(4).eval(input) + if (value5 != null) { + return nullSafeEval(value1, value2, value3, value4, value5) + } + } + } + } + } + null + } + + /** + * Called by default [[eval]] implementation. If subclass of QuinaryExpression keep the + * default nullability, they can override this method to save null-check code. If we need + * full control of evaluation process, we should override [[eval]]. + */ + protected def nullSafeEval(input1: Any, input2: Any, input3: Any, input4: Any, input5: Any): Any = + sys.error(s"QuinaryExpressions must override either eval or nullSafeEval") + + /** + * Short hand for generating quinary evaluation code. + * If either of the sub-expressions is null, the result of this computation + * is assumed to be null. + * + * @param f accepts five variable names and returns Java code to compute the output. + */ + protected def defineCodeGen(ctx: CodegenContext, ev: ExprCode, f: (String, String, String, String, String) => String): ExprCode = { + nullSafeCodeGen(ctx, ev, (eval1, eval2, eval3, eval4, eval5) => { + s"${ev.value} = ${f(eval1, eval2, eval3, eval4, eval5)};" + }) + } + + /** + * Short hand for generating quinary evaluation code. + * If either of the sub-expressions is null, the result of this computation + * is assumed to be null. + * + * @param f function that accepts the 5 non-null evaluation result names of children + * and returns Java code to compute the output. + */ + protected def nullSafeCodeGen(ctx: CodegenContext, ev: ExprCode, f: (String, String, String, String, String) => String): ExprCode = { + val firstGen = children(0).genCode(ctx) + val secondGen = children(1).genCode(ctx) + val thridGen = children(2).genCode(ctx) + val fourthGen = children(3).genCode(ctx) + val fifthGen = children(4).genCode(ctx) + val resultCode = f(firstGen.value, secondGen.value, thridGen.value, fourthGen.value, fifthGen.value) + + if (nullable) { + val nullSafeEval = + firstGen.code + ctx.nullSafeExec(children(0).nullable, firstGen.isNull) { + secondGen.code + ctx.nullSafeExec(children(1).nullable, secondGen.isNull) { + thridGen.code + ctx.nullSafeExec(children(2).nullable, thridGen.isNull) { + fourthGen.code + ctx.nullSafeExec(children(3).nullable, fourthGen.isNull) { + fifthGen.code + ctx.nullSafeExec(children(4).nullable, fifthGen.isNull) { + s""" + ${ev.isNull} = false; // resultCode could change nullability. + $resultCode + """ + } + } + } + } + } + + ev.copy(code = code""" + boolean ${ev.isNull} = true; + ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)}; + $nullSafeEval""") + } else { + ev.copy(code = code""" + ${firstGen.code} + ${secondGen.code} + ${thridGen.code} + ${fourthGen.code} + ${fifthGen.code} + ${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)}; + $resultCode""", isNull = FalseLiteral) + } + } +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala index 18cf1eea8..02cfde90f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala @@ -58,4 +58,11 @@ object SerializersCache { def rowSerialize[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): T => Row = { t => rowDeserializer[T](tag, encoder)(serializer[T](tag, encoder)(t)) } + + def clean(): Unit = { + cacheSerializer.remove() + cacheSerializerRow.remove() + cacheDeserializer.remove() + cacheDeserializerRow.remove() + } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala index 3301bca70..bdf5bb03f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala @@ -32,7 +32,7 @@ import org.apache.spark.sql.catalyst.util.QuantileSummaries import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders import org.locationtech.rasterframes.model.{CellContext, LongExtent, TileContext, TileDataContext} import frameless.TypedEncoder -import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood} +import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood, TargetCell} import java.net.URI import java.sql.Timestamp @@ -55,6 +55,7 @@ trait StandardEncoders extends SpatialEncoders with TypedEncoders { implicit lazy val uriEncoder: ExpressionEncoder[URI] = typedExpressionEncoder[URI] implicit lazy val neighborhoodEncoder: ExpressionEncoder[Neighborhood] = typedExpressionEncoder[Neighborhood] + implicit lazy val targetCellEncoder: ExpressionEncoder[TargetCell] = typedExpressionEncoder[TargetCell] implicit lazy val kernelEncoder: ExpressionEncoder[Kernel] = typedExpressionEncoder[Kernel] implicit lazy val quantileSummariesEncoder: ExpressionEncoder[QuantileSummaries] = typedExpressionEncoder[QuantileSummaries] implicit lazy val envelopeEncoder: ExpressionEncoder[Envelope] = typedExpressionEncoder diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala index dff2453b2..c4e56aa27 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala @@ -3,14 +3,14 @@ package org.locationtech.rasterframes.encoders import frameless._ import geotrellis.layer.{KeyBounds, LayoutDefinition, TileLayerMetadata} import geotrellis.proj4.CRS -import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood} +import geotrellis.raster.mapalgebra.focal.{Kernel, Neighborhood, TargetCell} import geotrellis.raster.{CellGrid, CellType, Dimensions, GridBounds, Raster, Tile} import geotrellis.vector.Extent import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.catalyst.util.QuantileSummaries import org.apache.spark.sql.rf.{CrsUDT, RasterSourceUDT, TileUDT} import org.locationtech.jts.geom.Envelope -import org.locationtech.rasterframes.util.{FocalNeighborhood, KryoSupport} +import org.locationtech.rasterframes.util.{FocalNeighborhood, FocalTargetCell, KryoSupport} import java.net.URI import java.nio.ByteBuffer @@ -37,6 +37,9 @@ trait TypedEncoders { implicit val neighborhoodInjection: Injection[Neighborhood, String] = Injection(FocalNeighborhood(_), FocalNeighborhood.fromString(_).get) implicit val neighborhoodTypedEncoder: TypedEncoder[Neighborhood] = TypedEncoder.usingInjection + implicit val targetCellInjection: Injection[TargetCell, String] = Injection(FocalTargetCell(_), FocalTargetCell.fromString) + implicit val targetCellTypedEncoder: TypedEncoder[TargetCell] = TypedEncoder.usingInjection + implicit val envelopeTypedEncoder: TypedEncoder[Envelope] = ManualTypedEncoder.newInstance[Envelope]( fields = List( diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala index 1dcd15ce6..9f337d226 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions import geotrellis.proj4.CRS -import geotrellis.raster.{CellGrid, Neighborhood, Raster, Tile} +import geotrellis.raster.{CellGrid, Neighborhood, Raster, TargetCell, Tile} import geotrellis.vector.Extent import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow @@ -38,7 +38,7 @@ import org.locationtech.rasterframes.model.{LazyCRS, LongExtent, TileContext} import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef} import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.apache.spark.sql.rf.CrsUDT -import org.locationtech.rasterframes.util.FocalNeighborhood +import org.locationtech.rasterframes.util.{FocalNeighborhood, FocalTargetCell} private[rasterframes] object DynamicExtractors { @@ -230,4 +230,9 @@ object DynamicExtractors { case _: StringType => (v: Any) => FocalNeighborhood.fromString(v.asInstanceOf[UTF8String].toString).get case n if n.conformsToSchema(neighborhoodEncoder.schema) => { case ir: InternalRow => ir.as[Neighborhood] } } + + lazy val targetCellExtractor: PartialFunction[DataType, Any => TargetCell] = { + case _: StringType => (v: Any) => FocalTargetCell.fromString(v.asInstanceOf[UTF8String].toString) + case n if n.conformsToSchema(targetCellEncoder.schema) => { case ir: InternalRow => ir.as[TargetCell] } + } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala index 10ba6727d..68083293b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -21,55 +21,61 @@ package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.{BufferTile, CellSize} +import geotrellis.raster.{BufferTile, CellSize, TargetCell, Tile} import org.apache.spark.sql.Column -import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, RasterResult, UnaryRasterFunction, row} +import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription} +import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.locationtech.rasterframes.encoders.syntax._ import org.locationtech.rasterframes.expressions.DynamicExtractors._ import org.locationtech.rasterframes.model.TileContext -import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType import org.slf4j.LoggerFactory import com.typesafe.scalalogging.Logger +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} @ExpressionDescription( - usage = "_FUNC_(tile) - Performs aspect on tile.", + usage = "_FUNC_(tile, target) - Performs aspect on tile.", arguments = """ Arguments: - * tile - a tile to apply operation""", + * tile - a tile to apply operation + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile); + > SELECT _FUNC_(tile, 'all'); ...""" ) -case class Aspect(child: Expression) extends UnaryRasterFunction with RasterResult with NullToValue with CodegenFallback { +case class Aspect(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - def na: Any = null + def dataType: DataType = left.dataType - def dataType: DataType = child.dataType + override def checkInputDataTypes(): TypeCheckResult = + if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") + else if(!targetCellExtractor.isDefinedAt(right.dataType)) TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a string TargetCell type.") + else TypeCheckSuccess - override protected def nullSafeEval(input: Any): Any = { - val (tile, ctx) = tileExtractor(child.dataType)(row(input)) - eval(extractBufferTile(tile), ctx) + override protected def nullSafeEval(tileInput: Any, targetCellInput: Any): Any = { + val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) + val target = targetCellExtractor(right.dataType)(targetCellInput) + eval(extractBufferTile(tile), ctx, target) } - protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ctx match { - case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow + protected def eval(tile: Tile, ctx: Option[TileContext], target: TargetCell): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, target)).toInternalRow case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") } override def nodeName: String = Aspect.name - def op(t: Tile, ctx: TileContext): Tile = t match { - case bt: BufferTile => bt.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) - case _ => t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) + def op(t: Tile, ctx: TileContext, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows), target = target) + case _ => t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows), target = target) } } object Aspect { def name: String = "rf_aspect" - def apply(tile: Column): Column = new Column(Aspect(tile.expr)) -} \ No newline at end of file + def apply(tile: Column, target: Column): Column = new Column(Aspect(tile.expr, target.expr)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala index 594c8b871..2d6cc1638 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -22,59 +22,62 @@ package org.locationtech.rasterframes.expressions.focalops import com.typesafe.scalalogging.Logger -import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.{BufferTile, TargetCell, Tile} import geotrellis.raster.mapalgebra.focal.Kernel import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression} import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes._ import org.locationtech.rasterframes.encoders._ import org.locationtech.rasterframes.encoders.syntax._ -import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor +import org.locationtech.rasterframes.expressions.DynamicExtractors.{targetCellExtractor, tileExtractor} import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.slf4j.LoggerFactory @ExpressionDescription( - usage = "_FUNC_(tile, kernel) - Performs convolve on tile in the neighborhood.", + usage = "_FUNC_(tile, kernel, target) - Performs convolve on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * kernel - a focal operation kernel""", + * kernel - a focal operation kernel + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, kernel); + > SELECT _FUNC_(tile, kernel, 'all'); ...""" ) -case class Convolve(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback { +case class Convolve(left: Expression, middle: Expression, right: Expression) extends TernaryExpression with RasterResult with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def nodeName: String = Convolve.name def dataType: DataType = left.dataType + val children: Seq[Expression] = Seq(left, middle, right) override def checkInputDataTypes(): TypeCheckResult = if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") - else if (!right.dataType.conformsToSchema(kernelEncoder.schema)) { - TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a kernel type.") - } else TypeCheckSuccess + else if (!middle.dataType.conformsToSchema(kernelEncoder.schema)) TypeCheckFailure(s"Input type '${middle.dataType}' does not conform to a Kernel type.") + else if (!targetCellExtractor.isDefinedAt(right.dataType)) TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a TargetCell type.") + else TypeCheckSuccess - override protected def nullSafeEval(tileInput: Any, kernelInput: Any): Any = { + override protected def nullSafeEval(tileInput: Any, kernelInput: Any, targetCellInput: Any): Any = { val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) val kernel = row(kernelInput).as[Kernel] - val result = op(extractBufferTile(tile), kernel) + val target = targetCellExtractor(right.dataType)(targetCellInput) + val result = op(extractBufferTile(tile), kernel, target) toInternalRow(result, ctx) } - protected def op(t: Tile, kernel: Kernel): Tile = t match { - case bt: BufferTile => bt.convolve(kernel) - case _ => t.convolve(kernel) + protected def op(t: Tile, kernel: Kernel, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.convolve(kernel, target = target) + case _ => t.convolve(kernel, target = target) } } object Convolve { def name: String = "rf_convolve" - def apply(tile: Column, kernel: Column): Column = new Column(Convolve(tile.expr, kernel.expr)) + def apply(tile: Column, kernel: Column, target: Column): Column = new Column(Convolve(tile.expr, kernel.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala index a7220f941..b8ad6d908 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -21,31 +21,32 @@ package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.{BufferTile, TargetCell, Tile} import geotrellis.raster.mapalgebra.focal.Neighborhood import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMax on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMax on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMax(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMax(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMax.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalMax(neighborhood) - case _ => t.focalMax(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalMax(neighborhood, target = target) + case _ => t.focalMax(neighborhood, target = target) } } object FocalMax { def name: String = "rf_focal_max" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMax(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMax(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala index b72019d2b..b6fb8ba0d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -21,31 +21,32 @@ package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.{BufferTile, TargetCell, Tile} import geotrellis.raster.mapalgebra.focal.Neighborhood import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMean on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMean on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMean(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMean(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMean.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalMean(neighborhood) - case _ => t.focalMean(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalMean(neighborhood, target = target) + case _ => t.focalMean(neighborhood, target = target) } } object FocalMean { def name:String = "rf_focal_mean" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMean(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMean(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala index 4dc11d029..b72a4ed8d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -21,31 +21,32 @@ package org.locationtech.rasterframes.expressions.focalops -import geotrellis.raster.{BufferTile, Tile} +import geotrellis.raster.{BufferTile, TargetCell, Tile} import geotrellis.raster.mapalgebra.focal.Neighborhood import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMedian on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMedian on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMedian(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMedian(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMedian.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalMedian(neighborhood) - case _ => t.focalMedian(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalMedian(neighborhood, target = target) + case _ => t.focalMedian(neighborhood, target = target) } } object FocalMedian { def name: String = "rf_focal_median" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMedian(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMedian(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala index fc5cfac70..439a8ae9f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -20,31 +20,33 @@ */ package org.locationtech.rasterframes.expressions.focalops + import geotrellis.raster.{BufferTile, Tile} -import geotrellis.raster.mapalgebra.focal.Neighborhood +import geotrellis.raster.mapalgebra.focal.{Neighborhood, TargetCell} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMin on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMin on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMin(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMin(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMin.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalMin(neighborhood) - case _ => t.focalMin(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalMin(neighborhood, target = target) + case _ => t.focalMin(neighborhood, target = target) } } object FocalMin { def name: String = "rf_focal_min" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMin(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMin(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala index af5ff14fd..6ea049cc6 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -22,30 +22,31 @@ package org.locationtech.rasterframes.expressions.focalops import geotrellis.raster.{BufferTile, Tile} -import geotrellis.raster.mapalgebra.focal.Neighborhood +import geotrellis.raster.mapalgebra.focal.{Neighborhood, TargetCell} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMode on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMode on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMode(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMode(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMode.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalMode(neighborhood) - case _ => t.focalMode(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalMode(neighborhood, target = target) + case _ => t.focalMode(neighborhood, target = target) } } object FocalMode { def name: String = "rf_focal_mode" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMode(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMode(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala index e09dd5681..d4db3192f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -22,30 +22,31 @@ package org.locationtech.rasterframes.expressions.focalops import geotrellis.raster.{BufferTile, Tile} -import geotrellis.raster.mapalgebra.focal.Neighborhood +import geotrellis.raster.mapalgebra.focal.{Neighborhood, TargetCell} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalMoransI on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalMoransI on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalMoransI(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalMoransI(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalMoransI.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.tileMoransI(neighborhood) - case _ => t.tileMoransI(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.tileMoransI(neighborhood, target = target) + case _ => t.tileMoransI(neighborhood, target = target) } } object FocalMoransI { def name: String = "rf_focal_moransi" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalMoransI(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalMoransI(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala index b73db0341..64bbd313e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOp.scala @@ -22,39 +22,43 @@ package org.locationtech.rasterframes.expressions.focalops import com.typesafe.scalalogging.Logger -import geotrellis.raster.{Neighborhood, Tile} +import geotrellis.raster.{Neighborhood, TargetCell, Tile} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression} +import org.apache.spark.sql.catalyst.expressions.{Expression, TernaryExpression} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.DynamicExtractors.{neighborhoodExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.DynamicExtractors.{neighborhoodExtractor, targetCellExtractor, tileExtractor} import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.slf4j.LoggerFactory -trait FocalNeighborhoodOp extends BinaryExpression with RasterResult with CodegenFallback { +trait FocalNeighborhoodOp extends TernaryExpression with RasterResult with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) - // tile + // Tile def left: Expression - // neighborhood + // Neighborhood + def middle: Expression + // TargetCell def right: Expression def dataType: DataType = left.dataType + def children: Seq[Expression] = Seq(left, middle, right) override def checkInputDataTypes(): TypeCheckResult = if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") - else if(!neighborhoodExtractor.isDefinedAt(right.dataType)) { - TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a string neighborhood type.") - } else TypeCheckSuccess + else if(!neighborhoodExtractor.isDefinedAt(middle.dataType)) TypeCheckFailure(s"Input type '${middle.dataType}' does not conform to a string Neighborhood type.") + else if(!targetCellExtractor.isDefinedAt(right.dataType)) TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a string TargetCell type.") + else TypeCheckSuccess - override protected def nullSafeEval(tileInput: Any, neighborhoodInput: Any): Any = { + override protected def nullSafeEval(tileInput: Any, neighborhoodInput: Any, targetCellInput: Any): Any = { val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) - val neighborhood = neighborhoodExtractor(right.dataType)(neighborhoodInput) - val result = op(extractBufferTile(tile), neighborhood) + val neighborhood = neighborhoodExtractor(middle.dataType)(neighborhoodInput) + val target = targetCellExtractor(right.dataType)(targetCellInput) + val result = op(extractBufferTile(tile), neighborhood, target) toInternalRow(result, ctx) } - protected def op(child: Tile, neighborhood: Neighborhood): Tile + protected def op(child: Tile, neighborhood: Neighborhood, target: TargetCell): Tile } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala index 7ec881544..ed05e077f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -22,30 +22,31 @@ package org.locationtech.rasterframes.expressions.focalops import geotrellis.raster.{BufferTile, Tile} -import geotrellis.raster.mapalgebra.focal.Neighborhood +import geotrellis.raster.mapalgebra.focal.{Neighborhood, TargetCell} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @ExpressionDescription( - usage = "_FUNC_(tile, neighborhood) - Performs focalStandardDeviation on tile in the neighborhood.", + usage = "_FUNC_(tile, neighborhood, target) - Performs focalStandardDeviation on tile in the neighborhood.", arguments = """ Arguments: * tile - a tile to apply operation - * neighborhood - a focal operation neighborhood""", + * neighborhood - a focal operation neighborhood + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 'square-1'); + > SELECT _FUNC_(tile, 'square-1', 'all'); ...""" ) -case class FocalStdDev(left: Expression, right: Expression) extends FocalNeighborhoodOp { +case class FocalStdDev(left: Expression, middle: Expression, right: Expression) extends FocalNeighborhoodOp { override def nodeName: String = FocalStdDev.name - protected def op(t: Tile, neighborhood: Neighborhood): Tile = t match { - case bt: BufferTile => bt.focalStandardDeviation(neighborhood) - case _ => t.focalStandardDeviation(neighborhood) + protected def op(t: Tile, neighborhood: Neighborhood, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.focalStandardDeviation(neighborhood, target = target) + case _ => t.focalStandardDeviation(neighborhood, target = target) } } object FocalStdDev { def name: String = "rf_focal_stddev" - def apply(tile: Column, neighborhood: Column): Column = new Column(FocalStdDev(tile.expr, neighborhood.expr)) + def apply(tile: Column, neighborhood: Column, target: Column): Column = new Column(FocalStdDev(tile.expr, neighborhood.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala index 256419435..3a917337b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -22,48 +22,53 @@ package org.locationtech.rasterframes.expressions.focalops import com.typesafe.scalalogging.Logger +import geotrellis.raster.mapalgebra.focal.TargetCell import geotrellis.raster.{BufferTile, CellSize, Tile} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, QuaternaryExpression} +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +import org.apache.spark.sql.rf.QuinaryExpression import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes.encoders.syntax._ -import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, targetCellExtractor, tileExtractor} import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.locationtech.rasterframes.model.TileContext import org.slf4j.LoggerFactory @ExpressionDescription( - usage = "_FUNC_(tile, azimuth, altitude, zFactor) - Performs hillshade on tile.", + usage = "_FUNC_(tile, azimuth, altitude, zFactor, target) - Performs hillshade on tile.", arguments = """ Arguments: * tile - a tile to apply operation * azimuth * altitude - * zFactor""", + * zFactor + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, azimuth, altitude, zFactor); + > SELECT _FUNC_(tile, azimuth, altitude, zFactor, 'all'); ...""" ) -case class Hillshade(first: Expression, second: Expression, third: Expression, fourth: Expression) extends QuaternaryExpression with RasterResult with CodegenFallback { +case class Hillshade(first: Expression, second: Expression, third: Expression, fourth: Expression, fifth: Expression) extends QuinaryExpression with RasterResult with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def nodeName: String = Hillshade.name def dataType: DataType = first.dataType - val children: Seq[Expression] = Seq(first, second, third, fourth) + val children: Seq[Expression] = Seq(first, second, third, fourth, fifth) + val numbers: Seq[Expression] = Seq(second, third, fourth) override def checkInputDataTypes(): TypeCheckResult = if (!tileExtractor.isDefinedAt(first.dataType)) TypeCheckFailure(s"Input type '${first.dataType}' does not conform to a raster type.") - else if (!children.tail.forall(expr => numberArgExtractor.isDefinedAt(expr.dataType))) { + else if (!numbers.forall(expr => numberArgExtractor.isDefinedAt(expr.dataType))) TypeCheckFailure(s"Input type '${second.dataType}', '${third.dataType}' or '${fourth.dataType}' do not conform to a numeric type.") - } else TypeCheckSuccess + else if(!targetCellExtractor.isDefinedAt(fifth.dataType)) TypeCheckFailure(s"Input type '${fifth.dataType}' does not conform to a string TargetCell type.") + else TypeCheckSuccess - override protected def nullSafeEval(tileInput: Any, azimuthInput: Any, altitudeInput: Any, zFactorInput: Any): Any = { + override protected def nullSafeEval(tileInput: Any, azimuthInput: Any, altitudeInput: Any, zFactorInput: Any, targetCellInput: Any): Any = { val (tile, ctx) = tileExtractor(first.dataType)(row(tileInput)) val List(azimuth, altitude, zFactor) = children @@ -73,22 +78,23 @@ case class Hillshade(first: Expression, second: Expression, third: Expression, f case DoubleArg(value) => value case IntegerArg(value) => value.toDouble } } - eval(extractBufferTile(tile), ctx, azimuth, altitude, zFactor) + val target = targetCellExtractor(fifth.dataType)(targetCellInput) + eval(extractBufferTile(tile), ctx, azimuth, altitude, zFactor, target) } - protected def eval(tile: Tile, ctx: Option[TileContext], azimuth: Double, altitude: Double, zFactor: Double): Any = ctx match { - case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, azimuth, altitude, zFactor)).toInternalRow + protected def eval(tile: Tile, ctx: Option[TileContext], azimuth: Double, altitude: Double, zFactor: Double, target: TargetCell): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, azimuth, altitude, zFactor, target)).toInternalRow case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") } - protected def op(t: Tile, ctx: TileContext, azimuth: Double, altitude: Double, zFactor: Double): Tile = t match { - case bt: BufferTile => bt.mapTile(_.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor)) - case _ => t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) + protected def op(t: Tile, ctx: TileContext, azimuth: Double, altitude: Double, zFactor: Double, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.mapTile(_.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor, target = target)) + case _ => t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor, target = target) } } object Hillshade { def name: String = "rf_hillshade" - def apply(tile: Column, azimuth: Column, altitude: Column, zFactor: Column): Column = - new Column(Hillshade(tile.expr, azimuth.expr, altitude.expr, zFactor.expr)) + def apply(tile: Column, azimuth: Column, altitude: Column, zFactor: Column, target: Column): Column = + new Column(Hillshade(tile.expr, azimuth.expr, altitude.expr, zFactor.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala index 9932b4406..2bd256ce2 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -22,63 +22,68 @@ package org.locationtech.rasterframes.expressions.focalops import com.typesafe.scalalogging.Logger +import geotrellis.raster.mapalgebra.focal.TargetCell import geotrellis.raster.{BufferTile, CellSize, Tile} import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.analysis.TypeCheckResult import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription} +import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression} import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes.encoders.syntax._ -import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, tileExtractor} +import org.locationtech.rasterframes.expressions.DynamicExtractors.{DoubleArg, IntegerArg, numberArgExtractor, targetCellExtractor, tileExtractor} import org.locationtech.rasterframes.expressions.{RasterResult, row} import org.locationtech.rasterframes.model.TileContext import org.slf4j.LoggerFactory @ExpressionDescription( - usage = "_FUNC_(tile, zFactor) - Performs slope on tile.", + usage = "_FUNC_(tile, zFactor, middle) - Performs slope on tile.", arguments = """ Arguments: * tile - a tile to apply operation - * zFactor - a slope operation zFactor""", + * zFactor - a slope operation zFactor + * target - the target cells to apply focal operation: data, nodata, all""", examples = """ Examples: - > SELECT _FUNC_(tile, 0.2); + > SELECT _FUNC_(tile, 0.2, 'all'); ...""" ) -case class Slope(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback { +case class Slope(left: Expression, middle: Expression, right: Expression) extends TernaryExpression with RasterResult with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def nodeName: String = Slope.name def dataType: DataType = left.dataType + val children: Seq[Expression] = Seq(left, middle, right) + override def checkInputDataTypes(): TypeCheckResult = if (!tileExtractor.isDefinedAt(left.dataType)) TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.") - else if (!numberArgExtractor.isDefinedAt(right.dataType)) { - TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a numeric type.") - } else TypeCheckSuccess + else if (!numberArgExtractor.isDefinedAt(middle.dataType)) TypeCheckFailure(s"Input type '${middle.dataType}' does not conform to a numeric type.") + else if (!targetCellExtractor.isDefinedAt(right.dataType)) TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a TargetCell type.") + else TypeCheckSuccess - override protected def nullSafeEval(tileInput: Any, zFactorInput: Any): Any = { + override protected def nullSafeEval(tileInput: Any, zFactorInput: Any, targetCellInput: Any): Any = { val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput)) - val zFactor = numberArgExtractor(right.dataType)(zFactorInput) match { + val zFactor = numberArgExtractor(middle.dataType)(zFactorInput) match { case DoubleArg(value) => value case IntegerArg(value) => value.toDouble } - eval(extractBufferTile(tile), ctx, zFactor) + val target = targetCellExtractor(right.dataType)(targetCellInput) + eval(extractBufferTile(tile), ctx, zFactor, target) } - protected def eval(tile: Tile, ctx: Option[TileContext], zFactor: Double): Any = ctx match { - case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, zFactor)).toInternalRow + protected def eval(tile: Tile, ctx: Option[TileContext], zFactor: Double, target: TargetCell): Any = ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx, zFactor, target)).toInternalRow case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") } - protected def op(t: Tile, ctx: TileContext, zFactor: Double): Tile = t match { - case bt: BufferTile => bt.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) - case _ => t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) + protected def op(t: Tile, ctx: TileContext, zFactor: Double, target: TargetCell): Tile = t match { + case bt: BufferTile => bt.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor, target = target) + case _ => t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor, target = target) } } object Slope { def name: String = "rf_slope" - def apply(tile: Column, zFactor: Column): Column = new Column(Slope(tile.expr, zFactor.expr)) + def apply(tile: Column, zFactor: Column, target: Column): Column = new Column(Slope(tile.expr, zFactor.expr, target.expr)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala index cdfe8e18d..bd9c9cc97 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.functions -import geotrellis.raster.Neighborhood +import geotrellis.raster.{Neighborhood, TargetCell} import geotrellis.raster.mapalgebra.focal.Kernel import org.apache.spark.sql.Column import org.apache.spark.sql.functions.lit @@ -31,65 +31,101 @@ import org.locationtech.rasterframes.expressions.focalops._ trait FocalFunctions { def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_mean(tileCol, serialized_literal(neighborhood)) + rf_focal_mean(tileCol, neighborhood, TargetCell.All) - def rf_focal_mean(tileCol: Column, neighborhoodCol: Column): Column = - FocalMean(tileCol, neighborhoodCol) + def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_mean(tileCol, serialized_literal(neighborhood), serialized_literal(target)) + + def rf_focal_mean(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMean(tileCol, neighborhoodCol, targetCol) def rf_focal_median(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_median(tileCol, serialized_literal(neighborhood)) + rf_focal_median(tileCol, neighborhood, TargetCell.All) + + def rf_focal_median(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_median(tileCol, serialized_literal(neighborhood), serialized_literal(target)) - def rf_focal_median(tileCol: Column, neighborhoodCol: Column): Column = - FocalMedian(tileCol, neighborhoodCol) + def rf_focal_median(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMedian(tileCol, neighborhoodCol, targetCol) def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_mode(tileCol, serialized_literal(neighborhood)) + rf_focal_mode(tileCol, neighborhood, TargetCell.All) + + def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_mode(tileCol, serialized_literal(neighborhood), serialized_literal(target)) - def rf_focal_mode(tileCol: Column, neighborhoodCol: Column): Column = - FocalMode(tileCol, neighborhoodCol) + def rf_focal_mode(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMode(tileCol, neighborhoodCol, targetCol) def rf_focal_max(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_max(tileCol, serialized_literal(neighborhood)) + rf_focal_max(tileCol, neighborhood, TargetCell.All) - def rf_focal_max(tileCol: Column, neighborhoodCol: Column): Column = - FocalMax(tileCol, neighborhoodCol) + def rf_focal_max(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_max(tileCol, serialized_literal(neighborhood), serialized_literal(target)) + + def rf_focal_max(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMax(tileCol, neighborhoodCol, targetCol) def rf_focal_min(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_min(tileCol, serialized_literal(neighborhood)) + rf_focal_min(tileCol, neighborhood, TargetCell.All) + + def rf_focal_min(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_min(tileCol, serialized_literal(neighborhood), serialized_literal(target)) - def rf_focal_min(tileCol: Column, neighborhoodCol: Column): Column = - FocalMin(tileCol, neighborhoodCol) + def rf_focal_min(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMin(tileCol, neighborhoodCol, targetCol) def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_stddev(tileCol, serialized_literal(neighborhood)) + rf_focal_stddev(tileCol, neighborhood, TargetCell.All) + + def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_stddev(tileCol, serialized_literal(neighborhood), serialized_literal(target)) - def rf_focal_stddev(tileCol: Column, neighborhoodCol: Column): Column = - FocalStdDev(tileCol, neighborhoodCol) + def rf_focal_stddev(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalStdDev(tileCol, neighborhoodCol, targetCol) def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood): Column = - rf_focal_moransi(tileCol, serialized_literal(neighborhood)) + rf_focal_moransi(tileCol, neighborhood, TargetCell.All) - def rf_focal_moransi(tileCol: Column, neighborhoodCol: Column): Column = - FocalMoransI(tileCol, neighborhoodCol) + def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood, target: TargetCell): Column = + rf_focal_moransi(tileCol, serialized_literal(neighborhood), serialized_literal(target)) + + def rf_focal_moransi(tileCol: Column, neighborhoodCol: Column, targetCol: Column): Column = + FocalMoransI(tileCol, neighborhoodCol, targetCol) def rf_convolve(tileCol: Column, kernel: Kernel): Column = - rf_convolve(tileCol, serialized_literal(kernel)) + rf_convolve(tileCol, kernel, TargetCell.All) + + def rf_convolve(tileCol: Column, kernel: Kernel, target: TargetCell): Column = + rf_convolve(tileCol, serialized_literal(kernel), serialized_literal(target)) + + def rf_convolve(tileCol: Column, kernelCol: Column, targetCol: Column): Column = + Convolve(tileCol, kernelCol, targetCol) - def rf_convolve(tileCol: Column, kernelCol: Column): Column = - Convolve(tileCol, kernelCol) + def rf_slope(tileCol: Column, zFactor: Double): Column = + rf_slope(tileCol, zFactor, TargetCell.All) - def rf_slope[T: Numeric](tileCol: Column, zFactor: T): Column = - rf_slope(tileCol, lit(zFactor)) + def rf_slope(tileCol: Column, zFactor: Double, target: TargetCell): Column = + rf_slope(tileCol, lit(zFactor), serialized_literal(target)) - def rf_slope(tileCol: Column, zFactorCol: Column): Column = - Slope(tileCol, zFactorCol) + def rf_slope(tileCol: Column, zFactorCol: Column, targetCol: Column): Column = + Slope(tileCol, zFactorCol, targetCol) def rf_aspect(tileCol: Column): Column = - Aspect(tileCol) + rf_aspect(tileCol, TargetCell.All) + + def rf_aspect(tileCol: Column, target: TargetCell): Column = + rf_aspect(tileCol, serialized_literal(target)) + + def rf_aspect(tileCol: Column, targetCol: Column): Column = + Aspect(tileCol, targetCol) + + def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + rf_hillshade(tileCol, azimuth, altitude, zFactor, TargetCell.All) - def rf_hillshade[T: Numeric](tileCol: Column, azimuth: T, altitude: T, zFactor: T): Column = - rf_hillshade(tileCol, lit(azimuth), lit(altitude), lit(zFactor)) + def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double, target: TargetCell): Column = + rf_hillshade(tileCol, lit(azimuth), lit(altitude), lit(zFactor), serialized_literal(target)) - def rf_hillshade(tileCol: Column, azimuth: Column, altitude: Column, zFactor: Column): Column = - Hillshade(tileCol, azimuth, altitude, zFactor) + def rf_hillshade(tileCol: Column, azimuthCol: Column, altitudeCol: Column, zFactorCol: Column, targetCol: Column): Column = + Hillshade(tileCol, azimuthCol, altitudeCol, zFactorCol, targetCol) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala index 2bcaa53a6..34bddc601 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/util/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/util/package.scala @@ -29,7 +29,7 @@ import geotrellis.raster.mask.TileMaskMethods import geotrellis.raster.merge.TileMergeMethods import geotrellis.raster.prototype.TilePrototypeMethods import geotrellis.raster.render.{ColorRamp, ColorRamps} -import geotrellis.raster.{CellGrid, Grid, GridBounds} +import geotrellis.raster.{CellGrid, Grid, GridBounds, TargetCell} import geotrellis.spark.tiling.TilerKeyMethods import geotrellis.util.GetComponent import org.apache.spark.sql._ @@ -267,11 +267,26 @@ package object util extends DataFrameRenderers { case Max => "max" case Min => "min" case Sum => "sum" - case _ => throw new IllegalArgumentException(s"Unrecognized ResampleMethod ${gtr.toString()}") + case _ => throw new IllegalArgumentException(s"Unrecognized ResampleMethod ${gtr.toString}") } } } + object FocalTargetCell { + def fromString(str: String): TargetCell = str.toLowerCase match { + case "nodata" => TargetCell.NoData + case "data" => TargetCell.Data + case "all" => TargetCell.All + case _ => throw new IllegalArgumentException(s"Unrecognized TargetCell $str") + } + + def apply(tc: TargetCell): String = tc match { + case TargetCell.NoData => "nodata" + case TargetCell.Data => "data" + case TargetCell.All => "all" + } + } + private[rasterframes] def toParquetFriendlyColumnName(name: String) = name.replaceAll("[ ,;{}()\n\t=]", "_") diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala index 1dce2a6ca..85f3ba044 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/RasterLayerSpec.scala @@ -33,10 +33,11 @@ import geotrellis.spark._ import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.functions._ import org.apache.spark.sql.{SQLContext, SparkSession} +import org.locationtech.rasterframes.encoders.SerializersCache import org.locationtech.rasterframes.ref.RFRasterSource import org.locationtech.rasterframes.tiles.ProjectedRasterTile import org.locationtech.rasterframes.util._ -import org.scalatest.BeforeAndAfterEach +import org.scalatest.BeforeAndAfterAll import scala.util.control.NonFatal @@ -45,17 +46,10 @@ import scala.util.control.NonFatal * * @since 7/10/17 */ -class RasterLayerSpec extends TestEnvironment with MetadataKeys - with BeforeAndAfterEach with TestData { +class RasterLayerSpec extends TestEnvironment with MetadataKeys with TestData { import TestData.randomTile import spark.implicits._ - override def beforeEach(): Unit = { - // Try to GC to avoid OOM on low memory instances. - // TODO: remove once we have a larger CI - System.gc() - } - describe("Runtime environment") { it("should provide build info") { //assert(RFBuildInfo.toMap.nonEmpty) diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala index 73271bb35..9ec4e46dc 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -67,7 +67,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_mean(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_mean(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -89,7 +89,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_median(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_median(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -111,7 +111,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_mode(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_mode(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -133,7 +133,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_max(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_max(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -155,7 +155,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_min(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_min(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -177,7 +177,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_stddev(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_stddev(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -199,7 +199,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_focal_moransi(proj_raster, 'square-1')") + .selectExpr(s"rf_focal_moransi(proj_raster, 'square-1', 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -222,7 +222,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df .withColumn("kernel", serialized_literal(Kernel(Circle(2d)))) - .selectExpr(s"rf_convolve(proj_raster, kernel)") + .selectExpr(s"rf_convolve(proj_raster, kernel, 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -244,7 +244,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_slope(proj_raster, 1)") + .selectExpr(s"rf_slope(proj_raster, 1, 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -266,7 +266,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_aspect(proj_raster)") + .selectExpr(s"rf_aspect(proj_raster, 'all')") .as[Option[ProjectedRasterTile]] .first() .get @@ -288,7 +288,7 @@ class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { val actualExpr = df - .selectExpr(s"rf_hillshade(proj_raster, 315, 45, 1)") + .selectExpr(s"rf_hillshade(proj_raster, 315, 45, 1, 'all')") .as[Option[ProjectedRasterTile]] .first() .get diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala index 41842cab1..1fc804c9e 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiPartition.scala @@ -15,11 +15,9 @@ import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, Par case class StacApiPartition(uri: Uri, searchFilters: SearchFilters) extends InputPartition class StacApiPartitionReaderFactory extends PartitionReaderFactory { - override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { - partition match { - case p: StacApiPartition => new StacApiPartitionReader(p) - case _ => throw new UnsupportedOperationException("Partition processing is unsupported by the reader.") - } + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = partition match { + case p: StacApiPartition => new StacApiPartitionReader(p) + case _ => throw new UnsupportedOperationException("Partition processing is unsupported by the reader.") } } diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala index a7886f81e..006b61c74 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/stac/api/StacApiScanBuilder.scala @@ -3,7 +3,6 @@ package org.locationtech.rasterframes.datasource.stac.api import org.locationtech.rasterframes.datasource.stac.api.encoders._ import com.azavea.stac4s.api.client.SearchFilters -import eu.timepit.refined.types.numeric.NonNegInt import org.apache.spark.sql.connector.read.{Batch, InputPartition, PartitionReaderFactory, Scan, ScanBuilder} import org.apache.spark.sql.types.StructType import sttp.model.Uri diff --git a/project/RFProjectPlugin.scala b/project/RFProjectPlugin.scala index c62ae3d02..c25d74855 100644 --- a/project/RFProjectPlugin.scala +++ b/project/RFProjectPlugin.scala @@ -46,8 +46,12 @@ object RFProjectPlugin extends AutoPlugin { publishMavenStyle := true, Compile / packageDoc / publishArtifact := true, Test / publishArtifact := false, - Test / fork := true, - Test / javaOptions := Seq("-Xmx1500m", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp"), + // don't fork it in tests to reduce memory usage + Test / fork := false, + // Test / javaOptions ++= Seq( + // "-XX:+HeapDumpOnOutOfMemoryError", + // "-XX:HeapDumpPath=/tmp" + // ), Test / parallelExecution := false, Test / testOptions += Tests.Argument("-oDF"), developers := List( diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index b9b67e247..108e28afb 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -780,59 +780,77 @@ def rf_identity(tile_col: Column_type) -> Column: """Pass tile through unchanged""" return _apply_column_function('rf_identity', tile_col) -def rf_focal_max(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_max(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the max value in its neighborhood of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_max', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_max', tile_col, neighborhood, target) -def rf_focal_mean(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_mean(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the mean value in its neighborhood of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_mean', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_mean', tile_col, neighborhood, target) -def rf_focal_median(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_median(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the max in its neighborhood value of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_median', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_median', tile_col, neighborhood, target) -def rf_focal_min(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_min(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the min value in its neighborhood of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_min', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_min', tile_col, neighborhood, target) -def rf_focal_mode(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_mode(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the mode value in its neighborhood of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_mode', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_mode', tile_col, neighborhood, target) -def rf_focal_std_dev(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_focal_std_dev(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute the standard deviation value in its neighborhood of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_std_dev', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_std_dev', tile_col, neighborhood, target) -def rf_moransI(tile_col: Column_type, neighborhood: Union[str, Column_type]) -> Column: +def rf_moransI(tile_col: Column_type, neighborhood: Union[str, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Compute moransI in its neighborhood value of each cell""" if isinstance(neighborhood, str): neighborhood = lit(neighborhood) - return _apply_column_function('rf_focal_moransi', tile_col, neighborhood) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_focal_moransi', tile_col, neighborhood, target) -def rf_aspect(tile_col: Column_type) -> Column: +def rf_aspect(tile_col: Column_type, target: Union[str, Column_type] = 'all') -> Column: """Calculates the aspect of each cell in an elevation raster""" - return _apply_column_function('rf_aspect', tile_col) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_aspect', tile_col, target) -def rf_slope(tile_col: Column_type, z_factor: Union[int, float, Column_type]) -> Column: +def rf_slope(tile_col: Column_type, z_factor: Union[int, float, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Calculates slope of each cell in an elevation raster""" if isinstance(z_factor, (int, float)): z_factor = lit(z_factor) - return _apply_column_function('rf_slope', tile_col, z_factor) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_slope', tile_col, z_factor, target) -def rf_hillshade(tile_col: Column_type, azimuth: Union[int, float, Column_type], altitude: Union[int, float, Column_type], z_factor: Union[int, float, Column_type]) -> Column: +def rf_hillshade(tile_col: Column_type, azimuth: Union[int, float, Column_type], altitude: Union[int, float, Column_type], z_factor: Union[int, float, Column_type], target: Union[str, Column_type] = 'all') -> Column: """Calculates the hillshade of each cell in an elevation raster""" if isinstance(azimuth, (int, float)): azimuth = lit(azimuth) @@ -840,7 +858,9 @@ def rf_hillshade(tile_col: Column_type, azimuth: Union[int, float, Column_type], altitude = lit(altitude) if isinstance(z_factor, (int, float)): z_factor = lit(z_factor) - return _apply_column_function('rf_hillshade', tile_col, azimuth, altitude, z_factor) + if isinstance(target, str): + target = lit(target) + return _apply_column_function('rf_hillshade', tile_col, azimuth, altitude, z_factor, target) def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile index f00dc5acb..30b7fdfb8 100644 --- a/rf-notebook/src/main/docker/Dockerfile +++ b/rf-notebook/src/main/docker/Dockerfile @@ -1,4 +1,4 @@ -# Python version compatible with Spark and GDAL 3.1.2 +# Python version compatible with Spark 3.1.x and GDAL 3.1.2 FROM jupyter/scipy-notebook:python-3.8.8 LABEL maintainer="Astraea, Inc. " diff --git a/rf-notebook/src/main/notebooks/STAC API Example.ipynb b/rf-notebook/src/main/notebooks/STAC API Example.ipynb index 3e5cf4e47..57c33ada6 100644 --- a/rf-notebook/src/main/notebooks/STAC API Example.ipynb +++ b/rf-notebook/src/main/notebooks/STAC API Example.ipynb @@ -17,21 +17,23 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "import pyrasterframes\n", "from pyrasterframes.utils import create_rf_spark_session\n", "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", "from pyrasterframes.rasterfunctions import *\n", - "import pyspark.sql.functions as F\n" + "import pyspark.sql.functions as F" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { - "scrolled": false + "scrolled": true }, "outputs": [ { @@ -45,7 +47,7 @@ "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n", "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n", "WARNING: All illegal access operations will be denied in a future release\n", - "21/10/01 00:25:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", + "21/10/02 03:12:39 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n", "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n", "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n" @@ -68,7 +70,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# read assets from the landsat-8-l1-c1 collection\n", @@ -81,7 +85,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [ { "name": "stdout", @@ -169,7 +175,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -196,7 +204,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -258,7 +268,9 @@ { "cell_type": "code", "execution_count": 7, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# select the first Landsat STAC Item\n", @@ -275,7 +287,9 @@ { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -294,7 +308,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -321,7 +337,9 @@ { "cell_type": "code", "execution_count": 10, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# read rasters from the exploded STAC Assets DataFrame\n", @@ -332,7 +350,9 @@ { "cell_type": "code", "execution_count": 11, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stderr", @@ -359,7 +379,9 @@ { "cell_type": "code", "execution_count": 12, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -397,6 +419,18 @@ "cell_type": "code", "execution_count": 13, "metadata": {}, + "outputs": [], + "source": [ + "# limit tiles, to work only with 10 DataFrame rows\n", + "rs = rs.limit(10)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": false + }, "outputs": [ { "name": "stderr", @@ -411,7 +445,7 @@ "\n", "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -426,7 +460,7 @@ "\n", "_Showing only top 5 rows_.\n", "\n", - "| rf_crs(band) | rf_extent(band) | rf_aspect(band) | rf_slope(band, 1) | rf_hillshade(band, 315, 45, 1) |\n", + "| rf_crs(band) | rf_extent(band) | rf_aspect(band, all) | rf_slope(band, 1, all) | rf_hillshade(band, 315, 45, 1, all) |\n", "|---|---|---|---|---|\n", "| utm-CS | {488445.0, -5335365.0, 503805.0, -5320005.0} | | | |\n", "| utm-CS | {657405.0, -5335365.0, 672765.0, -5320005.0} | | | |\n", @@ -435,22 +469,518 @@ "| utm-CS | {549885.0, -5366085.0, 565245.0, -5350725.0} | | | |" ], "text/plain": [ - "DataFrame[rf_crs(band): udt, rf_extent(band): struct, rf_aspect(band): struct,crs:udt>, rf_slope(band, 1): struct,crs:udt>, rf_hillshade(band, 315, 45, 1): struct,crs:udt>]" + "DataFrame[rf_crs(band): udt, rf_extent(band): struct, rf_aspect(band, all): struct,crs:udt>, rf_slope(band, 1, all): struct,crs:udt>, rf_hillshade(band, 315, 45, 1, all): struct,crs:udt>]" ] }, - "execution_count": 13, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "# apply focal operations to data cells only\n", "rs.select(\n", " rf_crs(rs.band), \n", " rf_extent(rs.band), \n", - " rf_aspect(rs.band), \n", + " rf_aspect(rs.band),\n", " rf_slope(rs.band, z_factor=1), \n", " rf_hillshade(rs.band, azimuth=315, altitude=45, z_factor=1))" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Focal operations above are applied to the entire rasters, and the NoData is not handled. Focal operations allow to specify the target cells type: \"data\", \"nodata\", \"all\"; and by default the \"all\" is used. The example below shows the NoData handling and applied focal operation only to Data cells." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Set LC 8 NoData to zero\n", + "rsnd = rs.select(rf_with_no_data(rs.band, 0).alias(\"band\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + }, + { + "data": { + "text/html": [ + "
Showing only top 5 rows
rf_crs(band)rf_extent(band)rf_aspect(band)rf_slope(band, 1)rf_hillshade(band, 315, 45, 1)
rf_crs(band)rf_extent(band)rf_aspect(band, all)rf_slope(band, 1, all)rf_hillshade(band, 315, 45, 1, all)
utm-CS{488445.0, -5335365.0, 503805.0, -5320005.0}
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
Showing only top 5 rows
rf_crs(band)rf_extent(band)rf_aspect(band, data)rf_slope(band, 1, data)rf_hillshade(band, 315, 45, 1, data)
utm-CS{488445.0, -5335365.0, 503805.0, -5320005.0}
utm-CS{657405.0, -5335365.0, 672765.0, -5320005.0}
utm-CS{688125.0, -5335365.0, 703485.0, -5320005.0}
utm-CS{642045.0, -5197125.0, 657405.0, -5181765.0}
utm-CS{549885.0, -5366085.0, 565245.0, -5350725.0}
" + ], + "text/markdown": [ + "\n", + "_Showing only top 5 rows_.\n", + "\n", + "| rf_crs(band) | rf_extent(band) | rf_aspect(band, data) | rf_slope(band, 1, data) | rf_hillshade(band, 315, 45, 1, data) |\n", + "|---|---|---|---|---|\n", + "| utm-CS | {488445.0, -5335365.0, 503805.0, -5320005.0} | | | |\n", + "| utm-CS | {657405.0, -5335365.0, 672765.0, -5320005.0} | | | |\n", + "| utm-CS | {688125.0, -5335365.0, 703485.0, -5320005.0} | | | |\n", + "| utm-CS | {642045.0, -5197125.0, 657405.0, -5181765.0} | | | |\n", + "| utm-CS | {549885.0, -5366085.0, 565245.0, -5350725.0} | | | |" + ], + "text/plain": [ + "DataFrame[rf_crs(band): udt, rf_extent(band): struct, rf_aspect(band, data): struct,crs:udt>, rf_slope(band, 1, data): struct,crs:udt>, rf_hillshade(band, 315, 45, 1, data): struct,crs:udt>]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# apply focal operations to data cells only\n", + "rsnd.select(\n", + " rf_crs(rsnd.band), \n", + " rf_extent(rsnd.band), \n", + " rf_aspect(rsnd.band, target=\"data\"),\n", + " rf_slope(rsnd.band, z_factor=1, target=\"data\"), \n", + " rf_hillshade(rsnd.band, azimuth=315, altitude=45, z_factor=1, target=\"data\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_boundary replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_coorddim replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_dimension replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_envelope replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_exteriorring replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geometryn replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geometrytype replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_interiorringn replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_isclosed replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_iscollection replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_isempty replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_isring replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_issimple replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_isvalid replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_numgeometries replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_numpoints replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_pointn replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_x replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_y replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_casttopoint replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_casttopolygon replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_casttolinestring replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_casttogeometry replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_bytearray replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_box2dfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geomfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geomfromgeojson replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geomfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geometryfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geomfromwkt replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geomfromwkb replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_linefromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_mlinefromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_mpointfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_mpolyfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makebbox replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makebox2d replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makeline replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makepoint replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makepointm replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_makepolygon replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_point replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_pointfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_pointfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_pointfromwkb replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_polygon replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_polygonfromtext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_asbinary replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_asgeojson replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_aslatlontext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_astext replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_geohash replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_antimeridiansafegeom replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_idlsafegeom replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_bufferpoint replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_translate replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_contains replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_covers replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_crosses replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_disjoint replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_equals replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_intersects replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_overlaps replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_touches replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_within replaced a previously registered function.\n", + "21/10/02 03:16:28 WARN SimpleFunctionRegistry: The function st_relate replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_relatebool replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_area replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_closestpoint replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_centroid replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_distance replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_length replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_distancesphere replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_aggregatedistancesphere replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_lengthsphere replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_convexhull replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_intersection replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_difference replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_make_constant_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_make_zeros_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_make_ones_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_cell_types replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_rasterize replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_array_to_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_add replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_subtract replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_assemble_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_explode_tiles replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_cell_type replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_convert_cell_type replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_interpret_cell_type_as replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_with_no_data replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_dimensions replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_geometry replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_geometry replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_extent replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_extent replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_crs replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_proj_raster replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_multiply replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_divide replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_normalized_difference replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_less replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_greater replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_less_equal replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_greater_equal replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_equal replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_unequal replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_is_in replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_no_data replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_data replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_min replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_max replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_clamp replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_where replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_standardize replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_rescale replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_sum replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_round replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_abs replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_log replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_log10 replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_log2 replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_log1p replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_exp replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_exp10 replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_exp2 replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_expm1 replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_sqrt replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_resample replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_resample_nearest replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_to_array_double replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_to_array_int replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_is_no_data_tile replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_exists replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_for_all replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_min replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_max replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_mean replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_stats replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_tile_histogram replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_stats replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_approx_histogram replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_stats replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_min replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_max replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_agg_local_mean replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_max replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_min replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_mean replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_mode replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_median replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_moransi replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_focal_stddev replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_convolve replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_slope replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_aspect replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_hillshade replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_mask replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_inverse_mask replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_mask_by_value replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_inverse_mask_by_value replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_mask_by_values replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_render_ascii replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_render_matrix replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_render_png replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_rgb_composite replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_xz2_index replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_z2_index replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function st_reproject replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_extract_bits replaced a previously registered function.\n", + "21/10/02 03:16:29 WARN SimpleFunctionRegistry: The function rf_local_extract_bit replaced a previously registered function.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "# save a hillshade raster to the disk as a tiff\n", + "rsnd \\\n", + " .limit(1) \\\n", + " .select(rf_hillshade(rsnd.band, azimuth=315, altitude=45, z_factor=1, target=\"data\")) \\\n", + " .write.geotiff(\"lc8-hillshade.tiff\", \"EPSG:32718\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_boundary replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_coorddim replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_dimension replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_envelope replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_exteriorring replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geometryn replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geometrytype replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_interiorringn replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_isclosed replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_iscollection replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_isempty replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_isring replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_issimple replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_isvalid replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_numgeometries replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_numpoints replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_pointn replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_x replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_y replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_casttopoint replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_casttopolygon replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_casttolinestring replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_casttogeometry replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_bytearray replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_box2dfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geomfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geomfromgeojson replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geomfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geometryfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geomfromwkt replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geomfromwkb replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_linefromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_mlinefromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_mpointfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_mpolyfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makebbox replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makebox2d replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makeline replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makepoint replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makepointm replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_makepolygon replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_point replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_pointfromgeohash replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_pointfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_pointfromwkb replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_polygon replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_polygonfromtext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_asbinary replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_asgeojson replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_aslatlontext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_astext replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geohash replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_antimeridiansafegeom replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_idlsafegeom replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_bufferpoint replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_translate replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_contains replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_covers replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_crosses replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_disjoint replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_equals replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_intersects replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_overlaps replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_touches replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_within replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_relate replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_relatebool replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_area replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_closestpoint replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_centroid replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_distance replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_length replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_distancesphere replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_aggregatedistancesphere replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_lengthsphere replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_convexhull replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_intersection replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_difference replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_make_constant_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_make_zeros_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_make_ones_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_cell_types replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_rasterize replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_array_to_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_add replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_subtract replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_assemble_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_explode_tiles replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_cell_type replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_convert_cell_type replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_interpret_cell_type_as replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_with_no_data replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_dimensions replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_geometry replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_geometry replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_extent replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_extent replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_crs replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_proj_raster replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_multiply replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_divide replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_normalized_difference replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_less replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_greater replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_less_equal replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_greater_equal replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_equal replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_unequal replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_is_in replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_no_data replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_data replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_min replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_max replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_clamp replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_where replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_standardize replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_rescale replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_sum replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_round replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_abs replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_log replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_log10 replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_log2 replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_log1p replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_exp replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_exp10 replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_exp2 replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_expm1 replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_sqrt replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_resample replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_resample_nearest replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_to_array_double replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_to_array_int replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_is_no_data_tile replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_exists replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_for_all replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_min replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_max replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_mean replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_stats replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_tile_histogram replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_stats replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_approx_histogram replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_stats replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_min replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_max replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_no_data_cells replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_agg_local_mean replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_max replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_min replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_mean replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_mode replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_median replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_moransi replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_focal_stddev replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_convolve replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_slope replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_aspect replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_hillshade replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_mask replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_inverse_mask replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_mask_by_value replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_inverse_mask_by_value replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_mask_by_values replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_render_ascii replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_render_matrix replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_render_png replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_rgb_composite replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_xz2_index replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_z2_index replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function st_reproject replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_extract_bits replaced a previously registered function.\n", + "21/10/02 03:16:51 WARN SimpleFunctionRegistry: The function rf_local_extract_bit replaced a previously registered function.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " \r" + ] + } + ], + "source": [ + "# save a hillshade raster to the disk as a tiff\n", + "rs \\\n", + " .limit(1) \\\n", + " .select(rf_hillshade(rs.band, azimuth=315, altitude=45, z_factor=1)) \\\n", + " .write.geotiff(\"lc8-hillshade-all.tiff\", \"EPSG:32718\")" + ] } ], "metadata": {