diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala index c7e36d985..adcda7c2e 100644 --- a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala @@ -46,7 +46,7 @@ class RasterRefBench extends SparkEnv with LazyLogging { val r2 = RFRasterSource(remoteCOGSingleband2) singleDF = Seq((r1, r2)).toDF("B1", "B2") - .select(RasterRefToTile(RasterSourceToRasterRefs(Some(r1.dimensions), Seq(0), $"B1", $"B2"))) + .select(RasterRefToTile(RasterSourceToRasterRefs(Some(r1.dimensions), Seq(0), 0: Short, $"B1", $"B2"))) expandedDF = Seq((r1, r2)).toDF("B1", "B2") .select(RasterRefToTile(RasterSourceToRasterRefs($"B1", $"B2"))) diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala index 5f9982307..63eaf2743 100644 --- a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala @@ -56,7 +56,7 @@ class TileEncodeBench extends SparkEnv { case "rasterRef" ⇒ val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF" val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0) - tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None)) + tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None, 0)) case _ ⇒ tile = randomTile(tileSize, tileSize, cellTypeName) } @@ -73,4 +73,3 @@ class TileEncodeBench extends SparkEnv { boundEncoder.fromRow(row) } } - diff --git a/build.sbt b/build.sbt index 7662baa5c..27efeabbb 100644 --- a/build.sbt +++ b/build.sbt @@ -24,6 +24,8 @@ addCommandAlias("previewSite", "docs/previewSite") addCommandAlias("ghpagesPushSite", "docs/ghpagesPushSite") addCommandAlias("console", "datasource/console") +clippyColorsEnabled := true + // Prefer our own IntegrationTest config definition, which inherits from Test. lazy val IntegrationTest = config("it") extend Test @@ -113,12 +115,17 @@ lazy val datasource = project spark("mllib").value % Provided, spark("sql").value % Provided ), + scalacOptions in (Compile, console) ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, + scalacOptions in (Test, console) ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, + clippyColorsEnabled := true, console / initialCommands := (console / initialCommands).value + """ |import org.locationtech.rasterframes.datasource.geotrellis._ |import org.locationtech.rasterframes.datasource.geotiff._ + |import org.locationtech.rasterframes.datasource.raster._ |""".stripMargin, IntegrationTest / fork := true, + Test / fork := true, IntegrationTest / javaOptions := Seq("-Xmx3g") ) @@ -180,4 +187,3 @@ lazy val docs = project lazy val bench = project .dependsOn(core % "compile->test") .settings(publish / skip := true) - diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala index 3ea7a65d0..f83fed807 100644 --- a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala +++ b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala @@ -39,9 +39,9 @@ class RasterRefIT extends TestEnvironment { // [west, south, east, north] val area = Extent(31.115, 29.963, 31.148, 29.99).reproject(LatLng, redScene.crs) - val red = RasterRef(redScene, 0, Some(area), None) - val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None) - val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None) + val red = RasterRef(redScene, 0, Some(area), None, 0) + val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None, 0) + val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None, 0) val rf = Seq((red, green, blue)).toDF("red", "green", "blue") val df = rf.select( @@ -54,7 +54,7 @@ class RasterRefIT extends TestEnvironment { stats should be ('defined) stats.get.dataCells shouldBe > (1000L) } - + import geotrellis.raster.io.geotiff.compression.DeflateCompression import geotrellis.raster.io.geotiff.tags.codes.ColorSpace import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled} diff --git a/core/src/main/scala/geotrellis/raster/BufferTile.scala b/core/src/main/scala/geotrellis/raster/BufferTile.scala new file mode 100644 index 000000000..3e655462b --- /dev/null +++ b/core/src/main/scala/geotrellis/raster/BufferTile.scala @@ -0,0 +1,434 @@ +/* + * Copyright 2021 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import spire.syntax.cfor._ + +/** + * When combined with another BufferTile the two tiles will be aligned on (0, 0) pixel of tile center. + * The operation will be carried over all overlapping pixels. + * For instance: combining a tile padded with 5 pixels on all sides with tile padded with 3 pixels on all sides will + * result in buffer tile with 3 pixel padding on all sides. + * + * When combined with another BufferTile the operation will be executed over the maximum shared in + * + * TODO: + * - toString method is friendly + * - toBytes needs to encode padding size? + */ +case class BufferTile( + sourceTile: Tile, + gridBounds: GridBounds[Int] +) extends Tile { + require( + gridBounds.colMin >= 0 && + gridBounds.rowMin >= 0 && + gridBounds.colMax < sourceTile.cols && + gridBounds.rowMax <= sourceTile.rows, + s"Tile center bounds $gridBounds exceed underlying tile dimensions ${sourceTile.dimensions}" + ) + + val cols = gridBounds.width + val rows = gridBounds.height + + val cellType = sourceTile.cellType + + private def colMin = gridBounds.colMin + private def rowMin = gridBounds.rowMin + private def sourceCols = sourceTile.cols + private def sourceRows = sourceTile.rows + + def bufferTop: Int = gridBounds.rowMin + def bufferLeft: Int = gridBounds.colMin + def bufferRight: Int = sourceTile.cols - gridBounds.colMin - gridBounds.colMax + def bufferBottom: Int = sourceTile.rows - gridBounds.rowMin - gridBounds.rowMax + + /** + * Returns a [[Tile]] equivalent to this tile, except with cells of + * the given type. + * + * @param targetCellType The type of cells that the result should have + * @return The new Tile + */ + def convert(targetCellType: CellType): Tile = + mutable(targetCellType) + + def withNoData(noDataValue: Option[Double]): BufferTile = + BufferTile(sourceTile.withNoData(noDataValue), gridBounds) + + def interpretAs(newCellType: CellType): BufferTile = + BufferTile(sourceTile.interpretAs(newCellType), gridBounds) + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Int datum found at the given location + */ + def get(col: Int, row: Int): Int = { + val c = col + gridBounds.colMin + val r = row + gridBounds.rowMin + if (c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") else sourceTile.get(c, r) + } + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Double datum found at the given location + */ + def getDouble(col: Int, row: Int): Double = { + val c = col + gridBounds.colMin + val r = row + gridBounds.rowMin + + if (c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") else sourceTile.getDouble(col + gridBounds.colMin, row + gridBounds.rowMin) + } + + /** + * Another name for the 'mutable' method on this class. + * + * @return An [[ArrayTile]] + */ + def toArrayTile: ArrayTile = mutable + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(): MutableArrayTile = + mutable(cellType) + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(targetCellType: CellType): MutableArrayTile = { + val tile = ArrayTile.alloc(targetCellType, cols, rows) + + if (!cellType.isFloatingPoint) cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, get(col, row)) + } + } else cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, getDouble(col, row)) + } + } + + tile + } + + /** + * Return the data behind this tile as an array of integers. + * + * @return The copy as an Array[Int] + */ + def toArray: Array[Int] = { + val arr = Array.ofDim[Int](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = get(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the data behind this tile as an array of doubles. + * + * @return The copy as an Array[Int] + */ + def toArrayDouble: Array[Double] = { + val arr = Array.ofDim[Double](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = getDouble(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the underlying data behind this tile as an array. + * + * @return An array of bytes + */ + def toBytes(): Array[Byte] = toArrayTile.toBytes + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Int to Unit + */ + def foreach(f: Int => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(get(col, row)) + } + } + } + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Double to Unit + */ + def foreachDouble(f: Double => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(getDouble(col, row)) + } + } + } + + /** + * Execute an [[IntTileVisitor]] at each cell of the present tile. + * + * @param visitor An IntTileVisitor + */ + def foreachIntVisitor(visitor: IntTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, get(col, row)) + } + } + } + + /** + * Execute an [[DoubleTileVisitor]] at each cell of the present tile. + * + * @param visitor An DoubleTileVisitor + */ + def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, getDouble(col, row)) + } + } + } + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Int to Int, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def map(f: Int => Int): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row))) + } + } + + tile + } + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Double to Double, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def mapDouble(f: Double => Double): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row))) + } + } + + tile + } + + /** + * Map an [[IntTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapIntMapper(mapper: IntTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, mapper(col, row, get(col, row))) + } + } + tile + } + + /** + * Map an [[DoubleTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapDoubleMapper(mapper: DoubleTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, mapper(col, row, getDouble(col, row))) + } + } + tile + } + + def combine(other: BufferTile)(f: (Int, Int) => Int): Tile = { + if ((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) throw new GeoAttrsError( + "Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.get(col, row) + val rightV = other.get(col, row) + tile.set(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile( + tile, + GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1)) + } + + def combineDouble(other: BufferTile)(f: (Double, Double) => Double): Tile = { + if ((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) throw new GeoAttrsError( + "Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.getDouble(col, row) + val rightV = other.getDouble(col, row) + tile.setDouble(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile( + tile, + GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1)) + } + + /** + * Combine two tiles' cells into new cells using the given integer + * function. For every (x, y) cell coordinate, get each of the + * tiles' integer values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combine(other: Tile)(f: (Int, Int) => Int): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => + this.combine(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row), other.get(col, row))) + } + } + tile + } + } + + /** + * Combine two tiles' cells into new cells using the given double + * function. For every (x, y) cell coordinate, get each of the + * tiles' double values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combineDouble(other: Tile)(f: (Double, Double) => Double): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => + this.combineDouble(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row), other.getDouble(col, row))) + } + } + tile + } + } +} + +object BufferTile { +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala index c8bfa3813..accca888d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala @@ -26,4 +26,4 @@ import org.locationtech.rasterframes.functions._ * Mix-in for UDFs for working with Tiles in Spark DataFrames. * @since 4/3/17 */ -trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions +trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions with FocalFunctions diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala similarity index 97% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala index 8d2b532c8..be1487963 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.UnaryExpression import org.locationtech.rasterframes.model.TileContext /** Boilerplate for expressions operating on a single Tile-like . */ -trait UnaryRasterOp extends UnaryExpression { +trait UnaryRasterFunction extends UnaryExpression { override def checkInputDataTypes(): TypeCheckResult = { if (!tileExtractor.isDefinedAt(child.dataType)) { TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") @@ -44,4 +44,3 @@ trait UnaryRasterOp extends UnaryExpression { protected def eval(tile: Tile, ctx: Option[TileContext]): Any } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala similarity index 60% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala index a410f47f8..257fd567f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala @@ -23,38 +23,26 @@ package org.locationtech.rasterframes.expressions import com.typesafe.scalalogging.Logger import geotrellis.raster.Tile -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.UnaryExpression import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.model.TileContext import org.slf4j.LoggerFactory /** Operation on a tile returning a tile. */ -trait UnaryLocalRasterOp extends UnaryExpression { +trait UnaryRasterOperator extends UnaryRasterFunction { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def dataType: DataType = child.dataType - override def checkInputDataTypes(): TypeCheckResult = { - if (!tileExtractor.isDefinedAt(child.dataType)) { - TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") - } - else TypeCheckSuccess - } - - override protected def nullSafeEval(input: Any): Any = { + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { implicit val tileSer = TileUDT.tileSerializer - val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - childCtx match { - case Some(ctx) => ctx.toProjectRasterTile(op(childTile)).toInternalRow - case None => op(childTile).toInternalRow + ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile)).toInternalRow + case None => op(tile).toInternalRow } } protected def op(child: Tile): Tile } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala index 4fc0a0374..e18471d3b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.accessors import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression @@ -35,7 +35,7 @@ import org.locationtech.rasterframes.tiles.InternalRowTile import org.locationtech.rasterframes._ /** Expression to extract at tile from several types that contain tiles.*/ -case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ExtractTile(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def dataType: DataType = TileType override def nodeName: String = "rf_extract_tile" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala index 6c9a3538a..4fe699d29 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala @@ -22,16 +22,16 @@ package org.locationtech.rasterframes.expressions.accessors import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext -case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class GetTileContext(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def dataType: DataType = schemaOf[TileContext] override def nodeName: String = "get_tile_context" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala new file mode 100644 index 000000000..c9e564ced --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Aspect(child: Expression) extends SurfaceOperation { + override def nodeName: String = Aspect.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) + } +} + +object Aspect { + def name: String = "rf_aspect" + def apply(tile: Column): Column = new Column(Aspect(tile.expr)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala new file mode 100644 index 000000000..fd70a7c48 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -0,0 +1,39 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} + +case class Convolve(child: Expression, kernel: Kernel) extends UnaryRasterOperator with NullToValue with CodegenFallback { + override def nodeName: String = Convolve.name + override def na: Any = null + override protected def op(t: Tile): Tile = t.convolve(kernel) +} + +object Convolve { + def name: String = "rf_convolve" + def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala new file mode 100644 index 000000000..e4b608f99 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMax.name + override protected def op(t: Tile): Tile = t.focalMax(neighborhood) +} + +object FocalMax { + def name: String = "rf_focal_max" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMax(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala new file mode 100644 index 000000000..588a3fcc3 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMean.name + override protected def op(t: Tile): Tile = t.focalMean(neighborhood) +} + +object FocalMean { + def name:String = "rf_focal_mean" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMean(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala new file mode 100644 index 000000000..dfe52ba5a --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMedian.name + override protected def op(t: Tile): Tile = t.focalMedian(neighborhood) +} + +object FocalMedian { + def name: String = "rf_focal_median" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMedian(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala new file mode 100644 index 000000000..2d6670c4f --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMin.name + override protected def op(t: Tile): Tile = t.focalMin(neighborhood) +} + +object FocalMin { + def name: String = "rf_focal_min" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMin(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala new file mode 100644 index 000000000..ecf7dfd0d --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMode.name + override protected def op(t: Tile): Tile = t.focalMode(neighborhood) +} + +object FocalMode { + def name: String = "rf_focal_mode" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMode(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala new file mode 100644 index 000000000..f89e39e0c --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalMoransI.name + override protected def op(t: Tile): Tile = t.tileMoransI(neighborhood) +} + +object FocalMoransI { + def name: String = "rf_focal_moransi" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMoransI(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala new file mode 100644 index 000000000..e4c1e3448 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala @@ -0,0 +1,31 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} + +trait FocalNeighborhoodOperator extends UnaryRasterOperator with NullToValue with CodegenFallback { + override def na: Any = null + def neighborhood: Neighborhood +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala new file mode 100644 index 000000000..c2dc89b57 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -0,0 +1,38 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = FocalStdDev.name + + override protected def op(t: Tile): Tile = t.focalStandardDeviation(neighborhood) +} + +object FocalStdDev { + def name: String = "rf_focal_stddevd" + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalStdDev(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala new file mode 100644 index 000000000..a5c73e993 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -0,0 +1,41 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOperation { + override def nodeName: String = Hillshade.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) + } +} + +object Hillshade { + def name: String = "rf_hillshade" + def apply(tile: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + new Column(Hillshade(tile.expr, azimuth, altitude, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala new file mode 100644 index 000000000..0054f01f7 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Slope(child: Expression, zFactor: Double) extends SurfaceOperation { + override def nodeName: String = Slope.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) + } +} + +object Slope { + def name: String = "rf_slope" + def apply(tile: Column, zFactor: Double): Column = new Column(Slope(tile.expr, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala new file mode 100644 index 000000000..99c542b13 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala @@ -0,0 +1,82 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import org.slf4j.LoggerFactory +import com.typesafe.scalalogging.Logger +import geotrellis.raster.Tile +import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.expressions.row +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.TileContext +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.locationtech.rasterframes.ref.RasterRef +import org.apache.spark.sql.catalyst.expressions.UnaryExpression + +import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.NullToValue +import org.locationtech.rasterframes.tiles.ProjectedRasterTile + +/** Operation on a tile returning a tile. */ +trait SurfaceOperation extends UnaryExpression with NullToValue with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + override def dataType: DataType = child.dataType + override def na: Any = null + + override def checkInputDataTypes(): TypeCheckResult = { + if (!tileExtractor.isDefinedAt(child.dataType)) { + TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") + } else TypeCheckSuccess + } + + override protected def nullSafeEval(input: Any): Any = { + val (tile, ctx) = tileExtractor(child.dataType)(row(input)) + + val literral = tile match { + case RasterRef.RasterRefTile(ref) => ref.realizedTile + case prt: ProjectedRasterTile => prt + } + eval(literral, ctx) + } + + protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { + ctx match { + case Some(ctx) => + val ret = op(tile, ctx) + ctx.toProjectRasterTile(ret).toInternalRow + + case None => + new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + } + + protected def op(t: Tile, ctx: TileContext): Tile +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala index 7022f75db..67d19e392 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala @@ -42,7 +42,7 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression +case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None, bufferSize: Short) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) @@ -55,7 +55,7 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[ } yield StructField(name, schemaOf[RasterRef], true)) private def band2ref(src: RFRasterSource, e: Option[(GridBounds[Int], Extent)])(b: Int): RasterRef = - if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1)) else null + if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1), bufferSize) else null override def eval(input: InternalRow): TraversableOnce[InternalRow] = { @@ -84,9 +84,9 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[ } object RasterSourceToRasterRefs { - def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), rrs: _*) - def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, RasterRef] = - new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[RasterRef] + def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), 0: Short, rrs: _*) + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, RasterRef] = + new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims, bufferSize)).as[RasterRef] private[rasterframes] def bandNames(basename: String, bandIndexes: Seq[Int]): Seq[String] = bandIndexes match { case Seq() => Seq.empty diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala index 309d306ae..cd391a230 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala @@ -45,7 +45,7 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression +case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], bufferSize: Short, subtileDims: Option[Dimensions[Int]] = None) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) @@ -83,9 +83,7 @@ case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], } object RasterSourceToTiles { - def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*) - def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = - new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile] + def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), 0: Short, rrs: _*) + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = + new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, bufferSize, subtileDims)).as[ProjectedRasterTile] } - - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala index d55860d29..e4b1fe6fa 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Compute the absolute value of each cell.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Abs(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Abs(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_abs" override def na: Any = null override protected def op(t: Tile): Tile = t.localAbs() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala index cdfcbaa63..ae99d339e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Defined(child: Expression) extends UnaryLocalRasterOp +case class Defined(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_data" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala index b1fb4d714..f4efef95e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Exp(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_exp" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E) @@ -60,7 +60,7 @@ object Exp { > SELECT _FUNC_(tile); ...""" ) -case class Exp10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp10(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(10.0) @@ -81,7 +81,7 @@ object Exp10 { > SELECT _FUNC_(tile); ...""" ) -case class Exp2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp2(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_exp2" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0) @@ -102,7 +102,7 @@ object Exp2{ > SELECT _FUNC_(tile); ...""" ) -case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class ExpM1(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_expm1" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0) @@ -112,4 +112,3 @@ case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallb object ExpM1{ def apply(tile: Column): Column = new Column(ExpM1(tile.expr)) } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala index ed9e4785f..abf2fd968 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return the given tile or projected raster unchanged. Useful in debugging round-trip serialization across various language and memory boundaries.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Identity(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Identity(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_identity" override def na: Any = null override protected def op(t: Tile): Tile = t diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala index a2249fa2a..285e2e20b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Log(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "log" override protected def op(tile: Tile): Tile = fpTile(tile).localLog() @@ -60,7 +60,7 @@ object Log { > SELECT _FUNC_(tile); ...""" ) -case class Log10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log10(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localLog10() @@ -81,7 +81,7 @@ object Log10 { > SELECT _FUNC_(tile); ...""" ) -case class Log2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log2(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log2" override protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0) @@ -102,7 +102,7 @@ object Log2{ > SELECT _FUNC_(tile); ...""" ) -case class Log1p(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log1p(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log1p" override protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala index 0d0cd036b..516e2d177 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Round cell values to the nearest integer without changing the cell type.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Round(child: Expression) extends UnaryLocalRasterOp +case class Round(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_round" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala index 3f5ea2d2e..1b95fec56 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( usage = "_FUNC_(tile) - Perform cell-wise square root", @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile) ... """ ) -case class Sqrt(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Sqrt(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_sqrt" override protected def op(tile: Tile): Tile = fpTile(tile).localPow(0.5) override def dataType: DataType = child.dataType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala index f91b54373..cb95f1b94 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Undefined(child: Expression) extends UnaryLocalRasterOp +case class Undefined(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_no_data" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala index 8bfda70d6..02f773a63 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala @@ -32,6 +32,7 @@ import org.locationtech.rasterframes.expressions.aggregates.CellCountAggregate.D import org.locationtech.rasterframes.expressions.aggregates._ import org.locationtech.rasterframes.expressions.generators._ import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.focalops._ import org.locationtech.rasterframes.expressions.tilestats._ import org.locationtech.rasterframes.expressions.transformers._ @@ -133,6 +134,19 @@ package object expressions { registry.registerExpression[LocalCountAggregate.LocalNoDataCellsUDAF]("rf_agg_local_no_data_cells") registry.registerExpression[LocalMeanAggregate]("rf_agg_local_mean") + registry.registerExpression[FocalMax](FocalMax.name) + registry.registerExpression[FocalMin](FocalMin.name) + registry.registerExpression[FocalMean](FocalMean.name) + registry.registerExpression[FocalMode](FocalMode.name) + registry.registerExpression[FocalMedian](FocalMedian.name) + registry.registerExpression[FocalMoransI](FocalMoransI.name) + registry.registerExpression[FocalStdDev](FocalStdDev.name) + registry.registerExpression[Convolve](Convolve.name) + + registry.registerExpression[Slope](Slope.name) + registry.registerExpression[Aspect](Aspect.name) + registry.registerExpression[Hillshade](Hillshade.name) + registry.registerExpression[Mask.MaskByDefined]("rf_mask") registry.registerExpression[Mask.InverseMaskByDefined]("rf_inverse_mask") registry.registerExpression[Mask.MaskByValue]("rf_mask_by_value") diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala index a18148db3..6167cc11b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 357""" ) -case class DataCells(child: Expression) extends UnaryRasterOp +case class DataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_data_cells" override def dataType: DataType = LongType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala index cd04b1467..02ebe2ebc 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala @@ -6,7 +6,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.isCellTrue -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -23,7 +23,7 @@ import spire.syntax.cfor.cfor true """ ) -case class Exists(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Exists(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "exists" override def dataType: DataType = BooleanType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala index a912a8a0b..3519c3a82 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala @@ -6,7 +6,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.isCellTrue -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -23,7 +23,7 @@ import spire.syntax.cfor.cfor true """ ) -case class ForAll(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ForAll(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "for_all" override def dataType: DataType = BooleanType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala index fd855cd39..fe547dff9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); false""" ) -case class IsNoDataTile(child: Expression) extends UnaryRasterOp +case class IsNoDataTile(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_is_no_data_tile" override def na: Any = true diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala index cf47ba14e..d492fc1f8 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 12""" ) -case class NoDataCells(child: Expression) extends UnaryRasterOp +case class NoDataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_no_data_cells" override def dataType: DataType = LongType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala index 096acdab6..78550a4f5 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster._ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile5); 2135.34""" ) -case class Sum(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Sum(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_sum" override def dataType: DataType = DoubleType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala index 96e3d3dcc..50d840d97 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.stats.CellHistogram import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -42,7 +42,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileHistogram(child: Expression) extends UnaryRasterOp +case class TileHistogram(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_histogram" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala index 3204f4aaf..29ca17e70 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 1""" ) -case class TileMax(child: Expression) extends UnaryRasterOp +case class TileMax(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_max" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala index 92c833f98..4a20bbc5b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMean(child: Expression) extends UnaryRasterOp +case class TileMean(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_mean" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala index 71fa0194a..cdf6c9d73 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMin(child: Expression) extends UnaryRasterOp +case class TileMin(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_min" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala index fac6d330e..50449b15b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.stats.CellStatistics import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -42,7 +42,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileStats(child: Expression) extends UnaryRasterOp +case class TileStats(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_stats" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala index 54201152e..cb37d3de9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala @@ -28,11 +28,11 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types.{DataType, StringType} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor -abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp +abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterFunction with CodegenFallback with Serializable { import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix override def dataType: DataType = StringType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala index 144a4abb6..4c41466bf 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{BinaryType, DataType} import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext /** @@ -35,7 +35,7 @@ import org.locationtech.rasterframes.model.TileContext * @param child tile column * @param ramp color ramp to use for non-composite tiles. */ -abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterOp with CodegenFallback with Serializable { +abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterFunction with CodegenFallback with Serializable { override def dataType: DataType = BinaryType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { val png = ramp.map(tile.renderPng).getOrElse(tile.renderPng()) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala index 5d7786f1c..b33faf249 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.transformers -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -36,7 +36,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayDouble(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayDouble(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_double" override def dataType: DataType = DataTypes.createArrayType(DoubleType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala index c299d57c7..3c50d5fdf 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala @@ -21,14 +21,14 @@ package org.locationtech.rasterframes.expressions.transformers -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType} import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayInt(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayInt(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_int" override def dataType: DataType = DataTypes.createArrayType(IntegerType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala new file mode 100644 index 000000000..f631da88e --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -0,0 +1,62 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions + +import geotrellis.raster.Neighborhood +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.locationtech.rasterframes.expressions.focalops._ + +trait FocalFunctions { + def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMean(tileCol, neighborhood) + + def rf_focal_median(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMedian(tileCol, neighborhood) + + def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMode(tileCol, neighborhood) + + def rf_focal_max(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMax(tileCol, neighborhood) + + def rf_focal_min(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMin(tileCol, neighborhood) + + def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood): Column = + FocalStdDev(tileCol, neighborhood) + + def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMoransI(tileCol, neighborhood) + + def rf_convolve(tileCol: Column, kernel: Kernel): Column = + Convolve(tileCol, kernel) + + def rf_aspect(tileCol: Column): Column = + Aspect(tileCol) + + def rf_slope(tileCol: Column, zFactor: Double): Column = + Slope(tileCol, zFactor) + + def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + Hillshade(tileCol, azimuth, altitude, zFactor) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala index 9d0d5f387..eba9e490d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala @@ -35,6 +35,7 @@ object TileDataContext { def apply(t: Tile): TileDataContext = { require(t.cols <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.cols}") require(t.rows <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.rows}") + // TODO EAC: match on BufferTile and add the information to TileDataContext TileDataContext( t.cellType, t.dimensions ) diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index 8c03c8427..436a9bc66 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -23,11 +23,11 @@ package org.locationtech.rasterframes.ref import com.typesafe.scalalogging.LazyLogging import geotrellis.proj4.CRS -import geotrellis.raster.{CellGrid, CellType, GridBounds, Tile} +import geotrellis.raster.{BufferTile, CellGrid, CellType, GridBounds, Tile} import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf.RasterSourceUDT -import org.apache.spark.sql.types.{IntegerType, StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, ShortType, StructField, StructType} import org.locationtech.rasterframes.RasterSourceType import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _} import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile * * @since 8/21/18 */ -case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[GridBounds[Int]]) +case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[GridBounds[Int]], bufferSize: Short) extends CellGrid[Int] with ProjectedRasterLike { def crs: CRS = source.crs def extent: Extent = subextent.getOrElse(source.extent) @@ -52,9 +52,21 @@ case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[E protected lazy val grid: GridBounds[Int] = subgrid.getOrElse(source.rasterExtent.gridBoundsFor(extent, true)) - protected lazy val realizedTile: Tile = { + lazy val realizedTile: Tile = { RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source") - source.read(grid, Seq(bandIndex)).tile.band(0) + // Pixel bounds we would like to read, including buffer + val bufferedGrid = grid.buffer(bufferSize) + // Pixel bounds we can read, including buffer + val possibleGrid = bufferedGrid.intersection(source.gridBounds).get + // Pixel bounds of center/non-buffer pixels in read tile + val tileCenterBounds = grid.offset( + colOffset = - possibleGrid.colMin, + rowOffset = - possibleGrid.rowMin) + + require(tileCenterBounds.colMin <= bufferSize) + require(tileCenterBounds.rowMin <= bufferSize) + val tile = source.read(possibleGrid, Seq(bandIndex)).tile.band(0) + BufferTile(tile, tileCenterBounds) } } @@ -64,7 +76,7 @@ object RasterRef extends LazyLogging { case class RasterRefTile(rr: RasterRef) extends ProjectedRasterTile { def extent: Extent = rr.extent def crs: CRS = rr.crs - override def cellType = rr.cellType + override def cellType: CellType = rr.cellType override def cols: Int = rr.cols override def rows: Int = rr.rows @@ -80,7 +92,8 @@ object RasterRef extends LazyLogging { StructField("source", RasterSourceType.sqlType, true), StructField("bandIndex", IntegerType, false), StructField("subextent", schemaOf[Extent], true), - StructField("subgrid", schemaOf[GridBounds[Int]], true) + StructField("subgrid", schemaOf[GridBounds[Int]], true), + StructField("bufferSize", ShortType, false) )) implicit val rasterRefSerializer: CatalystSerializer[RasterRef] = new CatalystSerializer[RasterRef] { @@ -88,24 +101,31 @@ object RasterRef extends LazyLogging { StructField("source", RasterSourceType, true), StructField("bandIndex", IntegerType, false), StructField("subextent", schemaOf[Extent], true), - StructField("subgrid", schemaOf[GridBounds[Int]], true) + StructField("subgrid", schemaOf[GridBounds[Int]], true), + StructField("bufferSize", ShortType, false) )) override def to[R](t: RasterRef, io: CatalystIO[R]): R = io.create( io.to(t.source)(RasterSourceUDT.rasterSourceSerializer), t.bandIndex, t.subextent.map(io.to[Extent]).orNull, - t.subgrid.map(io.to[GridBounds[Int]]).orNull + t.subgrid.map(io.to[GridBounds[Int]]).orNull, + t.bufferSize ) - override def from[R](row: R, io: CatalystIO[R]): RasterRef = RasterRef( - io.get[RFRasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer), - io.getInt(row, 1), - if (io.isNullAt(row, 2)) None - else Option(io.get[Extent](row, 2)), - if (io.isNullAt(row, 3)) None - else Option(io.get[GridBounds[Int]](row, 3)) - ) + override def from[R](row: R, io: CatalystIO[R]): RasterRef = { + val source = io.get[RFRasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer) + val bandIndex = io.getInt(row, 1) + val subextent = + if (io.isNullAt(row, 2)) None + else Option(io.get[Extent](row, 2)) + val subgrid = + if (io.isNullAt(row, 3)) None + else Option(io.get[GridBounds[Int]](row, 3)) + val bufferSize = io.getShort(row, 4) + + RasterRef( source, bandIndex, subextent, subgrid, bufferSize) + } } implicit def rrEncoder: ExpressionEncoder[RasterRef] = CatalystSerializerEncoder[RasterRef](true) diff --git a/core/src/test/scala/org/locationtech/rasterframes/TestData.scala b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala index 467f8a4da..3a739105a 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/TestData.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala @@ -182,7 +182,7 @@ trait TestData { TestData.randomTile(cols, rows, UByteConstantNoDataCellType) )).map(ProjectedRasterTile(_, extent, crs)) :+ null - def rasterRef = RasterRef(RFRasterSource(TestData.l8samplePath), 0, None, None) + def rasterRef = RasterRef(RFRasterSource(TestData.l8samplePath), 0, None, None, 0) def lazyPRT = rasterRef.tile diff --git a/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala index dc8a60f22..8ff82eb9a 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala @@ -107,7 +107,7 @@ class CatalystSerializerSpec extends TestEnvironment { // TODO: Decide if RasterRef should be encoded 'flat', non-'flat', or depends val src = RFRasterSource(remoteCOGSingleband1) val ext = src.extent.buffer(-3.0) - val value = RasterRef(src, 0, Some(ext), Some(src.rasterExtent.gridBoundsFor(ext))) + val value = RasterRef(src, 0, Some(ext), Some(src.rasterExtent.gridBoundsFor(ext)), 0) assertConsistent(value) assertInvertable(value) } diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala new file mode 100644 index 000000000..8eeda6ec9 --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -0,0 +1,57 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions +import geotrellis.raster.testkit.RasterMatchers +import org.locationtech.rasterframes.TestEnvironment + +class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { + describe("focal operations") { + it("should provide focal mean") { + checkDocs("rf_focal_mean") + fail() + } + it("should provide focal mode") { + checkDocs("rf_focal_mode") + fail() + } + it("should provide focal median") { + checkDocs("rf_focal_median") + fail() + } + it("should provide focal min") { + checkDocs("rf_focal_min") + fail() + } + it("should provide focal max") { + checkDocs("rf_focal_max") + fail() + } + it("should provide focal Moran's I") { + checkDocs("rf_focal_moransi") + fail() + } + it("should provide convolve") { + checkDocs("rf_convolve") + fail() + } + } +} \ No newline at end of file diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala index 9d5c4ea64..3ae8cb561 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala @@ -50,9 +50,9 @@ class RasterRefSpec extends TestEnvironment with TestData { trait Fixture { val src = RFRasterSource(remoteCOGSingleband1) - val fullRaster = RasterRef(src, 0, None, None) + val fullRaster = RasterRef(src, 0, None, None, 0) val subExtent = sub(src.extent) - val subRaster = RasterRef(src, 0, Some(subExtent), Some(src.rasterExtent.gridBoundsFor(subExtent))) + val subRaster = RasterRef(src, 0, Some(subExtent), Some(src.rasterExtent.gridBoundsFor(subExtent)), 0) } import spark.implicits._ @@ -174,7 +174,7 @@ class RasterRefSpec extends TestEnvironment with TestData { val src = RFRasterSource(remoteMODIS) val dims = src .layoutExtents(NOMINAL_TILE_DIMS) - .map(e => RasterRef(src, 0, Some(e), None)) + .map(e => RasterRef(src, 0, Some(e), None, 0)) .map(_.dimensions) .distinct @@ -185,12 +185,42 @@ class RasterRefSpec extends TestEnvironment with TestData { } } + describe("buffering") { + val src = RFRasterSource(remoteMODIS) + val refs = src + .layoutExtents(NOMINAL_TILE_DIMS) + .map(e => RasterRef(src, 0, Some(e), None, bufferSize = 3)) + val refTiles = refs.map(r => RasterRefTile(r)) + + it("should maintain reported tile size with buffering") { + val dims = refTiles + .map(_.dimensions) + .distinct + + forEvery(dims) { d => + d._1 should be <= NOMINAL_TILE_SIZE + d._2 should be <= NOMINAL_TILE_SIZE + } + } + + it("should read a buffered ref") { + val ref = refs.head + + val tile = RasterRefTile(ref) + // RasterRefTile is lazy on tile content + val v = tile.get(0, 0) + // I can't inspect the BufferTile because its hidden behind RasterRefTile.delegate + info(s"tile.get(max+1, max+1): ${tile.get(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE)}") + } + + } + describe("RasterSourceToRasterRefs") { it("should convert and expand RasterSource") { val src = RFRasterSource(remoteMODIS) import spark.implicits._ val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs(None, Seq(0), $"src")) + val refs = df.select(RasterSourceToRasterRefs(None, Seq(0), 0.toShort, $"src")) refs.count() should be (1) } @@ -198,7 +228,7 @@ class RasterRefSpec extends TestEnvironment with TestData { val src = RFRasterSource(remoteMODIS) import spark.implicits._ val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs(Some(NOMINAL_TILE_DIMS), Seq(0), $"src") as "proj_raster") + val refs = df.select(RasterSourceToRasterRefs(Some(NOMINAL_TILE_DIMS), Seq(0), 0.toShort, $"src") as "proj_raster") refs.count() shouldBe > (1L) diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala index 8fa511c48..3ccb3d113 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala @@ -39,12 +39,13 @@ class RasterSourceDataSource extends DataSourceRegister with RelationProvider { override def shortName(): String = SHORT_NAME override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { val bands = parameters.bandIndexes + val bufferSize = parameters.bufferSize val tiling = parameters.tileDims.orElse(Some(NOMINAL_TILE_DIMS)) val lazyTiles = parameters.lazyTiles val spatialIndex = parameters.spatialIndex val spec = parameters.pathSpec val catRef = spec.fold(_.registerAsTable(sqlContext), identity) - RasterSourceRelation(sqlContext, catRef, bands, tiling, lazyTiles, spatialIndex) + RasterSourceRelation(sqlContext, catRef, bands, tiling, bufferSize, lazyTiles, spatialIndex) } } @@ -54,6 +55,7 @@ object RasterSourceDataSource { final val PATHS_PARAM = "paths" final val BAND_INDEXES_PARAM = "band_indexes" final val TILE_DIMS_PARAM = "tile_dimensions" + final val BUFFER_SIZE_PARAM = "buffer_size" final val CATALOG_TABLE_PARAM = "catalog_table" final val CATALOG_TABLE_COLS_PARAM = "catalog_col_names" final val CATALOG_CSV_PARAM = "catalog_csv" @@ -122,6 +124,12 @@ object RasterSourceDataSource { def lazyTiles: Boolean = parameters .get(LAZY_TILES_PARAM).forall(_.toBoolean) + def bufferSize: Short = { + val size = parameters.get(BUFFER_SIZE_PARAM).map(_.toShort).getOrElse(0.toShort) + require(size >= 0, s"Invalid $BUFFER_SIZE_PARAM: $size") + size + } + def spatialIndex: Option[Int] = parameters .get(SPATIAL_INDEX_PARTITIONS_PARAM).flatMap(p => Try(p.toInt).toOption) @@ -192,6 +200,11 @@ object RasterSourceDataSource { reader.option(RasterSourceDataSource.TILE_DIMS_PARAM, s"$cols,$rows") ) + def withBufferSize(bufferSize: Short): TaggedReader = + tag[ReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.BUFFER_SIZE_PARAM, bufferSize.toShort) + ) + /** Indicate if tile reading should be delayed until cells are fetched. Defaults to `true`. */ def withLazyTiles(state: Boolean): TaggedReader = tag[ReaderTag][DataFrameReader]( diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala index dc6254b6b..02a689958 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala @@ -52,6 +52,7 @@ case class RasterSourceRelation( catalogTable: RasterSourceCatalogRef, bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]], + bufferSize: Short, lazyTiles: Boolean, spatialIndexPartitions: Option[Int] ) extends BaseRelation with TableScan { @@ -128,7 +129,7 @@ case class RasterSourceRelation( // Expand RasterSource into multiple columns per band, and multiple rows per tile // There's some unintentional fragility here in that the structure of the expression // is expected to line up with our column structure here. - val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, srcs: _*) as refColNames + val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, bufferSize, srcs: _*) as refColNames // RasterSourceToRasterRef is a generator, which means you have to do the Tile conversion // in a separate select statement (Query planner doesn't know how many columns ahead of time). @@ -141,7 +142,7 @@ case class RasterSourceRelation( .select(paths ++ refsToTiles ++ extras: _*) } else { - val tiles = RasterSourceToTiles(subtileDims, bandIndexes, srcs: _*) as tileColNames + val tiles = RasterSourceToTiles(subtileDims, bandIndexes, bufferSize, srcs: _*) as tileColNames withPaths .select((paths :+ tiles) ++ extras: _*) } diff --git a/datasource/src/test/scala/examples/BufferTiles.scala b/datasource/src/test/scala/examples/BufferTiles.scala new file mode 100644 index 000000000..6ba587ba2 --- /dev/null +++ b/datasource/src/test/scala/examples/BufferTiles.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package examples + +import org.apache.spark.sql._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ + +object BufferTiles extends App { + + implicit val spark = SparkSession.builder() + .master("local[*]").appName("RasterFrames") + .withKryoSerialization.getOrCreate().withRasterFrames + spark.sparkContext.setLogLevel("ERROR") + + import spark.implicits._ + + val example = "https://raw.githubusercontent.com/locationtech/rasterframes/develop/core/src/test/resources/LC08_B7_Memphis_COG.tiff" + spark.read.raster.from(example).withBufferSize(11).withTileDimensions(16, 16).load().select($"proj_raster").show(false) + // spark.stop() +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 488a48e4a..e4fa974d4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,5 +16,4 @@ addSbtPlugin("com.github.gseitz" %% "sbt-release" % "1.0.9") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") - - +addSbtPlugin("com.softwaremill.clippy" % "plugin-sbt" % "0.6.1") diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index add1c42da..a8f362412 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -118,6 +118,7 @@ def _raster_reader( catalog_col_names: Optional[List[str]] = None, band_indexes: Optional[List[int]] = None, tile_dimensions: Tuple[int] = (256, 256), + buffer_size: int = 0, lazy_tiles: bool = True, spatial_index_partitions=None, **options: str) -> DataFrame: @@ -134,6 +135,7 @@ def _raster_reader( :param catalog_col_names: required if `source` is a DataFrame or CSV string. It is a list of strings giving the names of columns containing URIs to read. :param band_indexes: list of integers indicating which bands, zero-based, to read from the raster files specified; default is to read only the first band. :param tile_dimensions: tuple or list of two indicating the default tile dimension as (columns, rows). + :param buffer_size: buffer each tile read by this many cells on all sides. :param lazy_tiles: If true (default) only generate minimal references to tile contents; if false, fetch tile cell values. :param spatial_index_partitions: If true, partitions read tiles by a Z2 spatial index using the default shuffle partitioning. If a values > 0, the given number of partitions are created instead of the default. @@ -176,7 +178,8 @@ def temp_name(): options.update({ "band_indexes": to_csv(band_indexes), "tile_dimensions": to_csv(tile_dimensions), - "lazy_tiles": str(lazy_tiles) + "lazy_tiles": str(lazy_tiles), + "buffer_size": int(buffer_size) }) # Parse the `source` argument diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index 0c48e91af..a7d98d276 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -65,6 +65,15 @@ def to_jvm(ct): elif isinstance(cell_type_arg, CellType): return to_jvm(cell_type_arg.cell_type_name) +def _parse_neighborhood(neighborhood_arg: str) -> JavaObject: + """ Convert the cell type representation to the expected JVM CellType object.""" + def to_jvm(n): + return _context_call('_parse_neighborhood', n) + + if isinstance(neighborhood_arg, str) and neighborhood_arg == "square-1": + return to_jvm(neighborhood_arg) + else: + raise NotImplementedError def rf_cell_types() -> List[CellType]: """Return a list of standard cell types""" @@ -781,6 +790,54 @@ def rf_identity(tile_col: Column_type) -> Column: """Pass tile through unchanged""" return _apply_column_function('rf_identity', tile_col) +def rf_focal_max(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mean(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mean value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mean') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_median(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_median') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_min(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the min value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_min') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mode(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mode value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mode') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_std_dev(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the standard deviation value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_std_dev') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_moransI(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_aspect(tile_col: Column_type) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + return _apply_column_function('rf_aspect', tile_col) + +def rf_slope(tile_col: Column_type, z_factor: float) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_slope') + return Column(jfcn(_to_java_column(tile_col), float(z_factor))) + +def rf_hillshade(tile_col: Column_type, azimuth: float, altitude: float, z_factor: float) -> Column: + """Calculates the hillshade of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_hillshade') + return Column(jfcn(_to_java_column(tile_col), float(azimuth), float(altitude), float(z_factor))) def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala index 906988691..eeea50ce4 100644 --- a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -38,6 +38,7 @@ import org.locationtech.rasterframes.{RasterFunctions, _} import spray.json._ import org.locationtech.rasterframes.util.JsonCodecs._ import scala.collection.JavaConverters._ +import geotrellis.raster.mapalgebra.focal.{Neighborhood, Square} /** * py4j access wrapper to RasterFrameLayer entry points. @@ -149,6 +150,8 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions */ def _parse_cell_type(name: String): CellType = CellType.fromName(name) + def _parse_neighborhood(name: String): Neighborhood = Square(1) + /** * Convenience list of valid cell type strings * @return Java List of String, which py4j can interpret as a python `list` @@ -255,7 +258,7 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions def _resolveRasterRef(srcBin: Array[Byte], bandIndex: jInt, xmin: jDouble, ymin: jDouble, xmax: jDouble, ymax: jDouble): AnyRef = { val src = KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(srcBin)) val extent = Extent(xmin, ymin, xmax, ymax) - RasterRef(src, bandIndex, Some(extent), None) + RasterRef(src, bandIndex, Some(extent), None, 0: Short) } def _dfToMarkdown(df: DataFrame, numRows: Int, truncate: Boolean): String = { diff --git a/rf-notebook/src/main/notebooks/FocalOperations.ipynb b/rf-notebook/src/main/notebooks/FocalOperations.ipynb new file mode 100644 index 000000000..464fe45c4 --- /dev/null +++ b/rf-notebook/src/main/notebooks/FocalOperations.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Focal Operations with RastrFrames Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Spark Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "from pyrasterframes.utils import create_rf_spark_session\n", + "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", + "from pyrasterframes.rasterfunctions import *\n", + "import pyspark.sql.functions as F" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "spark = create_rf_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get a PySpark DataFrame from elevation raster\n", + "\n", + "Read a single scene of elevation into DataFrame or raster tiles.\n", + "Each tile overlaps its neighbor by \"buffer_size\" of pixels, providing focal operations neighbor information around tile edges.\n", + "You can configure the default size of these tiles, by passing a tuple of desired columns and rows as: `raster(uri, tile_dimensions=(96, 96))`. The default is `(256, 256)`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "uri = 'https://geotrellis-demo.s3.us-east-1.amazonaws.com/cogs/harrisburg-pa/elevation.tif'\n", + "df = spark.read.raster(uri, tile_dimensions=(512, 512), buffer_size=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "root\n |-- proj_raster_path: string (nullable = false)\n |-- proj_raster: struct (nullable = true)\n | |-- tile_context: struct (nullable = true)\n | | |-- extent: struct (nullable = false)\n | | | |-- xmin: double (nullable = false)\n | | | |-- ymin: double (nullable = false)\n | | | |-- xmax: double (nullable = false)\n | | | |-- ymax: double (nullable = false)\n | | |-- crs: struct (nullable = false)\n | | | |-- crsProj4: string (nullable = false)\n | |-- tile: tile (nullable = false)\n\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The extent struct tells us where in the [CRS](https://spatialreference.org/ref/sr-org/6842/) the tile data covers. The granule is split into arbitrary sized chunks. Each row is a different chunk. Let's see how many." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "81" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], + "source": [ + "df.count()" + ] + }, + { + "source": [ + "## Focal Operations\n", + "Additional transformations are complished through use of column functions.\n", + "The functions used here are mapped to their Scala implementation and applied per row.\n", + "For each row the source elevation data is fetched only once before it's used as input." + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[rf_crs(proj_raster): struct, rf_extent(proj_raster): struct, rf_aspect(proj_raster): struct,crs:struct>,tile:udt>, rf_slope(proj_raster): struct,crs:struct>,tile:udt>, rf_hillshade(proj_raster): struct,crs:struct>,tile:udt>]" + ], + "text/html": "\n\n\n\n\n\n\n\n\n\n\n\n
Showing only top 5 rows
rf_crs(proj_raster)rf_extent(proj_raster)rf_aspect(proj_raster)rf_slope(proj_raster)rf_hillshade(proj_raster)
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][302369.2154, 4478399.0319, 317729.2154, 4493759.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4398599.0319, 287009.2154, 4401599.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][225569.2154, 4416959.0319, 240929.2154, 4432319.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4463039.0319, 287009.2154, 4478399.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][317729.2154, 4416959.0319, 333089.2154, 4432319.0319]
", + "text/markdown": "\n_Showing only top 5 rows_.\n\n| rf_crs(proj_raster) | rf_extent(proj_raster) | rf_aspect(proj_raster) | rf_slope(proj_raster) | rf_hillshade(proj_raster) |\n|---|---|---|---|---|\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[302369.2154, 4478399.0319, 317729.2154, 4493759.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4398599.0319, 287009.2154, 4401599.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[225569.2154, 4416959.0319, 240929.2154, 4432319.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4463039.0319, 287009.2154, 4478399.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[317729.2154, 4416959.0319, 333089.2154, 4432319.0319] | | | |" + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "df.select(\n", + " rf_crs(df.proj_raster), \n", + " rf_extent(df.proj_raster), \n", + " rf_aspect(df.proj_raster), \n", + " rf_slope(df.proj_raster, z_factor=1), \n", + " rf_hillshade(df.proj_raster, azimuth=315, altitude=45, z_factor=1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file