From 31e4383cae99efeb3860352198944fa4fb3a8a93 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Sun, 28 Mar 2021 14:50:28 -0400 Subject: [PATCH 01/11] initial commit RasterRef decoding sometimes produces buffers of 0 when non-zero buffer is requested from DataSource --- .../rasterframes/bench/RasterRefBench.scala | 2 +- .../rasterframes/bench/TileEncodeBench.scala | 3 +- build.sbt | 8 +- .../rasterframes/ref/RasterRefIT.scala | 8 +- .../scala/geotrellis/raster/BufferTile.scala | 434 ++++++++++++++++++ .../expressions/focalops/Convolve.scala | 53 +++ .../generators/RasterSourceToRasterRefs.scala | 11 +- .../generators/RasterSourceToTiles.scala | 10 +- .../rasterframes/model/TileDataContext.scala | 1 + .../rasterframes/ref/RasterRef.scala | 49 +- .../locationtech/rasterframes/TestData.scala | 2 +- .../encoders/CatalystSerializerSpec.scala | 2 +- .../rasterframes/ref/RasterRefSpec.scala | 40 +- .../raster/RasterSourceDataSource.scala | 11 +- .../raster/RasterSourceRelation.scala | 5 +- .../src/test/scala/examples/BufferTiles.scala | 40 ++ project/plugins.sbt | 3 +- .../rasterframes/py/PyRFContext.scala | 2 +- 18 files changed, 637 insertions(+), 47 deletions(-) create mode 100644 core/src/main/scala/geotrellis/raster/BufferTile.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala create mode 100644 datasource/src/test/scala/examples/BufferTiles.scala diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala index c7e36d985..adcda7c2e 100644 --- a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala @@ -46,7 +46,7 @@ class RasterRefBench extends SparkEnv with LazyLogging { val r2 = RFRasterSource(remoteCOGSingleband2) singleDF = Seq((r1, r2)).toDF("B1", "B2") - .select(RasterRefToTile(RasterSourceToRasterRefs(Some(r1.dimensions), Seq(0), $"B1", $"B2"))) + .select(RasterRefToTile(RasterSourceToRasterRefs(Some(r1.dimensions), Seq(0), 0: Short, $"B1", $"B2"))) expandedDF = Seq((r1, r2)).toDF("B1", "B2") .select(RasterRefToTile(RasterSourceToRasterRefs($"B1", $"B2"))) diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala index 5f9982307..63eaf2743 100644 --- a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala +++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala @@ -56,7 +56,7 @@ class TileEncodeBench extends SparkEnv { case "rasterRef" ⇒ val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF" val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0) - tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None)) + tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None, 0)) case _ ⇒ tile = randomTile(tileSize, tileSize, cellTypeName) } @@ -73,4 +73,3 @@ class TileEncodeBench extends SparkEnv { boundEncoder.fromRow(row) } } - diff --git a/build.sbt b/build.sbt index 7662baa5c..27efeabbb 100644 --- a/build.sbt +++ b/build.sbt @@ -24,6 +24,8 @@ addCommandAlias("previewSite", "docs/previewSite") addCommandAlias("ghpagesPushSite", "docs/ghpagesPushSite") addCommandAlias("console", "datasource/console") +clippyColorsEnabled := true + // Prefer our own IntegrationTest config definition, which inherits from Test. lazy val IntegrationTest = config("it") extend Test @@ -113,12 +115,17 @@ lazy val datasource = project spark("mllib").value % Provided, spark("sql").value % Provided ), + scalacOptions in (Compile, console) ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, + scalacOptions in (Test, console) ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) }, + clippyColorsEnabled := true, console / initialCommands := (console / initialCommands).value + """ |import org.locationtech.rasterframes.datasource.geotrellis._ |import org.locationtech.rasterframes.datasource.geotiff._ + |import org.locationtech.rasterframes.datasource.raster._ |""".stripMargin, IntegrationTest / fork := true, + Test / fork := true, IntegrationTest / javaOptions := Seq("-Xmx3g") ) @@ -180,4 +187,3 @@ lazy val docs = project lazy val bench = project .dependsOn(core % "compile->test") .settings(publish / skip := true) - diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala index 3ea7a65d0..f83fed807 100644 --- a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala +++ b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala @@ -39,9 +39,9 @@ class RasterRefIT extends TestEnvironment { // [west, south, east, north] val area = Extent(31.115, 29.963, 31.148, 29.99).reproject(LatLng, redScene.crs) - val red = RasterRef(redScene, 0, Some(area), None) - val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None) - val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None) + val red = RasterRef(redScene, 0, Some(area), None, 0) + val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None, 0) + val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None, 0) val rf = Seq((red, green, blue)).toDF("red", "green", "blue") val df = rf.select( @@ -54,7 +54,7 @@ class RasterRefIT extends TestEnvironment { stats should be ('defined) stats.get.dataCells shouldBe > (1000L) } - + import geotrellis.raster.io.geotiff.compression.DeflateCompression import geotrellis.raster.io.geotiff.tags.codes.ColorSpace import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled} diff --git a/core/src/main/scala/geotrellis/raster/BufferTile.scala b/core/src/main/scala/geotrellis/raster/BufferTile.scala new file mode 100644 index 000000000..3e655462b --- /dev/null +++ b/core/src/main/scala/geotrellis/raster/BufferTile.scala @@ -0,0 +1,434 @@ +/* + * Copyright 2021 Azavea + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package geotrellis.raster + +import spire.syntax.cfor._ + +/** + * When combined with another BufferTile the two tiles will be aligned on (0, 0) pixel of tile center. + * The operation will be carried over all overlapping pixels. + * For instance: combining a tile padded with 5 pixels on all sides with tile padded with 3 pixels on all sides will + * result in buffer tile with 3 pixel padding on all sides. + * + * When combined with another BufferTile the operation will be executed over the maximum shared in + * + * TODO: + * - toString method is friendly + * - toBytes needs to encode padding size? + */ +case class BufferTile( + sourceTile: Tile, + gridBounds: GridBounds[Int] +) extends Tile { + require( + gridBounds.colMin >= 0 && + gridBounds.rowMin >= 0 && + gridBounds.colMax < sourceTile.cols && + gridBounds.rowMax <= sourceTile.rows, + s"Tile center bounds $gridBounds exceed underlying tile dimensions ${sourceTile.dimensions}" + ) + + val cols = gridBounds.width + val rows = gridBounds.height + + val cellType = sourceTile.cellType + + private def colMin = gridBounds.colMin + private def rowMin = gridBounds.rowMin + private def sourceCols = sourceTile.cols + private def sourceRows = sourceTile.rows + + def bufferTop: Int = gridBounds.rowMin + def bufferLeft: Int = gridBounds.colMin + def bufferRight: Int = sourceTile.cols - gridBounds.colMin - gridBounds.colMax + def bufferBottom: Int = sourceTile.rows - gridBounds.rowMin - gridBounds.rowMax + + /** + * Returns a [[Tile]] equivalent to this tile, except with cells of + * the given type. + * + * @param targetCellType The type of cells that the result should have + * @return The new Tile + */ + def convert(targetCellType: CellType): Tile = + mutable(targetCellType) + + def withNoData(noDataValue: Option[Double]): BufferTile = + BufferTile(sourceTile.withNoData(noDataValue), gridBounds) + + def interpretAs(newCellType: CellType): BufferTile = + BufferTile(sourceTile.interpretAs(newCellType), gridBounds) + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Int datum found at the given location + */ + def get(col: Int, row: Int): Int = { + val c = col + gridBounds.colMin + val r = row + gridBounds.rowMin + if (c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") else sourceTile.get(c, r) + } + + /** + * Fetch the datum at the given column and row of the tile. + * + * @param col The column + * @param row The row + * @return The Double datum found at the given location + */ + def getDouble(col: Int, row: Int): Double = { + val c = col + gridBounds.colMin + val r = row + gridBounds.rowMin + + if (c < 0 || r < 0 || c >= sourceCols || r >= sourceRows) throw new IndexOutOfBoundsException(s"(col=$col, row=$row) is out of tile bounds") else sourceTile.getDouble(col + gridBounds.colMin, row + gridBounds.rowMin) + } + + /** + * Another name for the 'mutable' method on this class. + * + * @return An [[ArrayTile]] + */ + def toArrayTile: ArrayTile = mutable + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(): MutableArrayTile = + mutable(cellType) + + /** + * Return the [[MutableArrayTile]] equivalent of this tile. + * + * @return An MutableArrayTile + */ + def mutable(targetCellType: CellType): MutableArrayTile = { + val tile = ArrayTile.alloc(targetCellType, cols, rows) + + if (!cellType.isFloatingPoint) cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, get(col, row)) + } + } else cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, getDouble(col, row)) + } + } + + tile + } + + /** + * Return the data behind this tile as an array of integers. + * + * @return The copy as an Array[Int] + */ + def toArray: Array[Int] = { + val arr = Array.ofDim[Int](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = get(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the data behind this tile as an array of doubles. + * + * @return The copy as an Array[Int] + */ + def toArrayDouble: Array[Double] = { + val arr = Array.ofDim[Double](cols * rows) + + var i = 0 + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + arr(i) = getDouble(col, row) + i += 1 + } + } + + arr + } + + /** + * Return the underlying data behind this tile as an array. + * + * @return An array of bytes + */ + def toBytes(): Array[Byte] = toArrayTile.toBytes + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Int to Unit + */ + def foreach(f: Int => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(get(col, row)) + } + } + } + + /** + * Execute a function on each cell of the tile. The function + * returns Unit, so it presumably produces side-effects. + * + * @param f A function from Double to Unit + */ + def foreachDouble(f: Double => Unit): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + f(getDouble(col, row)) + } + } + } + + /** + * Execute an [[IntTileVisitor]] at each cell of the present tile. + * + * @param visitor An IntTileVisitor + */ + def foreachIntVisitor(visitor: IntTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, get(col, row)) + } + } + } + + /** + * Execute an [[DoubleTileVisitor]] at each cell of the present tile. + * + * @param visitor An DoubleTileVisitor + */ + def foreachDoubleVisitor(visitor: DoubleTileVisitor): Unit = { + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + visitor(col, row, getDouble(col, row)) + } + } + } + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Int to Int, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def map(f: Int => Int): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row))) + } + } + + tile + } + + /** + * Map each cell in the given tile to a new one, using the given + * function. + * + * @param f A function from Double to Double, executed at each point of the tile + * @return The result, a [[Tile]] + */ + def mapDouble(f: Double => Double): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row))) + } + } + + tile + } + + /** + * Map an [[IntTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapIntMapper(mapper: IntTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, mapper(col, row, get(col, row))) + } + } + tile + } + + /** + * Map an [[DoubleTileMapper]] over the present tile. + * + * @param mapper The mapper + * @return The result, a [[Tile]] + */ + def mapDoubleMapper(mapper: DoubleTileMapper): Tile = { + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, mapper(col, row, getDouble(col, row))) + } + } + tile + } + + def combine(other: BufferTile)(f: (Int, Int) => Int): Tile = { + if ((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) throw new GeoAttrsError( + "Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.get(col, row) + val rightV = other.get(col, row) + tile.set(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile( + tile, + GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1)) + } + + def combineDouble(other: BufferTile)(f: (Double, Double) => Double): Tile = { + if ((this.gridBounds.width != other.gridBounds.width) || (this.gridBounds.height != other.gridBounds.height)) throw new GeoAttrsError( + "Cannot combine rasters with different dimensions: " + + s"${this.gridBounds.width}x${this.gridBounds.height} != ${other.gridBounds.width}x${other.gridBounds.height}") + + val bufferTop = math.min(this.bufferTop, other.bufferTop) + val bufferLeft = math.min(this.bufferLeft, other.bufferLeft) + val bufferRight = math.min(this.bufferRight, other.bufferRight) + val bufferBottom = math.min(this.bufferBottom, other.bufferBottom) + val cols = bufferLeft + gridBounds.width + bufferRight + val rows = bufferTop + gridBounds.height + bufferBottom + + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + + // index both tiles relative to (0, 0) pixel + cfor(-bufferTop)(_ < gridBounds.height + bufferRight, _ + 1) { row => + cfor(-bufferLeft)(_ < gridBounds.width + bufferRight, _ + 1) { col => + val leftV = this.getDouble(col, row) + val rightV = other.getDouble(col, row) + tile.setDouble(col + bufferLeft, row + bufferTop, f(leftV, rightV)) + } + } + + if (bufferTop + bufferLeft + bufferRight + bufferBottom == 0) + tile + else + BufferTile( + tile, + GridBounds[Int]( + colMin = bufferLeft, + rowMin = bufferTop, + colMax = bufferLeft + gridBounds.width - 1, + rowMax = bufferTop + gridBounds.height - 1)) + } + + /** + * Combine two tiles' cells into new cells using the given integer + * function. For every (x, y) cell coordinate, get each of the + * tiles' integer values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combine(other: Tile)(f: (Int, Int) => Int): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => + this.combine(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType.union(other.cellType), cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.set(col, row, f(get(col, row), other.get(col, row))) + } + } + tile + } + } + + /** + * Combine two tiles' cells into new cells using the given double + * function. For every (x, y) cell coordinate, get each of the + * tiles' double values, map them to a new value, and assign it to + * the output's (x, y) cell. + * + * @param other The other Tile + * @param f A function from (Int, Int) to Int + * @return The result, an Tile + */ + def combineDouble(other: Tile)(f: (Double, Double) => Double): Tile = { + (this, other).assertEqualDimensions + + other match { + case bt: BufferTile => + this.combineDouble(bt)(f) + case _ => + val tile = ArrayTile.alloc(cellType, cols, rows) + cfor(0)(_ < rows, _ + 1) { row => + cfor(0)(_ < cols, _ + 1) { col => + tile.setDouble(col, row, f(getDouble(col, row), other.getDouble(col, row))) + } + } + tile + } + } +} + +object BufferTile { +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala new file mode 100644 index 000000000..25dc4fcd5 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -0,0 +1,53 @@ +///* +// * This software is licensed under the Apache 2 license, quoted below. +// * +// * Copyright 2020 Astraea, Inc. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); you may not +// * use this file except in compliance with the License. You may obtain a copy of +// * the License at +// * +// * [http://www.apache.org/licenses/LICENSE-2.0] +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// * License for the specific language governing permissions and limitations under +// * the License. +// * +// * SPDX-License-Identifier: Apache-2.0 +// * +// */ +// +//package org.locationtech.rasterframes.expressions.focalops +//import geotrellis.raster.Tile +//import geotrellis.raster.mapalgebra.focal.Kernel +//import org.apache.spark.sql.{Column, TypedColumn} +//import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} +//import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +//import org.apache.spark.sql.types.DataType +//import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +//import org.locationtech.rasterframes.model.TileContext +// +//@ExpressionDescription( +// usage = "_FUNC_(tile, kernel) - ", +// arguments = """ +// Arguments: +// * tile - +// * kernel - """, +// examples = "" +// Examples: +// > SELECT _FUNC_(tile, Square(1)); +// ...""" +//) +//case class Convolve(child: Expression, kernel: Kernel) extends UnaryRasterOp with NullToValue with CodegenFallback { +// override def nodeName: String = "rf_convolve" +// override def na: Any = null +// override def eval(tile: Tile, ctx: Option[TileContext]): Tile = tile.convolve(kernel) +// +// def dataType: DataType = ??? +//} +// +//object Convolve { +// def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) +//} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala index 7022f75db..014c0f788 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala @@ -42,8 +42,9 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression +case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None, bufferSize: Short) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { + require(bufferSize > 0) override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) override def nodeName: String = "rf_raster_source_to_raster_ref" @@ -55,7 +56,7 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[ } yield StructField(name, schemaOf[RasterRef], true)) private def band2ref(src: RFRasterSource, e: Option[(GridBounds[Int], Extent)])(b: Int): RasterRef = - if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1)) else null + if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1), bufferSize) else null override def eval(input: InternalRow): TraversableOnce[InternalRow] = { @@ -84,9 +85,9 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[ } object RasterSourceToRasterRefs { - def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), rrs: _*) - def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, RasterRef] = - new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[RasterRef] + def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), 0: Short, rrs: _*) + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, RasterRef] = + new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims, bufferSize)).as[RasterRef] private[rasterframes] def bandNames(basename: String, bandIndexes: Seq[Int]): Seq[String] = bandIndexes match { case Seq() => Seq.empty diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala index 309d306ae..cd391a230 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala @@ -45,7 +45,7 @@ import scala.util.control.NonFatal * * @since 9/6/18 */ -case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression +case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], bufferSize: Short, subtileDims: Option[Dimensions[Int]] = None) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) @@ -83,9 +83,7 @@ case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], } object RasterSourceToTiles { - def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*) - def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = - new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile] + def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), 0: Short, rrs: _*) + def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], bufferSize: Short, rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = + new Column(new RasterSourceToTiles(rrs.map(_.expr), bandIndexes, bufferSize, subtileDims)).as[ProjectedRasterTile] } - - diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala index 9d0d5f387..eba9e490d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala @@ -35,6 +35,7 @@ object TileDataContext { def apply(t: Tile): TileDataContext = { require(t.cols <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.cols}") require(t.rows <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.rows}") + // TODO EAC: match on BufferTile and add the information to TileDataContext TileDataContext( t.cellType, t.dimensions ) diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index 8c03c8427..67f100ba1 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -23,11 +23,11 @@ package org.locationtech.rasterframes.ref import com.typesafe.scalalogging.LazyLogging import geotrellis.proj4.CRS -import geotrellis.raster.{CellGrid, CellType, GridBounds, Tile} +import geotrellis.raster.{BufferTile, CellGrid, CellType, GridBounds, Tile} import geotrellis.vector.{Extent, ProjectedExtent} import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder import org.apache.spark.sql.rf.RasterSourceUDT -import org.apache.spark.sql.types.{IntegerType, StructField, StructType} +import org.apache.spark.sql.types.{IntegerType, ShortType, StructField, StructType} import org.locationtech.rasterframes.RasterSourceType import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _} import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile * * @since 8/21/18 */ -case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[GridBounds[Int]]) +case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[GridBounds[Int]], bufferSize: Short) extends CellGrid[Int] with ProjectedRasterLike { def crs: CRS = source.crs def extent: Extent = subextent.getOrElse(source.extent) @@ -54,7 +54,19 @@ case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[E protected lazy val realizedTile: Tile = { RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source") - source.read(grid, Seq(bandIndex)).tile.band(0) + // Pixel bounds we would like to read, including buffer + val bufferedGrid = grid.buffer(bufferSize) + // Pixel bounds we can read, including buffer + val possibleGrid = bufferedGrid.intersection(source.gridBounds).get + // Pixel bounds of center/non-buffer pixels in read tile + val tileCenterBounds = grid.offset( + colOffset = - possibleGrid.colMin, + rowOffset = - possibleGrid.rowMin) + + require(tileCenterBounds.colMin <= bufferSize) + require(tileCenterBounds.rowMin <= bufferSize) + val tile = source.read(possibleGrid, Seq(bandIndex)).tile.band(0) + BufferTile(tile, tileCenterBounds) } } @@ -64,7 +76,7 @@ object RasterRef extends LazyLogging { case class RasterRefTile(rr: RasterRef) extends ProjectedRasterTile { def extent: Extent = rr.extent def crs: CRS = rr.crs - override def cellType = rr.cellType + override def cellType: CellType = rr.cellType override def cols: Int = rr.cols override def rows: Int = rr.rows @@ -88,24 +100,31 @@ object RasterRef extends LazyLogging { StructField("source", RasterSourceType, true), StructField("bandIndex", IntegerType, false), StructField("subextent", schemaOf[Extent], true), - StructField("subgrid", schemaOf[GridBounds[Int]], true) + StructField("subgrid", schemaOf[GridBounds[Int]], true), + StructField("bufferSize", ShortType, false) )) override def to[R](t: RasterRef, io: CatalystIO[R]): R = io.create( io.to(t.source)(RasterSourceUDT.rasterSourceSerializer), t.bandIndex, t.subextent.map(io.to[Extent]).orNull, - t.subgrid.map(io.to[GridBounds[Int]]).orNull + t.subgrid.map(io.to[GridBounds[Int]]).orNull, + t.bufferSize ) - override def from[R](row: R, io: CatalystIO[R]): RasterRef = RasterRef( - io.get[RFRasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer), - io.getInt(row, 1), - if (io.isNullAt(row, 2)) None - else Option(io.get[Extent](row, 2)), - if (io.isNullAt(row, 3)) None - else Option(io.get[GridBounds[Int]](row, 3)) - ) + override def from[R](row: R, io: CatalystIO[R]): RasterRef = { + val source = io.get[RFRasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer) + val bandIndex = io.getInt(row, 1) + val subextent = + if (io.isNullAt(row, 2)) None + else Option(io.get[Extent](row, 2)) + val subgrid = + if (io.isNullAt(row, 3)) None + else Option(io.get[GridBounds[Int]](row, 3)) + val bufferSize = io.getShort(row, 4) + + RasterRef( source, bandIndex, subextent, subgrid, bufferSize) + } } implicit def rrEncoder: ExpressionEncoder[RasterRef] = CatalystSerializerEncoder[RasterRef](true) diff --git a/core/src/test/scala/org/locationtech/rasterframes/TestData.scala b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala index 467f8a4da..3a739105a 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/TestData.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/TestData.scala @@ -182,7 +182,7 @@ trait TestData { TestData.randomTile(cols, rows, UByteConstantNoDataCellType) )).map(ProjectedRasterTile(_, extent, crs)) :+ null - def rasterRef = RasterRef(RFRasterSource(TestData.l8samplePath), 0, None, None) + def rasterRef = RasterRef(RFRasterSource(TestData.l8samplePath), 0, None, None, 0) def lazyPRT = rasterRef.tile diff --git a/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala index dc8a60f22..8ff82eb9a 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/encoders/CatalystSerializerSpec.scala @@ -107,7 +107,7 @@ class CatalystSerializerSpec extends TestEnvironment { // TODO: Decide if RasterRef should be encoded 'flat', non-'flat', or depends val src = RFRasterSource(remoteCOGSingleband1) val ext = src.extent.buffer(-3.0) - val value = RasterRef(src, 0, Some(ext), Some(src.rasterExtent.gridBoundsFor(ext))) + val value = RasterRef(src, 0, Some(ext), Some(src.rasterExtent.gridBoundsFor(ext)), 0) assertConsistent(value) assertInvertable(value) } diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala index 9d5c4ea64..3ae8cb561 100644 --- a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala +++ b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala @@ -50,9 +50,9 @@ class RasterRefSpec extends TestEnvironment with TestData { trait Fixture { val src = RFRasterSource(remoteCOGSingleband1) - val fullRaster = RasterRef(src, 0, None, None) + val fullRaster = RasterRef(src, 0, None, None, 0) val subExtent = sub(src.extent) - val subRaster = RasterRef(src, 0, Some(subExtent), Some(src.rasterExtent.gridBoundsFor(subExtent))) + val subRaster = RasterRef(src, 0, Some(subExtent), Some(src.rasterExtent.gridBoundsFor(subExtent)), 0) } import spark.implicits._ @@ -174,7 +174,7 @@ class RasterRefSpec extends TestEnvironment with TestData { val src = RFRasterSource(remoteMODIS) val dims = src .layoutExtents(NOMINAL_TILE_DIMS) - .map(e => RasterRef(src, 0, Some(e), None)) + .map(e => RasterRef(src, 0, Some(e), None, 0)) .map(_.dimensions) .distinct @@ -185,12 +185,42 @@ class RasterRefSpec extends TestEnvironment with TestData { } } + describe("buffering") { + val src = RFRasterSource(remoteMODIS) + val refs = src + .layoutExtents(NOMINAL_TILE_DIMS) + .map(e => RasterRef(src, 0, Some(e), None, bufferSize = 3)) + val refTiles = refs.map(r => RasterRefTile(r)) + + it("should maintain reported tile size with buffering") { + val dims = refTiles + .map(_.dimensions) + .distinct + + forEvery(dims) { d => + d._1 should be <= NOMINAL_TILE_SIZE + d._2 should be <= NOMINAL_TILE_SIZE + } + } + + it("should read a buffered ref") { + val ref = refs.head + + val tile = RasterRefTile(ref) + // RasterRefTile is lazy on tile content + val v = tile.get(0, 0) + // I can't inspect the BufferTile because its hidden behind RasterRefTile.delegate + info(s"tile.get(max+1, max+1): ${tile.get(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE)}") + } + + } + describe("RasterSourceToRasterRefs") { it("should convert and expand RasterSource") { val src = RFRasterSource(remoteMODIS) import spark.implicits._ val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs(None, Seq(0), $"src")) + val refs = df.select(RasterSourceToRasterRefs(None, Seq(0), 0.toShort, $"src")) refs.count() should be (1) } @@ -198,7 +228,7 @@ class RasterRefSpec extends TestEnvironment with TestData { val src = RFRasterSource(remoteMODIS) import spark.implicits._ val df = Seq(src).toDF("src") - val refs = df.select(RasterSourceToRasterRefs(Some(NOMINAL_TILE_DIMS), Seq(0), $"src") as "proj_raster") + val refs = df.select(RasterSourceToRasterRefs(Some(NOMINAL_TILE_DIMS), Seq(0), 0.toShort, $"src") as "proj_raster") refs.count() shouldBe > (1L) diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala index 8fa511c48..e7f8b35fe 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala @@ -39,12 +39,13 @@ class RasterSourceDataSource extends DataSourceRegister with RelationProvider { override def shortName(): String = SHORT_NAME override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { val bands = parameters.bandIndexes + val bufferSize = parameters.bufferSize val tiling = parameters.tileDims.orElse(Some(NOMINAL_TILE_DIMS)) val lazyTiles = parameters.lazyTiles val spatialIndex = parameters.spatialIndex val spec = parameters.pathSpec val catRef = spec.fold(_.registerAsTable(sqlContext), identity) - RasterSourceRelation(sqlContext, catRef, bands, tiling, lazyTiles, spatialIndex) + RasterSourceRelation(sqlContext, catRef, bands, tiling, bufferSize, lazyTiles, spatialIndex) } } @@ -54,6 +55,7 @@ object RasterSourceDataSource { final val PATHS_PARAM = "paths" final val BAND_INDEXES_PARAM = "band_indexes" final val TILE_DIMS_PARAM = "tile_dimensions" + final val BUFFER_SIZE_PARAM = "buffer_size" final val CATALOG_TABLE_PARAM = "catalog_table" final val CATALOG_TABLE_COLS_PARAM = "catalog_col_names" final val CATALOG_CSV_PARAM = "catalog_csv" @@ -122,6 +124,8 @@ object RasterSourceDataSource { def lazyTiles: Boolean = parameters .get(LAZY_TILES_PARAM).forall(_.toBoolean) + def bufferSize: Short = parameters.get(BUFFER_SIZE_PARAM).map(_.toShort).getOrElse(-1.toShort) + def spatialIndex: Option[Int] = parameters .get(SPATIAL_INDEX_PARTITIONS_PARAM).flatMap(p => Try(p.toInt).toOption) @@ -192,6 +196,11 @@ object RasterSourceDataSource { reader.option(RasterSourceDataSource.TILE_DIMS_PARAM, s"$cols,$rows") ) + def withBufferSize(bufferSize: Short): TaggedReader = + tag[ReaderTag][DataFrameReader]( + reader.option(RasterSourceDataSource.BUFFER_SIZE_PARAM, bufferSize.toShort) + ) + /** Indicate if tile reading should be delayed until cells are fetched. Defaults to `true`. */ def withLazyTiles(state: Boolean): TaggedReader = tag[ReaderTag][DataFrameReader]( diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala index dc6254b6b..02a689958 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala @@ -52,6 +52,7 @@ case class RasterSourceRelation( catalogTable: RasterSourceCatalogRef, bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]], + bufferSize: Short, lazyTiles: Boolean, spatialIndexPartitions: Option[Int] ) extends BaseRelation with TableScan { @@ -128,7 +129,7 @@ case class RasterSourceRelation( // Expand RasterSource into multiple columns per band, and multiple rows per tile // There's some unintentional fragility here in that the structure of the expression // is expected to line up with our column structure here. - val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, srcs: _*) as refColNames + val refs = RasterSourceToRasterRefs(subtileDims, bandIndexes, bufferSize, srcs: _*) as refColNames // RasterSourceToRasterRef is a generator, which means you have to do the Tile conversion // in a separate select statement (Query planner doesn't know how many columns ahead of time). @@ -141,7 +142,7 @@ case class RasterSourceRelation( .select(paths ++ refsToTiles ++ extras: _*) } else { - val tiles = RasterSourceToTiles(subtileDims, bandIndexes, srcs: _*) as tileColNames + val tiles = RasterSourceToTiles(subtileDims, bandIndexes, bufferSize, srcs: _*) as tileColNames withPaths .select((paths :+ tiles) ++ extras: _*) } diff --git a/datasource/src/test/scala/examples/BufferTiles.scala b/datasource/src/test/scala/examples/BufferTiles.scala new file mode 100644 index 000000000..6ba587ba2 --- /dev/null +++ b/datasource/src/test/scala/examples/BufferTiles.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package examples + +import org.apache.spark.sql._ +import org.locationtech.rasterframes._ +import org.locationtech.rasterframes.datasource.raster._ + +object BufferTiles extends App { + + implicit val spark = SparkSession.builder() + .master("local[*]").appName("RasterFrames") + .withKryoSerialization.getOrCreate().withRasterFrames + spark.sparkContext.setLogLevel("ERROR") + + import spark.implicits._ + + val example = "https://raw.githubusercontent.com/locationtech/rasterframes/develop/core/src/test/resources/LC08_B7_Memphis_COG.tiff" + spark.read.raster.from(example).withBufferSize(11).withTileDimensions(16, 16).load().select($"proj_raster").show(false) + // spark.stop() +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 488a48e4a..e4fa974d4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -16,5 +16,4 @@ addSbtPlugin("com.github.gseitz" %% "sbt-release" % "1.0.9") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") - - +addSbtPlugin("com.softwaremill.clippy" % "plugin-sbt" % "0.6.1") diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala index 906988691..3f524173e 100644 --- a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -255,7 +255,7 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions def _resolveRasterRef(srcBin: Array[Byte], bandIndex: jInt, xmin: jDouble, ymin: jDouble, xmax: jDouble, ymax: jDouble): AnyRef = { val src = KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(srcBin)) val extent = Extent(xmin, ymin, xmax, ymax) - RasterRef(src, bandIndex, Some(extent), None) + RasterRef(src, bandIndex, Some(extent), None, 0: Short) } def _dfToMarkdown(df: DataFrame, numRows: Int, truncate: Boolean): String = { From b3144f3b02e447cf1888d6a6ad8ccb334b326360 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Sun, 28 Mar 2021 16:45:40 -0400 Subject: [PATCH 02/11] fix: embedded schema knows about buffer size --- .../expressions/generators/RasterSourceToRasterRefs.scala | 1 - .../scala/org/locationtech/rasterframes/ref/RasterRef.scala | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala index 014c0f788..67d19e392 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala @@ -44,7 +44,6 @@ import scala.util.control.NonFatal */ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None, bufferSize: Short) extends Expression with Generator with CodegenFallback with ExpectsInputTypes { - require(bufferSize > 0) override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType) override def nodeName: String = "rf_raster_source_to_raster_ref" diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index 67f100ba1..f23f330d4 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -92,7 +92,8 @@ object RasterRef extends LazyLogging { StructField("source", RasterSourceType.sqlType, true), StructField("bandIndex", IntegerType, false), StructField("subextent", schemaOf[Extent], true), - StructField("subgrid", schemaOf[GridBounds[Int]], true) + StructField("subgrid", schemaOf[GridBounds[Int]], true), + StructField("bufferSize", ShortType, false) )) implicit val rasterRefSerializer: CatalystSerializer[RasterRef] = new CatalystSerializer[RasterRef] { From ef65fc3f79a1f451f37a02e87033ed54269c04e8 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Sun, 28 Mar 2021 18:35:29 -0400 Subject: [PATCH 03/11] Factor out UnaryRasterFunction/Operator --- ...sterOp.scala => UnaryRasterFunction.scala} | 3 +-- ...sterOp.scala => UnaryRasterOperator.scala} | 24 +++++-------------- .../expressions/accessors/ExtractTile.scala | 4 ++-- .../accessors/GetTileContext.scala | 6 ++--- .../expressions/localops/Abs.scala | 4 ++-- .../expressions/localops/Defined.scala | 4 ++-- .../expressions/localops/Exp.scala | 11 ++++----- .../expressions/localops/Identity.scala | 4 ++-- .../expressions/localops/Log.scala | 10 ++++---- .../expressions/localops/Round.scala | 4 ++-- .../expressions/localops/Sqrt.scala | 4 ++-- .../expressions/localops/Undefined.scala | 4 ++-- .../expressions/tilestats/DataCells.scala | 4 ++-- .../expressions/tilestats/Exists.scala | 4 ++-- .../expressions/tilestats/ForAll.scala | 4 ++-- .../expressions/tilestats/IsNoDataTile.scala | 4 ++-- .../expressions/tilestats/NoDataCells.scala | 4 ++-- .../expressions/tilestats/Sum.scala | 4 ++-- .../expressions/tilestats/TileHistogram.scala | 6 ++--- .../expressions/tilestats/TileMax.scala | 4 ++-- .../expressions/tilestats/TileMean.scala | 4 ++-- .../expressions/tilestats/TileMin.scala | 4 ++-- .../expressions/tilestats/TileStats.scala | 6 ++--- .../transformers/DebugRender.scala | 4 ++-- .../expressions/transformers/RenderPNG.scala | 4 ++-- .../transformers/TileToArrayDouble.scala | 4 ++-- .../transformers/TileToArrayInt.scala | 6 ++--- 27 files changed, 67 insertions(+), 81 deletions(-) rename core/src/main/scala/org/locationtech/rasterframes/expressions/{UnaryRasterOp.scala => UnaryRasterFunction.scala} (97%) rename core/src/main/scala/org/locationtech/rasterframes/expressions/{UnaryLocalRasterOp.scala => UnaryRasterOperator.scala} (60%) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala similarity index 97% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala index 8d2b532c8..be1487963 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterFunction.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.UnaryExpression import org.locationtech.rasterframes.model.TileContext /** Boilerplate for expressions operating on a single Tile-like . */ -trait UnaryRasterOp extends UnaryExpression { +trait UnaryRasterFunction extends UnaryExpression { override def checkInputDataTypes(): TypeCheckResult = { if (!tileExtractor.isDefinedAt(child.dataType)) { TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") @@ -44,4 +44,3 @@ trait UnaryRasterOp extends UnaryExpression { protected def eval(tile: Tile, ctx: Option[TileContext]): Any } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala similarity index 60% rename from core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala rename to core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala index a410f47f8..257fd567f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterOperator.scala @@ -23,38 +23,26 @@ package org.locationtech.rasterframes.expressions import com.typesafe.scalalogging.Logger import geotrellis.raster.Tile -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult -import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} -import org.apache.spark.sql.catalyst.expressions.UnaryExpression import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.DataType import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import org.locationtech.rasterframes.model.TileContext import org.slf4j.LoggerFactory /** Operation on a tile returning a tile. */ -trait UnaryLocalRasterOp extends UnaryExpression { +trait UnaryRasterOperator extends UnaryRasterFunction { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def dataType: DataType = child.dataType - override def checkInputDataTypes(): TypeCheckResult = { - if (!tileExtractor.isDefinedAt(child.dataType)) { - TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") - } - else TypeCheckSuccess - } - - override protected def nullSafeEval(input: Any): Any = { + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { implicit val tileSer = TileUDT.tileSerializer - val (childTile, childCtx) = tileExtractor(child.dataType)(row(input)) - childCtx match { - case Some(ctx) => ctx.toProjectRasterTile(op(childTile)).toInternalRow - case None => op(childTile).toInternalRow + ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile)).toInternalRow + case None => op(tile).toInternalRow } } protected def op(child: Tile): Tile } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala index 4fc0a0374..e18471d3b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala @@ -22,7 +22,7 @@ package org.locationtech.rasterframes.expressions.accessors import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression @@ -35,7 +35,7 @@ import org.locationtech.rasterframes.tiles.InternalRowTile import org.locationtech.rasterframes._ /** Expression to extract at tile from several types that contain tiles.*/ -case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ExtractTile(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def dataType: DataType = TileType override def nodeName: String = "rf_extract_tile" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala index 6c9a3538a..4fe699d29 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala @@ -22,16 +22,16 @@ package org.locationtech.rasterframes.expressions.accessors import org.locationtech.rasterframes.encoders.CatalystSerializer._ -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext -case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class GetTileContext(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def dataType: DataType = schemaOf[TileContext] override def nodeName: String = "get_tile_context" diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala index d55860d29..e4b1fe6fa 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Compute the absolute value of each cell.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Abs(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Abs(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_abs" override def na: Any = null override protected def op(t: Tile): Tile = t.localAbs() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala index cdfcbaa63..ae99d339e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Defined(child: Expression) extends UnaryLocalRasterOp +case class Defined(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_data" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala index b1fb4d714..f4efef95e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Exp(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_exp" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E) @@ -60,7 +60,7 @@ object Exp { > SELECT _FUNC_(tile); ...""" ) -case class Exp10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp10(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(10.0) @@ -81,7 +81,7 @@ object Exp10 { > SELECT _FUNC_(tile); ...""" ) -case class Exp2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Exp2(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_exp2" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0) @@ -102,7 +102,7 @@ object Exp2{ > SELECT _FUNC_(tile); ...""" ) -case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class ExpM1(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_expm1" override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0) @@ -112,4 +112,3 @@ case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallb object ExpM1{ def apply(tile: Column): Column = new Column(ExpM1(tile.expr)) } - diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala index ed9e4785f..abf2fd968 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return the given tile or projected raster unchanged. Useful in debugging round-trip serialization across various language and memory boundaries.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Identity(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback { +case class Identity(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_identity" override def na: Any = null override protected def op(t: Tile): Tile = t diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala index a2249fa2a..285e2e20b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile); ...""" ) -case class Log(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "log" override protected def op(tile: Tile): Tile = fpTile(tile).localLog() @@ -60,7 +60,7 @@ object Log { > SELECT _FUNC_(tile); ...""" ) -case class Log10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log10(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log10" override protected def op(tile: Tile): Tile = fpTile(tile).localLog10() @@ -81,7 +81,7 @@ object Log10 { > SELECT _FUNC_(tile); ...""" ) -case class Log2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log2(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log2" override protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0) @@ -102,7 +102,7 @@ object Log2{ > SELECT _FUNC_(tile); ...""" ) -case class Log1p(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Log1p(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_log1p" override protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog() diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala index 0d0cd036b..516e2d177 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Round cell values to the nearest integer without changing the cell type.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Round(child: Expression) extends UnaryLocalRasterOp +case class Round(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_round" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala index 3f5ea2d2e..1b95fec56 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType -import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} +import org.locationtech.rasterframes.expressions.{UnaryRasterOperator, fpTile} @ExpressionDescription( usage = "_FUNC_(tile) - Perform cell-wise square root", @@ -40,7 +40,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile} > SELECT _FUNC_(tile) ... """ ) -case class Sqrt(child: Expression) extends UnaryLocalRasterOp with CodegenFallback { +case class Sqrt(child: Expression) extends UnaryRasterOperator with CodegenFallback { override val nodeName: String = "rf_sqrt" override protected def op(tile: Tile): Tile = fpTile(tile).localPow(0.5) override def dataType: DataType = child.dataType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala index f91b54373..cb95f1b94 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala @@ -25,7 +25,7 @@ import geotrellis.raster.Tile import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} @ExpressionDescription( usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.", @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO > SELECT _FUNC_(tile); ...""" ) -case class Undefined(child: Expression) extends UnaryLocalRasterOp +case class Undefined(child: Expression) extends UnaryRasterOperator with NullToValue with CodegenFallback { override def nodeName: String = "rf_local_no_data" override def na: Any = null diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala index a18148db3..6167cc11b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 357""" ) -case class DataCells(child: Expression) extends UnaryRasterOp +case class DataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_data_cells" override def dataType: DataType = LongType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala index cd04b1467..02ebe2ebc 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala @@ -6,7 +6,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.isCellTrue -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -23,7 +23,7 @@ import spire.syntax.cfor.cfor true """ ) -case class Exists(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Exists(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "exists" override def dataType: DataType = BooleanType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala index a912a8a0b..3519c3a82 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala @@ -6,7 +6,7 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types._ import org.apache.spark.sql.{Column, TypedColumn} import org.locationtech.rasterframes.isCellTrue -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor @@ -23,7 +23,7 @@ import spire.syntax.cfor.cfor true """ ) -case class ForAll(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class ForAll(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "for_all" override def dataType: DataType = BooleanType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala index fd855cd39..fe547dff9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); false""" ) -case class IsNoDataTile(child: Expression) extends UnaryRasterOp +case class IsNoDataTile(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_is_no_data_tile" override def na: Any = true diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala index cf47ba14e..d492fc1f8 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster._ import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 12""" ) -case class NoDataCells(child: Expression) extends UnaryRasterOp +case class NoDataCells(child: Expression) extends UnaryRasterFunction with CodegenFallback with NullToValue { override def nodeName: String = "rf_no_data_cells" override def dataType: DataType = LongType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala index 096acdab6..78550a4f5 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster._ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile5); 2135.34""" ) -case class Sum(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class Sum(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_sum" override def dataType: DataType = DoubleType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala index 96e3d3dcc..50d840d97 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.stats.CellHistogram import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -42,7 +42,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileHistogram(child: Expression) extends UnaryRasterOp +case class TileHistogram(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_histogram" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala index 3204f4aaf..29ca17e70 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); 1""" ) -case class TileMax(child: Expression) extends UnaryRasterOp +case class TileMax(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_max" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala index 92c833f98..4a20bbc5b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMean(child: Expression) extends UnaryRasterOp +case class TileMean(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_mean" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala index 71fa0194a..cdf6c9d73 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} import geotrellis.raster.{Tile, isData} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} @@ -39,7 +39,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); -1""" ) -case class TileMin(child: Expression) extends UnaryRasterOp +case class TileMin(child: Expression) extends UnaryRasterFunction with NullToValue with CodegenFallback { override def nodeName: String = "rf_tile_min" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala index fac6d330e..50449b15b 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.tilestats -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.stats.CellStatistics import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.CatalystTypeConverters @@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.DataType import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -42,7 +42,7 @@ import org.locationtech.rasterframes.model.TileContext > SELECT _FUNC_(tile); ...""" ) -case class TileStats(child: Expression) extends UnaryRasterOp +case class TileStats(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_stats" override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala index 54201152e..cb37d3de9 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala @@ -28,11 +28,11 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript import org.apache.spark.sql.types.{DataType, StringType} import org.apache.spark.sql.{Column, TypedColumn} import org.apache.spark.unsafe.types.UTF8String -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext import spire.syntax.cfor.cfor -abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp +abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterFunction with CodegenFallback with Serializable { import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix override def dataType: DataType = StringType diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala index 144a4abb6..4c41466bf 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.types.{BinaryType, DataType} import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext /** @@ -35,7 +35,7 @@ import org.locationtech.rasterframes.model.TileContext * @param child tile column * @param ramp color ramp to use for non-composite tiles. */ -abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterOp with CodegenFallback with Serializable { +abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterFunction with CodegenFallback with Serializable { override def dataType: DataType = BinaryType override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { val png = ramp.map(tile.renderPng).getOrElse(tile.renderPng()) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala index 5d7786f1c..b33faf249 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala @@ -21,7 +21,7 @@ package org.locationtech.rasterframes.expressions.transformers -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback @@ -36,7 +36,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayDouble(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayDouble(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_double" override def dataType: DataType = DataTypes.createArrayType(DoubleType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala index c299d57c7..3c50d5fdf 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala @@ -21,14 +21,14 @@ package org.locationtech.rasterframes.expressions.transformers -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import geotrellis.raster.Tile import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} import org.apache.spark.sql.catalyst.util.ArrayData import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType} import org.apache.spark.sql.{Column, TypedColumn} -import org.locationtech.rasterframes.expressions.UnaryRasterOp +import org.locationtech.rasterframes.expressions.UnaryRasterFunction import org.locationtech.rasterframes.model.TileContext @ExpressionDescription( @@ -37,7 +37,7 @@ import org.locationtech.rasterframes.model.TileContext Arguments: * tile - tile to convert""" ) -case class TileToArrayInt(child: Expression) extends UnaryRasterOp with CodegenFallback { +case class TileToArrayInt(child: Expression) extends UnaryRasterFunction with CodegenFallback { override def nodeName: String = "rf_tile_to_array_int" override def dataType: DataType = DataTypes.createArrayType(IntegerType, false) override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { From 342385890de5edfd22e7127b6a564f9f04c3f596 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Sun, 28 Mar 2021 18:56:03 -0400 Subject: [PATCH 04/11] Add Focal Functions --- .../expressions/focalops/Convolve.scala | 91 ++++++++----------- .../expressions/focalops/FocalMax.scala | 36 ++++++++ .../expressions/focalops/FocalMean.scala | 36 ++++++++ .../expressions/focalops/FocalMedian.scala | 36 ++++++++ .../expressions/focalops/FocalMin.scala | 35 +++++++ .../expressions/focalops/FocalMode.scala | 36 ++++++++ .../expressions/focalops/FocalMoransI.scala | 36 ++++++++ .../focalops/FocalNeighborhoodOperator.scala | 31 +++++++ .../expressions/focalops/FocalStdDev.scala | 37 ++++++++ .../functions/FocalFunctions.scala | 53 +++++++++++ .../functions/FocalFunctionsSpec.scala | 57 ++++++++++++ 11 files changed, 431 insertions(+), 53 deletions(-) create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala create mode 100644 core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala index 25dc4fcd5..e3b5edc3e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -1,53 +1,38 @@ -///* -// * This software is licensed under the Apache 2 license, quoted below. -// * -// * Copyright 2020 Astraea, Inc. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); you may not -// * use this file except in compliance with the License. You may obtain a copy of -// * the License at -// * -// * [http://www.apache.org/licenses/LICENSE-2.0] -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// * License for the specific language governing permissions and limitations under -// * the License. -// * -// * SPDX-License-Identifier: Apache-2.0 -// * -// */ -// -//package org.locationtech.rasterframes.expressions.focalops -//import geotrellis.raster.Tile -//import geotrellis.raster.mapalgebra.focal.Kernel -//import org.apache.spark.sql.{Column, TypedColumn} -//import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription} -//import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -//import org.apache.spark.sql.types.DataType -//import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp} -//import org.locationtech.rasterframes.model.TileContext -// -//@ExpressionDescription( -// usage = "_FUNC_(tile, kernel) - ", -// arguments = """ -// Arguments: -// * tile - -// * kernel - """, -// examples = "" -// Examples: -// > SELECT _FUNC_(tile, Square(1)); -// ...""" -//) -//case class Convolve(child: Expression, kernel: Kernel) extends UnaryRasterOp with NullToValue with CodegenFallback { -// override def nodeName: String = "rf_convolve" -// override def na: Any = null -// override def eval(tile: Tile, ctx: Option[TileContext]): Tile = tile.convolve(kernel) -// -// def dataType: DataType = ??? -//} -// -//object Convolve { -// def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) -//} \ No newline at end of file +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} + +case class Convolve(child: Expression, kernel: Kernel) extends UnaryRasterOperator with NullToValue with CodegenFallback { + override def nodeName: String = "rf_convolve" + override def na: Any = null + override protected def op(t: Tile): Tile = t.convolve(kernel) +} + +object Convolve { + def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala new file mode 100644 index 000000000..7c4d7abcb --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_max" + override protected def op(t: Tile): Tile = t.focalMax(neighborhood) +} + +object FocalMax { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMax(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala new file mode 100644 index 000000000..7f0e94c39 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_mean" + override protected def op(t: Tile): Tile = t.focalMean(neighborhood) +} + +object FocalMean { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMean(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala new file mode 100644 index 000000000..c4d505971 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_median" + override protected def op(t: Tile): Tile = t.focalMedian(neighborhood) +} + +object FocalMedian { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMedian(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala new file mode 100644 index 000000000..9de6ef598 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -0,0 +1,35 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_min" + override protected def op(t: Tile): Tile = t.focalMin(neighborhood) +} + +object FocalMin { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMin(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala new file mode 100644 index 000000000..1504035ec --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_mode" + override protected def op(t: Tile): Tile = t.focalMode(neighborhood) +} + +object FocalMode { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMode(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala new file mode 100644 index 000000000..ec756ce49 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -0,0 +1,36 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_moransi" + override protected def op(t: Tile): Tile = t.tileMoransI(neighborhood) +} + +object FocalMoransI { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMoransI(tile.expr, neighborhood)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala new file mode 100644 index 000000000..e4c1e3448 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalNeighborhoodOperator.scala @@ -0,0 +1,31 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} + +trait FocalNeighborhoodOperator extends UnaryRasterOperator with NullToValue with CodegenFallback { + override def na: Any = null + def neighborhood: Neighborhood +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala new file mode 100644 index 000000000..ddc90e61a --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -0,0 +1,37 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import geotrellis.raster.mapalgebra.focal.Neighborhood +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression + +case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { + override def nodeName: String = "rf_focal_stddev" + + override protected def op(t: Tile): Tile = t.focalStandardDeviation(neighborhood) +} + +object FocalStdDev { + def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalStdDev(tile.expr, neighborhood)) +} \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala new file mode 100644 index 000000000..235225645 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -0,0 +1,53 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions + +import geotrellis.raster.Neighborhood +import geotrellis.raster.mapalgebra.focal.Kernel +import org.apache.spark.sql.Column +import org.locationtech.rasterframes.expressions.focalops._ + +trait FocalFunctions { + def rf_focal_mean(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMean(tileCol, neighborhood) + + def rf_focal_median(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMedian(tileCol, neighborhood) + + def rf_focal_mode(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMode(tileCol, neighborhood) + + def rf_focal_max(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMax(tileCol, neighborhood) + + def rf_focal_min(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMin(tileCol, neighborhood) + + def rf_focal_stddev(tileCol: Column, neighborhood: Neighborhood): Column = + FocalStdDev(tileCol, neighborhood) + + def rf_focal_moransi(tileCol: Column, neighborhood: Neighborhood): Column = + FocalMoransI(tileCol, neighborhood) + + def rf_convolve(tileCol: Column, kernel: Kernel): Column = + Convolve(tileCol, kernel) +} diff --git a/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala new file mode 100644 index 000000000..8eeda6ec9 --- /dev/null +++ b/core/src/test/scala/org/locationtech/rasterframes/functions/FocalFunctionsSpec.scala @@ -0,0 +1,57 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2020 Astraea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.functions +import geotrellis.raster.testkit.RasterMatchers +import org.locationtech.rasterframes.TestEnvironment + +class FocalFunctionsSpec extends TestEnvironment with RasterMatchers { + describe("focal operations") { + it("should provide focal mean") { + checkDocs("rf_focal_mean") + fail() + } + it("should provide focal mode") { + checkDocs("rf_focal_mode") + fail() + } + it("should provide focal median") { + checkDocs("rf_focal_median") + fail() + } + it("should provide focal min") { + checkDocs("rf_focal_min") + fail() + } + it("should provide focal max") { + checkDocs("rf_focal_max") + fail() + } + it("should provide focal Moran's I") { + checkDocs("rf_focal_moransi") + fail() + } + it("should provide convolve") { + checkDocs("rf_convolve") + fail() + } + } +} \ No newline at end of file From e6fd5a8c401fc52e4cf684f955d09a1205663a9c Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Sun, 28 Mar 2021 20:29:46 -0400 Subject: [PATCH 05/11] include import --- .../scala/org/locationtech/rasterframes/RasterFunctions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala index c8bfa3813..accca888d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala @@ -26,4 +26,4 @@ import org.locationtech.rasterframes.functions._ * Mix-in for UDFs for working with Tiles in Spark DataFrames. * @since 4/3/17 */ -trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions +trait RasterFunctions extends TileFunctions with LocalFunctions with SpatialFunctions with AggregateFunctions with FocalFunctions From 917b8f8a8e99e4ab2edb20378f0f718dcc144201 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Mon, 29 Mar 2021 16:04:13 -0400 Subject: [PATCH 06/11] Expose focal neighborhood methods to pythong only square(1) neighborhood is possible with this --- .../expressions/focalops/Convolve.scala | 3 +- .../expressions/focalops/FocalMax.scala | 3 +- .../expressions/focalops/FocalMean.scala | 3 +- .../expressions/focalops/FocalMedian.scala | 3 +- .../expressions/focalops/FocalMin.scala | 3 +- .../expressions/focalops/FocalMode.scala | 3 +- .../expressions/focalops/FocalMoransI.scala | 3 +- .../expressions/focalops/FocalStdDev.scala | 3 +- .../rasterframes/expressions/package.scala | 10 +++++ .../raster/RasterSourceDataSource.scala | 6 ++- .../python/pyrasterframes/rasterfunctions.py | 43 +++++++++++++++++++ .../rasterframes/py/PyRFContext.scala | 3 ++ 12 files changed, 77 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala index e3b5edc3e..fd70a7c48 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Convolve.scala @@ -28,11 +28,12 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOperator} case class Convolve(child: Expression, kernel: Kernel) extends UnaryRasterOperator with NullToValue with CodegenFallback { - override def nodeName: String = "rf_convolve" + override def nodeName: String = Convolve.name override def na: Any = null override protected def op(t: Tile): Tile = t.convolve(kernel) } object Convolve { + def name: String = "rf_convolve" def apply(tile: Column, kernel: Kernel): Column = new Column(Convolve(tile.expr, kernel)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala index 7c4d7abcb..e4b608f99 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMax.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMax(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_max" + override def nodeName: String = FocalMax.name override protected def op(t: Tile): Tile = t.focalMax(neighborhood) } object FocalMax { + def name: String = "rf_focal_max" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMax(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala index 7f0e94c39..588a3fcc3 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMean.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMean(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_mean" + override def nodeName: String = FocalMean.name override protected def op(t: Tile): Tile = t.focalMean(neighborhood) } object FocalMean { + def name:String = "rf_focal_mean" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMean(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala index c4d505971..dfe52ba5a 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMedian.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMedian(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_median" + override def nodeName: String = FocalMedian.name override protected def op(t: Tile): Tile = t.focalMedian(neighborhood) } object FocalMedian { + def name: String = "rf_focal_median" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMedian(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala index 9de6ef598..2d6670c4f 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMin.scala @@ -26,10 +26,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMin(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_min" + override def nodeName: String = FocalMin.name override protected def op(t: Tile): Tile = t.focalMin(neighborhood) } object FocalMin { + def name: String = "rf_focal_min" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMin(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala index 1504035ec..ecf7dfd0d 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMode.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMode(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_mode" + override def nodeName: String = FocalMode.name override protected def op(t: Tile): Tile = t.focalMode(neighborhood) } object FocalMode { + def name: String = "rf_focal_mode" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMode(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala index ec756ce49..f89e39e0c 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalMoransI.scala @@ -27,10 +27,11 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalMoransI(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_moransi" + override def nodeName: String = FocalMoransI.name override protected def op(t: Tile): Tile = t.tileMoransI(neighborhood) } object FocalMoransI { + def name: String = "rf_focal_moransi" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalMoransI(tile.expr, neighborhood)) } diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala index ddc90e61a..c2dc89b57 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/FocalStdDev.scala @@ -27,11 +27,12 @@ import org.apache.spark.sql.Column import org.apache.spark.sql.catalyst.expressions.Expression case class FocalStdDev(child: Expression, neighborhood: Neighborhood) extends FocalNeighborhoodOperator { - override def nodeName: String = "rf_focal_stddev" + override def nodeName: String = FocalStdDev.name override protected def op(t: Tile): Tile = t.focalStandardDeviation(neighborhood) } object FocalStdDev { + def name: String = "rf_focal_stddevd" def apply(tile: Column, neighborhood: Neighborhood): Column = new Column(FocalStdDev(tile.expr, neighborhood)) } \ No newline at end of file diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala index 8bfda70d6..e5ce11b06 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala @@ -32,6 +32,7 @@ import org.locationtech.rasterframes.expressions.aggregates.CellCountAggregate.D import org.locationtech.rasterframes.expressions.aggregates._ import org.locationtech.rasterframes.expressions.generators._ import org.locationtech.rasterframes.expressions.localops._ +import org.locationtech.rasterframes.expressions.focalops._ import org.locationtech.rasterframes.expressions.tilestats._ import org.locationtech.rasterframes.expressions.transformers._ @@ -133,6 +134,15 @@ package object expressions { registry.registerExpression[LocalCountAggregate.LocalNoDataCellsUDAF]("rf_agg_local_no_data_cells") registry.registerExpression[LocalMeanAggregate]("rf_agg_local_mean") + registry.registerExpression[FocalMax](FocalMax.name) + registry.registerExpression[FocalMin](FocalMin.name) + registry.registerExpression[FocalMean](FocalMean.name) + registry.registerExpression[FocalMode](FocalMode.name) + registry.registerExpression[FocalMedian](FocalMedian.name) + registry.registerExpression[FocalMoransI](FocalMoransI.name) + registry.registerExpression[FocalStdDev](FocalStdDev.name) + registry.registerExpression[Convolve](Convolve.name) + registry.registerExpression[Mask.MaskByDefined]("rf_mask") registry.registerExpression[Mask.InverseMaskByDefined]("rf_inverse_mask") registry.registerExpression[Mask.MaskByValue]("rf_mask_by_value") diff --git a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala index e7f8b35fe..3ccb3d113 100644 --- a/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala +++ b/datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala @@ -124,7 +124,11 @@ object RasterSourceDataSource { def lazyTiles: Boolean = parameters .get(LAZY_TILES_PARAM).forall(_.toBoolean) - def bufferSize: Short = parameters.get(BUFFER_SIZE_PARAM).map(_.toShort).getOrElse(-1.toShort) + def bufferSize: Short = { + val size = parameters.get(BUFFER_SIZE_PARAM).map(_.toShort).getOrElse(0.toShort) + require(size >= 0, s"Invalid $BUFFER_SIZE_PARAM: $size") + size + } def spatialIndex: Option[Int] = parameters .get(SPATIAL_INDEX_PARTITIONS_PARAM).flatMap(p => Try(p.toInt).toOption) diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index 0c48e91af..e68f042e4 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -65,6 +65,15 @@ def to_jvm(ct): elif isinstance(cell_type_arg, CellType): return to_jvm(cell_type_arg.cell_type_name) +def _parse_neighborhood(neighborhood_arg: str) -> JavaObject: + """ Convert the cell type representation to the expected JVM CellType object.""" + def to_jvm(n): + return _context_call('_parse_neighborhood', n) + + if isinstance(neighborhood_arg, str) and neighborhood_arg == "square-1": + return to_jvm(neighborhood_arg) + else: + raise NotImplementedError def rf_cell_types() -> List[CellType]: """Return a list of standard cell types""" @@ -781,6 +790,40 @@ def rf_identity(tile_col: Column_type) -> Column: """Pass tile through unchanged""" return _apply_column_function('rf_identity', tile_col) +def rf_focal_max(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mean(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mean value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mean') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_median(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_median') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_min(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the min value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_min') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_mode(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the mode value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_mode') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_focal_std_dev(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the standard deviation value in its neighborhood of each cell""" + jfcn = RFContext.active().lookup('rf_focal_std_dev') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) + +def rf_moransI(tile_col: Column_type, neighborhood: str) -> Column: + """Compute the max in its neighborhood value of each cell""" + jfcn = RFContext.active().lookup('rf_focal_max') + return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala index 3f524173e..eeea50ce4 100644 --- a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala +++ b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala @@ -38,6 +38,7 @@ import org.locationtech.rasterframes.{RasterFunctions, _} import spray.json._ import org.locationtech.rasterframes.util.JsonCodecs._ import scala.collection.JavaConverters._ +import geotrellis.raster.mapalgebra.focal.{Neighborhood, Square} /** * py4j access wrapper to RasterFrameLayer entry points. @@ -149,6 +150,8 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions */ def _parse_cell_type(name: String): CellType = CellType.fromName(name) + def _parse_neighborhood(name: String): Neighborhood = Square(1) + /** * Convenience list of valid cell type strings * @return Java List of String, which py4j can interpret as a python `list` From acad3fb7aca75e513f033c76718e292441176102 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Mon, 29 Mar 2021 17:36:45 -0400 Subject: [PATCH 07/11] Add surface calculations --- .../expressions/focalops/Aspect.scala | 40 +++++++++++++++ .../expressions/focalops/Hillshade.scala | 41 +++++++++++++++ .../expressions/focalops/Slope.scala | 40 +++++++++++++++ .../focalops/SurfaceOperation.scala | 51 +++++++++++++++++++ .../rasterframes/expressions/package.scala | 4 ++ .../functions/FocalFunctions.scala | 9 ++++ .../python/pyrasterframes/rasterfunctions.py | 14 +++++ 7 files changed, 199 insertions(+) create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala create mode 100644 core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala new file mode 100644 index 000000000..c9e564ced --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Aspect.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Aspect(child: Expression) extends SurfaceOperation { + override def nodeName: String = Aspect.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.aspect(CellSize(ctx.extent, cols = t.cols, rows = t.rows)) + } +} + +object Aspect { + def name: String = "rf_aspect" + def apply(tile: Column): Column = new Column(Aspect(tile.expr)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala new file mode 100644 index 000000000..ca42af899 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -0,0 +1,41 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOperation { + override def nodeName: String = Slope.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) + } +} + +object Hillshade { + def name: String = "rf_hillshade" + def apply(tile: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + new Column(Hillshade(tile.expr, azimuth, altitude, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala new file mode 100644 index 000000000..0054f01f7 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Slope.scala @@ -0,0 +1,40 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import geotrellis.raster.Tile +import org.apache.spark.sql.Column +import org.apache.spark.sql.catalyst.expressions.Expression +import org.locationtech.rasterframes.model.TileContext +import geotrellis.raster.CellSize + +case class Slope(child: Expression, zFactor: Double) extends SurfaceOperation { + override def nodeName: String = Slope.name + override protected def op(t: Tile, ctx: TileContext): Tile = { + t.slope(CellSize(ctx.extent, cols = t.cols, rows = t.rows), zFactor) + } +} + +object Slope { + def name: String = "rf_slope" + def apply(tile: Column, zFactor: Double): Column = new Column(Slope(tile.expr, zFactor)) +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala new file mode 100644 index 000000000..86b55f8f9 --- /dev/null +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala @@ -0,0 +1,51 @@ +/* + * This software is licensed under the Apache 2 license, quoted below. + * + * Copyright 2021 Azavea, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * [http://www.apache.org/licenses/LICENSE-2.0] + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package org.locationtech.rasterframes.expressions.focalops + +import org.slf4j.LoggerFactory +import com.typesafe.scalalogging.Logger +import geotrellis.raster.Tile +import org.apache.spark.sql.rf.TileUDT +import org.apache.spark.sql.types.DataType +import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.encoders.CatalystSerializer._ +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} + +/** Operation on a tile returning a tile. */ +trait SurfaceOperation extends UnaryRasterFunction with NullToValue with CodegenFallback { + @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) + + override def dataType: DataType = child.dataType + override def na: Any = null + + override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { + implicit val tileSer = TileUDT.tileSerializer + + ctx match { + case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow + case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + } + } + + protected def op(t: Tile, ctx: TileContext): Tile +} diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala index e5ce11b06..02f773a63 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala @@ -143,6 +143,10 @@ package object expressions { registry.registerExpression[FocalStdDev](FocalStdDev.name) registry.registerExpression[Convolve](Convolve.name) + registry.registerExpression[Slope](Slope.name) + registry.registerExpression[Aspect](Aspect.name) + registry.registerExpression[Hillshade](Hillshade.name) + registry.registerExpression[Mask.MaskByDefined]("rf_mask") registry.registerExpression[Mask.InverseMaskByDefined]("rf_inverse_mask") registry.registerExpression[Mask.MaskByValue]("rf_mask_by_value") diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala index 235225645..f631da88e 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/functions/FocalFunctions.scala @@ -50,4 +50,13 @@ trait FocalFunctions { def rf_convolve(tileCol: Column, kernel: Kernel): Column = Convolve(tileCol, kernel) + + def rf_aspect(tileCol: Column): Column = + Aspect(tileCol) + + def rf_slope(tileCol: Column, zFactor: Double): Column = + Slope(tileCol, zFactor) + + def rf_hillshade(tileCol: Column, azimuth: Double, altitude: Double, zFactor: Double): Column = + Hillshade(tileCol, azimuth, altitude, zFactor) } diff --git a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py index e68f042e4..a7d98d276 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py +++ b/pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py @@ -825,6 +825,20 @@ def rf_moransI(tile_col: Column_type, neighborhood: str) -> Column: jfcn = RFContext.active().lookup('rf_focal_max') return Column(jfcn(_to_java_column(tile_col), _parse_neighborhood(neighborhood))) +def rf_aspect(tile_col: Column_type) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + return _apply_column_function('rf_aspect', tile_col) + +def rf_slope(tile_col: Column_type, z_factor: float) -> Column: + """Calculates the aspect of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_slope') + return Column(jfcn(_to_java_column(tile_col), float(z_factor))) + +def rf_hillshade(tile_col: Column_type, azimuth: float, altitude: float, z_factor: float) -> Column: + """Calculates the hillshade of each cell in an elevation raster""" + jfcn = RFContext.active().lookup('rf_hillshade') + return Column(jfcn(_to_java_column(tile_col), float(azimuth), float(altitude), float(z_factor))) + def rf_resample(tile_col: Column_type, scale_factor: Union[int, float, Column_type]) -> Column: """Resample tile to different size based on scalar factor or tile whose dimension to match Scalar less than one will downsample tile; greater than one will upsample. Uses nearest-neighbor.""" From b61aa6330371d1b0787464da3fa26828c1aaaa74 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Mon, 29 Mar 2021 17:54:32 -0400 Subject: [PATCH 08/11] Expose buffer_size raster read parameter in python --- pyrasterframes/src/main/python/pyrasterframes/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyrasterframes/src/main/python/pyrasterframes/__init__.py b/pyrasterframes/src/main/python/pyrasterframes/__init__.py index add1c42da..a8f362412 100644 --- a/pyrasterframes/src/main/python/pyrasterframes/__init__.py +++ b/pyrasterframes/src/main/python/pyrasterframes/__init__.py @@ -118,6 +118,7 @@ def _raster_reader( catalog_col_names: Optional[List[str]] = None, band_indexes: Optional[List[int]] = None, tile_dimensions: Tuple[int] = (256, 256), + buffer_size: int = 0, lazy_tiles: bool = True, spatial_index_partitions=None, **options: str) -> DataFrame: @@ -134,6 +135,7 @@ def _raster_reader( :param catalog_col_names: required if `source` is a DataFrame or CSV string. It is a list of strings giving the names of columns containing URIs to read. :param band_indexes: list of integers indicating which bands, zero-based, to read from the raster files specified; default is to read only the first band. :param tile_dimensions: tuple or list of two indicating the default tile dimension as (columns, rows). + :param buffer_size: buffer each tile read by this many cells on all sides. :param lazy_tiles: If true (default) only generate minimal references to tile contents; if false, fetch tile cell values. :param spatial_index_partitions: If true, partitions read tiles by a Z2 spatial index using the default shuffle partitioning. If a values > 0, the given number of partitions are created instead of the default. @@ -176,7 +178,8 @@ def temp_name(): options.update({ "band_indexes": to_csv(band_indexes), "tile_dimensions": to_csv(tile_dimensions), - "lazy_tiles": str(lazy_tiles) + "lazy_tiles": str(lazy_tiles), + "buffer_size": int(buffer_size) }) # Parse the `source` argument From db2a1f76dbae63b8766fa8ed425dc11acb878673 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Tue, 30 Mar 2021 11:46:42 -0400 Subject: [PATCH 09/11] fix: hillshade function name --- .../rasterframes/expressions/focalops/Hillshade.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala index ca42af899..a5c73e993 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/Hillshade.scala @@ -28,7 +28,7 @@ import org.locationtech.rasterframes.model.TileContext import geotrellis.raster.CellSize case class Hillshade(child: Expression, azimuth: Double, altitude: Double, zFactor: Double) extends SurfaceOperation { - override def nodeName: String = Slope.name + override def nodeName: String = Hillshade.name override protected def op(t: Tile, ctx: TileContext): Tile = { t.hillshade(CellSize(ctx.extent, cols = t.cols, rows = t.rows), azimuth, altitude, zFactor) } From 02b653d6e9e58f4b43dc7292e5ec7873800375ce Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Tue, 30 Mar 2021 11:47:19 -0400 Subject: [PATCH 10/11] Focal operations demo / test notebook Useful for development, will likely move during final review --- .../src/main/notebooks/FocalOperations.ipynb | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 rf-notebook/src/main/notebooks/FocalOperations.ipynb diff --git a/rf-notebook/src/main/notebooks/FocalOperations.ipynb b/rf-notebook/src/main/notebooks/FocalOperations.ipynb new file mode 100644 index 000000000..464fe45c4 --- /dev/null +++ b/rf-notebook/src/main/notebooks/FocalOperations.ipynb @@ -0,0 +1,172 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Focal Operations with RastrFrames Notebook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Spark Environment" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pyrasterframes\n", + "from pyrasterframes.utils import create_rf_spark_session\n", + "import pyrasterframes.rf_ipython # enables nicer visualizations of pandas DF\n", + "from pyrasterframes.rasterfunctions import *\n", + "import pyspark.sql.functions as F" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "spark = create_rf_spark_session()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get a PySpark DataFrame from elevation raster\n", + "\n", + "Read a single scene of elevation into DataFrame or raster tiles.\n", + "Each tile overlaps its neighbor by \"buffer_size\" of pixels, providing focal operations neighbor information around tile edges.\n", + "You can configure the default size of these tiles, by passing a tuple of desired columns and rows as: `raster(uri, tile_dimensions=(96, 96))`. The default is `(256, 256)`" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "uri = 'https://geotrellis-demo.s3.us-east-1.amazonaws.com/cogs/harrisburg-pa/elevation.tif'\n", + "df = spark.read.raster(uri, tile_dimensions=(512, 512), buffer_size=2)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "root\n |-- proj_raster_path: string (nullable = false)\n |-- proj_raster: struct (nullable = true)\n | |-- tile_context: struct (nullable = true)\n | | |-- extent: struct (nullable = false)\n | | | |-- xmin: double (nullable = false)\n | | | |-- ymin: double (nullable = false)\n | | | |-- xmax: double (nullable = false)\n | | | |-- ymax: double (nullable = false)\n | | |-- crs: struct (nullable = false)\n | | | |-- crsProj4: string (nullable = false)\n | |-- tile: tile (nullable = false)\n\n" + ] + } + ], + "source": [ + "df.printSchema()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The extent struct tells us where in the [CRS](https://spatialreference.org/ref/sr-org/6842/) the tile data covers. The granule is split into arbitrary sized chunks. Each row is a different chunk. Let's see how many." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "81" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], + "source": [ + "df.count()" + ] + }, + { + "source": [ + "## Focal Operations\n", + "Additional transformations are complished through use of column functions.\n", + "The functions used here are mapped to their Scala implementation and applied per row.\n", + "For each row the source elevation data is fetched only once before it's used as input." + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "DataFrame[rf_crs(proj_raster): struct, rf_extent(proj_raster): struct, rf_aspect(proj_raster): struct,crs:struct>,tile:udt>, rf_slope(proj_raster): struct,crs:struct>,tile:udt>, rf_hillshade(proj_raster): struct,crs:struct>,tile:udt>]" + ], + "text/html": "\n\n\n\n\n\n\n\n\n\n\n\n
Showing only top 5 rows
rf_crs(proj_raster)rf_extent(proj_raster)rf_aspect(proj_raster)rf_slope(proj_raster)rf_hillshade(proj_raster)
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][302369.2154, 4478399.0319, 317729.2154, 4493759.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4398599.0319, 287009.2154, 4401599.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][225569.2154, 4416959.0319, 240929.2154, 4432319.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][271649.2154, 4463039.0319, 287009.2154, 4478399.0319]
[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ][317729.2154, 4416959.0319, 333089.2154, 4432319.0319]
", + "text/markdown": "\n_Showing only top 5 rows_.\n\n| rf_crs(proj_raster) | rf_extent(proj_raster) | rf_aspect(proj_raster) | rf_slope(proj_raster) | rf_hillshade(proj_raster) |\n|---|---|---|---|---|\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[302369.2154, 4478399.0319, 317729.2154, 4493759.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4398599.0319, 287009.2154, 4401599.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[225569.2154, 4416959.0319, 240929.2154, 4432319.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[271649.2154, 4463039.0319, 287009.2154, 4478399.0319] | | | |\n| \\[+proj=utm +zone=18 +datum=NAD83 +units=m +no_defs ] | \\[317729.2154, 4416959.0319, 333089.2154, 4432319.0319] | | | |" + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "df.select(\n", + " rf_crs(df.proj_raster), \n", + " rf_extent(df.proj_raster), \n", + " rf_aspect(df.proj_raster), \n", + " rf_slope(df.proj_raster, z_factor=1), \n", + " rf_hillshade(df.proj_raster, azimuth=315, altitude=45, z_factor=1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.10-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file From fecb0087962fe78fe8f4152ee590fdf340b0a379 Mon Sep 17 00:00:00 2001 From: Eugene Cheipesh Date: Tue, 4 May 2021 10:31:36 -0400 Subject: [PATCH 11/11] SurfaceOperation works over BufferTiles We need to be able to unwrap RasterRefTile and call realizedTile before wrapping it in ProjectedRasterTile. This allows current version of BufferTile to do its work. ProjectedRasterTile can be a base trait for any kind of eventual tile representation. --- .../focalops/SurfaceOperation.scala | 45 ++++++++++++++++--- .../rasterframes/ref/RasterRef.scala | 2 +- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala index 86b55f8f9..99c542b13 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/focalops/SurfaceOperation.scala @@ -24,26 +24,57 @@ package org.locationtech.rasterframes.expressions.focalops import org.slf4j.LoggerFactory import com.typesafe.scalalogging.Logger import geotrellis.raster.Tile -import org.apache.spark.sql.rf.TileUDT import org.apache.spark.sql.types.DataType +import org.locationtech.rasterframes.expressions.row import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback +import org.locationtech.rasterframes.expressions.DynamicExtractors._ import org.locationtech.rasterframes.encoders.CatalystSerializer._ import org.locationtech.rasterframes.model.TileContext -import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterFunction} +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.locationtech.rasterframes.ref.RasterRef +import org.apache.spark.sql.catalyst.expressions.UnaryExpression + +import org.locationtech.rasterframes.expressions.DynamicExtractors._ +import geotrellis.raster.Tile +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess} +import org.apache.spark.sql.catalyst.expressions.UnaryExpression +import org.locationtech.rasterframes.model.TileContext +import org.locationtech.rasterframes.expressions.NullToValue +import org.locationtech.rasterframes.tiles.ProjectedRasterTile /** Operation on a tile returning a tile. */ -trait SurfaceOperation extends UnaryRasterFunction with NullToValue with CodegenFallback { +trait SurfaceOperation extends UnaryExpression with NullToValue with CodegenFallback { @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName)) override def dataType: DataType = child.dataType override def na: Any = null - override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { - implicit val tileSer = TileUDT.tileSerializer + override def checkInputDataTypes(): TypeCheckResult = { + if (!tileExtractor.isDefinedAt(child.dataType)) { + TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a raster type.") + } else TypeCheckSuccess + } + + override protected def nullSafeEval(input: Any): Any = { + val (tile, ctx) = tileExtractor(child.dataType)(row(input)) + val literral = tile match { + case RasterRef.RasterRefTile(ref) => ref.realizedTile + case prt: ProjectedRasterTile => prt + } + eval(literral, ctx) + } + + protected def eval(tile: Tile, ctx: Option[TileContext]): Any = { ctx match { - case Some(ctx) => ctx.toProjectRasterTile(op(tile, ctx)).toInternalRow - case None => new NotImplementedError("Surface operation requires ProjectedRasterTile") + case Some(ctx) => + val ret = op(tile, ctx) + ctx.toProjectRasterTile(ret).toInternalRow + + case None => + new NotImplementedError("Surface operation requires ProjectedRasterTile") } } diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala index f23f330d4..436a9bc66 100644 --- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala +++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala @@ -52,7 +52,7 @@ case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[E protected lazy val grid: GridBounds[Int] = subgrid.getOrElse(source.rasterExtent.gridBoundsFor(extent, true)) - protected lazy val realizedTile: Tile = { + lazy val realizedTile: Tile = { RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source") // Pixel bounds we would like to read, including buffer val bufferedGrid = grid.buffer(bufferSize)