diff --git a/.gitignore b/.gitignore
index e5020d283..838c6abec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,13 @@
+# Operating System Files
+
+*.DS_Store
+Thumbs.db
+
*.class
*.log
# sbt specific
+.bsp
.cache
.history
.lib/
diff --git a/.java-version b/.java-version
deleted file mode 100644
index 625934097..000000000
--- a/.java-version
+++ /dev/null
@@ -1 +0,0 @@
-1.8
diff --git a/.scalafmt.conf b/.scalafmt.conf
index ca5e10394..82235ad30 100644
--- a/.scalafmt.conf
+++ b/.scalafmt.conf
@@ -1,16 +1,11 @@
-maxColumn = 138
+version = 3.0.3
+runner.dialect = scala212
+indent.main = 2
+indent.significant = 2
+maxColumn = 150
continuationIndent.defnSite = 2
-binPack.parentConstructors = true
-binPack.literalArgumentLists = false
-newlines.penalizeSingleSelectMultiArgList = false
-newlines.sometimesBeforeColonInMethodReturnType = false
-align.openParenCallSite = false
-align.openParenDefnSite = false
-docstrings = JavaDoc
-rewriteTokens {
- "⇒" = "=>"
- "←" = "<-"
-}
-optIn.selfAnnotationNewline = false
-optIn.breakChainOnFirstMethodDot = true
-importSelectors = BinPack
\ No newline at end of file
+assumeStandardLibraryStripMargin = true
+danglingParentheses.preset = true
+rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers]
+docstrings.style = Asterisk
+# align.preset = more
diff --git a/.sdkmanrc b/.sdkmanrc
new file mode 100644
index 000000000..0262a2610
--- /dev/null
+++ b/.sdkmanrc
@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=11.0.11.hs-adpt
diff --git a/bench/build.sbt b/bench/build.sbt
index 36eb61323..e01ea663d 100644
--- a/bench/build.sbt
+++ b/bench/build.sbt
@@ -11,7 +11,7 @@ libraryDependencies ++= Seq(
jmhIterations := Some(5)
jmhWarmupIterations := Some(8)
jmhTimeUnit := None
-javaOptions in Jmh := Seq("-Xmx4g")
+Jmh / javaOptions := Seq("-Xmx4g")
// To enable profiling:
// jmhExtraOptions := Some("-prof jmh.extras.JFR")
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala
deleted file mode 100644
index b24042b56..000000000
--- a/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2019 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.bench
-
-import java.util.concurrent.TimeUnit
-
-import geotrellis.proj4.{CRS, LatLng, Sinusoidal}
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, StandardEncoders}
-import org.openjdk.jmh.annotations._
-
-@BenchmarkMode(Array(Mode.AverageTime))
-@State(Scope.Benchmark)
-@OutputTimeUnit(TimeUnit.MICROSECONDS)
-class CatalystSerializerBench extends SparkEnv {
-
- val serde = CatalystSerializer[CRS]
-
- val epsg: CRS = LatLng
- val epsgEnc: Row = serde.toRow(epsg)
- val proj4: CRS = Sinusoidal
- val proj4Enc: Row = serde.toRow(proj4)
-
- var crsEnc: ExpressionEncoder[CRS] = _
-
- @Setup(Level.Trial)
- def setupData(): Unit = {
- crsEnc = StandardEncoders.crsSparkEncoder.resolveAndBind()
- }
-
- @Benchmark
- def encodeEpsg(): Row = {
- serde.toRow(epsg)
- }
-
- @Benchmark
- def encodeProj4(): Row = {
- serde.toRow(proj4)
- }
-
- @Benchmark
- def decodeEpsg(): CRS = {
- serde.fromRow(epsgEnc)
- }
-
- @Benchmark
- def decodeProj4(): CRS = {
- serde.fromRow(proj4Enc)
- }
-
- @Benchmark
- def exprEncodeEpsg(): InternalRow = {
- crsEnc.toRow(epsg)
- }
-
- @Benchmark
- def exprEncodeProj4(): InternalRow = {
- crsEnc.toRow(proj4)
- }
-
-// @Benchmark
-// def exprDecodeEpsg(): CRS = {
-//
-// }
-//
-// @Benchmark
-// def exprDecodeProj4(): CRS = {
-//
-// }
-
-}
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala
index dfc88f855..3a4d9f3f1 100644
--- a/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala
+++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala
@@ -21,10 +21,9 @@
package org.locationtech.rasterframes.bench
import java.util.concurrent.TimeUnit
-
import geotrellis.raster.{CellType, DoubleUserDefinedNoDataCellType, IntUserDefinedNoDataCellType}
import org.apache.spark.sql.catalyst.InternalRow
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.StandardEncoders
import org.openjdk.jmh.annotations._
@BenchmarkMode(Array(Mode.AverageTime))
@@ -37,16 +36,12 @@ class CellTypeBench {
def setupData(): Unit = {
ct = IntUserDefinedNoDataCellType(scala.util.Random.nextInt())
val o: CellType = DoubleUserDefinedNoDataCellType(scala.util.Random.nextDouble())
- row = o.toInternalRow
+ row = StandardEncoders.cellTypeEncoder.createSerializer()(o)
}
@Benchmark
- def fromRow(): CellType = {
- row.to[CellType]
- }
+ def fromRow(): CellType = StandardEncoders.cellTypeEncoder.createDeserializer()(row)
@Benchmark
- def intoRow(): InternalRow = {
- ct.toInternalRow
- }
+ def intoRow(): InternalRow = StandardEncoders.cellTypeEncoder.createSerializer()(ct)
}
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala
index 8de95f56c..737e0c9b2 100644
--- a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala
+++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala
@@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit
import geotrellis.raster.Dimensions
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.rf.TileUDT
-import org.locationtech.rasterframes.tiles.InternalRowTile
import org.openjdk.jmh.annotations._
@BenchmarkMode(Array(Mode.AverageTime))
@@ -62,15 +61,4 @@ class TileCellScanBench extends SparkEnv {
tile.getDouble(cols/2, rows/2) +
tile.getDouble(0, 0)
}
-
- @Benchmark
- def internalRowRead(): Double = {
- val tile = new InternalRowTile(tileRow)
- val cols = tile.cols
- val rows = tile.rows
- tile.getDouble(cols - 1, rows - 1) +
- tile.getDouble(cols/2, rows/2) +
- tile.getDouble(0, 0)
- }
}
-
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala
index 5f9982307..d49027206 100644
--- a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala
+++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala
@@ -24,8 +24,6 @@ package org.locationtech.rasterframes.bench
import java.net.URI
import java.util.concurrent.TimeUnit
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
-import org.locationtech.rasterframes.ref.RasterRef
import geotrellis.raster.Tile
import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.InternalRow
@@ -53,24 +51,23 @@ class TileEncodeBench extends SparkEnv {
@Setup(Level.Trial)
def setupData(): Unit = {
cellTypeName match {
- case "rasterRef" ⇒
+ case "rasterRef" =>
val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF"
val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0)
- tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None))
- case _ ⇒
+ tile = RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None)
+ case _ =>
tile = randomTile(tileSize, tileSize, cellTypeName)
}
}
@Benchmark
def encode(): InternalRow = {
- tileEncoder.toRow(tile)
+ tileEncoder.createSerializer.apply(tile)
}
@Benchmark
def roundTrip(): Tile = {
- val row = tileEncoder.toRow(tile)
- boundEncoder.fromRow(row)
+ val row = tileEncoder.createSerializer().apply(tile)
+ boundEncoder.createDeserializer().apply(row)
}
}
-
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala
index 65d8ab88f..8296cad37 100644
--- a/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala
+++ b/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala
@@ -37,10 +37,10 @@ package object bench {
val cellType = CellType.fromName(cellTypeName)
val tile = ArrayTile.alloc(cellType, cols, rows)
if(cellType.isFloatingPoint) {
- tile.mapDouble(_ ⇒ rnd.nextGaussian())
+ tile.mapDouble(_ => rnd.nextGaussian())
}
else {
- tile.map(_ ⇒ {
+ tile.map(_ => {
var c = NODATA
do {
c = rnd.nextInt(255)
diff --git a/build.sbt b/build.sbt
index 7662baa5c..bc4f3cceb 100644
--- a/build.sbt
+++ b/build.sbt
@@ -30,7 +30,7 @@ lazy val IntegrationTest = config("it") extend Test
lazy val root = project
.in(file("."))
.withId("RasterFrames")
- .aggregate(core, datasource, pyrasterframes, experimental)
+ .aggregate(core, datasource, pyrasterframes)
.enablePlugins(RFReleasePlugin)
.settings(
publish / skip := true,
@@ -52,6 +52,7 @@ lazy val core = project
libraryDependencies ++= Seq(
`slf4j-api`,
shapeless,
+ frameless excludeAll ExclusionRule("com.github.mpilquist", "simulacrum"),
`jts-core`,
`spray-json`,
geomesa("z3").value,
@@ -59,12 +60,15 @@ lazy val core = project
spark("core").value % Provided,
spark("mllib").value % Provided,
spark("sql").value % Provided,
- geotrellis("spark").value,
- geotrellis("raster").value,
- geotrellis("s3").value,
+ // TODO: scala-uri brings an outdated simulacrum dep
+ // Fix it in GT
+ geotrellis("spark").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+ geotrellis("raster").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+ geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
geotrellis("spark-testkit").value % Test excludeAll (
ExclusionRule(organization = "org.scalastic"),
- ExclusionRule(organization = "org.scalatest")
+ ExclusionRule(organization = "org.scalatest"),
+ ExclusionRule(organization = "com.github.mpilquist")
),
scaffeine,
scalatest,
@@ -73,8 +77,8 @@ lazy val core = project
libraryDependencies ++= {
val gv = rfGeoTrellisVersion.value
if (gv.startsWith("3")) Seq[ModuleID](
- geotrellis("gdal").value,
- geotrellis("s3-spark").value
+ geotrellis("gdal").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+ geotrellis("s3-spark").value excludeAll ExclusionRule(organization = "com.github.mpilquist")
)
else Seq.empty[ModuleID]
},
@@ -90,11 +94,11 @@ lazy val core = project
)
lazy val pyrasterframes = project
- .dependsOn(core, datasource, experimental)
+ .dependsOn(core, datasource)
.enablePlugins(RFAssemblyPlugin, PythonBuildPlugin)
.settings(
libraryDependencies ++= Seq(
- geotrellis("s3").value,
+ geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
spark("core").value % Provided,
spark("mllib").value % Provided,
spark("sql").value % Provided
@@ -108,11 +112,16 @@ lazy val datasource = project
.settings(
moduleName := "rasterframes-datasource",
libraryDependencies ++= Seq(
- geotrellis("s3").value,
+ compilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full),
+ sttpCatsCe2,
+ stac4s,
+ geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
spark("core").value % Provided,
spark("mllib").value % Provided,
spark("sql").value % Provided
),
+ Compile / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
+ Test / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
console / initialCommands := (console / initialCommands).value +
"""
|import org.locationtech.rasterframes.datasource.geotrellis._
@@ -130,7 +139,7 @@ lazy val experimental = project
.settings(
moduleName := "rasterframes-experimental",
libraryDependencies ++= Seq(
- geotrellis("s3").value,
+ geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
spark("core").value % Provided,
spark("mllib").value % Provided,
spark("sql").value % Provided
@@ -180,4 +189,3 @@ lazy val docs = project
lazy val bench = project
.dependsOn(core % "compile->test")
.settings(publish / skip := true)
-
diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
index 61a5b5b6b..824bb4094 100644
--- a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
+++ b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
@@ -117,7 +117,7 @@ class RasterSourceIT extends TestEnvironment with TestData {
private def expectedTileCountAndBands(x:Int, y:Int, bandCount:Int = 1) = {
val imageDimensions = Seq(x.toDouble, y.toDouble)
- val tilesPerBand = imageDimensions.map(x ⇒ ceil(x / NOMINAL_TILE_SIZE)).product
+ val tilesPerBand = imageDimensions.map(x => ceil(x / NOMINAL_TILE_SIZE)).product
val bands = Range(0, bandCount)
val expectedTileCount = tilesPerBand * bands.length
(expectedTileCount, bands)
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/CrsUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/CrsUDT.scala
new file mode 100644
index 000000000..74b9941c0
--- /dev/null
+++ b/core/src/main/scala/org/apache/spark/sql/rf/CrsUDT.scala
@@ -0,0 +1,63 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2021 Azavea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.apache.spark.sql.rf
+import geotrellis.proj4.CRS
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.rasterframes.model.LazyCRS
+import org.apache.spark.sql.catalyst.InternalRow
+
+
+@SQLUserDefinedType(udt = classOf[CrsUDT])
+class CrsUDT extends UserDefinedType[CRS] {
+ override def typeName: String = CrsUDT.typeName
+
+ override def pyUDT: String = "pyrasterframes.rf_types.CrsUDT"
+
+ def userClass: Class[CRS] = classOf[CRS]
+
+ def sqlType: DataType = StringType
+
+ override def serialize(obj: CRS): UTF8String =
+ Option(obj)
+ .map { crs => UTF8String.fromString(crs.toProj4String) }
+ .orNull
+
+ override def deserialize(datum: Any): CRS =
+ Option(datum)
+ .collect {
+ case ir: InternalRow => LazyCRS(ir.getString(0))
+ case s: UTF8String => LazyCRS(s.toString)
+ }
+ .orNull
+
+ override def acceptsType(dataType: DataType): Boolean = dataType match {
+ case _: CrsUDT => true
+ case _ => super.acceptsType(dataType)
+ }
+}
+
+case object CrsUDT {
+ UDTRegistration.register(classOf[CRS].getName, classOf[CrsUDT].getName)
+
+ final val typeName: String = "crs"
+}
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala b/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala
index 6433ef8d3..d7a183796 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/FilterTranslator.scala
@@ -20,7 +20,6 @@ package org.apache.spark.sql.rf
import java.sql.{Date, Timestamp}
import org.locationtech.rasterframes.expressions.SpatialRelation.{Contains, Intersects}
-import org.locationtech.rasterframes.rules._
import org.apache.spark.sql.catalyst.CatalystTypeConverters.{convertToScala, createToScalaConverter}
import org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.catalyst.expressions.{Attribute, EmptyRow, Expression, Literal}
@@ -30,9 +29,11 @@ import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.{DateType, StringType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
import org.locationtech.geomesa.spark.jts.rules.GeometryLiteral
-import org.locationtech.rasterframes.rules.{SpatialFilters, TemporalFilters}
+import org.locationtech.rasterframes.rules.TemporalFilters
/**
+ * TODO: fix it, how to implement these filters as ScalaUDFs?
+ * Why do we need them?
* This is a copy of [[org.apache.spark.sql.execution.datasources.DataSourceStrategy.translateFilter]], modified to add our spatial predicates.
*
* @since 1/11/18
@@ -46,55 +47,61 @@ object FilterTranslator {
*/
def translateFilter(predicate: Expression): Option[Filter] = {
predicate match {
- case Intersects(a: Attribute, Literal(geom, udt: AbstractGeometryUDT[_])) ⇒
- Some(SpatialFilters.Intersects(a.name, udt.deserialize(geom)))
+ case Intersects(a: Attribute, Literal(geom, udt: AbstractGeometryUDT[_])) =>
+ // Some(SpatialFilters.Intersects(a.name, udt.deserialize(geom)))
+ ???
- case Contains(a: Attribute, Literal(geom, udt: AbstractGeometryUDT[_])) ⇒
- Some(SpatialFilters.Contains(a.name, udt.deserialize(geom)))
+ case Contains(a: Attribute, Literal(geom, udt: AbstractGeometryUDT[_])) =>
+ // Some(SpatialFilters.Contains(a.name, udt.deserialize(geom)))
+ ???
- case Intersects(a: Attribute, GeometryLiteral(_, geom)) ⇒
- Some(SpatialFilters.Intersects(a.name, geom))
+ case Intersects(a: Attribute, GeometryLiteral(_, geom)) =>
+ // Some(SpatialFilters.Intersects(a.name, geom))
+ ???
- case Contains(a: Attribute, GeometryLiteral(_, geom)) ⇒
- Some(SpatialFilters.Contains(a.name, geom))
+ case Contains(a: Attribute, GeometryLiteral(_, geom)) =>
+ // Some(SpatialFilters.Contains(a.name, geom))
+ ???
case expressions.And(
expressions.GreaterThanOrEqual(a: Attribute, Literal(start, TimestampType)),
expressions.LessThanOrEqual(b: Attribute, Literal(end, TimestampType))
- ) if a.name == b.name ⇒
+ ) if a.name == b.name =>
val toScala = createToScalaConverter(TimestampType)(_: Any).asInstanceOf[Timestamp]
- Some(TemporalFilters.BetweenTimes(a.name, toScala(start), toScala(end)))
+ // Some(TemporalFilters.BetweenTimes(a.name, toScala(start), toScala(end)))
+ ???
case expressions.And(
expressions.GreaterThanOrEqual(a: Attribute, Literal(start, DateType)),
expressions.LessThanOrEqual(b: Attribute, Literal(end, DateType))
- ) if a.name == b.name ⇒
+ ) if a.name == b.name =>
val toScala = createToScalaConverter(DateType)(_: Any).asInstanceOf[Date]
- Some(TemporalFilters.BetweenDates(a.name, toScala(start), toScala(end)))
+ // Some(TemporalFilters.BetweenDates(a.name, toScala(start), toScala(end)))
+ ???
// TODO: Need to figure out how to generalize over capturing right-hand pairs
case expressions.And(expressions.And(left,
expressions.GreaterThanOrEqual(a: Attribute, Literal(start, TimestampType))),
expressions.LessThanOrEqual(b: Attribute, Literal(end, TimestampType))
- ) if a.name == b.name ⇒
+ ) if a.name == b.name =>
val toScala = createToScalaConverter(TimestampType)(_: Any).asInstanceOf[Timestamp]
for {
- leftFilter ← translateFilter(left)
+ leftFilter <- translateFilter(left)
rightFilter = TemporalFilters.BetweenTimes(a.name, toScala(start), toScala(end))
- } yield sources.And(leftFilter, rightFilter)
+ } yield sources.And(leftFilter, ???)
// TODO: Ditto as above
case expressions.And(expressions.And(left,
expressions.GreaterThanOrEqual(a: Attribute, Literal(start, DateType))),
expressions.LessThanOrEqual(b: Attribute, Literal(end, DateType))
- ) if a.name == b.name ⇒
+ ) if a.name == b.name =>
val toScala = createToScalaConverter(DateType)(_: Any).asInstanceOf[Date]
for {
- leftFilter ← translateFilter(left)
+ leftFilter <- translateFilter(left)
rightFilter = TemporalFilters.BetweenDates(a.name, toScala(start), toScala(end))
- } yield sources.And(leftFilter, rightFilter)
+ } yield sources.And(leftFilter, ???)
case expressions.EqualTo(a: Attribute, Literal(v, t)) =>
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala
index 5ccb621f9..4715609b2 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala
@@ -21,46 +21,51 @@
package org.apache.spark.sql.rf
-import java.nio.ByteBuffer
-
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.types.{DataType, UDTRegistration, UserDefinedType, _}
-import org.locationtech.rasterframes.encoders.CatalystSerializer
+import org.apache.spark.sql.types._
import org.locationtech.rasterframes.ref.RFRasterSource
import org.locationtech.rasterframes.util.KryoSupport
+import java.nio.ByteBuffer
+
/**
* Catalyst representation of a RasterSource.
*
* @since 9/5/18
*/
+// TODO: remove it
@SQLUserDefinedType(udt = classOf[RasterSourceUDT])
class RasterSourceUDT extends UserDefinedType[RFRasterSource] {
- import RasterSourceUDT._
override def typeName = "rastersource"
override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT"
def userClass: Class[RFRasterSource] = classOf[RFRasterSource]
- override def sqlType: DataType = schemaOf[RFRasterSource]
+ def sqlType: DataType = StructType(Seq(
+ StructField("raster_source_kryo", BinaryType, false)
+ ))
- override def serialize(obj: RFRasterSource): InternalRow =
+ def serialize(obj: RFRasterSource): InternalRow =
Option(obj)
- .map(_.toInternalRow)
+ .map { rs => InternalRow(KryoSupport.serialize(rs).array()) }
.orNull
- override def deserialize(datum: Any): RFRasterSource =
+ def deserialize(datum: Any): RFRasterSource =
Option(datum)
.collect {
- case ir: InternalRow ⇒ ir.to[RFRasterSource]
+ case ir: InternalRow =>
+ val bytes = ir.getBinary(0)
+ KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(bytes))
+ case bytes: Array[Byte] =>
+ KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(bytes))
+
}
.orNull
- private[sql] override def acceptsType(dataType: DataType) = dataType match {
- case _: RasterSourceUDT ⇒ true
- case _ ⇒ super.acceptsType(dataType)
+ private[sql] override def acceptsType(dataType: DataType): Boolean = dataType match {
+ case _: RasterSourceUDT => true
+ case _ => super.acceptsType(dataType)
}
}
@@ -68,21 +73,6 @@ object RasterSourceUDT {
UDTRegistration.register(classOf[RFRasterSource].getName, classOf[RasterSourceUDT].getName)
/** Deserialize a byte array, also used inside the Python API */
- def from(byteArray: Array[Byte]): RFRasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RFRasterSource]
-
- implicit val rasterSourceSerializer: CatalystSerializer[RFRasterSource] = new CatalystSerializer[RFRasterSource] {
-
- override val schema: StructType = StructType(Seq(
- StructField("raster_source_kryo", BinaryType, false)
- ))
-
- override def to[R](t: RFRasterSource, io: CatalystIO[R]): R = {
- val buf = KryoSupport.serialize(t)
- io.create(buf.array())
- }
-
- override def from[R](row: R, io: CatalystIO[R]): RFRasterSource = {
- KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(io.getByteArray(row, 0)))
- }
- }
+ def from(byteArray: Array[Byte]): RFRasterSource =
+ KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(byteArray))
}
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala
index a424869ac..2c2077fe4 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala
@@ -22,13 +22,14 @@
package org.apache.spark.sql.rf
import geotrellis.raster._
import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.execution.datasources.parquet.ParquetReadSupport
import org.apache.spark.sql.types.{DataType, _}
-import org.locationtech.rasterframes.encoders.CatalystSerializer
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.model.{Cells, TileDataContext}
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
-import org.locationtech.rasterframes.tiles.InternalRowTile
+import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.rasterframes.encoders.syntax._
+import org.locationtech.rasterframes.ref.RasterRef
+import org.locationtech.rasterframes.tiles.{ProjectedRasterTile, ShowableTile}
+import scala.util.Try
/**
* UDT for singleband tiles.
@@ -37,66 +38,87 @@ import org.locationtech.rasterframes.tiles.InternalRowTile
*/
@SQLUserDefinedType(udt = classOf[TileUDT])
class TileUDT extends UserDefinedType[Tile] {
- import TileUDT._
override def typeName = TileUDT.typeName
override def pyUDT: String = "pyrasterframes.rf_types.TileUDT"
def userClass: Class[Tile] = classOf[Tile]
- def sqlType: StructType = schemaOf[Tile]
-
- override def serialize(obj: Tile): InternalRow =
- Option(obj)
- .map(_.toInternalRow)
- .orNull
+ def sqlType: StructType = StructType(Seq(
+ StructField("cell_type", StringType, false),
+ StructField("cols", IntegerType, false),
+ StructField("rows", IntegerType, false),
+ StructField("cells", BinaryType, true),
+ // make it parquet compliant, only expanded UDTs can be in a UDT schema
+ StructField("ref", ParquetReadSupport.expandUDT(RasterRef.rasterRefEncoder.schema), true)
+ ))
+
+ def serialize(obj: Tile): InternalRow = {
+ if (obj == null) return null
+ obj match {
+ // TODO: review matches there
+ case ref: RasterRef =>
+ val ct = UTF8String.fromString(ref.cellType.toString())
+ InternalRow(ct, ref.cols, ref.rows, null, ref.toInternalRow)
+ case ProjectedRasterTile(ref: RasterRef, _, _) =>
+ val ct = UTF8String.fromString(ref.cellType.toString())
+ InternalRow(ct, ref.cols, ref.rows, null, ref.toInternalRow)
+ case prt: ProjectedRasterTile =>
+ val tile = prt.tile
+ val ct = UTF8String.fromString(tile.cellType.toString())
+ InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null)
+ case const: ConstantTile =>
+ // Must expand constant tiles so they can be interpreted properly in catalyst and Python.
+ val tile = const.toArrayTile()
+ val ct = UTF8String.fromString(tile.cellType.toString())
+ InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null)
+ case tile =>
+ val ct = UTF8String.fromString(tile.cellType.toString())
+ InternalRow(ct, tile.cols, tile.rows, tile.toBytes(), null)
+ }
+ }
- override def deserialize(datum: Any): Tile =
- Option(datum)
- .collect {
- case ir: InternalRow ⇒ ir.to[Tile]
- }
- .map {
- case realIRT: InternalRowTile ⇒ realIRT.realizedTile
- case other ⇒ other
+ def deserialize(datum: Any): Tile = {
+ if (datum == null) return null
+ val row = datum.asInstanceOf[InternalRow]
+
+ /** TODO: a compatible encoder for the ProjectedRasterTile */
+ val tile: Tile =
+ if (! row.isNullAt(4)) {
+ Try {
+ val ir = row.getStruct(4, 4)
+ val ref = ir.as[RasterRef]
+ ref
+ }/*.orElse {
+ Try(
+ ProjectedRasterTile
+ .projectedRasterTileEncoder
+ .resolveAndBind()
+ .createDeserializer()(row)
+ .tile
+ )
+ }*/.get
+ } else {
+ val ct = CellType.fromName(row.getString(0))
+ val cols = row.getInt(1)
+ val rows = row.getInt(2)
+ val bytes = row.getBinary(3)
+ ArrayTile.fromBytes(bytes, ct, cols, rows)
}
- .orNull
+
+ if (TileUDT.showableTiles) new ShowableTile(tile) else tile
+ }
override def acceptsType(dataType: DataType): Boolean = dataType match {
- case _: TileUDT ⇒ true
- case _ ⇒ super.acceptsType(dataType)
+ case _: TileUDT => true
+ case _ => super.acceptsType(dataType)
}
}
-case object TileUDT {
+case object TileUDT {
+ private val showableTiles = org.locationtech.rasterframes.rfConfig.getBoolean("showable-tiles")
+
UDTRegistration.register(classOf[Tile].getName, classOf[TileUDT].getName)
final val typeName: String = "tile"
-
- implicit val tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
-
- override val schema: StructType = StructType(Seq(
- StructField("cell_context", schemaOf[TileDataContext], true),
- StructField("cell_data", schemaOf[Cells], false)
- ))
-
- override def to[R](t: Tile, io: CatalystIO[R]): R = io.create(
- t match {
- case _: RasterRefTile => null
- case o => io.to(TileDataContext(o))
- },
- io.to(Cells(t))
- )
-
- override def from[R](row: R, io: CatalystIO[R]): Tile = {
- val cells = io.get[Cells](row, 1)
-
- row match {
- case ir: InternalRow if !cells.isRef ⇒ new InternalRowTile(ir)
- case _ ⇒
- val ctx = io.get[TileDataContext](row, 0)
- cells.toTile(ctx)
- }
- }
- }
}
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala
index a75932886..bb05573d1 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala
@@ -39,7 +39,7 @@ object VersionShims {
// relation: BaseRelation,
// output: Seq[AttributeReference],
// catalogTable: Option[CatalogTable])
- case 3 ⇒
+ case 3 =>
val arg2: Seq[AttributeReference] = lr.output
val arg3: Option[CatalogTable] = lr.catalogTable
if(ctor.getParameterTypes()(1).isAssignableFrom(classOf[Option[_]])) {
@@ -57,13 +57,13 @@ object VersionShims {
// catalogTable: Option[CatalogTable],
// override val isStreaming: Boolean)
// extends LeafNode with MultiInstanceRelation {
- case 4 ⇒
+ case 4 =>
val arg2: Seq[AttributeReference] = lr.output
val arg3: Option[CatalogTable] = lr.catalogTable
val arg4 = lrClazz.getMethod("isStreaming").invoke(lr)
ctor.newInstance(base, arg2, arg3, arg4)
- case _ ⇒
+ case _ =>
throw new NotImplementedError("LogicalRelation constructor has unexpected shape")
}
}
@@ -83,7 +83,7 @@ object VersionShims {
// dataType: DataType,
// arguments: Seq[Expression] = Nil,
// propagateNull: Boolean = true) extends InvokeLike
- case 5 ⇒
+ case 5 =>
ctor.newInstance(targetObject, functionName, dataType, Nil, TRUE).asInstanceOf[InvokeLike]
// In spark 2.2.0 the signature looks like this:
//
@@ -94,10 +94,10 @@ object VersionShims {
// arguments: Seq[Expression] = Nil,
// propagateNull: Boolean = true,
// returnNullable : Boolean = true) extends InvokeLike
- case 6 ⇒
+ case 6 =>
ctor.newInstance(targetObject, functionName, dataType, Nil, TRUE, TRUE).asInstanceOf[InvokeLike]
- case _ ⇒
+ case _ =>
throw new NotImplementedError("Invoke constructor has unexpected shape")
}
}
@@ -108,8 +108,8 @@ object VersionShims {
// Spark 2.3 introduced a new way of specifying Functions
val spark23FI = "org.apache.spark.sql.catalyst.FunctionIdentifier"
registry.getClass.getDeclaredMethods
- .filter(m ⇒ m.getName == "registerFunction" && m.getParameterCount == 2)
- .foreach { m ⇒
+ .filter(m => m.getName == "registerFunction" && m.getParameterCount == 2)
+ .foreach { m =>
val firstParam = m.getParameterTypes()(0)
if(firstParam == classOf[String])
m.invoke(registry, name, builder)
@@ -133,7 +133,7 @@ object VersionShims {
val df = clazz.getAnnotation(classOf[ExpressionDescription])
if (df != null) {
if (df.extended().isEmpty) {
- new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.arguments(), df.examples(), df.note(), df.since())
+ new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.arguments(), df.examples(), df.note(), df.group(), df.since(), df.deprecated())
} else {
// This exists for the backward compatibility with old `ExpressionDescription`s defining
// the extended description in `extended()`.
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/package.scala b/core/src/main/scala/org/apache/spark/sql/rf/package.scala
index 4035b60c4..7a708924c 100644
--- a/core/src/main/scala/org/apache/spark/sql/rf/package.scala
+++ b/core/src/main/scala/org/apache/spark/sql/rf/package.scala
@@ -43,6 +43,7 @@ package object rf {
// which is where the registration actually happens. The ordering matters!
RasterSourceUDT
TileUDT
+ CrsUDT
}
def registry(sqlContext: SQLContext): FunctionRegistry = {
@@ -65,7 +66,6 @@ package object rf {
implicit class WithPPrint[T](enc: ExpressionEncoder[T]) {
def pprint(): Unit = {
println(enc.getClass.getSimpleName + "{")
- println("\tflat=" + enc.flat)
println("\tschema=" + enc.schema)
println("\tserializers=" + enc.serializer)
println("\tnamedExpressions=" + enc.namedExpressions)
diff --git a/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala b/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala
index b4a2fe8f0..14a754ec5 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala
@@ -80,14 +80,14 @@ object PairRDDConverter {
def toDataFrame(rdd: RDD[(SpaceTimeKey, Tile)])(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
- rdd.map{ case (k, v) ⇒ (k.spatialKey, k.temporalKey, v)}.toDF(schema.fields.map(_.name): _*)
+ rdd.map{ case (k, v) => (k.spatialKey, k.temporalKey, v)}.toDF(schema.fields.map(_.name): _*)
}
}
/** Enables conversion of `RDD[(SpatialKey, TileFeature[Tile, D])]` to DataFrame. */
implicit def spatialTileFeatureConverter[D: Encoder] = new PairRDDConverter[SpatialKey, TileFeature[Tile, D]] {
implicit val featureEncoder = implicitly[Encoder[D]]
- implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, singlebandTileEncoder, featureEncoder)
+ implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, tileEncoder, featureEncoder)
val schema: StructType = {
val base = spatialTileConverter.schema
@@ -96,14 +96,14 @@ object PairRDDConverter {
def toDataFrame(rdd: RDD[(SpatialKey, TileFeature[Tile, D])])(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
- rdd.map{ case (k, v) ⇒ (k, v.tile, v.data)}.toDF(schema.fields.map(_.name): _*)
+ rdd.map{ case (k, v) => (k, v.tile, v.data)}.toDF(schema.fields.map(_.name): _*)
}
}
/** Enables conversion of `RDD[(SpaceTimeKey, TileFeature[Tile, D])]` to DataFrame. */
implicit def spaceTimeTileFeatureConverter[D: Encoder] = new PairRDDConverter[SpaceTimeKey, TileFeature[Tile, D]] {
implicit val featureEncoder = implicitly[Encoder[D]]
- implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, temporalKeyEncoder, singlebandTileEncoder, featureEncoder)
+ implicit val rowEncoder = Encoders.tuple(spatialKeyEncoder, temporalKeyEncoder, tileEncoder, featureEncoder)
val schema: StructType = {
val base = spaceTimeTileConverter.schema
@@ -112,7 +112,7 @@ object PairRDDConverter {
def toDataFrame(rdd: RDD[(SpaceTimeKey, TileFeature[Tile, D])])(implicit spark: SparkSession): DataFrame = {
import spark.implicits._
- val tupRDD = rdd.map { case (k, v) ⇒ (k.spatialKey, k.temporalKey, v.tile, v.data) }
+ val tupRDD = rdd.map { case (k, v) => (k.spatialKey, k.temporalKey, v.tile, v.data) }
rddToDatasetHolder(tupRDD)
tupRDD.toDF(schema.fields.map(_.name): _*)
@@ -126,7 +126,7 @@ object PairRDDConverter {
val basename = TILE_COLUMN.columnName
- val tiles = for(i ← 1 to bands) yield {
+ val tiles = for(i <- 1 to bands) yield {
val name = if(bands <= 1) basename else s"${basename}_$i"
StructField(name , serializableTileUDT, nullable = false)
}
@@ -136,20 +136,20 @@ object PairRDDConverter {
def toDataFrame(rdd: RDD[(SpatialKey, MultibandTile)])(implicit spark: SparkSession): DataFrame = {
spark.createDataFrame(
- rdd.map { case (k, v) ⇒ Row(Row(k.col, k.row) +: v.bands: _*) },
+ rdd.map { case (k, v) => Row(Row(k.col, k.row) +: v.bands: _*) },
schema
)
}
}
/** Enables conversion of `RDD[(SpaceTimeKey, MultibandTile)]` to DataFrame. */
- def forSpaceTimeMultiband(bands: Int) = new PairRDDConverter[SpaceTimeKey, MultibandTile] {
+ def forSpaceTimeMultiband(bands: Int): PairRDDConverter[SpaceTimeKey, MultibandTile] = new PairRDDConverter[SpaceTimeKey, MultibandTile] {
val schema: StructType = {
val base = spaceTimeTileConverter.schema
val basename = TILE_COLUMN.columnName
- val tiles = for(i ← 1 to bands) yield {
+ val tiles = for(i <- 1 to bands) yield {
StructField(s"${basename}_$i" , serializableTileUDT, nullable = false)
}
@@ -158,7 +158,7 @@ object PairRDDConverter {
def toDataFrame(rdd: RDD[(SpaceTimeKey, MultibandTile)])(implicit spark: SparkSession): DataFrame = {
spark.createDataFrame(
- rdd.map { case (k, v) ⇒ Row(Seq(Row(k.spatialKey.col, k.spatialKey.row), Row(k.temporalKey)) ++ v.bands: _*) },
+ rdd.map { case (k, v) => Row(Seq(Row(k.spatialKey.col, k.spatialKey.row), Row(k.temporalKey)) ++ v.bands: _*) },
schema
)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala b/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala
index 4ae29f1d3..cd4e9580a 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala
@@ -29,8 +29,8 @@ import geotrellis.layer._
import geotrellis.vector.{Extent, ProjectedExtent}
import org.apache.spark.sql.functions.col
import org.locationtech.jts.geom.{Point => jtsPoint, Polygon => jtsPolygon}
-import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
/**
* Constants identifying column in most RasterFrames.
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala
deleted file mode 100644
index 831411557..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializer.scala
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2018 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import CatalystSerializer.CatalystIO
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.util.ArrayData
-import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
-
-/**
- * Typeclass for converting to/from JVM object to catalyst encoding. The reason this exists is that
- * instantiating and binding `ExpressionEncoder[T]` is *very* expensive, and not suitable for
- * operations internal to an `Expression`.
- *
- * @since 10/19/18
- */
-trait CatalystSerializer[T] extends Serializable {
- def schema: StructType
- protected def to[R](t: T, io: CatalystIO[R]): R
- protected def from[R](t: R, io: CatalystIO[R]): T
-
- final def toRow(t: T): Row = to(t, CatalystIO[Row])
- final def fromRow(row: Row): T = from(row, CatalystIO[Row])
-
- final def toInternalRow(t: T): InternalRow = to(t, CatalystIO[InternalRow])
- final def fromInternalRow(row: InternalRow): T = from(row, CatalystIO[InternalRow])
-}
-
-object CatalystSerializer extends StandardSerializers {
- def apply[T: CatalystSerializer]: CatalystSerializer[T] = implicitly
-
- def schemaOf[T: CatalystSerializer]: StructType = apply[T].schema
-
- /**
- * For some reason `Row` and `InternalRow` share no common base type. Instead of using
- * structural types (which use reflection), this typeclass is used to normalize access
- * to the underlying storage construct.
- *
- * @tparam R row storage type
- */
- trait CatalystIO[R] extends Serializable {
- def create(values: Any*): R
- def to[T: CatalystSerializer](t: T): R = CatalystSerializer[T].to(t, this)
- def toSeq[T: CatalystSerializer](t: Seq[T]): AnyRef
- def get[T >: Null: CatalystSerializer](d: R, ordinal: Int): T
- def getSeq[T >: Null: CatalystSerializer](d: R, ordinal: Int): Seq[T]
- def isNullAt(d: R, ordinal: Int): Boolean
- def getBoolean(d: R, ordinal: Int): Boolean
- def getByte(d: R, ordinal: Int): Byte
- def getShort(d: R, ordinal: Int): Short
- def getInt(d: R, ordinal: Int): Int
- def getLong(d: R, ordinal: Int): Long
- def getFloat(d: R, ordinal: Int): Float
- def getDouble(d: R, ordinal: Int): Double
- def getString(d: R, ordinal: Int): String
- def getByteArray(d: R, ordinal: Int): Array[Byte]
- def encode(str: String): AnyRef
- }
-
- object CatalystIO {
- def apply[R: CatalystIO]: CatalystIO[R] = implicitly
-
- trait AbstractRowEncoder[R <: Row] extends CatalystIO[R] {
- override def isNullAt(d: R, ordinal: Int): Boolean = d.isNullAt(ordinal)
- override def getBoolean(d: R, ordinal: Int): Boolean = d.getBoolean(ordinal)
- override def getByte(d: R, ordinal: Int): Byte = d.getByte(ordinal)
- override def getShort(d: R, ordinal: Int): Short = d.getShort(ordinal)
- override def getInt(d: R, ordinal: Int): Int = d.getInt(ordinal)
- override def getLong(d: R, ordinal: Int): Long = d.getLong(ordinal)
- override def getFloat(d: R, ordinal: Int): Float = d.getFloat(ordinal)
- override def getDouble(d: R, ordinal: Int): Double = d.getDouble(ordinal)
- override def getString(d: R, ordinal: Int): String = d.getString(ordinal)
- override def getByteArray(d: R, ordinal: Int): Array[Byte] =
- d.get(ordinal).asInstanceOf[Array[Byte]]
- override def get[T >: Null: CatalystSerializer](d: R, ordinal: Int): T = {
- d.getAs[Any](ordinal) match {
- case r: Row => r.to[T]
- case o => o.asInstanceOf[T]
- }
- }
- override def toSeq[T: CatalystSerializer](t: Seq[T]): AnyRef = t.map(_.toRow)
- override def getSeq[T >: Null: CatalystSerializer](d: R, ordinal: Int): Seq[T] =
- d.getSeq[Row](ordinal).map(_.to[T])
- override def encode(str: String): String = str
- }
-
- implicit val rowIO: CatalystIO[Row] = new AbstractRowEncoder[Row] {
- override def create(values: Any*): Row = Row(values: _*)
- }
-
- implicit val internalRowIO: CatalystIO[InternalRow] = new CatalystIO[InternalRow] {
- override def isNullAt(d: InternalRow, ordinal: Int): Boolean = d.isNullAt(ordinal)
- override def getBoolean(d: InternalRow, ordinal: Int): Boolean = d.getBoolean(ordinal)
- override def getByte(d: InternalRow, ordinal: Int): Byte = d.getByte(ordinal)
- override def getShort(d: InternalRow, ordinal: Int): Short = d.getShort(ordinal)
- override def getInt(d: InternalRow, ordinal: Int): Int = d.getInt(ordinal)
- override def getLong(d: InternalRow, ordinal: Int): Long = d.getLong(ordinal)
- override def getFloat(d: InternalRow, ordinal: Int): Float = d.getFloat(ordinal)
- override def getDouble(d: InternalRow, ordinal: Int): Double = d.getDouble(ordinal)
- override def getString(d: InternalRow, ordinal: Int): String = d.getString(ordinal)
- override def getByteArray(d: InternalRow, ordinal: Int): Array[Byte] = d.getBinary(ordinal)
- override def get[T >: Null: CatalystSerializer](d: InternalRow, ordinal: Int): T = {
- val ser = CatalystSerializer[T]
- val struct = d.getStruct(ordinal, ser.schema.size)
- struct.to[T]
- }
- override def create(values: Any*): InternalRow = InternalRow(values: _*)
- override def toSeq[T: CatalystSerializer](t: Seq[T]): ArrayData =
- ArrayData.toArrayData(t.map(_.toInternalRow).toArray)
-
- override def getSeq[T >: Null: CatalystSerializer](d: InternalRow, ordinal: Int): Seq[T] = {
- val ad = d.getArray(ordinal)
- val result = Array.ofDim[Any](ad.numElements()).asInstanceOf[Array[T]]
- ad.foreach(
- CatalystSerializer[T].schema,
- (i, v) => result(i) = v.asInstanceOf[InternalRow].to[T]
- )
- result.toSeq
- }
- override def encode(str: String): UTF8String = UTF8String.fromString(str)
- }
- }
-
- implicit class WithToRow[T: CatalystSerializer](t: T) {
- def toInternalRow: InternalRow = if (t == null) null else CatalystSerializer[T].toInternalRow(t)
- def toRow: Row = if (t == null) null else CatalystSerializer[T].toRow(t)
- }
-
- implicit class WithFromInternalRow(val r: InternalRow) extends AnyVal {
- def to[T >: Null: CatalystSerializer]: T = if (r == null) null else CatalystSerializer[T].fromInternalRow(r)
- }
-
- implicit class WithFromRow(val r: Row) extends AnyVal {
- def to[T >: Null: CatalystSerializer]: T = if (r == null) null else CatalystSerializer[T].fromRow(r)
- }
-
- implicit class WithTypeConformity(val left: DataType) extends AnyVal {
- def conformsTo[T >: Null: CatalystSerializer]: Boolean =
- org.apache.spark.sql.rf.WithTypeConformity(left).conformsTo(schemaOf[T])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala
deleted file mode 100644
index 792b74165..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/CatalystSerializerEncoder.scala
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2019 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
-import org.apache.spark.sql.types.{DataType, ObjectType, StructField, StructType}
-
-import scala.reflect.runtime.universe.TypeTag
-
-object CatalystSerializerEncoder {
-
- case class CatSerializeToRow[T](child: Expression, serde: CatalystSerializer[T])
- extends UnaryExpression {
- override def dataType: DataType = serde.schema
- override protected def nullSafeEval(input: Any): Any = {
- val value = input.asInstanceOf[T]
- serde.toInternalRow(value)
- }
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val cs = ctx.addReferenceObj("serde", serde, serde.getClass.getName)
- nullSafeCodeGen(ctx, ev, input => s"${ev.value} = $cs.toInternalRow($input);")
- }
- }
- case class CatDeserializeFromRow[T](child: Expression, serde: CatalystSerializer[T], outputType: DataType)
- extends UnaryExpression {
- override def dataType: DataType = outputType
-
- private def objType = outputType match {
- case ot: ObjectType => ot.cls.getName
- case o => s"java.lang.Object /* $o */" // not sure what to do here... hopefully shouldn't happen
- }
- override protected def nullSafeEval(input: Any): Any = {
- val row = input.asInstanceOf[InternalRow]
- serde.fromInternalRow(row)
- }
- override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val cs = ctx.addReferenceObj("serde", serde, classOf[CatalystSerializer[_]].getName)
- nullSafeCodeGen(ctx, ev, input => s"${ev.value} = ($objType) $cs.fromInternalRow($input);")
- }
- }
- def apply[T: TypeTag: CatalystSerializer](flat: Boolean = false): ExpressionEncoder[T] = {
- val serde = CatalystSerializer[T]
-
- val schema = if (flat)
- StructType(Seq(
- StructField("value", serde.schema, true)
- ))
- else serde.schema
-
- val parentType: DataType = ScalaReflection.dataTypeFor[T]
-
- val inputObject = BoundReference(0, parentType, nullable = true)
-
- val serializer = CatSerializeToRow(inputObject, serde)
-
- val deserializer: Expression = CatDeserializeFromRow(GetColumnByOrdinal(0, schema), serde, parentType)
-
- ExpressionEncoder(schema, flat = flat, Seq(serializer), deserializer, typeToClassTag[T])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala
deleted file mode 100644
index ea01d4143..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/CellTypeEncoder.scala
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import geotrellis.raster.{CellType, DataType}
-import org.apache.spark.sql.catalyst.ScalaReflection
-import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.rf.VersionShims.InvokeSafely
-import org.apache.spark.sql.types.{ObjectType, StringType}
-import org.apache.spark.unsafe.types.UTF8String
-import CatalystSerializer._
-import scala.reflect.classTag
-
-/**
- * Custom encoder for GT [[CellType]]. It's necessary since [[CellType]] is a type alias of
- * a type intersection.
- * @since 7/21/17
- */
-object CellTypeEncoder {
- def apply(): ExpressionEncoder[CellType] = {
- // We can't use StringBackedEncoder due to `CellType` being a type alias,
- // and Spark doesn't like that.
- import org.apache.spark.sql.catalyst.expressions._
- import org.apache.spark.sql.catalyst.expressions.objects._
- val ctType = ScalaReflection.dataTypeFor[DataType]
- val schema = schemaOf[CellType]
- val inputObject = BoundReference(0, ctType, nullable = false)
-
- val intermediateType = ObjectType(classOf[String])
- val serializer: Expression =
- StaticInvoke(
- classOf[UTF8String],
- StringType,
- "fromString",
- InvokeSafely(inputObject, "name", intermediateType) :: Nil
- )
-
- val inputRow = GetColumnByOrdinal(0, schema)
- val deserializer: Expression =
- StaticInvoke(CellType.getClass, ctType, "fromName", InvokeSafely(inputRow, "toString", intermediateType) :: Nil)
-
- ExpressionEncoder[CellType](schema, flat = false, Seq(serializer), deserializer, classTag[CellType])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala
deleted file mode 100644
index cf4c2e5ac..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/DelegatingSubfieldEncoder.scala
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.apache.spark.sql.catalyst.ScalaReflection
-import org.apache.spark.sql.catalyst.analysis.{GetColumnByOrdinal, UnresolvedAttribute, UnresolvedExtractValue}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.objects.NewInstance
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.apache.spark.sql.rf.VersionShims.InvokeSafely
-
-import scala.reflect.runtime.universe.TypeTag
-
-/**
- * Encoder builder for types composed of other fields with {{ExpressionEncoder}}s.
- *
- * @since 8/2/17
- */
-object DelegatingSubfieldEncoder {
- def apply[T: TypeTag](
- fieldEncoders: (String, ExpressionEncoder[_])*): ExpressionEncoder[T] = {
- val schema = StructType(fieldEncoders.map {
- case (name, encoder) ⇒
- StructField(name, encoder.schema, false)
- })
-
- val parentType = ScalaReflection.dataTypeFor[T]
-
- val inputObject = BoundReference(0, parentType, nullable = false)
- val serializer = CreateNamedStruct(fieldEncoders.flatMap {
- case (name, encoder) ⇒
- val enc = encoder.serializer.map(_.transform {
- case r: BoundReference if r != inputObject ⇒
- InvokeSafely(inputObject, name, r.dataType)
- })
- Literal(name) :: CreateStruct(enc) :: Nil
- })
-
- val fieldDeserializers = fieldEncoders.map(_._2).zipWithIndex.map {
- case (enc, index) ⇒
- val input = GetColumnByOrdinal(index, enc.schema)
- val deserialized = enc.deserializer.transformUp {
- case UnresolvedAttribute(nameParts) ⇒
- UnresolvedExtractValue(input, Literal(nameParts.head))
- case GetColumnByOrdinal(ordinal, _) ⇒ GetStructField(input, ordinal)
- }
- If(IsNull(input), Literal.create(null, deserialized.dataType), deserialized)
- }
-
- val deserializer: Expression = NewInstance(runtimeClass[T], fieldDeserializers, parentType, propagateNull = false)
-
- ExpressionEncoder(schema, flat = false, serializer.flatten, deserializer, typeToClassTag[T])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala
deleted file mode 100644
index 50d66f3e0..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/EnvelopeEncoder.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2019 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.locationtech.jts.geom.Envelope
-import org.apache.spark.sql.catalyst.ScalaReflection
-import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.objects.NewInstance
-import org.apache.spark.sql.catalyst.expressions.{BoundReference, CreateNamedStruct, Literal}
-import org.apache.spark.sql.rf.VersionShims.InvokeSafely
-import org.apache.spark.sql.types._
-import CatalystSerializer._
-import scala.reflect.classTag
-
-/**
- * Spark DataSet codec for JTS Envelope.
- *
- * @since 2/22/18
- */
-object EnvelopeEncoder {
-
- val schema = schemaOf[Envelope]
-
- val dataType: DataType = ScalaReflection.dataTypeFor[Envelope]
-
- def apply(): ExpressionEncoder[Envelope] = {
- val inputObject = BoundReference(0, ObjectType(classOf[Envelope]), nullable = true)
-
- val invokers = schema.flatMap { f ⇒
- val getter = "get" + f.name.head.toUpper + f.name.tail
- Literal(f.name) :: InvokeSafely(inputObject, getter, DoubleType) :: Nil
- }
-
- val serializer = CreateNamedStruct(invokers)
- val deserializer = NewInstance(classOf[Envelope],
- (0 to 3).map(GetColumnByOrdinal(_, DoubleType)),
- dataType, false
- )
-
- new ExpressionEncoder[Envelope](schema, flat = false, serializer.flatten, deserializer, classTag[Envelope])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala
deleted file mode 100644
index d366adbd2..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.locationtech.rasterframes._
-import geotrellis.vector.ProjectedExtent
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-
-/**
- * Custom encoder for [[ProjectedExtent]]. Necessary because CRS isn't a case class.
- *
- * @since 8/2/17
- */
-object ProjectedExtentEncoder {
- def apply(): ExpressionEncoder[ProjectedExtent] = {
- DelegatingSubfieldEncoder("extent" -> extentEncoder, "crs" -> crsSparkEncoder)
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala
new file mode 100644
index 000000000..18cf1eea8
--- /dev/null
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/SerializersCache.scala
@@ -0,0 +1,61 @@
+package org.locationtech.rasterframes.encoders
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder}
+
+import scala.collection.mutable
+import scala.reflect.runtime.universe.TypeTag
+
+object SerializersCache {
+ /**
+ * Spark partitions are executed on a blocking thread pool.
+ * We can keep the cache of (De)Serializers (every serializer instance creation is pretty expensive),
+ * but the cache should be local per thread.
+ *
+ * When used from multiple threads (De)Serializers tend to corrupt data and / or fail at runtime.
+ * The alternative can be to use global locks or to use a separate executor per each (De)Serializer.
+ */
+ private class ThreadLocalHashMap[K, V] extends ThreadLocal[mutable.HashMap[K, V]] {
+ override def initialValue(): mutable.HashMap[K, V] = mutable.HashMap.empty
+ }
+ private object ThreadLocalHashMap {
+ def empty[K, V]: ThreadLocalHashMap[K, V] = new ThreadLocalHashMap
+ }
+
+ /** SerializerSafe ensures that all Serializers from the pool call copy after application. */
+ case class SerializerSafe[T](underlying: ExpressionEncoder.Serializer[T]) {
+ def apply(t: T): InternalRow = underlying.apply(t).copy()
+ }
+
+ // T => InternalRow
+ private val cacheSerializer: ThreadLocalHashMap[TypeTag[_], SerializerSafe[_]] = ThreadLocalHashMap.empty
+ // Row with Schema T => InternalRow
+ private val cacheSerializerRow: ThreadLocalHashMap[TypeTag[_], SerializerSafe[Row]] = ThreadLocalHashMap.empty
+ // InternalRow => T
+ private val cacheDeserializer: ThreadLocalHashMap[TypeTag[_], ExpressionEncoder.Deserializer[_]] = ThreadLocalHashMap.empty
+ // InternalRow => Row with Schema T
+ private val cacheDeserializerRow: ThreadLocalHashMap[TypeTag[_], ExpressionEncoder.Deserializer[Row]] = ThreadLocalHashMap.empty
+
+ def serializer[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): SerializerSafe[T] =
+ cacheSerializer.get.getOrElseUpdate(tag, SerializerSafe(encoder.createSerializer())).asInstanceOf[SerializerSafe[T]]
+
+ def rowSerializer[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): SerializerSafe[Row] =
+ cacheSerializerRow.get.getOrElseUpdate(tag, SerializerSafe(RowEncoder(encoder.schema).createSerializer()))
+
+ def deserializer[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): ExpressionEncoder.Deserializer[T] =
+ cacheDeserializer.get.getOrElseUpdate(tag, encoder.resolveAndBind().createDeserializer()).asInstanceOf[ExpressionEncoder.Deserializer[T]]
+
+ def rowDeserializer[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): ExpressionEncoder.Deserializer[Row] =
+ cacheDeserializerRow.get.getOrElseUpdate(tag, RowEncoder(encoder.schema).resolveAndBind().createDeserializer())
+
+ /**
+ * https://jaceklaskowski.gitbooks.io/mastering-spark-sql/content/spark-sql-RowEncoder.html
+ * https://github.com/apache/spark/blob/93cec49212fe82816fcadf69f429cebaec60e058/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala#L75-L86
+ */
+ def rowDeserialize[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): Row => T =
+ { row => deserializer[T](tag, encoder)(rowSerializer[T](tag, encoder)(row)) }
+
+ def rowSerialize[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): T => Row =
+ { t => rowDeserializer[T](tag, encoder)(serializer[T](tag, encoder)(t)) }
+}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala
index e2830f7f1..b1257b6bd 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/SparkBasicEncoders.scala
@@ -28,11 +28,13 @@ import scala.reflect.runtime.universe._
/**
* Container for primitive Spark encoders, pulled into implicit scope.
+ * Be careful with these imports, it may conflict with spark.implicits._ when is in the same scope.
*
* @since 12/28/17
*/
private[rasterframes] trait SparkBasicEncoders {
implicit def arrayEnc[T: TypeTag]: Encoder[Array[T]] = ExpressionEncoder()
+ implicit def seqEnc[T: TypeTag]: Encoder[Seq[T]] = ExpressionEncoder()
implicit val intEnc: Encoder[Int] = Encoders.scalaInt
implicit val longEnc: Encoder[Long] = Encoders.scalaLong
implicit val stringEnc: Encoder[String] = Encoders.STRING
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala
index 302262768..6dd645320 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala
@@ -21,56 +21,73 @@
package org.locationtech.rasterframes.encoders
-import java.net.URI
-import java.sql.Timestamp
-
import org.locationtech.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics}
import org.locationtech.jts.geom.Envelope
import geotrellis.proj4.CRS
import geotrellis.raster.{CellSize, CellType, Dimensions, Raster, Tile, TileLayout}
import geotrellis.layer._
import geotrellis.vector.{Extent, ProjectedExtent}
-import org.apache.spark.sql.{Encoder, Encoders}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.util.QuantileSummaries
import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
-import org.locationtech.rasterframes.model.{CellContext, Cells, TileContext, TileDataContext}
+import org.locationtech.rasterframes.model.{CellContext, LongExtent, TileContext, TileDataContext}
+import frameless.TypedEncoder
+
+import java.net.URI
+import java.sql.Timestamp
+import scala.reflect.ClassTag
import scala.reflect.runtime.universe._
+/**
+ * TODO: move this overload to GeoTrellis, the reason is in the generic method invocation and Integral in implicits
+ */
+object DimensionsInt {
+ def apply(cols: Int, rows: Int): Dimensions[Int] = new Dimensions(cols, rows)
+}
+
+object EnvelopeLocal {
+ def apply(minx: Double, maxx: Double, miny: Double, maxy: Double): Envelope = new Envelope(minx, maxx, miny, miny)
+}
+
/**
* Implicit encoder definitions for RasterFrameLayer types.
*/
-trait StandardEncoders extends SpatialEncoders {
- object PrimitiveEncoders extends SparkBasicEncoders
+trait StandardEncoders extends SpatialEncoders with TypedEncoders {
def expressionEncoder[T: TypeTag]: ExpressionEncoder[T] = ExpressionEncoder()
- implicit def spatialKeyEncoder: ExpressionEncoder[SpatialKey] = ExpressionEncoder()
- implicit def temporalKeyEncoder: ExpressionEncoder[TemporalKey] = ExpressionEncoder()
- implicit def spaceTimeKeyEncoder: ExpressionEncoder[SpaceTimeKey] = ExpressionEncoder()
- implicit def layoutDefinitionEncoder: ExpressionEncoder[LayoutDefinition] = ExpressionEncoder()
- implicit def stkBoundsEncoder: ExpressionEncoder[KeyBounds[SpaceTimeKey]] = ExpressionEncoder()
- implicit def extentEncoder: ExpressionEncoder[Extent] = ExpressionEncoder[Extent]()
- implicit def singlebandTileEncoder: ExpressionEncoder[Tile] = ExpressionEncoder()
- implicit def rasterEncoder: ExpressionEncoder[Raster[Tile]] = ExpressionEncoder()
- implicit def tileLayerMetadataEncoder[K: TypeTag]: ExpressionEncoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder()
- implicit def crsSparkEncoder: ExpressionEncoder[CRS] = CRSEncoder()
- implicit def projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ProjectedExtentEncoder()
- implicit def temporalProjectedExtentEncoder: ExpressionEncoder[TemporalProjectedExtent] = TemporalProjectedExtentEncoder()
- implicit def cellTypeEncoder: ExpressionEncoder[CellType] = CellTypeEncoder()
- implicit def cellSizeEncoder: ExpressionEncoder[CellSize] = ExpressionEncoder()
- implicit def uriEncoder: ExpressionEncoder[URI] = URIEncoder()
- implicit def envelopeEncoder: ExpressionEncoder[Envelope] = EnvelopeEncoder()
- implicit def timestampEncoder: ExpressionEncoder[Timestamp] = ExpressionEncoder()
- implicit def strMapEncoder: ExpressionEncoder[Map[String, String]] = ExpressionEncoder()
- implicit def cellStatsEncoder: ExpressionEncoder[CellStatistics] = ExpressionEncoder()
- implicit def cellHistEncoder: ExpressionEncoder[CellHistogram] = ExpressionEncoder()
- implicit def localCellStatsEncoder: ExpressionEncoder[LocalCellStatistics] = ExpressionEncoder()
- implicit def tilelayoutEncoder: ExpressionEncoder[TileLayout] = ExpressionEncoder()
- implicit def cellContextEncoder: ExpressionEncoder[CellContext] = CellContext.encoder
- implicit def cellsEncoder: ExpressionEncoder[Cells] = Cells.encoder
- implicit def tileContextEncoder: ExpressionEncoder[TileContext] = TileContext.encoder
- implicit def tileDataContextEncoder: ExpressionEncoder[TileDataContext] = TileDataContext.encoder
- implicit def extentTilePairEncoder: Encoder[(ProjectedExtent, Tile)] = Encoders.tuple(projectedExtentEncoder, singlebandTileEncoder)
- implicit def tileDimensionsEncoder: Encoder[Dimensions[Int]] = CatalystSerializerEncoder[Dimensions[Int]](true)
+
+ implicit lazy val crsExpressionEncoder: ExpressionEncoder[CRS] = ExpressionEncoder()
+ implicit lazy val projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ExpressionEncoder()
+ implicit lazy val temporalProjectedExtentEncoder: ExpressionEncoder[TemporalProjectedExtent] = ExpressionEncoder()
+ implicit lazy val timestampEncoder: ExpressionEncoder[Timestamp] = ExpressionEncoder()
+ implicit lazy val strMapEncoder: ExpressionEncoder[Map[String, String]] = ExpressionEncoder()
+ implicit lazy val cellStatsEncoder: ExpressionEncoder[CellStatistics] = ExpressionEncoder()
+ implicit lazy val cellHistEncoder: ExpressionEncoder[CellHistogram] = ExpressionEncoder()
+ implicit lazy val localCellStatsEncoder: ExpressionEncoder[LocalCellStatistics] = ExpressionEncoder()
+ implicit lazy val uriEncoder: ExpressionEncoder[URI] = typedExpressionEncoder[URI]
+ implicit lazy val quantileSummariesEncoder: ExpressionEncoder[QuantileSummaries] = typedExpressionEncoder[QuantileSummaries]
+
+ implicit lazy val envelopeEncoder: ExpressionEncoder[Envelope] = typedExpressionEncoder
+ implicit lazy val longExtentEncoder: ExpressionEncoder[LongExtent] = typedExpressionEncoder
+ implicit lazy val extentEncoder: ExpressionEncoder[Extent] = typedExpressionEncoder
+ implicit lazy val cellSizeEncoder: ExpressionEncoder[CellSize] = typedExpressionEncoder
+ implicit lazy val tileLayoutEncoder: ExpressionEncoder[TileLayout] = typedExpressionEncoder
+ implicit lazy val spatialKeyEncoder: ExpressionEncoder[SpatialKey] = typedExpressionEncoder
+ implicit lazy val temporalKeyEncoder: ExpressionEncoder[TemporalKey] = typedExpressionEncoder
+ implicit lazy val spaceTimeKeyEncoder: ExpressionEncoder[SpaceTimeKey] = typedExpressionEncoder
+ implicit def keyBoundsEncoder[K: TypedEncoder]: ExpressionEncoder[KeyBounds[K]] = typedExpressionEncoder[KeyBounds[K]]
+ implicit def boundsEncoder[K: TypedEncoder]: ExpressionEncoder[Bounds[K]] = keyBoundsEncoder[KeyBounds[K]].asInstanceOf[ExpressionEncoder[Bounds[K]]]
+ implicit lazy val cellTypeEncoder: ExpressionEncoder[CellType] = typedExpressionEncoder[CellType]
+ implicit lazy val dimensionsEncoder: ExpressionEncoder[Dimensions[Int]] = typedExpressionEncoder
+ implicit lazy val layoutDefinitionEncoder: ExpressionEncoder[LayoutDefinition] = typedExpressionEncoder
+ implicit def tileLayerMetadataEncoder[K: TypedEncoder: ClassTag]: ExpressionEncoder[TileLayerMetadata[K]] = typedExpressionEncoder[TileLayerMetadata[K]]
+ implicit lazy val tileContextEncoder: ExpressionEncoder[TileContext] = typedExpressionEncoder
+ implicit lazy val tileDataContextEncoder: ExpressionEncoder[TileDataContext] = typedExpressionEncoder
+ implicit lazy val cellContextEncoder: ExpressionEncoder[CellContext] = typedExpressionEncoder
+
+ implicit lazy val tileEncoder: ExpressionEncoder[Tile] = typedExpressionEncoder
+ implicit lazy val optionalTileEncoder: ExpressionEncoder[Option[Tile]] = typedExpressionEncoder
+ implicit lazy val rasterEncoder: ExpressionEncoder[Raster[Tile]] = typedExpressionEncoder
}
object StandardEncoders extends StandardEncoders
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala
deleted file mode 100644
index c2a0972f1..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala
+++ /dev/null
@@ -1,349 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2019 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import java.nio.ByteBuffer
-
-import com.github.blemale.scaffeine.Scaffeine
-import geotrellis.proj4.CRS
-import geotrellis.raster._
-import geotrellis.layer._
-import geotrellis.vector._
-import org.apache.spark.sql.catalyst.util.QuantileSummaries
-import org.apache.spark.sql.types._
-import org.locationtech.jts.geom.Envelope
-import org.locationtech.rasterframes.TileType
-import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _}
-import org.locationtech.rasterframes.model.LazyCRS
-import org.locationtech.rasterframes.util.KryoSupport
-
-/** Collection of CatalystSerializers for third-party types. */
-trait StandardSerializers {
-
- implicit val envelopeSerializer: CatalystSerializer[Envelope] = new CatalystSerializer[Envelope] {
- override val schema: StructType = StructType(Seq(
- StructField("minX", DoubleType, false),
- StructField("maxX", DoubleType, false),
- StructField("minY", DoubleType, false),
- StructField("maxY", DoubleType, false)
- ))
-
- override protected def to[R](t: Envelope, io: CatalystIO[R]): R = io.create(
- t.getMinX, t.getMaxX, t.getMinY, t.getMaxY
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): Envelope = new Envelope(
- io.getDouble(t, 0), io.getDouble(t, 1), io.getDouble(t, 2), io.getDouble(t, 3)
- )
- }
-
- implicit val extentSerializer: CatalystSerializer[Extent] = new CatalystSerializer[Extent] {
- override val schema: StructType = StructType(Seq(
- StructField("xmin", DoubleType, false),
- StructField("ymin", DoubleType, false),
- StructField("xmax", DoubleType, false),
- StructField("ymax", DoubleType, false)
- ))
-
- override def to[R](t: Extent, io: CatalystIO[R]): R = io.create(
- t.xmin, t.ymin, t.xmax, t.ymax
- )
-
- override def from[R](row: R, io: CatalystIO[R]): Extent = Extent(
- io.getDouble(row, 0),
- io.getDouble(row, 1),
- io.getDouble(row, 2),
- io.getDouble(row, 3)
- )
- }
-
- implicit val gridBoundsSerializer: CatalystSerializer[GridBounds[Int]] = new CatalystSerializer[GridBounds[Int]] {
- override val schema: StructType = StructType(Seq(
- StructField("colMin", IntegerType, false),
- StructField("rowMin", IntegerType, false),
- StructField("colMax", IntegerType, false),
- StructField("rowMax", IntegerType, false)
- ))
-
- override protected def to[R](t: GridBounds[Int], io: CatalystIO[R]): R = io.create(
- t.colMin, t.rowMin, t.colMax, t.rowMax
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): GridBounds[Int] = GridBounds[Int](
- io.getInt(t, 0),
- io.getInt(t, 1),
- io.getInt(t, 2),
- io.getInt(t, 3)
- )
- }
-
- implicit val crsSerializer: CatalystSerializer[CRS] = new CatalystSerializer[CRS] {
- override val schema: StructType = StructType(Seq(
- StructField("crsProj4", StringType, false)
- ))
-
- override def to[R](t: CRS, io: CatalystIO[R]): R = io.create(
- io.encode(
- // Don't do this... it's 1000x slower to decode.
- //t.epsgCode.map(c => "EPSG:" + c).getOrElse(t.toProj4String)
- t.toProj4String
- )
- )
-
- override def from[R](row: R, io: CatalystIO[R]): CRS =
- LazyCRS(io.getString(row, 0))
- }
-
- implicit val cellTypeSerializer: CatalystSerializer[CellType] = new CatalystSerializer[CellType] {
-
- import StandardSerializers._
-
- override val schema: StructType = StructType(Seq(
- StructField("cellTypeName", StringType, false)
- ))
-
- override def to[R](t: CellType, io: CatalystIO[R]): R = io.create(
- io.encode(ct2sCache.get(t))
- )
-
- override def from[R](row: R, io: CatalystIO[R]): CellType =
- s2ctCache.get(io.getString(row, 0))
- }
-
- implicit val projectedExtentSerializer: CatalystSerializer[ProjectedExtent] = new CatalystSerializer[ProjectedExtent] {
- override val schema: StructType = StructType(Seq(
- StructField("extent", schemaOf[Extent], false),
- StructField("crs", schemaOf[CRS], false)
- ))
-
- override protected def to[R](t: ProjectedExtent, io: CatalystSerializer.CatalystIO[R]): R = io.create(
- io.to(t.extent),
- io.to(t.crs)
- )
-
- override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): ProjectedExtent = ProjectedExtent(
- io.get[Extent](t, 0),
- io.get[CRS](t, 1)
- )
- }
-
- implicit val spatialKeySerializer: CatalystSerializer[SpatialKey] = new CatalystSerializer[SpatialKey] {
- override val schema: StructType = StructType(Seq(
- StructField("col", IntegerType, false),
- StructField("row", IntegerType, false)
- ))
-
- override protected def to[R](t: SpatialKey, io: CatalystIO[R]): R = io.create(
- t.col,
- t.row
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): SpatialKey = SpatialKey(
- io.getInt(t, 0),
- io.getInt(t, 1)
- )
- }
-
- implicit val spacetimeKeySerializer: CatalystSerializer[SpaceTimeKey] = new CatalystSerializer[SpaceTimeKey] {
- override val schema: StructType = StructType(Seq(
- StructField("col", IntegerType, false),
- StructField("row", IntegerType, false),
- StructField("instant", LongType, false)
- ))
-
- override protected def to[R](t: SpaceTimeKey, io: CatalystIO[R]): R = io.create(
- t.col,
- t.row,
- t.instant
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): SpaceTimeKey = SpaceTimeKey(
- io.getInt(t, 0),
- io.getInt(t, 1),
- io.getLong(t, 2)
- )
- }
-
- implicit val cellSizeSerializer: CatalystSerializer[CellSize] = new CatalystSerializer[CellSize] {
- override val schema: StructType = StructType(Seq(
- StructField("width", DoubleType, false),
- StructField("height", DoubleType, false)
- ))
-
- override protected def to[R](t: CellSize, io: CatalystIO[R]): R = io.create(
- t.width,
- t.height
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): CellSize = CellSize(
- io.getDouble(t, 0),
- io.getDouble(t, 1)
- )
- }
-
- implicit val tileLayoutSerializer: CatalystSerializer[TileLayout] = new CatalystSerializer[TileLayout] {
- override val schema: StructType = StructType(Seq(
- StructField("layoutCols", IntegerType, false),
- StructField("layoutRows", IntegerType, false),
- StructField("tileCols", IntegerType, false),
- StructField("tileRows", IntegerType, false)
- ))
-
- override protected def to[R](t: TileLayout, io: CatalystIO[R]): R = io.create(
- t.layoutCols,
- t.layoutRows,
- t.tileCols,
- t.tileRows
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): TileLayout = TileLayout(
- io.getInt(t, 0),
- io.getInt(t, 1),
- io.getInt(t, 2),
- io.getInt(t, 3)
- )
- }
-
- implicit val layoutDefinitionSerializer = new CatalystSerializer[LayoutDefinition] {
- override val schema: StructType = StructType(Seq(
- StructField("extent", schemaOf[Extent], true),
- StructField("tileLayout", schemaOf[TileLayout], true)
- ))
-
- override protected def to[R](t: LayoutDefinition, io: CatalystIO[R]): R = io.create(
- io.to(t.extent),
- io.to(t.tileLayout)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): LayoutDefinition = LayoutDefinition(
- io.get[Extent](t, 0),
- io.get[TileLayout](t, 1)
- )
- }
-
- implicit def boundsSerializer[T >: Null : CatalystSerializer]: CatalystSerializer[KeyBounds[T]] = new CatalystSerializer[KeyBounds[T]] {
- override val schema: StructType = StructType(Seq(
- StructField("minKey", schemaOf[T], true),
- StructField("maxKey", schemaOf[T], true)
- ))
-
- override protected def to[R](t: KeyBounds[T], io: CatalystIO[R]): R = io.create(
- io.to(t.get.minKey),
- io.to(t.get.maxKey)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): KeyBounds[T] = KeyBounds(
- io.get[T](t, 0),
- io.get[T](t, 1)
- )
- }
-
- def tileLayerMetadataSerializer[T >: Null : CatalystSerializer]: CatalystSerializer[TileLayerMetadata[T]] = new CatalystSerializer[TileLayerMetadata[T]] {
- override val schema: StructType = StructType(Seq(
- StructField("cellType", schemaOf[CellType], false),
- StructField("layout", schemaOf[LayoutDefinition], false),
- StructField("extent", schemaOf[Extent], false),
- StructField("crs", schemaOf[CRS], false),
- StructField("bounds", schemaOf[KeyBounds[T]], false)
- ))
-
- override protected def to[R](t: TileLayerMetadata[T], io: CatalystIO[R]): R = io.create(
- io.to(t.cellType),
- io.to(t.layout),
- io.to(t.extent),
- io.to(t.crs),
- io.to(t.bounds.head)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): TileLayerMetadata[T] = TileLayerMetadata(
- io.get[CellType](t, 0),
- io.get[LayoutDefinition](t, 1),
- io.get[Extent](t, 2),
- io.get[CRS](t, 3),
- io.get[KeyBounds[T]](t, 4)
- )
- }
-
- implicit def rasterSerializer: CatalystSerializer[Raster[Tile]] = new CatalystSerializer[Raster[Tile]] {
-
- import org.apache.spark.sql.rf.TileUDT.tileSerializer
-
- override val schema: StructType = StructType(Seq(
- StructField("tile", TileType, false),
- StructField("extent", schemaOf[Extent], false)
- ))
-
- override protected def to[R](t: Raster[Tile], io: CatalystIO[R]): R = io.create(
- io.to(t.tile),
- io.to(t.extent)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): Raster[Tile] = Raster(
- io.get[Tile](t, 0),
- io.get[Extent](t, 1)
- )
- }
-
- implicit val spatialKeyTLMSerializer = tileLayerMetadataSerializer[SpatialKey]
- implicit val spaceTimeKeyTLMSerializer = tileLayerMetadataSerializer[SpaceTimeKey]
-
- implicit val tileDimensionsSerializer: CatalystSerializer[Dimensions[Int]] = new CatalystSerializer[Dimensions[Int]] {
- override val schema: StructType = StructType(Seq(
- StructField("cols", IntegerType, false),
- StructField("rows", IntegerType, false)
- ))
-
- override protected def to[R](t: Dimensions[Int], io: CatalystIO[R]): R = io.create(
- t.cols,
- t.rows
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): Dimensions[Int] = Dimensions[Int](
- io.getInt(t, 0),
- io.getInt(t, 1)
- )
- }
-
- implicit val quantileSerializer: CatalystSerializer[QuantileSummaries] = new CatalystSerializer[QuantileSummaries] {
- override val schema: StructType = StructType(Seq(
- StructField("quantile_serializer_kryo", BinaryType, false)
- ))
-
- override protected def to[R](t: QuantileSummaries, io: CatalystSerializer.CatalystIO[R]): R = {
- val buf = KryoSupport.serialize(t)
- io.create(buf.array())
- }
-
- override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): QuantileSummaries = {
- KryoSupport.deserialize[QuantileSummaries](ByteBuffer.wrap(io.getByteArray(t, 0)))
- }
- }
-}
-
-object StandardSerializers extends StandardSerializers {
- private val s2ctCache = Scaffeine().build[String, CellType](
- (s: String) => CellType.fromName(s)
- )
- private val ct2sCache = Scaffeine().build[CellType, String](
- (ct: CellType) => ct.toString()
- )
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala
deleted file mode 100644
index 2ec265ccc..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/StringBackedEncoder.scala
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2018 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.apache.spark.sql.catalyst.ScalaReflection
-import org.apache.spark.sql.catalyst.analysis.GetColumnByOrdinal
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
-import org.apache.spark.sql.catalyst.expressions.{BoundReference, Expression}
-import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
-import org.apache.spark.sql.rf.VersionShims.InvokeSafely
-
-import scala.reflect.runtime.universe._
-
-/**
- * Generalized operations for creating an encoder when the type can be represented as a Catalyst string.
- *
- * @since 1/16/18
- */
-object StringBackedEncoder {
- def apply[T: TypeTag](
- fieldName: String,
- toStringMethod: String,
- fromStringStatic: (Class[_], String)): ExpressionEncoder[T] = {
-
- val sparkType = ScalaReflection.dataTypeFor[T]
- val schema = StructType(Seq(StructField(fieldName, StringType, false)))
- val inputObject = BoundReference(0, sparkType, nullable = false)
-
- val intermediateType = ObjectType(classOf[String])
- val serializer: Expression =
- StaticInvoke(
- classOf[UTF8String],
- StringType,
- "fromString",
- InvokeSafely(inputObject, toStringMethod, intermediateType) :: Nil
- )
-
- val inputRow = GetColumnByOrdinal(0, schema)
- val deserializer: Expression =
- StaticInvoke(
- fromStringStatic._1,
- sparkType,
- fromStringStatic._2,
- InvokeSafely(inputRow, "toString", intermediateType) :: Nil
- )
-
- ExpressionEncoder[T](schema, flat = false, Seq(serializer), deserializer, typeToClassTag[T])
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala
deleted file mode 100644
index 5d41e6386..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import org.locationtech.rasterframes._
-import geotrellis.layer._
-import org.apache.spark.sql.Encoders
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-
-/**
- * Custom encoder for `TemporalProjectedExtent`. Necessary because `geotrellis.proj4.CRS` within
- * `ProjectedExtent` isn't a case class, and `ZonedDateTime` doesn't have a natural encoder.
- *
- * @since 8/2/17
- */
-object TemporalProjectedExtentEncoder {
- def apply(): ExpressionEncoder[TemporalProjectedExtent] = {
- import StandardEncoders.crsSparkEncoder
- DelegatingSubfieldEncoder(
- "extent" -> extentEncoder,
- "crs" -> crsSparkEncoder,
- "instant" -> Encoders.scalaLong.asInstanceOf[ExpressionEncoder[Long]]
- )
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala
deleted file mode 100644
index 56f845db3..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2017 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import geotrellis.layer._
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-
-import scala.reflect.runtime.universe._
-
-/**
- * Specialized encoder for [[TileLayerMetadata]], necessary to be able to delegate to the
- * specialized cell type and crs encoders.
- *
- * @since 7/21/17
- */
-object TileLayerMetadataEncoder {
- import org.locationtech.rasterframes._
-
- private def fieldEncoders = Seq[(String, ExpressionEncoder[_])](
- "cellType" -> cellTypeEncoder,
- "layout" -> layoutDefinitionEncoder,
- "extent" -> extentEncoder,
- "crs" -> crsSparkEncoder
- )
-
- def apply[K: TypeTag](): ExpressionEncoder[TileLayerMetadata[K]] = {
- val boundsEncoder = ExpressionEncoder[KeyBounds[K]]()
- val fEncoders = fieldEncoders :+ ("bounds" -> boundsEncoder)
- DelegatingSubfieldEncoder(fEncoders: _*)
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala
new file mode 100644
index 000000000..ca614ee28
--- /dev/null
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/TypedEncoders.scala
@@ -0,0 +1,280 @@
+package org.locationtech.rasterframes.encoders
+
+import frameless._
+import geotrellis.layer.{KeyBounds, LayoutDefinition, TileLayerMetadata}
+import geotrellis.proj4.CRS
+import geotrellis.raster.{CellType, Dimensions, GridBounds, Raster, Tile}
+import geotrellis.vector.Extent
+import org.apache.spark.sql.FramelessInternals
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, NewInstance, StaticInvoke}
+import org.apache.spark.sql.catalyst.expressions.{CreateNamedStruct, Expression, GetStructField, If, IsNull, Literal}
+import org.apache.spark.sql.catalyst.util.QuantileSummaries
+import org.apache.spark.sql.rf.{CrsUDT, RasterSourceUDT, TileUDT}
+import org.apache.spark.sql.types.{DataType, Metadata, StructField, StructType}
+import org.locationtech.jts.geom.Envelope
+import org.locationtech.rasterframes.util.KryoSupport
+
+import java.net.URI
+import java.nio.ByteBuffer
+import scala.reflect.ClassTag
+
+trait TypedEncoders {
+ def typedExpressionEncoder[T: TypedEncoder]: ExpressionEncoder[T] = TypedExpressionEncoder[T].asInstanceOf[ExpressionEncoder[T]]
+
+ implicit val crsUDT = new CrsUDT
+ implicit val tileUDT = new TileUDT
+ implicit val rasterSourceUDT = new RasterSourceUDT
+
+ implicit val cellTypeInjection: Injection[CellType, String] = Injection(_.toString, CellType.fromName)
+ implicit val cellTypeTypedEncoder: TypedEncoder[CellType] = TypedEncoder.usingInjection[CellType, String]
+
+ implicit val quantileSummariesInjection: Injection[QuantileSummaries, Array[Byte]] =
+ Injection(KryoSupport.serialize(_).array(), array => KryoSupport.deserialize[QuantileSummaries](ByteBuffer.wrap(array)))
+
+ implicit val quantileSummariesTypedEncoder: TypedEncoder[QuantileSummaries] = TypedEncoder.usingInjection
+
+ implicit val uriInjection: Injection[URI, String] = Injection(_.toString, new URI(_))
+ implicit val uriTypedEncoder: TypedEncoder[URI] = TypedEncoder.usingInjection
+
+ implicit val envelopeTypedEncoder: TypedEncoder[Envelope] = new TypedEncoder[Envelope] {
+ val fields: List[RecordEncoderField] = List(
+ RecordEncoderField(0, "minX", TypedEncoder[Double]),
+ RecordEncoderField(1, "maxX", TypedEncoder[Double]),
+ RecordEncoderField(2, "minY", TypedEncoder[Double]),
+ RecordEncoderField(3, "maxY", TypedEncoder[Double])
+ )
+
+ def nullable: Boolean = true
+
+ def jvmRepr: DataType = FramelessInternals.objectTypeFor[Envelope]
+
+ def catalystRepr: DataType = {
+ val structFields = fields.map { field =>
+ StructField(
+ name = field.name,
+ dataType = field.encoder.catalystRepr,
+ nullable = field.encoder.nullable,
+ metadata = Metadata.empty
+ )
+ }
+
+ StructType(structFields)
+ }
+
+ def fromCatalyst(path: Expression): Expression = {
+ val newArgs = fields.map { field =>
+ field.encoder.fromCatalyst( GetStructField(path, field.ordinal, Some(field.name)) )
+ }
+ // TODO: sounds like we should abstract this
+ val newExpr = NewInstance(classTag.runtimeClass, newArgs, jvmRepr, propagateNull = true)
+ // val newExpr = StaticInvoke(EnvelopeLocal.getClass, jvmRepr, "apply", newArgs, propagateNull = true, returnNullable = false)
+
+ val nullExpr = Literal.create(null, jvmRepr)
+ If(IsNull(path), nullExpr, newExpr)
+ }
+
+ def toCatalyst(path: Expression): Expression = {
+ val nameExprs = fields.map { field =>
+ Literal(field.name)
+ }
+
+ val valueExprs = fields.map { field =>
+ val fieldPath = Invoke(path, s"get${field.name.capitalize}", field.encoder.jvmRepr, Nil)
+ field.encoder.toCatalyst(fieldPath)
+ }
+
+ // the way exprs are encoded in CreateNamedStruct
+ val exprs = nameExprs.zip(valueExprs).flatMap {
+ case (nameExpr, valueExpr) => nameExpr :: valueExpr :: Nil
+ }
+
+ val createExpr = CreateNamedStruct(exprs)
+ val nullExpr = Literal.create(null, createExpr.dataType)
+ If(IsNull(path), nullExpr, createExpr)
+ }
+ }
+
+ implicit val dimensionsTypedEncoder: TypedEncoder[Dimensions[Int]] = new TypedEncoder[Dimensions[Int]] {
+ val fields: List[RecordEncoderField] = List(
+ RecordEncoderField(0, "cols", TypedEncoder[Int]),
+ RecordEncoderField(1, "rows", TypedEncoder[Int]))
+
+ def nullable: Boolean = true
+
+ def jvmRepr: DataType = FramelessInternals.objectTypeFor[Dimensions[Int]]
+
+ def catalystRepr: DataType = {
+ val structFields = fields.map { field =>
+ StructField(
+ name = field.name,
+ dataType = field.encoder.catalystRepr,
+ nullable = field.encoder.nullable,
+ metadata = Metadata.empty
+ )
+ }
+
+ StructType(structFields)
+ }
+
+ def fromCatalyst(path: Expression): Expression = {
+ val newArgs = fields.map { field =>
+ field.encoder.fromCatalyst( GetStructField(path, field.ordinal, Some(field.name)) )
+ }
+ // TODO: sounds like we should abstract this
+ //val newExpr = NewInstance(classTag.runtimeClass, newArgs, jvmRepr, propagateNull = true)
+ val newExpr = StaticInvoke(DimensionsInt.getClass, jvmRepr, "apply", newArgs, propagateNull = true, returnNullable = false)
+
+ val nullExpr = Literal.create(null, jvmRepr)
+ If(IsNull(path), nullExpr, newExpr)
+ }
+
+ def toCatalyst(path: Expression): Expression = {
+ val nameExprs = fields.map { field =>
+ Literal(field.name)
+ }
+
+ val valueExprs = fields.map { field =>
+ val fieldPath = Invoke(path, field.name, field.encoder.jvmRepr, Nil)
+ field.encoder.toCatalyst(fieldPath)
+ }
+
+ // the way exprs are encoded in CreateNamedStruct
+ val exprs = nameExprs.zip(valueExprs).flatMap {
+ case (nameExpr, valueExpr) => nameExpr :: valueExpr :: Nil
+ }
+
+ val createExpr = CreateNamedStruct(exprs)
+ val nullExpr = Literal.create(null, createExpr.dataType)
+ If(IsNull(path), nullExpr, createExpr)
+ }
+ }
+
+ /**
+ * @note
+ * Frameless cannot derive encoder for GridBounds because it lacks constructor from (int, int, int int)
+ * Defining Injection is not suitable because Injection is used in derivation of encoder fields but is not an encoder.
+ * Additionally Injection to Tuple4[Int, Int, Int, Int] would not have correct fields.
+ */
+ implicit val gridBoundsTypedEncoder: TypedEncoder[GridBounds[Int]] = new TypedEncoder[GridBounds[Int]]() {
+ val fields: List[RecordEncoderField] = List(
+ RecordEncoderField(0, "colMin", TypedEncoder[Int]),
+ RecordEncoderField(1, "rowMin", TypedEncoder[Int]),
+ RecordEncoderField(2, "colMax", TypedEncoder[Int]),
+ RecordEncoderField(3, "rowMax", TypedEncoder[Int]))
+
+ def nullable: Boolean = true
+
+ def jvmRepr: DataType = FramelessInternals.objectTypeFor[GridBounds[Int]]
+
+ def catalystRepr: DataType = {
+ val structFields = fields.map { field =>
+ StructField(
+ name = field.name,
+ dataType = field.encoder.catalystRepr,
+ nullable = field.encoder.nullable,
+ metadata = Metadata.empty
+ )
+ }
+
+ StructType(structFields)
+ }
+
+ def fromCatalyst(path: Expression): Expression = {
+ val newArgs = fields.map { field =>
+ field.encoder.fromCatalyst( GetStructField(path, field.ordinal, Some(field.name)) )
+ }
+ // TODO: sounds like we should abstract this
+ //val newExpr = NewInstance(classTag.runtimeClass, newArgs, jvmRepr, propagateNull = true)
+ val newExpr = StaticInvoke(classTag.runtimeClass, jvmRepr, "apply", newArgs, propagateNull = true, returnNullable = false)
+
+ val nullExpr = Literal.create(null, jvmRepr)
+ If(IsNull(path), nullExpr, newExpr)
+ }
+
+ def toCatalyst(path: Expression): Expression = {
+ val nameExprs = fields.map { field =>
+ Literal(field.name)
+ }
+
+ val valueExprs = fields.map { field =>
+ val fieldPath = Invoke(path, field.name, field.encoder.jvmRepr, Nil)
+ field.encoder.toCatalyst(fieldPath)
+ }
+
+ // the way exprs are encoded in CreateNamedStruct
+ val exprs = nameExprs.zip(valueExprs).flatMap {
+ case (nameExpr, valueExpr) => nameExpr :: valueExpr :: Nil
+ }
+
+ val createExpr = CreateNamedStruct(exprs)
+ val nullExpr = Literal.create(null, createExpr.dataType)
+ If(IsNull(path), nullExpr, createExpr)
+ }
+ }
+
+ implicit def tileLayerMetadataTypedEncoder[K: TypedEncoder: ClassTag]: TypedEncoder[TileLayerMetadata[K]] = new TypedEncoder[TileLayerMetadata[K]] {
+ val fields: List[RecordEncoderField] = List(
+ RecordEncoderField(0, "cellType", cellTypeTypedEncoder),
+ RecordEncoderField(1, "layout", TypedEncoder[LayoutDefinition]),
+ RecordEncoderField(2, "extent", TypedEncoder[Extent]),
+ RecordEncoderField(3, "crs", TypedEncoder[CRS]),
+ RecordEncoderField(4, "bounds", TypedEncoder[KeyBounds[K]])
+ )
+
+ def nullable: Boolean = true
+
+ def jvmRepr: DataType = FramelessInternals.objectTypeFor[TileLayerMetadata[K]]
+
+ def catalystRepr: DataType = {
+ val structFields = fields.map { field =>
+ StructField(
+ name = field.name,
+ dataType = field.encoder.catalystRepr,
+ nullable = field.encoder.nullable,
+ metadata = Metadata.empty
+ )
+ }
+
+ StructType(structFields)
+ }
+
+ def fromCatalyst(path: Expression): Expression = {
+ val newArgs = fields.map { field =>
+ field.encoder.fromCatalyst( GetStructField(path, field.ordinal, Some(field.name)) )
+ }
+ // TODO: sounds like we should abstract this
+ // val newExpr = NewInstance(classTag.runtimeClass, newArgs, jvmRepr, propagateNull = true)
+ val newExpr = StaticInvoke(classTag.runtimeClass, jvmRepr, "apply", newArgs, propagateNull = true, returnNullable = false)
+
+ val nullExpr = Literal.create(null, jvmRepr)
+ If(IsNull(path), nullExpr, newExpr)
+ }
+
+ def toCatalyst(path: Expression): Expression = {
+ val nameExprs = fields.map { field =>
+ Literal(field.name)
+ }
+
+ val valueExprs = fields.map { field =>
+ val fieldPath = Invoke(path, field.name, field.encoder.jvmRepr, Nil)
+ field.encoder.toCatalyst(fieldPath)
+ }
+
+ // the way exprs are encoded in CreateNamedStruct
+ val exprs = nameExprs.zip(valueExprs).flatMap {
+ case (nameExpr, valueExpr) => nameExpr :: valueExpr :: Nil
+ }
+
+ val createExpr = CreateNamedStruct(exprs)
+ val nullExpr = Literal.create(null, createExpr.dataType)
+ If(IsNull(path), nullExpr, createExpr)
+ }
+ }
+
+ implicit val tileTypedEncoder: TypedEncoder[Tile] = TypedEncoder.usingUserDefinedType[Tile]
+ implicit val rasterTileTypedEncoder: TypedEncoder[Raster[Tile]] = TypedEncoder.usingDerivation
+
+}
+
+object TypedEncoders extends TypedEncoders
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala
deleted file mode 100644
index bbbcf25ea..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/URIEncoder.scala
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2018 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.encoders
-
-import java.net.URI
-
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-
-/**
- * Custom Encoder for allowing friction-free use of URIs in DataFrames.
- *
- * @since 1/16/18
- */
-object URIEncoder {
- def apply(): ExpressionEncoder[URI] =
- StringBackedEncoder[URI]("uri", "toASCIIString", (URIEncoder.getClass, "fromString"))
- // Not sure why this delegate is necessary, but doGenCode fails without it.
- def fromString(str: String): URI = URI.create(str)
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala
index 8cb5a6f85..6851a56f6 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/package.scala
@@ -21,12 +21,17 @@
package org.locationtech.rasterframes
-import org.apache.spark.sql.rf._
+import org.locationtech.rasterframes.encoders.syntax._
+
import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.Literal
import scala.reflect.ClassTag
-import scala.reflect.runtime.universe.{Literal => _, _}
+import scala.reflect.runtime.universe._
+import frameless.TypedEncoder
+import org.apache.spark.sql.types.{DataType, StructType}
+import org.apache.spark.sql.rf.WithTypeConformity
/**
* Module utilities
@@ -34,6 +39,17 @@ import scala.reflect.runtime.universe.{Literal => _, _}
* @since 9/25/17
*/
package object encoders {
+ /** High priority specific product encoder derivation. Without it, the default spark would be used. */
+ implicit def productTypedToExpressionEncoder[T <: Product: TypedEncoder]: ExpressionEncoder[T] = TypedEncoders.typedExpressionEncoder
+
+ implicit class WithTypeConformityToEncoder(val left: DataType) extends AnyVal {
+ def conformsToSchema(schema: StructType): Boolean =
+ WithTypeConformity(left).conformsTo(schema)
+
+ def conformsToDataType(dataType: DataType): Boolean =
+ WithTypeConformity(left).conformsTo(dataType)
+ }
+
private[rasterframes] def runtimeClass[T: TypeTag]: Class[T] =
typeTag[T].mirror.runtimeClass(typeTag[T].tpe).asInstanceOf[Class[T]]
@@ -41,18 +57,26 @@ package object encoders {
ClassTag[T](typeTag[T].mirror.runtimeClass(typeTag[T].tpe))
}
- /** Constructs a catalyst literal expression from anything with a serializer. */
- def SerializedLiteral[T >: Null: CatalystSerializer](t: T): Literal = {
- val ser = CatalystSerializer[T]
- val schema = ser.schema match {
- case s if s.conformsTo(TileType.sqlType) => TileType
- case s if s.conformsTo(RasterSourceType.sqlType) => RasterSourceType
+ /** Constructs a catalyst literal expression from anything with a serializer.
+ * Using this serializer avoids using lit() function wich will defer to ScalaReflection to derive encoder.
+ * Therefore, this should be used when literal value can not be handled by Spark ScalaReflection.
+ */
+ def SerializedLiteral[T >: Null](t: T)(implicit tag: TypeTag[T], enc: ExpressionEncoder[T]): Literal = {
+ val schema = enc.schema match {
+ case s if s.conformsTo(tileUDT.sqlType) => tileUDT
+ case s if s.conformsTo(rasterSourceUDT.sqlType) => rasterSourceUDT
case s => s
}
- Literal.create(ser.toInternalRow(t), schema)
+ // we need to convert to Literal right here because otherwise ScalaReflection takes over
+ val ir = t.toInternalRow
+ Literal.create(ir, schema)
}
- /** Constructs a Dataframe literal column from anything with a serializer. */
- def serialized_literal[T >: Null: CatalystSerializer](t: T): Column =
+ /**
+ * Constructs a Dataframe literal column from anything with a serializer.
+ * TODO: review its usage.
+ */
+ def serialized_literal[T >: Null: ExpressionEncoder: TypeTag](t: T): Column =
new Column(SerializedLiteral(t))
+
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/syntax/package.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/syntax/package.scala
new file mode 100644
index 000000000..eb4ea931c
--- /dev/null
+++ b/core/src/main/scala/org/locationtech/rasterframes/encoders/syntax/package.scala
@@ -0,0 +1,35 @@
+package org.locationtech.rasterframes.encoders
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+
+import scala.reflect.runtime.universe.TypeTag
+
+package object syntax {
+ implicit class CachedExpressionOps[T](val self: T) extends AnyVal {
+ def toInternalRow(implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): InternalRow = {
+ val toRow = SerializersCache.serializer[T]
+ toRow(self)
+ }
+
+ def toRow(implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): Row = {
+ val toRow = SerializersCache.rowSerialize[T]
+ toRow(self)
+ }
+ }
+
+ implicit class CachedExpressionRowOps(val self: Row) extends AnyVal {
+ def as[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): T = {
+ val fromRow = SerializersCache.rowDeserialize[T]
+ fromRow(self)
+ }
+ }
+
+ implicit class CachedInternalRowOps(val self: InternalRow) extends AnyVal {
+ def as[T](implicit tag: TypeTag[T], encoder: ExpressionEncoder[T]): T = {
+ val fromRow = SerializersCache.deserializer[T]
+ fromRow(self)
+ }
+ }
+}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala
index 9994fdef1..18d337bdc 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala
@@ -26,18 +26,15 @@ import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.BinaryExpression
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.slf4j.LoggerFactory
/** Operation combining two tiles or a tile and a scalar into a new tile. */
-trait BinaryLocalRasterOp extends BinaryExpression {
+trait BinaryLocalRasterOp extends BinaryExpression with RasterResult {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
-
override def dataType: DataType = left.dataType
override def checkInputDataTypes(): TypeCheckResult = {
@@ -51,7 +48,6 @@ trait BinaryLocalRasterOp extends BinaryExpression {
}
override protected def nullSafeEval(input1: Any, input2: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
val result = tileOrNumberExtractor(right.dataType)(input2) match {
case TileArg(rightTile, rightCtx) =>
@@ -67,11 +63,7 @@ trait BinaryLocalRasterOp extends BinaryExpression {
case DoubleArg(d) => op(fpTile(leftTile), d)
case IntegerArg(i) => op(leftTile, i)
}
-
- leftCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, leftCtx)
}
@@ -79,4 +71,3 @@ trait BinaryLocalRasterOp extends BinaryExpression {
protected def op(left: Tile, right: Double): Tile
protected def op(left: Tile, right: Int): Tile
}
-
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala
index 2c33eae12..26e5138aa 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala
@@ -26,17 +26,15 @@ import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.BinaryExpression
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
import org.slf4j.LoggerFactory
/** Operation combining two tiles into a new tile. */
-trait BinaryRasterOp extends BinaryExpression {
+trait BinaryRasterOp extends BinaryExpression with RasterResult {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
- override def dataType: DataType = left.dataType
+ def dataType: DataType = left.dataType
override def checkInputDataTypes(): TypeCheckResult = {
if (!tileExtractor.isDefinedAt(left.dataType)) {
@@ -51,7 +49,6 @@ trait BinaryRasterOp extends BinaryExpression {
protected def op(left: Tile, right: Tile): Tile
override protected def nullSafeEval(input1: Any, input2: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
val (rightTile, rightCtx) = tileExtractor(right.dataType)(row(input2))
@@ -65,9 +62,6 @@ trait BinaryRasterOp extends BinaryExpression {
val result = op(leftTile, rightTile)
- leftCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, leftCtx)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala
index dfced6c14..447d51954 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala
@@ -31,76 +31,86 @@ import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.locationtech.jts.geom.{Envelope, Point}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.model.{LazyCRS, LongExtent, TileContext}
-import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RFRasterSource, RasterRef}
+import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef}
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+import org.apache.spark.sql.rf.CrsUDT
private[rasterframes]
object DynamicExtractors {
/** Partial function for pulling a tile and its context from an input row. */
lazy val tileExtractor: PartialFunction[DataType, InternalRow => (Tile, Option[TileContext])] = {
case _: TileUDT =>
- (row: InternalRow) =>
- (row.to[Tile](TileUDT.tileSerializer), None)
- case t if t.conformsTo[ProjectedRasterTile] =>
+ (row: InternalRow) => (tileUDT.deserialize(row), None)
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
(row: InternalRow) => {
- val prt = row.to[ProjectedRasterTile]
+ val prt = row.as[ProjectedRasterTile]
(prt, Some(TileContext(prt)))
}
}
lazy val rasterRefExtractor: PartialFunction[DataType, InternalRow => RasterRef] = {
- case t if t.conformsTo[RasterRef] =>
- (row: InternalRow) => row.to[RasterRef]
+ case t if t.conformsToSchema(RasterRef.rasterRefEncoder.schema) =>
+ (row: InternalRow) => row.as[RasterRef]
}
lazy val tileableExtractor: PartialFunction[DataType, InternalRow => Tile] =
tileExtractor.andThen(_.andThen(_._1)).orElse(rasterRefExtractor.andThen(_.andThen(_.tile)))
- lazy val rowTileExtractor: PartialFunction[DataType, Row => (Tile, Option[TileContext])] = {
- case _: TileUDT =>
- (row: Row) => (row.to[Tile](TileUDT.tileSerializer), None)
- case t if t.conformsTo[Raster[Tile]] =>
- (row: Row) => {
- val rt = row.to[Raster[Tile]]
- (rt.tile, None)
+ lazy val internalRowTileExtractor: PartialFunction[DataType, InternalRow => (Tile, Option[TileContext])] = {
+ case _: TileUDT => (row: Any) => (new TileUDT().deserialize(row), None)
+ case t if t.conformsToSchema(rasterEncoder.schema) =>
+ (row: InternalRow) =>(row.as[Raster[Tile]].tile, None)
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
+ (row: InternalRow) => {
+ val prt = row.as[ProjectedRasterTile]
+ (prt, Some(TileContext(prt)))
}
- case t if t.conformsTo[ProjectedRasterTile] =>
+ }
+
+ lazy val rowTileExtractor: PartialFunction[DataType, Row => (Tile, Option[TileContext])] = {
+ case _: TileUDT => (row: Row) => (row.as[Tile], None)
+ case t if t.conformsToSchema(rasterEncoder.schema) => (row: Row) => (row.as[Raster[Tile]].tile, None)
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
(row: Row) => {
- val prt = row.to[ProjectedRasterTile]
+ val prt = row.as[ProjectedRasterTile]
(prt, Some(TileContext(prt)))
}
}
/** Partial function for pulling a ProjectedRasterLike an input row. */
- lazy val projectedRasterLikeExtractor: PartialFunction[DataType, Any ⇒ ProjectedRasterLike] = {
- case _: RasterSourceUDT ⇒
- (input: Any) => input.asInstanceOf[InternalRow].to[RFRasterSource](RasterSourceUDT.rasterSourceSerializer)
- case t if t.conformsTo[ProjectedRasterTile] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[ProjectedRasterTile]
- case t if t.conformsTo[RasterRef] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[RasterRef]
+ lazy val projectedRasterLikeExtractor: PartialFunction[DataType, Any => ProjectedRasterLike] = {
+ case _: RasterSourceUDT =>
+ (input: Any) =>
+ val row = input.asInstanceOf[InternalRow]
+ rasterSourceUDT.deserialize(row)
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[ProjectedRasterTile]
+ case t if t.conformsToSchema(RasterRef.rasterRefEncoder.schema) =>
+ (row: Any) => row.asInstanceOf[InternalRow].as[RasterRef]
}
/** Partial function for pulling a CellGrid from an input row. */
- lazy val gridExtractor: PartialFunction[DataType, InternalRow ⇒ CellGrid[Int]] = {
+ lazy val gridExtractor: PartialFunction[DataType, InternalRow => CellGrid[Int]] = {
case _: TileUDT =>
- (row: InternalRow) => row.to[Tile](TileUDT.tileSerializer)
- case _: RasterSourceUDT =>
- (row: InternalRow) => row.to[RFRasterSource](RasterSourceUDT.rasterSourceSerializer)
- case t if t.conformsTo[RasterRef] ⇒
- (row: InternalRow) => row.to[RasterRef]
- case t if t.conformsTo[ProjectedRasterTile] =>
- (row: InternalRow) => row.to[ProjectedRasterTile]
+ // TODO EAC: is there way to extract grid from TileUDT without reading the cells with an expression?
+ (row: InternalRow) => tileUDT.deserialize(row)
+ case _: RasterSourceUDT => (row: InternalRow) => rasterSourceUDT.deserialize(row)
+ case t if t.conformsToSchema(RasterRef.rasterRefEncoder.schema) =>
+ (row: InternalRow) => row.as[RasterRef]
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
+ (row: InternalRow) => row.as[ProjectedRasterTile]
}
lazy val crsExtractor: PartialFunction[DataType, Any => CRS] = {
- val base: PartialFunction[DataType, Any ⇒ CRS] = {
- case _: StringType =>
- (v: Any) => LazyCRS(v.asInstanceOf[UTF8String].toString)
- case t if t.conformsTo[CRS] =>
- (v: Any) => v.asInstanceOf[InternalRow].to[CRS]
+ val base: PartialFunction[DataType, Any => CRS] = {
+ case _: StringType => (v: Any) => LazyCRS(v.asInstanceOf[UTF8String].toString)
+ case _: CrsUDT => (v: Any) => LazyCRS(v.asInstanceOf[UTF8String].toString)
+ case t if t.conformsToSchema(crsExpressionEncoder.schema) =>
+ (v: Any) => v.asInstanceOf[InternalRow].as[CRS]
}
val fromPRL = projectedRasterLikeExtractor.andThen(_.andThen(_.crs))
@@ -109,8 +119,7 @@ object DynamicExtractors {
/** This is necessary because extents created from Python Rows will reorder field names. */
object ExtentLike {
-
- def rightShape(struct: StructType) =
+ def rightShape(struct: StructType): Boolean =
struct.size == 4 && {
val n = struct.fieldNames.map(_.toLowerCase).toSet
n == Set("xmin", "ymin", "xmax", "ymax")|| n == Set("minx", "miny", "maxx", "maxy")
@@ -143,16 +152,16 @@ object DynamicExtractors {
}
}
- lazy val extentExtractor: PartialFunction[DataType, Any ⇒ Extent] = {
- val base: PartialFunction[DataType, Any ⇒ Extent] = {
+ lazy val extentExtractor: PartialFunction[DataType, Any => Extent] = {
+ val base: PartialFunction[DataType, Any => Extent] = {
case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
(input: Any) => Extent(JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal)
- case t if t.conformsTo[Extent] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[Extent]
- case t if t.conformsTo[Envelope] =>
- (input: Any) => Extent(input.asInstanceOf[InternalRow].to[Envelope])
- case t if t.conformsTo[LongExtent] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[LongExtent].toExtent
+ case t if t.conformsToSchema(StandardEncoders.extentEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[Extent]
+ case t if t.conformsToSchema(StandardEncoders.envelopeEncoder.schema) =>
+ (input: Any) => Extent(input.asInstanceOf[InternalRow].as[Envelope])
+ case t if t.conformsToSchema(StandardEncoders.longExtentEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[LongExtent].toExtent
case ExtentLike(e) => e
}
@@ -161,22 +170,22 @@ object DynamicExtractors {
}
lazy val envelopeExtractor: PartialFunction[DataType, Any => Envelope] = {
- val base = PartialFunction[DataType, Any => Envelope] {
- case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
+ val base: PartialFunction[DataType, Any => Envelope] = {
+ case t if t.conformsToDataType(JTSTypes.GeometryTypeInstance) =>
(input: Any) => JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal
- case t if t.conformsTo[Extent] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[Extent].jtsEnvelope
- case t if t.conformsTo[LongExtent] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[LongExtent].toExtent.jtsEnvelope
- case t if t.conformsTo[Envelope] =>
- (input: Any) => input.asInstanceOf[InternalRow].to[Envelope]
+ case t if t.conformsToSchema(StandardEncoders.extentEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[Extent].jtsEnvelope
+ case t if t.conformsToSchema(StandardEncoders.longExtentEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[LongExtent].toExtent.jtsEnvelope
+ case t if t.conformsToSchema(StandardEncoders.envelopeEncoder.schema) =>
+ (input: Any) => input.asInstanceOf[InternalRow].as[Envelope]
}
val fromPRL = projectedRasterLikeExtractor.andThen(_.andThen(_.extent.jtsEnvelope))
fromPRL orElse base
}
- lazy val centroidExtractor: PartialFunction[DataType, Any ⇒ Point] = {
+ lazy val centroidExtractor: PartialFunction[DataType, Any => Point] = {
extentExtractor.andThen(_.andThen(_.center))
}
@@ -215,5 +224,4 @@ object DynamicExtractors {
case c: Char => IntegerArg(c.toInt)
}
}
-
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala
index 8bc98c1e2..31e3f39b5 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/NullToValue.scala
@@ -25,18 +25,12 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.UnaryExpression
trait NullToValue { self: UnaryExpression =>
-
def na: Any
-
- override def eval(input: InternalRow): Any = {
+ override def eval(input: InternalRow): Any =
if (input == null) na
else {
val value = child.eval(input)
- if (value == null) {
- na
- } else {
- nullSafeEval(value)
- }
+ if (value == null) na
+ else nullSafeEval(value)
}
- }
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala
index 62dac78c1..741a85a8e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnCellGridExpression.scala
@@ -35,6 +35,12 @@ import org.apache.spark.sql.catalyst.expressions.UnaryExpression
* @since 11/4/18
*/
trait OnCellGridExpression extends UnaryExpression {
+
+ private lazy val fromRow: InternalRow => CellGrid[Int] = {
+ if (child.resolved) gridExtractor(child.dataType)
+ else throw new IllegalStateException(s"Child expression unbound: ${child}")
+ }
+
override def checkInputDataTypes(): TypeCheckResult = {
if (!gridExtractor.isDefinedAt(child.dataType)) {
TypeCheckFailure(s"Input type '${child.dataType}' does not conform to `Grid`.")
@@ -44,10 +50,8 @@ trait OnCellGridExpression extends UnaryExpression {
final override protected def nullSafeEval(input: Any): Any = {
input match {
- case row: InternalRow ⇒
- val g = gridExtractor(child.dataType)(row)
- eval(g)
- case o ⇒ throw new IllegalArgumentException(s"Unsupported input type: $o")
+ case row: InternalRow => eval(fromRow(row))
+ case o => throw new IllegalArgumentException(s"Unsupported input type: $o")
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala
index 78ebd1f5b..3767b4d0f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/OnTileContextExpression.scala
@@ -45,10 +45,10 @@ trait OnTileContextExpression extends UnaryExpression {
final override protected def nullSafeEval(input: Any): Any = {
input match {
- case row: InternalRow ⇒
+ case row: InternalRow =>
val prl = projectedRasterLikeExtractor(child.dataType)(row)
eval(TileContext(prl.extent, prl.crs))
- case o ⇒ throw new IllegalArgumentException(s"Unsupported input type: $o")
+ case o => throw new IllegalArgumentException(s"Unsupported input type: $o")
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/RasterResult.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/RasterResult.scala
new file mode 100644
index 000000000..7afd49ba9
--- /dev/null
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/RasterResult.scala
@@ -0,0 +1,19 @@
+package org.locationtech.rasterframes.expressions
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.model.TileContext
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+
+trait RasterResult { self: Expression =>
+ private lazy val tileSer: Tile => InternalRow = tileUDT.serialize
+ private lazy val prtSer: ProjectedRasterTile => InternalRow = SerializersCache.serializer[ProjectedRasterTile].apply
+
+ def toInternalRow(result: Tile, tileContext: Option[TileContext] = None): InternalRow =
+ tileContext.fold(tileSer(result))(ctx => prtSer(ProjectedRasterTile(result, ctx.extent, ctx.crs)))
+
+ def toInternalRow(result: ProjectedRasterTile): InternalRow = prtSer(result)
+}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala
index 9f4d19725..bc6249d1d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/SpatialRelation.scala
@@ -21,14 +21,15 @@
package org.locationtech.rasterframes.expressions
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.expressions.SpatialRelation.RelationPredicate
import geotrellis.vector.Extent
import org.locationtech.jts.geom._
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.catalyst.expressions.{ScalaUDF, _}
+import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.jts.AbstractGeometryUDT
import org.apache.spark.sql.types._
import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions._
@@ -38,27 +39,23 @@ import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions._
*
* @since 12/28/17
*/
-abstract class SpatialRelation extends BinaryExpression
- with CodegenFallback {
+abstract class SpatialRelation extends BinaryExpression with CodegenFallback {
def extractGeometry(expr: Expression, input: Any): Geometry = {
input match {
- case g: Geometry ⇒ g
- case r: InternalRow ⇒
+ case g: Geometry => g
+ case r: InternalRow =>
expr.dataType match {
- case udt: AbstractGeometryUDT[_] ⇒ udt.deserialize(r)
- case dt if dt.conformsTo[Extent] =>
- val extent = r.to[Extent]
- extent.toPolygon()
+ case udt: AbstractGeometryUDT[_] => udt.deserialize(r)
+ case dt if dt.conformsToSchema(extentEncoder.schema) =>
+ r.as[Extent].toPolygon()
}
}
}
- // TODO: replace with serializer.
- lazy val jtsPointEncoder = ExpressionEncoder[Point]()
override def toString: String = s"$nodeName($left, $right)"
- override def dataType: DataType = BooleanType
+ def dataType: DataType = BooleanType
override def nullable: Boolean = left.nullable || right.nullable
@@ -72,7 +69,7 @@ abstract class SpatialRelation extends BinaryExpression
}
object SpatialRelation {
- type RelationPredicate = (Geometry, Geometry) ⇒ java.lang.Boolean
+ type RelationPredicate = (Geometry, Geometry) => java.lang.Boolean
case class Intersects(left: Expression, right: Expression) extends SpatialRelation {
override def nodeName = "intersects"
@@ -118,11 +115,9 @@ object SpatialRelation {
ST_Within -> Within
)
- def fromUDF(udf: ScalaUDF) = {
+ def fromUDF(udf: ScalaUDF): Option[SpatialRelation] =
udf.function match {
- case rp: RelationPredicate @unchecked ⇒
- predicateMap.get(rp).map(_.apply(udf.children.head, udf.children.last))
- case _ ⇒ None
+ case rp: RelationPredicate @unchecked => predicateMap.get(rp).map(_.apply(udf.children.head, udf.children.last))
+ case _ => None
}
- }
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala
index c3fe0e17b..ea187e662 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/TileAssembler.scala
@@ -21,8 +21,7 @@
package org.locationtech.rasterframes.expressions
-import java.nio.ByteBuffer
-
+import org.locationtech.rasterframes.encoders.StandardEncoders._
import org.locationtech.rasterframes.expressions.TileAssembler.TileBuffer
import org.locationtech.rasterframes.util._
import geotrellis.raster.{DataType => _, _}
@@ -32,7 +31,8 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, TypedColumn}
import spire.syntax.cfor._
-import org.locationtech.rasterframes.TileType
+
+import java.nio.{ByteBuffer, DoubleBuffer}
/**
* Aggregator for reassembling tiles from from exploded form
@@ -64,43 +64,37 @@ case class TileAssembler(
tileCols: Expression,
tileRows: Expression,
mutableAggBufferOffset: Int = 0,
- inputAggBufferOffset: Int = 0)
- extends TypedImperativeAggregate[TileBuffer] with ImplicitCastInputTypes {
+ inputAggBufferOffset: Int = 0
+) extends TypedImperativeAggregate[TileBuffer] with ImplicitCastInputTypes {
- def this(colIndex: Expression,
- rowIndex: Expression,
- cellValue: Expression,
- tileCols: Expression,
- tileRows: Expression) = this(colIndex, rowIndex, cellValue, tileCols, tileRows, 0, 0)
+ def this(colIndex: Expression, rowIndex: Expression, cellValue: Expression, tileCols: Expression, tileRows: Expression) = this(colIndex, rowIndex, cellValue, tileCols, tileRows, 0, 0)
- override def children: Seq[Expression] = Seq(colIndex, rowIndex, cellValue, tileCols, tileRows)
+ def children: Seq[Expression] = Seq(colIndex, rowIndex, cellValue, tileCols, tileRows)
- override def inputTypes = Seq(ShortType, ShortType, DoubleType, ShortType, ShortType)
+ def inputTypes = Seq(ShortType, ShortType, DoubleType, ShortType, ShortType)
override def prettyName: String = "rf_assemble_tiles"
- override def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
+ def withNewMutableAggBufferOffset(newMutableAggBufferOffset: Int): ImperativeAggregate =
copy(mutableAggBufferOffset = newMutableAggBufferOffset)
- override def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
+ def withNewInputAggBufferOffset(newInputAggBufferOffset: Int): ImperativeAggregate =
copy(inputAggBufferOffset = newInputAggBufferOffset)
- override def nullable: Boolean = true
+ def nullable: Boolean = true
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
- override def createAggregationBuffer(): TileBuffer = new TileBuffer(Array.empty)
+ def createAggregationBuffer(): TileBuffer = new TileBuffer(Array.empty)
@inline
private def toIndex(col: Int, row: Int, tileCols: Short): Int = row * tileCols + col
- override def update(inBuf: TileBuffer, input: InternalRow): TileBuffer = {
+ def update(inBuf: TileBuffer, input: InternalRow): TileBuffer = {
val tc = tileCols.eval(input).asInstanceOf[Short]
val tr = tileRows.eval(input).asInstanceOf[Short]
- val buffer = if (inBuf.isEmpty) {
- TileBuffer(tc, tr)
- } else inBuf
+ val buffer = if (inBuf.isEmpty) TileBuffer(tc, tr) else inBuf
val col = colIndex.eval(input).asInstanceOf[Short]
require(col < tc, s"`tileCols` is $tc, but received index value $col")
@@ -112,7 +106,7 @@ case class TileAssembler(
buffer
}
- override def merge(inBuf: TileBuffer, input: TileBuffer): TileBuffer = {
+ def merge(inBuf: TileBuffer, input: TileBuffer): TileBuffer = {
val buffer = if (inBuf.isEmpty) {
val (cols, rows) = input.tileSize
@@ -133,7 +127,7 @@ case class TileAssembler(
buffer
}
- override def eval(buffer: TileBuffer): InternalRow = {
+ def eval(buffer: TileBuffer): InternalRow = {
// TODO: figure out how to eliminate copies here.
val result = buffer.cellBuffer
val length = result.capacity()
@@ -141,25 +135,23 @@ case class TileAssembler(
result.get(cells)
val (tileCols, tileRows) = buffer.tileSize
val tile = ArrayTile(cells, tileCols.toInt, tileRows.toInt)
- TileType.serialize(tile)
+ tileUDT.serialize(tile)
}
- override def serialize(buffer: TileBuffer): Array[Byte] = buffer.serialize()
- override def deserialize(storageFormat: Array[Byte]): TileBuffer = new TileBuffer(storageFormat)
+ def serialize(buffer: TileBuffer): Array[Byte] = buffer.serialize()
+ def deserialize(storageFormat: Array[Byte]): TileBuffer = new TileBuffer(storageFormat)
}
object TileAssembler {
- import org.locationtech.rasterframes.encoders.StandardEncoders._
-
def apply(
columnIndex: Column,
rowIndex: Column,
cellData: Column,
tileCols: Column,
- tileRows: Column): TypedColumn[Any, Tile] =
- new Column(new TileAssembler(columnIndex.expr, rowIndex.expr, cellData.expr, tileCols.expr,
- tileRows.expr)
- .toAggregateExpression())
+ tileRows: Column
+ ): TypedColumn[Any, Tile] =
+ new Column(new TileAssembler(columnIndex.expr, rowIndex.expr, cellData.expr, tileCols.expr, tileRows.expr)
+ .toAggregateExpression())
.as(cellData.columnName)
.as[Tile]
@@ -167,11 +159,10 @@ object TileAssembler {
class TileBuffer(val storage: Array[Byte]) {
- def isEmpty = storage.isEmpty
+ def isEmpty: Boolean = storage.isEmpty
- def cellBuffer = ByteBuffer.wrap(storage, 0, storage.length - indexPad).asDoubleBuffer()
- private def indexBuffer =
- ByteBuffer.wrap(storage, storage.length - indexPad, indexPad).asShortBuffer()
+ def cellBuffer: DoubleBuffer = ByteBuffer.wrap(storage, 0, storage.length - indexPad).asDoubleBuffer()
+ private def indexBuffer = ByteBuffer.wrap(storage, storage.length - indexPad, indexPad).asShortBuffer()
def reset(): Unit = {
val cells = cellBuffer
@@ -198,5 +189,4 @@ object TileAssembler {
buf
}
}
-
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala
index a410f47f8..2904fe57d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala
@@ -26,17 +26,15 @@ import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.UnaryExpression
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.slf4j.LoggerFactory
/** Operation on a tile returning a tile. */
-trait UnaryLocalRasterOp extends UnaryExpression {
+trait UnaryLocalRasterOp extends UnaryExpression with RasterResult {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
- override def dataType: DataType = child.dataType
+ def dataType: DataType = child.dataType
override def checkInputDataTypes(): TypeCheckResult = {
if (!tileExtractor.isDefinedAt(child.dataType)) {
@@ -46,13 +44,9 @@ trait UnaryLocalRasterOp extends UnaryExpression {
}
override protected def nullSafeEval(input: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (childTile, childCtx) = tileExtractor(child.dataType)(row(input))
-
- childCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(op(childTile)).toInternalRow
- case None => op(childTile).toInternalRow
- }
+ val result = op(childTile)
+ toInternalRow(result, childCtx)
}
protected def op(child: Tile): Tile
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala
index d2f36c39c..42f886b65 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryRasterAggregate.scala
@@ -21,11 +21,15 @@
package org.locationtech.rasterframes.expressions
-import org.locationtech.rasterframes.expressions.DynamicExtractors.rowTileExtractor
import geotrellis.raster.Tile
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF}
import org.apache.spark.sql.catalyst.expressions.aggregate.DeclarativeAggregate
+import org.apache.spark.sql.types.DataType
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.syntax._
+
import scala.reflect.runtime.universe._
/** Mixin providing boilerplate for DeclarativeAggrates over tile-conforming columns. */
@@ -37,11 +41,14 @@ trait UnaryRasterAggregate extends DeclarativeAggregate {
def children = Seq(child)
protected def tileOpAsExpression[R: TypeTag](name: String, op: Tile => R): Expression => ScalaUDF =
- udfexpr[R, Any](name, (a: Any) => if(a == null) null.asInstanceOf[R] else op(extractTileFromAny(a)))
+ udfiexpr[R, Any](name, (dataType: DataType) => (a: Any) => if(a == null) null.asInstanceOf[R] else op(UnaryRasterAggregate.extractTileFromAny(dataType, a)))
+}
- protected val extractTileFromAny = (a: Any) => a match {
+object UnaryRasterAggregate {
+ val extractTileFromAny: (DataType, Any) => Tile = (dt: DataType, row: Any) => row match {
case t: Tile => t
- case r: Row => rowTileExtractor(child.dataType)(r)._1
- case null => null
+ case r: Row => r.as[Tile]
+ case i: InternalRow => DynamicExtractors.internalRowTileExtractor(dt)(i)._1
+ case r => throw new Exception(s"UnaryRasterAggregate.extractFromAny unsupported row: $r")
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala
index 4fc0a0374..529c88996 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/ExtractTile.scala
@@ -21,34 +21,31 @@
package org.locationtech.rasterframes.expressions.accessors
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
-import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.rasterframes.model.TileContext
-import org.locationtech.rasterframes.tiles.InternalRowTile
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import org.locationtech.rasterframes._
/** Expression to extract at tile from several types that contain tiles.*/
case class ExtractTile(child: Expression) extends UnaryRasterOp with CodegenFallback {
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
override def nodeName: String = "rf_extract_tile"
- implicit val tileSer = TileUDT.tileSerializer
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile match {
- case irt: InternalRowTile => irt.mem
- case tile: ConcreteProjectedRasterTile => tile.t.toInternalRow
- case tile: Tile => tile.toInternalRow
+
+ private lazy val tileSer = tileUDT.serialize _
+
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile match {
+ case prt: ProjectedRasterTile => tileUDT.serialize(prt.tile)
+ case tile: Tile => tileSer(tile)
}
}
object ExtractTile {
- import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder
def apply(input: Column): TypedColumn[Any, Tile] =
new Column(new ExtractTile(input.expr)).as[Tile]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala
index 68784b2c2..0ffc0d78e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala
@@ -27,13 +27,18 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.rf.CrsUDT
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.encoders.syntax._
+import org.locationtech.rasterframes.expressions.DynamicExtractors
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+import org.apache.spark.sql.rf.RasterSourceUDT
+import org.locationtech.rasterframes.ref.RasterRef
import org.apache.spark.unsafe.types.UTF8String
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.StandardEncoders.crsSparkEncoder
-import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.model.LazyCRS
+import org.apache.spark.sql.types.StringType
/**
* Expression to extract the CRS out of a RasterRef or ProjectedRasterTile column.
@@ -48,22 +53,47 @@ import org.locationtech.rasterframes.model.LazyCRS
....
""")
case class GetCRS(child: Expression) extends UnaryExpression with CodegenFallback {
- override def dataType: DataType = schemaOf[CRS]
+ override def dataType: DataType = crsUDT
override def nodeName: String = "rf_crs"
override def checkInputDataTypes(): TypeCheckResult = {
- if (!crsExtractor.isDefinedAt(child.dataType) )
+ if (!DynamicExtractors.crsExtractor.isDefinedAt(child.dataType))
TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a CRS or something with one.")
else TypeCheckSuccess
}
override protected def nullSafeEval(input: Any): Any = {
- input match {
- case s: UTF8String => LazyCRS(s.toString).toInternalRow
- case row: InternalRow ⇒
- val crs = crsExtractor(child.dataType)(row)
- crs.toInternalRow
- case o ⇒ throw new IllegalArgumentException(s"Unsupported input type: $o")
+ // TODO: move construction of this function to checkInputDataType as dataType is constant per instance of this exp.
+ child.dataType match {
+ case _: CrsUDT =>
+ val str = input.asInstanceOf[UTF8String]
+ val crs = crsUDT.deserialize(str)
+ crsUDT.serialize(crs)
+
+ case _: StringType =>
+ val str = input.asInstanceOf[UTF8String]
+ val crs = crsUDT.deserialize(str)
+ crsUDT.serialize(crs)
+
+ case t if t.conformsToSchema(crsExpressionEncoder.schema) =>
+ crsUDT.serialize(input.asInstanceOf[InternalRow].as[CRS])
+
+ case t if t.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema) =>
+ val idx = ProjectedRasterTile.projectedRasterTileEncoder.schema.fieldIndex("crs")
+ input.asInstanceOf[InternalRow].get(idx, crsUDT).asInstanceOf[UTF8String]
+
+ case _: RasterSourceUDT =>
+ val rs = rasterSourceUDT.deserialize(input)
+ val crs = rs.crs
+ crsUDT.serialize(crs)
+
+ case t if t.conformsToSchema(RasterRef.rasterRefEncoder.schema) =>
+ val row = input.asInstanceOf[InternalRow]
+ val idx = RasterRef.rasterRefEncoder.schema.fieldIndex("source")
+ val rsc = row.get(idx, rasterSourceUDT)
+ val rs = rasterSourceUDT.deserialize(rsc)
+ val crs = rs.crs
+ crsUDT.serialize(crs)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala
index bb7cdf233..b5966733c 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCellType.scala
@@ -21,29 +21,42 @@
package org.locationtech.rasterframes.expressions.accessors
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
import org.locationtech.rasterframes.expressions.OnCellGridExpression
import geotrellis.raster.{CellGrid, CellType}
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
+import org.apache.spark.sql.catalyst.InternalRow
/**
* Extract a Tile's cell type
* @since 12/21/17
*/
case class GetCellType(child: Expression) extends OnCellGridExpression with CodegenFallback {
-
override def nodeName: String = "rf_cell_type"
- def dataType: DataType = schemaOf[CellType]
+ def dataType: DataType =
+ if (cellTypeEncoder.isSerializedAsStructForTopLevel) cellTypeEncoder.schema
+ else cellTypeEncoder.schema.fields(0).dataType
+
+ private lazy val resultConverter: Any => Any = {
+ val ser = SerializersCache.serializer[CellType].apply _
+ val toRow = ser.asInstanceOf[Any => Any]
+ // TODO: wather encoder is top level or not should be constant, so this check is overly general
+ if (cellTypeEncoder.isSerializedAsStructForTopLevel) {
+ value: Any =>if (value == null) null else toRow(value).asInstanceOf[InternalRow]
+ } else {
+ value: Any => if (value == null) null else toRow(value).asInstanceOf[InternalRow].get(0, dataType)
+ }
+ }
+
/** Implemented by subtypes to process incoming ProjectedRasterLike entity. */
- override def eval(cg: CellGrid[Int]): Any = cg.cellType.toInternalRow
+ def eval(cg: CellGrid[Int]): Any = resultConverter(cg.cellType)
}
object GetCellType {
- import org.locationtech.rasterframes.encoders.StandardEncoders._
- def apply(col: Column): TypedColumn[Any, CellType] =
- new Column(new GetCellType(col.expr)).as[CellType]
+ def apply(col: Column): TypedColumn[Any, CellType] = new Column(new GetCellType(col.expr)).as[CellType]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala
index 2e8c71ded..d28b80a44 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetDimensions.scala
@@ -21,12 +21,13 @@
package org.locationtech.rasterframes.expressions.accessors
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.OnCellGridExpression
import geotrellis.raster.{CellGrid, Dimensions}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.syntax._
/**
* Extract a raster's dimensions
@@ -42,13 +43,11 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
case class GetDimensions(child: Expression) extends OnCellGridExpression with CodegenFallback {
override def nodeName: String = "rf_dimensions"
- def dataType = schemaOf[Dimensions[Int]]
+ def dataType = dimensionsEncoder.schema
- override def eval(grid: CellGrid[Int]): Any = Dimensions[Int](grid.cols, grid.rows).toInternalRow
+ def eval(grid: CellGrid[Int]): Any = Dimensions[Int](grid.cols, grid.rows).toInternalRow
}
object GetDimensions {
- import org.locationtech.rasterframes.encoders.StandardEncoders.tileDimensionsEncoder
- def apply(col: Column): TypedColumn[Any, Dimensions[Int]] =
- new Column(new GetDimensions(col.expr)).as[Dimensions[Int]]
+ def apply(col: Column): TypedColumn[Any, Dimensions[Int]] = new Column(new GetDimensions(col.expr)).as[Dimensions[Int]]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala
index d0c14491b..00ba62e83 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetEnvelope.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.accessors
+import org.locationtech.rasterframes._
import org.locationtech.jts.geom.{Envelope, Geometry}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -29,7 +30,6 @@ import org.apache.spark.sql.jts.AbstractGeometryUDT
import org.apache.spark.sql.rf._
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.encoders.EnvelopeEncoder
/**
* Extracts the bounding box (envelope) of arbitrary JTS Geometry.
@@ -56,11 +56,9 @@ case class GetEnvelope(child: Expression) extends UnaryExpression with CodegenFa
InternalRow(env.getMinX, env.getMaxX, env.getMinY, env.getMaxY)
}
- def dataType: DataType = EnvelopeEncoder.schema
+ def dataType: DataType = envelopeEncoder.schema
}
object GetEnvelope {
- import org.locationtech.rasterframes.encoders.StandardEncoders._
- def apply(col: Column): TypedColumn[Any, Envelope] =
- new GetEnvelope(col.expr).asColumn.as[Envelope]
+ def apply(col: Column): TypedColumn[Any, Envelope] = new GetEnvelope(col.expr).asColumn.as[Envelope]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala
index 2266c69b5..5dfb6781a 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetExtent.scala
@@ -21,8 +21,7 @@
package org.locationtech.rasterframes.expressions.accessors
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.StandardEncoders.extentEncoder
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.OnTileContextExpression
import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.InternalRow
@@ -30,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.model.TileContext
/**
@@ -45,12 +45,12 @@ import org.locationtech.rasterframes.model.TileContext
....
""")
case class GetExtent(child: Expression) extends OnTileContextExpression with CodegenFallback {
- override def dataType: DataType = schemaOf[Extent]
+ def dataType: DataType = extentEncoder.schema
+
override def nodeName: String = "rf_extent"
- override def eval(ctx: TileContext): InternalRow = ctx.extent.toInternalRow
+ def eval(ctx: TileContext): InternalRow = ctx.extent.toInternalRow
}
object GetExtent {
- def apply(col: Column): TypedColumn[Any, Extent] =
- new Column(new GetExtent(col.expr)).as[Extent]
+ def apply(col: Column): TypedColumn[Any, Extent] = new Column(new GetExtent(col.expr)).as[Extent]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala
index e099cca04..760263292 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetGeometry.scala
@@ -45,9 +45,9 @@ import org.locationtech.rasterframes.model.TileContext
....
""")
case class GetGeometry(child: Expression) extends OnTileContextExpression with CodegenFallback {
- override def dataType: DataType = JTSTypes.GeometryTypeInstance
+ def dataType: DataType = JTSTypes.GeometryTypeInstance
override def nodeName: String = "rf_geometry"
- override def eval(ctx: TileContext): InternalRow =
+ def eval(ctx: TileContext): InternalRow =
JTSTypes.GeometryTypeInstance.serialize(ctx.extent.toPolygon())
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala
index 6c9a3538a..52bc4074e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetTileContext.scala
@@ -21,27 +21,25 @@
package org.locationtech.rasterframes.expressions.accessors
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.UnaryRasterOp
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
case class GetTileContext(child: Expression) extends UnaryRasterOp with CodegenFallback {
- override def dataType: DataType = schemaOf[TileContext]
+ def dataType: DataType = tileContextEncoder.schema
override def nodeName: String = "get_tile_context"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
- ctx.map(_.toInternalRow).orNull
+
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
+ ctx.map(SerializersCache.serializer[TileContext].apply).orNull
}
object GetTileContext {
- import org.locationtech.rasterframes.encoders.StandardEncoders.tileContextEncoder
-
- def apply(input: Column): TypedColumn[Any, TileContext] =
- new Column(new GetTileContext(input.expr)).as[TileContext]
+ def apply(input: Column): TypedColumn[Any, TileContext] = new Column(new GetTileContext(input.expr)).as[TileContext]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala
index 34c794d92..e5d9f9f45 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala
@@ -26,11 +26,9 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, UnaryExpression}
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions._
@@ -42,24 +40,24 @@ import org.locationtech.rasterframes.expressions._
....
""")
case class RealizeTile(child: Expression) extends UnaryExpression with CodegenFallback {
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
override def nodeName: String = "rf_tile"
- override def checkInputDataTypes(): TypeCheckResult = {
+ private lazy val tileSer = tileUDT.serialize _
+
+ override def checkInputDataTypes(): TypeCheckResult =
if (!tileableExtractor.isDefinedAt(child.dataType)) {
TypeCheckFailure(s"Input type '${child.dataType}' does not conform to a tiled raster type.")
} else TypeCheckSuccess
- }
- implicit val tileSer = TileUDT.tileSerializer
+
override protected def nullSafeEval(input: Any): Any = {
val in = row(input)
val tile = tileableExtractor(child.dataType)(in)
- (tile.toArrayTile(): Tile).toInternalRow
+ tileSer(tile.toArrayTile())
}
}
object RealizeTile {
- def apply(col: Column): TypedColumn[Any, Tile] =
- new Column(new RealizeTile(col.expr)).as[Tile]
+ def apply(col: Column): TypedColumn[Any, Tile] = new Column(new RealizeTile(col.expr)).as[Tile]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ApproxCellQuantilesAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ApproxCellQuantilesAggregate.scala
index e2ab3f899..00d3bd2c9 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ApproxCellQuantilesAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ApproxCellQuantilesAggregate.scala
@@ -22,66 +22,66 @@
package org.locationtech.rasterframes.expressions.aggregates
import geotrellis.raster.{Tile, isNoData}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.util.QuantileSummaries
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
-import org.apache.spark.sql.{Column, Encoder, Row, TypedColumn, types}
+import org.apache.spark.sql.{Column, Row, TypedColumn, types}
import org.apache.spark.sql.types.{DataTypes, StructField, StructType}
-import org.locationtech.rasterframes.TileType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.syntax._
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
-
case class ApproxCellQuantilesAggregate(probabilities: Seq[Double], relativeError: Double) extends UserDefinedAggregateFunction {
- import org.locationtech.rasterframes.encoders.StandardSerializers.quantileSerializer
-
- override def inputSchema: StructType = StructType(Seq(
- StructField("value", TileType, true)
+ def inputSchema: StructType = StructType(Seq(
+ StructField("value", tileUDT, true)
))
- override def bufferSchema: StructType = StructType(Seq(
- StructField("buffer", schemaOf[QuantileSummaries], false)
+ def bufferSchema: StructType = StructType(Seq(
+ StructField("buffer", quantileSummariesEncoder.schema, false)
))
- override def dataType: types.DataType = DataTypes.createArrayType(DataTypes.DoubleType)
+ def dataType: types.DataType = DataTypes.createArrayType(DataTypes.DoubleType)
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit =
- buffer.update(0, new QuantileSummaries(QuantileSummaries.defaultCompressThreshold, relativeError).toRow)
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
+ val qs = new QuantileSummaries(QuantileSummaries.defaultCompressThreshold, relativeError)
+ buffer.update(0, qs.toRow)
+ }
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val qs = buffer.getStruct(0).to[QuantileSummaries]
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ val qs = buffer.getStruct(0).as[QuantileSummaries]
if (!input.isNullAt(0)) {
val tile = input.getAs[Tile](0)
var result = qs
tile.foreachDouble(d => if (!isNoData(d)) result = result.insert(d))
+
buffer.update(0, result.toRow)
}
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- val left = buffer1.getStruct(0).to[QuantileSummaries]
- val right = buffer2.getStruct(0).to[QuantileSummaries]
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ val left = buffer1.getStruct(0).as[QuantileSummaries]
+ val right = buffer2.getStruct(0).as[QuantileSummaries]
val merged = left.compress().merge(right.compress())
- buffer1.update(0, merged.toRow)
+
+ val mergedRow = merged.toRow
+ buffer1.update(0, mergedRow)
}
- override def evaluate(buffer: Row): Seq[Double] = {
- val summaries = buffer.getStruct(0).to[QuantileSummaries]
+ def evaluate(buffer: Row): Seq[Double] = {
+ val summaries = buffer.getStruct(0).as[QuantileSummaries]
probabilities.flatMap(summaries.query)
}
}
object ApproxCellQuantilesAggregate {
- private implicit def doubleSeqEncoder: Encoder[Seq[Double]] = ExpressionEncoder()
-
def apply(
tile: Column,
probabilities: Seq[Double],
- relativeError: Double = 0.00001): TypedColumn[Any, Seq[Double]] = {
+ relativeError: Double = 0.00001
+ ): TypedColumn[Any, Seq[Double]] =
new ApproxCellQuantilesAggregate(probabilities, relativeError)(ExtractTile(tile))
.as(s"rf_agg_approx_quantiles")
.as[Seq[Double]]
- }
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala
index 82c2d3f93..7e845f409 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellCountAggregate.scala
@@ -21,11 +21,12 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterAggregate
import org.locationtech.rasterframes.expressions.tilestats.{DataCells, NoDataCells}
import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, _}
-import org.apache.spark.sql.types.{LongType, Metadata}
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.types.{DataType, LongType, Metadata}
import org.apache.spark.sql.{Column, TypedColumn}
/**
@@ -35,37 +36,26 @@ import org.apache.spark.sql.{Column, TypedColumn}
* @param isData true if count should be of non-NoData cells, false if count should be of NoData cells.
*/
abstract class CellCountAggregate(isData: Boolean) extends UnaryRasterAggregate {
- private lazy val count =
- AttributeReference("count", LongType, false, Metadata.empty)()
+ private lazy val count = AttributeReference("count", LongType, false, Metadata.empty)()
- override lazy val aggBufferAttributes = Seq(
- count
- )
+ override lazy val aggBufferAttributes = Seq(count)
- val initialValues = Seq(
- Literal(0L)
- )
+ val initialValues = Seq(Literal(0L))
- private def CellTest =
+ private def CellTest: Expression => ScalaUDF =
if (isData) tileOpAsExpression("rf_data_cells", DataCells.op)
else tileOpAsExpression("rf_no_data_cells", NoDataCells.op)
- val updateExpressions = Seq(
- If(IsNull(child), count, Add(count, CellTest(child)))
- )
+ val updateExpressions = Seq(If(IsNull(child), count, Add(count, CellTest(child))))
- val mergeExpressions = Seq(
- count.left + count.right
- )
+ val mergeExpressions = Seq(count.left + count.right)
- val evaluateExpression = count
+ val evaluateExpression: AttributeReference = count
- def dataType = LongType
+ def dataType: DataType = LongType
}
object CellCountAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Count the total data (non-no-data) cells in a tile column.",
arguments = """
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala
index 009a46cf3..38b2e453f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellMeanAggregate.scala
@@ -21,11 +21,12 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterAggregate
import org.locationtech.rasterframes.expressions.tilestats.{DataCells, Sum}
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, _}
-import org.apache.spark.sql.types.{DoubleType, LongType, Metadata}
+import org.apache.spark.sql.types.{DataType, DoubleType, LongType, Metadata}
import org.apache.spark.sql.{Column, TypedColumn}
/**
@@ -43,17 +44,12 @@ import org.apache.spark.sql.{Column, TypedColumn}
case class CellMeanAggregate(child: Expression) extends UnaryRasterAggregate {
override def nodeName: String = "rf_agg_mean"
- private lazy val sum =
- AttributeReference("sum", DoubleType, false, Metadata.empty)()
- private lazy val count =
- AttributeReference("count", LongType, false, Metadata.empty)()
+ private lazy val sum = AttributeReference("sum", DoubleType, false, Metadata.empty)()
+ private lazy val count = AttributeReference("count", LongType, false, Metadata.empty)()
- override lazy val aggBufferAttributes = Seq(sum, count)
+ lazy val aggBufferAttributes = Seq(sum, count)
- override val initialValues = Seq(
- Literal(0.0),
- Literal(0L)
- )
+ val initialValues = Seq(Literal(0.0), Literal(0L))
// Cant' figure out why we can't just use the Expression directly
// this is necessary to properly handle null rows. For example,
@@ -61,25 +57,21 @@ case class CellMeanAggregate(child: Expression) extends UnaryRasterAggregate {
private val DataCellCounts = tileOpAsExpression("rf_data_cells", DataCells.op)
private val SumCells = tileOpAsExpression("sum_cells", Sum.op)
- override val updateExpressions = Seq(
+ val updateExpressions = Seq(
// TODO: Figure out why this doesn't work. See above.
- //If(IsNull(child), sum , Add(sum, Sum(child))),
+ // If(IsNull(child), sum , Add(sum, Sum(child))),
If(IsNull(child), sum , Add(sum, SumCells(child))),
If(IsNull(child), count, Add(count, DataCellCounts(child)))
)
- override val mergeExpressions = Seq(
- sum.left + sum.right,
- count.left + count.right
- )
+ val mergeExpressions = Seq(sum.left + sum.right, count.left + count.right)
- override val evaluateExpression = sum / new Cast(count, DoubleType)
+ val evaluateExpression = sum / new Cast(count, DoubleType)
- override def dataType = DoubleType
+ def dataType: DataType = DoubleType
}
object CellMeanAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc
/** Computes the column aggregate mean. */
def apply(tile: Column): TypedColumn[Any, Double] =
new Column(new CellMeanAggregate(tile.expr).toAggregateExpression()).as[Double]
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala
index c9acf4ed4..f5bd68d7e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala
@@ -21,10 +21,10 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
import org.locationtech.rasterframes.stats.CellStatistics
-import org.locationtech.rasterframes.TileType
-import geotrellis.raster.{Tile, _}
+import geotrellis.raster._
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, AggregateFunction, AggregateMode, Complete}
import org.apache.spark.sql.catalyst.expressions.{ExprId, Expression, ExpressionDescription, NamedExpression}
import org.apache.spark.sql.execution.aggregate.ScalaUDAF
@@ -40,9 +40,9 @@ import org.apache.spark.sql.{Column, Row, TypedColumn}
case class CellStatsAggregate() extends UserDefinedAggregateFunction {
import CellStatsAggregate.C
// TODO: rewrite as a DeclarativeAggregate
- override def inputSchema: StructType = StructType(StructField("value", TileType) :: Nil)
+ def inputSchema: StructType = StructType(StructField("value", tileUDT) :: Nil)
- override def dataType: DataType = StructType(Seq(
+ def dataType: DataType = StructType(Seq(
StructField("data_cells", LongType),
StructField("no_data_cells", LongType),
StructField("min", DoubleType),
@@ -51,7 +51,7 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
StructField("variance", DoubleType)
))
- override def bufferSchema: StructType = StructType(Seq(
+ def bufferSchema: StructType = StructType(Seq(
StructField("data_cells", LongType),
StructField("no_data_cells", LongType),
StructField("min", DoubleType),
@@ -60,9 +60,9 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
StructField("sumSqr", DoubleType)
))
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
+ def initialize(buffer: MutableAggregationBuffer): Unit = {
buffer(C.COUNT) = 0L
buffer(C.NODATA) = 0L
buffer(C.MIN) = Double.MaxValue
@@ -71,7 +71,7 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
buffer(C.SUM_SQRS) = 0.0
}
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit =
if (!input.isNullAt(0)) {
val tile = input.getAs[Tile](0)
var count = buffer.getLong(C.COUNT)
@@ -98,9 +98,8 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
buffer(C.SUM) = sum
buffer(C.SUM_SQRS) = sumSqr
}
- }
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
buffer1(C.COUNT) = buffer1.getLong(C.COUNT) + buffer2.getLong(C.COUNT)
buffer1(C.NODATA) = buffer1.getLong(C.NODATA) + buffer2.getLong(C.NODATA)
buffer1(C.MIN) = math.min(buffer1.getDouble(C.MIN), buffer2.getDouble(C.MIN))
@@ -109,7 +108,7 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
buffer1(C.SUM_SQRS) = buffer1.getDouble(C.SUM_SQRS) + buffer2.getDouble(C.SUM_SQRS)
}
- override def evaluate(buffer: Row): Any = {
+ def evaluate(buffer: Row): Any = {
val count = buffer.getLong(C.COUNT)
val sum = buffer.getDouble(C.SUM)
val sumSqr = buffer.getDouble(C.SUM_SQRS)
@@ -120,8 +119,6 @@ case class CellStatsAggregate() extends UserDefinedAggregateFunction {
}
object CellStatsAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.cellStatsEncoder
-
def apply(col: Column): TypedColumn[Any, CellStatistics] =
new CellStatsAggregate()(ExtractTile(col))
.as(s"rf_agg_stats($col)")
@@ -142,9 +139,9 @@ object CellStatsAggregate {
|960 |40 |1.0|255.0|127.175|5441.704791666667|
+----------+-------------+---+-----+-------+-----------------+"""
)
- class CellStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId)
- extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new CellStatsAggregate()), Complete, false, NamedExpression.newExprId)
+ class CellStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId)
+ extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new CellStatsAggregate()), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_stats"
}
object CellStatsAggregateUDAF {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala
index e3fe80679..0fcb2f1e6 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala
@@ -21,8 +21,7 @@
package org.locationtech.rasterframes.expressions.aggregates
-import java.nio.ByteBuffer
-
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
import org.locationtech.rasterframes.functions.safeEval
import org.locationtech.rasterframes.stats.CellHistogram
@@ -35,7 +34,8 @@ import org.apache.spark.sql.execution.aggregate.ScalaUDAF
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, Row, TypedColumn}
-import org.locationtech.rasterframes.TileType
+
+import java.nio.ByteBuffer
/**
* Histogram aggregation function for a full column of tiles.
@@ -46,13 +46,13 @@ case class HistogramAggregate(numBuckets: Int) extends UserDefinedAggregateFunct
def this() = this(StreamingHistogram.DEFAULT_NUM_BUCKETS)
// TODO: rewrite as TypedAggregateExpression or similar.
- override def inputSchema: StructType = StructType(StructField("value", TileType) :: Nil)
+ def inputSchema: StructType = StructType(StructField("value", tileUDT) :: Nil)
- override def bufferSchema: StructType = StructType(StructField("buffer", BinaryType) :: Nil)
+ def bufferSchema: StructType = StructType(StructField("buffer", BinaryType) :: Nil)
- override def dataType: DataType = CellHistogram.schema
+ def dataType: DataType = CellHistogram.schema
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
@transient
private lazy val ser = KryoSerializer.ser.newInstance()
@@ -63,17 +63,17 @@ case class HistogramAggregate(numBuckets: Int) extends UserDefinedAggregateFunct
@inline
private def unmarshall(blob: Array[Byte]): Histogram[Double] = ser.deserialize(ByteBuffer.wrap(blob))
- override def initialize(buffer: MutableAggregationBuffer): Unit =
+ def initialize(buffer: MutableAggregationBuffer): Unit =
buffer(0) = marshall(StreamingHistogram(numBuckets))
- private val safeMerge = (h1: Histogram[Double], h2: Histogram[Double]) ⇒ (h1, h2) match {
+ private val safeMerge = (h1: Histogram[Double], h2: Histogram[Double]) => (h1, h2) match {
case (null, null) => null
case (l, null) => l
case (null, r) => r
case (l, r) => l merge r
}
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val tile = input.getAs[Tile](0)
val hist1 = unmarshall(buffer.getAs[Array[Byte]](0))
val hist2 = safeEval(StreamingHistogram.fromTile(_: Tile, numBuckets))(tile)
@@ -81,22 +81,20 @@ case class HistogramAggregate(numBuckets: Int) extends UserDefinedAggregateFunct
buffer(0) = marshall(updatedHist)
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
val hist1 = unmarshall(buffer1.getAs[Array[Byte]](0))
val hist2 = unmarshall(buffer2.getAs[Array[Byte]](0))
val updatedHist = safeMerge(hist1, hist2)
buffer1(0) = marshall(updatedHist)
}
- override def evaluate(buffer: Row): Any = {
+ def evaluate(buffer: Row): Any = {
val hist = unmarshall(buffer.getAs[Array[Byte]](0))
CellHistogram(hist)
}
}
object HistogramAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.cellHistEncoder
-
def apply(col: Column): TypedColumn[Any, CellHistogram] =
apply(col, StreamingHistogram.DEFAULT_NUM_BUCKETS)
@@ -116,9 +114,9 @@ object HistogramAggregate {
> SELECT _FUNC_(tile);
..."""
)
- class HistogramAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId)
- extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new HistogramAggregate()), Complete, false, NamedExpression.newExprId)
+ class HistogramAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId)
+ extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new HistogramAggregate()), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_approx_histogram"
}
object HistogramAggregateUDAF {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala
index 2fd65700d..1db81ed3e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
import org.locationtech.rasterframes.functions.safeBinaryOp
import geotrellis.raster.mapalgebra.local.{Add, Defined, Undefined}
@@ -31,7 +32,6 @@ import org.apache.spark.sql.execution.aggregate.ScalaUDAF
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{Column, Row, TypedColumn}
-import org.locationtech.rasterframes.TileType
/**
* Catalyst aggregate function that counts `NoData` values in a cell-wise fashion.
@@ -42,31 +42,29 @@ import org.locationtech.rasterframes.TileType
class LocalCountAggregate(isData: Boolean) extends UserDefinedAggregateFunction {
private val incCount =
- if (isData) safeBinaryOp((t1: Tile, t2: Tile) ⇒ Add(t1, Defined(t2)))
- else safeBinaryOp((t1: Tile, t2: Tile) ⇒ Add(t1, Undefined(t2)))
+ if (isData) safeBinaryOp((t1: Tile, t2: Tile) => Add(t1, Defined(t2)))
+ else safeBinaryOp((t1: Tile, t2: Tile) => Add(t1, Undefined(t2)))
private val add = safeBinaryOp(Add.apply(_: Tile, _: Tile))
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
- override def inputSchema: StructType = StructType(Seq(
- StructField("value", TileType, true)
+ def inputSchema: StructType = StructType(Seq(
+ StructField("value", tileUDT, true)
))
- override def bufferSchema: StructType = inputSchema
+ def bufferSchema: StructType = inputSchema
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit =
+ def initialize(buffer: MutableAggregationBuffer): Unit =
buffer(0) = null
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val right = input.getAs[Tile](0)
if (right != null) {
if (buffer(0) == null) {
- buffer(0) = (
- if (isData) Defined(right) else Undefined(right)
- ).convert(IntConstantNoDataCellType)
+ buffer(0) = (if (isData) Defined(right) else Undefined(right)).convert(IntConstantNoDataCellType)
} else {
val left = buffer.getAs[Tile](0)
buffer(0) = incCount(left, right)
@@ -74,19 +72,17 @@ class LocalCountAggregate(isData: Boolean) extends UserDefinedAggregateFunction
}
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit =
buffer1(0) = add(buffer1.getAs[Tile](0), buffer2.getAs[Tile](0))
- }
- override def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
+ def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
}
object LocalCountAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder
@ExpressionDescription(
usage = "_FUNC_(tile) - Compute cell-wise count of non-no-data values."
)
- class LocalDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(true)), Complete, false, NamedExpression.newExprId)
+ class LocalDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(true)), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_local_data_cells"
}
object LocalDataCellsUDAF {
@@ -100,8 +96,8 @@ object LocalCountAggregate {
@ExpressionDescription(
usage = "_FUNC_(tile) - Compute cell-wise count of no-data values."
)
- class LocalNoDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(false)), Complete, false, NamedExpression.newExprId)
+ class LocalNoDataCellsUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalCountAggregate(false)), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_local_no_data_cells"
}
object LocalNoDataCellsUDAF {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala
index 0bb23cb9e..c749b1b8f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala
@@ -21,45 +21,42 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.UnaryRasterAggregate
import org.locationtech.rasterframes.expressions.localops.{BiasedAdd, Divide => DivideTiles}
import org.locationtech.rasterframes.expressions.transformers.SetCellType
import geotrellis.raster.Tile
import geotrellis.raster.mapalgebra.local
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionDescription, If, IsNull, Literal}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionDescription, If, IsNull, Literal, ScalaUDF}
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.TileType
-import org.locationtech.rasterframes.expressions.accessors.RealizeTile
+import org.locationtech.rasterframes.expressions.accessors.{ExtractTile, RealizeTile}
@ExpressionDescription(
usage = "_FUNC_(tile) - Computes a new tile contining the mean cell values across all tiles in column.",
- note = "All tiles in the column must be the same size."
+ note = """"
+ All tiles in the column must be the same size.
+ """
)
case class LocalMeanAggregate(child: Expression) extends UnaryRasterAggregate {
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
override def nodeName: String = "rf_agg_local_mean"
- private lazy val count =
- AttributeReference("count", TileType, true)()
- private lazy val sum =
- AttributeReference("sum", TileType, true)()
+ private lazy val count = AttributeReference("count", dataType, true)()
+ private lazy val sum = AttributeReference("sum", dataType, true)()
- override def aggBufferAttributes: Seq[AttributeReference] = Seq(
- count,
- sum
- )
+ def aggBufferAttributes: Seq[AttributeReference] = Seq(count, sum)
- private lazy val Defined = tileOpAsExpression("defined_cells", local.Defined.apply)
+ private lazy val Defined: Expression => ScalaUDF = tileOpAsExpression("defined_cells", local.Defined.apply)
- override lazy val initialValues: Seq[Expression] = Seq(
- Literal.create(null, TileType),
- Literal.create(null, TileType)
+ lazy val initialValues: Seq[Expression] = Seq(
+ Literal.create(null, dataType),
+ Literal.create(null, dataType)
)
- override lazy val updateExpressions: Seq[Expression] = Seq(
+ lazy val updateExpressions: Seq[Expression] = Seq(
If(IsNull(count),
- SetCellType(RealizeTile(Defined(child)), Literal("int32")),
+ SetCellType(RealizeTile(Defined(ExtractTile(child))), Literal("int32")),
If(IsNull(child), count, BiasedAdd(count, Defined(RealizeTile(child))))
),
If(IsNull(sum),
@@ -67,15 +64,13 @@ case class LocalMeanAggregate(child: Expression) extends UnaryRasterAggregate {
If(IsNull(child), sum, BiasedAdd(sum, child))
)
)
- override val mergeExpressions: Seq[Expression] = Seq(
+ val mergeExpressions: Seq[Expression] = Seq(
BiasedAdd(count.left, count.right),
BiasedAdd(sum.left, sum.right)
)
- override lazy val evaluateExpression: Expression = DivideTiles(sum, count)
+ lazy val evaluateExpression: Expression = DivideTiles(sum, count)
}
object LocalMeanAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder
-
def apply(tile: Column): TypedColumn[Any, Tile] =
new Column(new LocalMeanAggregate(tile.expr).toAggregateExpression()).as[Tile]
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala
index 080579633..3d7a52862 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalStatsAggregate.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.aggregates
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
import org.locationtech.rasterframes.functions.safeBinaryOp
import org.locationtech.rasterframes.stats.LocalCellStatistics
@@ -33,7 +34,6 @@ import org.apache.spark.sql.execution.aggregate.ScalaUDAF
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, Row, TypedColumn}
-import org.locationtech.rasterframes.TileType
/**
@@ -44,71 +44,68 @@ import org.locationtech.rasterframes.TileType
class LocalStatsAggregate() extends UserDefinedAggregateFunction {
import LocalStatsAggregate.C
- override def inputSchema: StructType = StructType(Seq(
- StructField("value", TileType, true)
+ def inputSchema: StructType = StructType(Seq(
+ StructField("value", tileUDT, true)
))
- override def dataType: DataType =
+ def dataType: DataType =
StructType(
Seq(
- StructField("count", TileType),
- StructField("min", TileType),
- StructField("max", TileType),
- StructField("mean", TileType),
- StructField("variance", TileType)
+ StructField("count", tileUDT),
+ StructField("min", tileUDT),
+ StructField("max", tileUDT),
+ StructField("mean", tileUDT),
+ StructField("variance", tileUDT)
)
)
- override def bufferSchema: StructType =
+ def bufferSchema: StructType =
StructType(
Seq(
- StructField("count", TileType),
- StructField("min", TileType),
- StructField("max", TileType),
- StructField("sum", TileType),
- StructField("sumSqr", TileType)
+ StructField("count", tileUDT),
+ StructField("min", tileUDT),
+ StructField("max", tileUDT),
+ StructField("sum", tileUDT),
+ StructField("sumSqr", tileUDT)
)
)
private val initFunctions = Seq(
- (t: Tile) ⇒ Defined(t).convert(IntConstantNoDataCellType),
- (t: Tile) ⇒ t,
- (t: Tile) ⇒ t,
- (t: Tile) ⇒ t.convert(DoubleConstantNoDataCellType),
- (t: Tile) ⇒ { val d = t.convert(DoubleConstantNoDataCellType); Multiply(d, d) }
+ (t: Tile) => Defined(t).convert(IntConstantNoDataCellType),
+ (t: Tile) => t,
+ (t: Tile) => t,
+ (t: Tile) => t.convert(DoubleConstantNoDataCellType),
+ (t: Tile) => { val d = t.convert(DoubleConstantNoDataCellType); Multiply(d, d) }
)
private val updateFunctions = Seq(
- safeBinaryOp((agg: Tile, t: Tile) ⇒ BiasedAdd(agg, Defined(t))),
- safeBinaryOp((agg: Tile, t: Tile) ⇒ BiasedMin(agg, t)),
- safeBinaryOp((agg: Tile, t: Tile) ⇒ BiasedMax(agg, t)),
- safeBinaryOp((agg: Tile, t: Tile) ⇒ BiasedAdd(agg, t)),
- safeBinaryOp((agg: Tile, t: Tile) ⇒ {
+ safeBinaryOp((agg: Tile, t: Tile) => BiasedAdd(agg, Defined(t))),
+ safeBinaryOp((agg: Tile, t: Tile) => BiasedMin(agg, t)),
+ safeBinaryOp((agg: Tile, t: Tile) => BiasedMax(agg, t)),
+ safeBinaryOp((agg: Tile, t: Tile) => BiasedAdd(agg, t)),
+ safeBinaryOp((agg: Tile, t: Tile) => {
val d = t.convert(DoubleConstantNoDataCellType)
BiasedAdd(agg, Multiply(d, d))
})
)
private val mergeFunctions = Seq(
- safeBinaryOp((t1: Tile, t2: Tile) ⇒ BiasedAdd(t1, t2)),
+ safeBinaryOp((t1: Tile, t2: Tile) => BiasedAdd(t1, t2)),
updateFunctions(C.MIN),
updateFunctions(C.MAX),
updateFunctions(C.SUM),
- safeBinaryOp((t1: Tile, t2: Tile) ⇒ BiasedAdd(t1, t2))
+ safeBinaryOp((t1: Tile, t2: Tile) => BiasedAdd(t1, t2))
)
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
- for(i ← initFunctions.indices) {
- buffer(i) = null
- }
- }
+ def initialize(buffer: MutableAggregationBuffer): Unit =
+ for(i <- initFunctions.indices) { buffer(i) = null }
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
val right = input.getAs[Tile](0)
if (right != null) {
- for (i ← initFunctions.indices) {
+ for (i <- initFunctions.indices) {
if (buffer.isNullAt(i)) {
buffer(i) = initFunctions(i)(right)
}
@@ -120,8 +117,8 @@ class LocalStatsAggregate() extends UserDefinedAggregateFunction {
}
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- for (i ← mergeFunctions.indices) {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ for (i <- mergeFunctions.indices) {
val left = buffer1.getAs[Tile](i)
val right = buffer2.getAs[Tile](i)
val merged = mergeFunctions(i)(left, right)
@@ -161,9 +158,9 @@ object LocalStatsAggregate {
> SELECT _FUNC_(tile);
..."""
)
- class LocalStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId)
- extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalStatsAggregate()), Complete, false, NamedExpression.newExprId)
+ class LocalStatsAggregateUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId)
+ extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalStatsAggregate()), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_local_stats"
}
object LocalStatsAggregateUDAF {
@@ -179,4 +176,3 @@ object LocalStatsAggregate {
val SUM_SQRS = 4
}
}
-
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala
index bd48f3981..98c9d9180 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalTileOpAggregate.scala
@@ -21,7 +21,7 @@
package org.locationtech.rasterframes.expressions.aggregates
-import org.locationtech.rasterframes.TileType
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
import org.locationtech.rasterframes.functions.safeBinaryOp
import org.locationtech.rasterframes.util.DataBiasedOp.{BiasedMax, BiasedMin}
@@ -43,20 +43,19 @@ class LocalTileOpAggregate(op: LocalTileBinaryOp) extends UserDefinedAggregateFu
private val safeOp = safeBinaryOp(op.apply(_: Tile, _: Tile))
- override def inputSchema: StructType = StructType(Seq(
- StructField("value", TileType, true)
+ def inputSchema: StructType = StructType(Seq(
+ StructField("value", dataType, true)
))
- override def bufferSchema: StructType = inputSchema
+ def bufferSchema: StructType = inputSchema
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit =
- buffer(0) = null
+ def initialize(buffer: MutableAggregationBuffer): Unit = buffer(0) = null
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit =
if (buffer(0) == null) {
buffer(0) = input(0)
} else {
@@ -64,21 +63,18 @@ class LocalTileOpAggregate(op: LocalTileBinaryOp) extends UserDefinedAggregateFu
val t2 = input.getAs[Tile](0)
buffer(0) = safeOp(t1, t2)
}
- }
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = update(buffer1, buffer2)
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = update(buffer1, buffer2)
- override def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
+ def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
}
object LocalTileOpAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Compute cell-wise minimum value from a tile column."
)
- class LocalMinUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMin)), Complete, false, NamedExpression.newExprId)
+ class LocalMinUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMin)), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_local_min"
}
object LocalMinUDAF {
@@ -92,8 +88,8 @@ object LocalTileOpAggregate {
@ExpressionDescription(
usage = "_FUNC_(tile) - Compute cell-wise maximum value from a tile column."
)
- class LocalMaxUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, resultId) {
- def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMax)), Complete, false, NamedExpression.newExprId)
+ class LocalMaxUDAF(aggregateFunction: AggregateFunction, mode: AggregateMode, isDistinct: Boolean, filter: Option[Expression], resultId: ExprId) extends AggregateExpression(aggregateFunction, mode, isDistinct, filter, resultId) {
+ def this(child: Expression) = this(ScalaUDAF(Seq(ExtractTile(child)), new LocalTileOpAggregate(BiasedMax)), Complete, false, None, NamedExpression.newExprId)
override def nodeName: String = "rf_agg_local_max"
}
object LocalMaxUDAF {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala
index ca9c8f58f..363de4505 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala
@@ -22,62 +22,58 @@
package org.locationtech.rasterframes.expressions.aggregates
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.syntax._
import geotrellis.proj4.{CRS, Transform}
import geotrellis.raster._
import geotrellis.raster.reproject.{Reproject, ReprojectRasterExtent}
import geotrellis.layer._
import geotrellis.vector.Extent
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
-import org.apache.spark.sql.types.{DataType, StructField, StructType}
+import org.apache.spark.sql.types.{DataType, StructType}
import org.apache.spark.sql.{Column, Row, TypedColumn}
class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: Dimensions[Int]) extends UserDefinedAggregateFunction {
import ProjectedLayerMetadataAggregate._
- override def inputSchema: StructType = CatalystSerializer[InputRecord].schema
+ def inputSchema: StructType = InputRecord.inputRecordEncoder.schema
- override def bufferSchema: StructType = CatalystSerializer[BufferRecord].schema
+ def bufferSchema: StructType = BufferRecord.bufferRecordEncoder.schema
- override def dataType: DataType = CatalystSerializer[TileLayerMetadata[SpatialKey]].schema
+ def dataType: DataType = tileLayerMetadataEncoder[SpatialKey].schema
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def initialize(buffer: MutableAggregationBuffer): Unit = ()
+ def initialize(buffer: MutableAggregationBuffer): Unit = ()
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
if(!input.isNullAt(0)) {
- val in = input.to[InputRecord]
+ val in = input.as[InputRecord]
if(buffer.isNullAt(0)) {
in.toBufferRecord(destCRS).write(buffer)
- }
- else {
- val br = buffer.to[BufferRecord]
+ } else {
+ val br = buffer.as[BufferRecord]
br.merge(in.toBufferRecord(destCRS)).write(buffer)
}
+
}
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit =
(buffer1.isNullAt(0), buffer2.isNullAt(0)) match {
- case (false, false) ⇒
- val left = buffer1.to[BufferRecord]
- val right = buffer2.to[BufferRecord]
+ case (false, false) =>
+ val left = buffer1.as[BufferRecord]
+ val right = buffer2.as[BufferRecord]
+
left.merge(right).write(buffer1)
- case (true, false) ⇒ buffer2.to[BufferRecord].write(buffer1)
- case _ ⇒ ()
+ case (true, false) => buffer2.as[BufferRecord].write(buffer1)
+ case _ => ()
}
- }
- override def evaluate(buffer: Row): Any = {
- import org.locationtech.rasterframes.encoders.CatalystSerializer._
- val buf = buffer.to[BufferRecord]
-
- if (buf.isEmpty) {
- throw new IllegalArgumentException("Can not collect metadata from empty data frame.")
- }
+ def evaluate(buffer: Row): Any = {
+ val buf = buffer.as[BufferRecord]
+ if (buf.isEmpty) throw new IllegalArgumentException("Can not collect metadata from empty data frame.")
val re = RasterExtent(buf.extent, buf.cellSize)
val layout = LayoutDefinition(re, destDims.cols, destDims.rows)
@@ -88,17 +84,17 @@ class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: Dimensions[Int]) e
}
object ProjectedLayerMetadataAggregate {
- import org.locationtech.rasterframes.encoders.StandardEncoders._
-
/** Primary user facing constructor */
- def apply(destCRS: CRS, extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] =
- // Ordering must match InputRecord schema
+ def apply(destCRS: CRS, extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] =
+ // Ordering must match InputRecord schema
new ProjectedLayerMetadataAggregate(destCRS, Dimensions(NOMINAL_TILE_SIZE, NOMINAL_TILE_SIZE))(extent, crs, cellType, tileSize).as[TileLayerMetadata[SpatialKey]]
- def apply(destCRS: CRS, destDims: Dimensions[Int], extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] =
- // Ordering must match InputRecord schema
+ def apply(destCRS: CRS, destDims: Dimensions[Int], extent: Column, crs: Column, cellType: Column, tileSize: Column): TypedColumn[Any, TileLayerMetadata[SpatialKey]] = {
+ // Ordering must match InputRecord schema
new ProjectedLayerMetadataAggregate(destCRS, destDims)(extent, crs, cellType, tileSize).as[TileLayerMetadata[SpatialKey]]
+ }
+
private[expressions]
case class InputRecord(extent: Extent, crs: CRS, cellType: CellType, tileSize: Dimensions[Int]) {
def toBufferRecord(destCRS: CRS): BufferRecord = {
@@ -119,24 +115,7 @@ object ProjectedLayerMetadataAggregate {
private[expressions]
object InputRecord {
- implicit val serializer: CatalystSerializer[InputRecord] = new CatalystSerializer[InputRecord]{
- override val schema: StructType = StructType(Seq(
- StructField("extent", CatalystSerializer[Extent].schema, false),
- StructField("crs", CatalystSerializer[CRS].schema, false),
- StructField("cellType", CatalystSerializer[CellType].schema, false),
- StructField("tileSize", CatalystSerializer[Dimensions[Int]].schema, false)
- ))
-
- override protected def to[R](t: InputRecord, io: CatalystIO[R]): R =
- throw new IllegalStateException("InputRecord is input only.")
-
- override protected def from[R](t: R, io: CatalystIO[R]): InputRecord = InputRecord(
- io.get[Extent](t, 0),
- io.get[CRS](t, 1),
- io.get[CellType](t, 2),
- io.get[Dimensions[Int]](t, 3)
- )
- }
+ implicit lazy val inputRecordEncoder: ExpressionEncoder[InputRecord] = typedExpressionEncoder[InputRecord]
}
private[expressions]
@@ -149,7 +128,7 @@ object ProjectedLayerMetadataAggregate {
}
def write(buffer: MutableAggregationBuffer): Unit = {
- val encoded = this.toRow
+ val encoded: Row = this.toRow
for(i <- 0 until encoded.size) {
buffer(i) = encoded(i)
}
@@ -160,24 +139,6 @@ object ProjectedLayerMetadataAggregate {
private[expressions]
object BufferRecord {
- implicit val serializer: CatalystSerializer[BufferRecord] = new CatalystSerializer[BufferRecord] {
- override val schema: StructType = StructType(Seq(
- StructField("extent", CatalystSerializer[Extent].schema, true),
- StructField("cellType", CatalystSerializer[CellType].schema, true),
- StructField("cellSize", CatalystSerializer[CellSize].schema, true)
- ))
-
- override protected def to[R](t: BufferRecord, io: CatalystIO[R]): R = io.create(
- io.to(t.extent),
- io.to(t.cellType),
- io.to(t.cellSize)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): BufferRecord = BufferRecord(
- io.get[Extent](t, 0),
- io.get[CellType](t, 1),
- io.get[CellSize](t, 2)
- )
- }
+ implicit lazy val bufferRecordEncoder: ExpressionEncoder[BufferRecord] = typedExpressionEncoder
}
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala
index 7fa0eb71e..446e7aeb2 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala
@@ -31,7 +31,7 @@ import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAg
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{Column, DataFrame, Row, TypedColumn}
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate.ProjectedRasterDefinition
import org.locationtech.rasterframes.util._
import org.slf4j.LoggerFactory
@@ -45,27 +45,26 @@ class TileRasterizerAggregate(prd: ProjectedRasterDefinition) extends UserDefine
val projOpts = Reproject.Options.DEFAULT.copy(method = prd.sampler)
- override def deterministic: Boolean = true
+ def deterministic: Boolean = true
- override def inputSchema: StructType = StructType(Seq(
- StructField("crs", schemaOf[CRS], false),
- StructField("extent", schemaOf[Extent], false),
- StructField("tile", TileType)
+ def inputSchema: StructType = StructType(Seq(
+ StructField("crs", crsUDT, false),
+ StructField("extent", extentEncoder.schema, false),
+ StructField("tile", tileUDT)
))
- override def bufferSchema: StructType = StructType(Seq(
- StructField("tile_buffer", TileType)
+ def bufferSchema: StructType = StructType(Seq(
+ StructField("tile_buffer", tileUDT)
))
- override def dataType: DataType = TileType
+ def dataType: DataType = tileUDT
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
+ def initialize(buffer: MutableAggregationBuffer): Unit =
buffer(0) = ArrayTile.empty(prd.destinationCellType, prd.totalCols, prd.totalRows)
- }
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val crs = input.getAs[Row](0).to[CRS]
- val extent = input.getAs[Row](1).to[Extent]
+ def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
+ val crs: CRS = input.getAs[CRS](0)
+ val extent: Extent = input.getAs[Row](1).as[Extent]
val localExtent = extent.reproject(crs, prd.destinationCRS)
@@ -77,13 +76,13 @@ class TileRasterizerAggregate(prd: ProjectedRasterDefinition) extends UserDefine
}
}
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
+ def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
val leftTile = buffer1.getAs[Tile](0)
val rightTile = buffer2.getAs[Tile](0)
buffer1(0) = leftTile.merge(rightTile)
}
- override def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
+ def evaluate(buffer: Row): Tile = buffer.getAs[Tile](0)
}
object TileRasterizerAggregate {
@@ -134,32 +133,31 @@ object TileRasterizerAggregate {
}
// Scan table and construct what the TileLayerMetadata would be in the specified destination CRS.
- val tlm: TileLayerMetadata[SpatialKey] = df
- .select(
- ProjectedLayerMetadataAggregate(
- destCRS,
- extCol,
- crsCol,
- rf_cell_type(tileCol),
- rf_dimensions(tileCol)
- ))
- .first()
+ val tlm: TileLayerMetadata[SpatialKey] =
+ df
+ .select(
+ ProjectedLayerMetadataAggregate(
+ destCRS,
+ extCol,
+ rf_crs(crsCol),
+ rf_cell_type(tileCol),
+ rf_dimensions(tileCol)
+ )
+ )
+ .first()
+
logger.debug(s"Collected TileLayerMetadata: ${tlm.toString}")
val c = ProjectedRasterDefinition(tlm, Bilinear)
- val config = rasterDims
- .map { dims =>
- c.copy(totalCols = dims.cols, totalRows = dims.rows)
- }
- .getOrElse(c)
+ val config =
+ rasterDims
+ .map { dims => c.copy(totalCols = dims.cols, totalRows = dims.rows) }
+ .getOrElse(c)
- destExtent.map { ext =>
- c.copy(destinationExtent = ext)
- }
+ destExtent.map { ext => c.copy(destinationExtent = ext) }
- val aggs = tileCols
- .map(t => TileRasterizerAggregate(config, crsCol, extCol, rf_tile(t)).as(t.columnName))
+ val aggs = tileCols.map(t => TileRasterizerAggregate(config, rf_crs(crsCol), extCol, rf_tile(t)).as(t.columnName))
val agg = df.select(aggs: _*)
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala
index ef1c51400..7ebbad7cc 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/ExplodeTiles.scala
@@ -37,40 +37,34 @@ import spire.syntax.cfor.cfor
*
* @since 4/12/17
*/
-case class ExplodeTiles(
- sampleFraction: Double , seed: Option[Long], override val children: Seq[Expression])
- extends Expression with Generator with CodegenFallback {
+case class ExplodeTiles(sampleFraction: Double , seed: Option[Long], override val children: Seq[Expression]) extends Expression with Generator with CodegenFallback {
def this(children: Seq[Expression]) = this(1.0, None, children)
+
override def nodeName: String = "rf_explode_tiles"
- override def elementSchema: StructType = {
- val names =
- if (children.size == 1) Seq("cell")
- else children.indices.map(i ⇒ s"cell_$i")
+ def elementSchema: StructType = {
+ val names = if (children.size == 1) Seq("cell") else children.indices.map(i => s"cell_$i")
StructType(
- Seq(
- StructField(COLUMN_INDEX_COLUMN.columnName, IntegerType, false),
- StructField(ROW_INDEX_COLUMN.columnName, IntegerType, false)) ++ names
- .map(n ⇒ StructField(n, DoubleType, false)))
+ Seq(StructField(COLUMN_INDEX_COLUMN.columnName, IntegerType, false), StructField(ROW_INDEX_COLUMN.columnName, IntegerType, false)) ++
+ names.map(n => StructField(n, DoubleType, false))
+ )
}
- private def sample[T](things: Seq[T]) = {
+ private def sample[T](things: Seq[T]): Seq[T] = {
// Apply random seed if provided
- seed.foreach(s ⇒ scala.util.Random.setSeed(s))
+ seed.foreach(s => scala.util.Random.setSeed(s))
scala.util.Random.shuffle(things)
.take(math.ceil(things.length * sampleFraction).toInt)
}
- override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
+ def eval(input: InternalRow): TraversableOnce[InternalRow] = {
val tiles = Array.ofDim[Tile](children.length)
cfor(0)(_ < tiles.length, _ + 1) { index =>
val c = children(index)
val row = c.eval(input).asInstanceOf[InternalRow]
- tiles(index) = if(row != null)
- DynamicExtractors.tileExtractor(c.dataType)(row)._1
- else null
+ tiles(index) = if(row != null) DynamicExtractors.tileExtractor(c.dataType)(row)._1 else null
}
val dims = tiles.filter(_ != null).map(_.dimensions)
if(dims.isEmpty) Seq.empty[InternalRow]
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala
index 7022f75db..e29966854 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala
@@ -28,11 +28,13 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames
import org.locationtech.rasterframes.ref.{RFRasterSource, RasterRef}
import org.locationtech.rasterframes.util._
-import org.locationtech.rasterframes.RasterSourceType
+import org.locationtech.rasterframes.ref.Subgrid
+import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import scala.util.Try
import scala.util.control.NonFatal
@@ -45,48 +47,47 @@ import scala.util.control.NonFatal
case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression
with Generator with CodegenFallback with ExpectsInputTypes {
- override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType)
+ def inputTypes: Seq[DataType] = Seq.fill(children.size)(rasterSourceUDT)
override def nodeName: String = "rf_raster_source_to_raster_ref"
- override def elementSchema: StructType = StructType(for {
+ def elementSchema: StructType = StructType(for {
child <- children
basename = child.name + "_ref"
name <- bandNames(basename, bandIndexes)
- } yield StructField(name, schemaOf[RasterRef], true))
+ } yield StructField(name, RasterRef.rasterRefEncoder.schema, true))
- private def band2ref(src: RFRasterSource, e: Option[(GridBounds[Int], Extent)])(b: Int): RasterRef =
- if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1)) else null
+ private def band2ref(src: RFRasterSource, grid: Option[GridBounds[Int]], extent: Option[Extent])(b: Int): RasterRef =
+ if (b < src.bandCount) RasterRef(src, b, extent, grid.map(Subgrid.apply)) else null
-
- override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
+ def eval(input: InternalRow): TraversableOnce[InternalRow] =
try {
- val refs = children.map { child ⇒
- val src = RasterSourceType.deserialize(child.eval(input))
+ val refs = children.map { child =>
+ // TODO: we're using the UDT here ... which is what we should do ?
+ // what would have serialized it, UDT?
+ val src = rasterSourceUDT.deserialize(child.eval(input))
val srcRE = src.rasterExtent
- subtileDims.map(dims => {
+ subtileDims.map({ dims =>
val subGB = src.layoutBounds(dims)
val subs = subGB.map(gb => (gb, srcRE.extentFor(gb, clamp = true)))
-
- subs.map(p => bandIndexes.map(band2ref(src, Some(p))))
- })
- .getOrElse(Seq(bandIndexes.map(band2ref(src, None))))
+ subs.map{ case (grid, extent) => bandIndexes.map(band2ref(src, Some(grid), Some(extent))) }
+ }).getOrElse(Seq(bandIndexes.map(band2ref(src, None, None))))
}
- refs.transpose.map(ts ⇒ InternalRow(ts.flatMap(_.map(_.toInternalRow)): _*))
+
+ refs.transpose.map(ts => InternalRow(ts.flatMap(_.map((_: RasterRef).toInternalRow)): _*))
}
catch {
- case NonFatal(ex) ⇒
+ case NonFatal(ex) =>
val description = "Error fetching data for one of: " +
- Try(children.map(c => RasterSourceType.deserialize(c.eval(input))))
+ Try(children.map(c => rasterSourceUDT.deserialize(c.eval(input))))
.toOption.toSeq.flatten.mkString(", ")
throw new java.lang.IllegalArgumentException(description, ex)
}
- }
}
object RasterSourceToRasterRefs {
- def apply(rrs: Column*): TypedColumn[Any, RasterRef] = apply(None, Seq(0), rrs: _*)
- def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, RasterRef] =
- new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[RasterRef]
+ def apply(rrs: Column*): TypedColumn[Any, ProjectedRasterTile] = apply(None, Seq(0), rrs: _*)
+ def apply(subtileDims: Option[Dimensions[Int]], bandIndexes: Seq[Int], rrs: Column*): TypedColumn[Any, ProjectedRasterTile] =
+ new Column(new RasterSourceToRasterRefs(rrs.map(_.expr), bandIndexes, subtileDims)).as[ProjectedRasterTile]
private[rasterframes] def bandNames(basename: String, bandIndexes: Seq[Int]): Seq[String] = bandIndexes match {
case Seq() => Seq.empty
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala
index 309d306ae..85a7be8f9 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToTiles.scala
@@ -29,8 +29,8 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types.{DataType, StructField, StructType}
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.rasterframes
-import org.locationtech.rasterframes.RasterSourceType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.expressions.RasterResult
import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import org.locationtech.rasterframes.util._
@@ -45,24 +45,24 @@ import scala.util.control.NonFatal
*
* @since 9/6/18
*/
-case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None) extends Expression
- with Generator with CodegenFallback with ExpectsInputTypes {
+case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int], subtileDims: Option[Dimensions[Int]] = None)
+ extends Expression with RasterResult with Generator with CodegenFallback with ExpectsInputTypes {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
- override def inputTypes: Seq[DataType] = Seq.fill(children.size)(RasterSourceType)
+ def inputTypes: Seq[DataType] = Seq.fill(children.size)(rasterSourceUDT)
override def nodeName: String = "rf_raster_source_to_tiles"
- override def elementSchema: StructType = StructType(for {
+ def elementSchema: StructType = StructType(for {
child <- children
basename = child.name
name <- bandNames(basename, bandIndexes)
- } yield StructField(name, schemaOf[ProjectedRasterTile], true))
+ } yield StructField(name, ProjectedRasterTile.projectedRasterTileEncoder.schema, true))
- override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
+ def eval(input: InternalRow): TraversableOnce[InternalRow] = {
try {
- val tiles = children.map { child ⇒
- val src = RasterSourceType.deserialize(child.eval(input))
+ val tiles = children.map { child =>
+ val src = rasterSourceUDT.deserialize(child.eval(input))
val maxBands = src.bandCount
val allowedBands = bandIndexes.filter(_ < maxBands)
src.readAll(subtileDims.getOrElse(rasterframes.NOMINAL_TILE_DIMS), allowedBands)
@@ -71,11 +71,15 @@ case class RasterSourceToTiles(children: Seq[Expression], bandIndexes: Seq[Int],
case _ => null
})
}
- tiles.transpose.map(ts ⇒ InternalRow(ts.flatMap(_.map(_.toInternalRow)): _*))
+ tiles
+ .transpose
+ .map { ts =>
+ InternalRow(ts.flatMap(_.map { prt => if (prt != null) toInternalRow(prt) else null }): _*)
+ }
}
catch {
- case NonFatal(ex) ⇒
- val payload = Try(children.map(c => RasterSourceType.deserialize(c.eval(input)))).toOption.toSeq.flatten
+ case NonFatal(ex) =>
+ val payload = Try(children.map(c => rasterSourceUDT.deserialize(c.eval(input)))).toOption.toSeq.flatten
logger.error("Error fetching data for one of: " + payload.mkString(", "), ex)
Traversable.empty
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala
index d55860d29..153eeb5fa 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Abs.scala
@@ -39,8 +39,8 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO
)
case class Abs(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_abs"
- override def na: Any = null
- override protected def op(t: Tile): Tile = t.localAbs()
+ def na: Any = null
+ protected def op(t: Tile): Tile = t.localAbs()
}
object Abs {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala
index 883900815..ff23eb646 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.functions.lit
import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
-import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
+import org.locationtech.rasterframes.expressions.DynamicExtractors
@ExpressionDescription(
usage = "_FUNC_(tile, rhs) - Performs cell-wise addition between two tiles or a tile and a scalar.",
@@ -46,9 +46,9 @@ import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
case class Add(left: Expression, right: Expression) extends BinaryLocalRasterOp
with CodegenFallback {
override val nodeName: String = "rf_local_add"
- override protected def op(left: Tile, right: Tile): Tile = left.localAdd(right)
- override protected def op(left: Tile, right: Double): Tile = left.localAdd(right)
- override protected def op(left: Tile, right: Int): Tile = left.localAdd(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localAdd(right)
+ protected def op(left: Tile, right: Double): Tile = left.localAdd(right)
+ protected def op(left: Tile, right: Int): Tile = left.localAdd(right)
override def eval(input: InternalRow): Any = {
if(input == null) null
@@ -57,16 +57,14 @@ case class Add(left: Expression, right: Expression) extends BinaryLocalRasterOp
val r = right.eval(input)
if (l == null && r == null) null
else if (l == null) r
- else if (r == null && tileExtractor.isDefinedAt(right.dataType)) l
+ else if (r == null && DynamicExtractors.tileExtractor.isDefinedAt(right.dataType)) l
else if (r == null) null
else nullSafeEval(l, r)
}
}
}
object Add {
- def apply(left: Column, right: Column): Column =
- new Column(Add(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Add(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Add(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Add(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala
index 412081467..e31dd17eb 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala
@@ -20,6 +20,7 @@
*/
package org.locationtech.rasterframes.expressions.localops
+
import geotrellis.raster.Tile
import org.apache.spark.sql.Column
import org.apache.spark.sql.catalyst.InternalRow
@@ -47,9 +48,9 @@ import org.locationtech.rasterframes.util.DataBiasedOp
case class BiasedAdd(left: Expression, right: Expression) extends BinaryLocalRasterOp
with CodegenFallback {
override val nodeName: String = "rf_local_biased_add"
- override protected def op(left: Tile, right: Tile): Tile = DataBiasedOp.BiasedAdd(left, right)
- override protected def op(left: Tile, right: Double): Tile = DataBiasedOp.BiasedAdd(left, right)
- override protected def op(left: Tile, right: Int): Tile = DataBiasedOp.BiasedAdd(left, right)
+ protected def op(left: Tile, right: Tile): Tile = DataBiasedOp.BiasedAdd(left, right)
+ protected def op(left: Tile, right: Double): Tile = DataBiasedOp.BiasedAdd(left, right)
+ protected def op(left: Tile, right: Int): Tile = DataBiasedOp.BiasedAdd(left, right)
override def eval(input: InternalRow): Any = {
if(input == null) null
@@ -65,9 +66,7 @@ case class BiasedAdd(left: Expression, right: Expression) extends BinaryLocalRas
}
}
object BiasedAdd {
- def apply(left: Column, right: Column): Column =
- new Column(BiasedAdd(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(BiasedAdd(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(BiasedAdd(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(BiasedAdd(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Clamp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Clamp.scala
index 68b3ee516..0b974e230 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Clamp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Clamp.scala
@@ -6,11 +6,9 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
@ExpressionDescription(
usage = "_FUNC_(tile, min, max) - Return the tile with its values limited to a range defined by min and max," +
@@ -21,11 +19,10 @@ import org.locationtech.rasterframes.expressions.row
* min - scalar or tile setting the minimum value for each cell
* max - scalar or tile setting the maximum value for each cell"""
)
-case class Clamp(left: Expression, middle: Expression, right: Expression)
- extends TernaryExpression with CodegenFallback with Serializable {
- override def dataType: DataType = left.dataType
+case class Clamp(left: Expression, middle: Expression, right: Expression) extends TernaryExpression with CodegenFallback with RasterResult with Serializable {
+ def dataType: DataType = left.dataType
- override def children: Seq[Expression] = Seq(left, middle, right)
+ def children: Seq[Expression] = Seq(left, middle, right)
override val nodeName = "rf_local_clamp"
@@ -41,27 +38,23 @@ case class Clamp(left: Expression, middle: Expression, right: Expression)
}
override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (targetTile, targetCtx) = tileExtractor(left.dataType)(row(input1))
val minVal = tileOrNumberExtractor(middle.dataType)(input2)
val maxVal = tileOrNumberExtractor(right.dataType)(input3)
val result = (minVal, maxVal) match {
- case (mn: TileArg, mx: TileArg) ⇒ targetTile.localMin(mx.tile).localMax(mn.tile)
- case (mn: TileArg, mx: IntegerArg) ⇒ targetTile.localMin(mx.value).localMax(mn.tile)
- case (mn: TileArg, mx: DoubleArg) ⇒ targetTile.localMin(mx.value).localMax(mn.tile)
- case (mn: IntegerArg, mx: TileArg) ⇒ targetTile.localMin(mx.tile).localMax(mn.value)
- case (mn: IntegerArg, mx: IntegerArg) ⇒ targetTile.localMin(mx.value).localMax(mn.value)
- case (mn: IntegerArg, mx: DoubleArg) ⇒ targetTile.localMin(mx.value).localMax(mn.value)
- case (mn: DoubleArg, mx: TileArg) ⇒ targetTile.localMin(mx.tile).localMax(mn.value)
- case (mn: DoubleArg, mx: IntegerArg) ⇒ targetTile.localMin(mx.value).localMax(mn.value)
- case (mn: DoubleArg, mx: DoubleArg) ⇒ targetTile.localMin(mx.value).localMax(mn.value)
+ case (mn: TileArg, mx: TileArg) => targetTile.localMin(mx.tile).localMax(mn.tile)
+ case (mn: TileArg, mx: IntegerArg) => targetTile.localMin(mx.value).localMax(mn.tile)
+ case (mn: TileArg, mx: DoubleArg) => targetTile.localMin(mx.value).localMax(mn.tile)
+ case (mn: IntegerArg, mx: TileArg) => targetTile.localMin(mx.tile).localMax(mn.value)
+ case (mn: IntegerArg, mx: IntegerArg) => targetTile.localMin(mx.value).localMax(mn.value)
+ case (mn: IntegerArg, mx: DoubleArg) => targetTile.localMin(mx.value).localMax(mn.value)
+ case (mn: DoubleArg, mx: TileArg) => targetTile.localMin(mx.tile).localMax(mn.value)
+ case (mn: DoubleArg, mx: IntegerArg) => targetTile.localMin(mx.value).localMax(mn.value)
+ case (mn: DoubleArg, mx: DoubleArg) => targetTile.localMin(mx.value).localMax(mn.value)
}
- targetCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, targetCtx)
}
}
@@ -70,5 +63,4 @@ object Clamp {
def apply[N: Numeric](tile: Column, min: N, max: Column): Column = new Column(Clamp(tile.expr, lit(min).expr, max.expr))
def apply[N: Numeric](tile: Column, min: Column, max: N): Column = new Column(Clamp(tile.expr, min.expr, lit(max).expr))
def apply[N: Numeric](tile: Column, min: N, max: N): Column = new Column(Clamp(tile.expr, lit(min).expr, lit(max).expr))
-
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala
index cdfcbaa63..1a7af9b25 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala
@@ -40,8 +40,8 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO
case class Defined(child: Expression) extends UnaryLocalRasterOp
with NullToValue with CodegenFallback {
override def nodeName: String = "rf_local_data"
- override def na: Any = null
- override protected def op(child: Tile): Tile = child.localDefined()
+ def na: Any = null
+ protected def op(child: Tile): Tile = child.localDefined()
}
object Defined{
def apply(tile: Column): Column = new Column(Defined(tile.expr))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala
index 7c9dbce75..f90fb4225 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Divide.scala
@@ -43,14 +43,12 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Divide(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_divide"
- override protected def op(left: Tile, right: Tile): Tile = left.localDivide(right)
- override protected def op(left: Tile, right: Double): Tile = left.localDivide(right)
- override protected def op(left: Tile, right: Int): Tile = left.localDivide(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localDivide(right)
+ protected def op(left: Tile, right: Double): Tile = left.localDivide(right)
+ protected def op(left: Tile, right: Int): Tile = left.localDivide(right)
}
object Divide {
- def apply(left: Column, right: Column): Column =
- new Column(Divide(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Divide(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Divide(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Divide(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala
index 9504bdcee..c1804708f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Equal.scala
@@ -41,15 +41,13 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Equal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_equal"
- override protected def op(left: Tile, right: Tile): Tile = left.localEqual(right)
- override protected def op(left: Tile, right: Double): Tile = left.localEqual(right)
- override protected def op(left: Tile, right: Int): Tile = left.localEqual(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localEqual(right)
+ protected def op(left: Tile, right: Double): Tile = left.localEqual(right)
+ protected def op(left: Tile, right: Int): Tile = left.localEqual(right)
}
object Equal {
- def apply(left: Column, right: Column): Column =
- new Column(Equal(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Equal(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Equal(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Equal(tile.expr, lit(value).expr))
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala
index b1fb4d714..01d45e19d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Exp.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript
import org.apache.spark.sql.types.DataType
import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile}
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Performs cell-wise exponential.",
arguments = """
@@ -42,7 +41,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile}
case class Exp(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_exp"
- override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E)
+ protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E)
override def dataType: DataType = child.dataType
}
@@ -84,11 +83,11 @@ object Exp10 {
case class Exp2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_exp2"
- override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0)
+ protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(2.0)
override def dataType: DataType = child.dataType
}
-object Exp2{
+object Exp2 {
def apply(tile: Column): Column = new Column(Exp2(tile.expr))
}
@@ -105,11 +104,11 @@ object Exp2{
case class ExpM1(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_expm1"
- override protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0)
+ protected def op(tile: Tile): Tile = fpTile(tile).localPowValue(math.E).localSubtract(1.0)
override def dataType: DataType = child.dataType
}
-object ExpM1{
+object ExpM1 {
def apply(tile: Column): Column = new Column(ExpM1(tile.expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala
index ac32e1155..b318329fc 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Greater.scala
@@ -40,15 +40,13 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Greater(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_greater"
- override protected def op(left: Tile, right: Tile): Tile = left.localGreater(right)
- override protected def op(left: Tile, right: Double): Tile = left.localGreater(right)
- override protected def op(left: Tile, right: Int): Tile = left.localGreater(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localGreater(right)
+ protected def op(left: Tile, right: Double): Tile = left.localGreater(right)
+ protected def op(left: Tile, right: Int): Tile = left.localGreater(right)
}
object Greater {
- def apply(left: Column, right: Column): Column =
- new Column(Greater(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Greater(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Greater(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Greater(tile.expr, lit(value).expr))
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala
index b963959bc..e4d1dcfc1 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/GreaterEqual.scala
@@ -41,15 +41,13 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class GreaterEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_greater_equal"
- override protected def op(left: Tile, right: Tile): Tile = left.localGreaterOrEqual(right)
- override protected def op(left: Tile, right: Double): Tile = left.localGreaterOrEqual(right)
- override protected def op(left: Tile, right: Int): Tile = left.localGreaterOrEqual(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localGreaterOrEqual(right)
+ protected def op(left: Tile, right: Double): Tile = left.localGreaterOrEqual(right)
+ protected def op(left: Tile, right: Int): Tile = left.localGreaterOrEqual(right)
}
object GreaterEqual {
- def apply(left: Column, right: Column): Column =
- new Column(GreaterEqual(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(GreaterEqual(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(GreaterEqual(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(GreaterEqual(tile.expr, lit(value).expr))
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala
index ed9e4785f..001688a1c 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Identity.scala
@@ -39,8 +39,8 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO
)
case class Identity(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_identity"
- override def na: Any = null
- override protected def op(t: Tile): Tile = t
+ def na: Any = null
+ protected def op(t: Tile): Tile = t
}
object Identity {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/IsIn.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/IsIn.scala
index 1707aff60..bf1d9d7aa 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/IsIn.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/IsIn.scala
@@ -30,9 +30,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
import org.apache.spark.sql.catalyst.util.ArrayData
-import org.apache.spark.sql.rf.TileUDT
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.DynamicExtractors._
+import org.locationtech.rasterframes.expressions.DynamicExtractors
import org.locationtech.rasterframes.expressions._
@ExpressionDescription(
@@ -47,37 +45,31 @@ import org.locationtech.rasterframes.expressions._
> SELECT _FUNC_(tile, array(lit(33), lit(66), lit(99)));
..."""
)
-case class IsIn(left: Expression, right: Expression) extends BinaryExpression with CodegenFallback {
+case class IsIn(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback {
override val nodeName: String = "rf_local_is_in"
- override def dataType: DataType = left.dataType
+ def dataType: DataType = left.dataType
@transient private lazy val elementType: DataType = right.dataType.asInstanceOf[ArrayType].elementType
override def checkInputDataTypes(): TypeCheckResult =
- if(!tileExtractor.isDefinedAt(left.dataType)) {
+ if(!DynamicExtractors.tileExtractor.isDefinedAt(left.dataType)) {
TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.")
} else right.dataType match {
- case _: ArrayType ⇒ TypeCheckSuccess
- case _ ⇒ TypeCheckFailure(s"Input type '${right.dataType}' does not conform to ArrayType.")
+ case _: ArrayType => TypeCheckSuccess
+ case _ => TypeCheckFailure(s"Input type '${right.dataType}' does not conform to ArrayType.")
}
override protected def nullSafeEval(input1: Any, input2: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
- val (childTile, childCtx) = tileExtractor(left.dataType)(row(input1))
-
+ val (childTile, childCtx) = DynamicExtractors.tileExtractor(left.dataType)(row(input1))
val arr = input2.asInstanceOf[ArrayData].toArray[AnyRef](elementType)
-
- childCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(op(childTile, arr)).toInternalRow
- case None => op(childTile, arr).toInternalRow
- }
-
+ val result = op(childTile, arr)
+ toInternalRow(result, childCtx)
}
protected def op(left: Tile, right: IndexedSeq[AnyRef]): Tile = {
def fn(i: Int): Boolean = right.contains(i)
- IfCell(left, fn(_), 1, 0)
+ IfCell(left, fn(_: Int), 1, 0)
}
}
@@ -92,5 +84,4 @@ object IsIn {
val arrayExpr = array(right.map(lit):_*).expr
new Column(IsIn(left.expr, arrayExpr))
}
-
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala
index 087ac7b45..76543e34e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Less.scala
@@ -40,14 +40,12 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Less(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_less"
- override protected def op(left: Tile, right: Tile): Tile = left.localLess(right)
- override protected def op(left: Tile, right: Double): Tile = left.localLess(right)
- override protected def op(left: Tile, right: Int): Tile = left.localLess(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localLess(right)
+ protected def op(left: Tile, right: Double): Tile = left.localLess(right)
+ protected def op(left: Tile, right: Int): Tile = left.localLess(right)
}
object Less {
- def apply(left: Column, right: Column): Column =
- new Column(Less(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Less(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Less(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Less(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala
index 8a13f6fc8..116b3c712 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/LessEqual.scala
@@ -41,14 +41,12 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class LessEqual(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_less_equal"
- override protected def op(left: Tile, right: Tile): Tile = left.localLessOrEqual(right)
- override protected def op(left: Tile, right: Double): Tile = left.localLessOrEqual(right)
- override protected def op(left: Tile, right: Int): Tile = left.localLessOrEqual(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localLessOrEqual(right)
+ protected def op(left: Tile, right: Double): Tile = left.localLessOrEqual(right)
+ protected def op(left: Tile, right: Int): Tile = left.localLessOrEqual(right)
}
object LessEqual {
- def apply(left: Column, right: Column): Column =
- new Column(LessEqual(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(LessEqual(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(LessEqual(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(LessEqual(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala
index a2249fa2a..c428cc922 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Log.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript
import org.apache.spark.sql.types.DataType
import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile}
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Performs cell-wise natural logarithm.",
arguments = """
@@ -42,7 +41,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile}
case class Log(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "log"
- override protected def op(tile: Tile): Tile = fpTile(tile).localLog()
+ protected def op(tile: Tile): Tile = fpTile(tile).localLog()
override def dataType: DataType = child.dataType
}
@@ -63,7 +62,7 @@ object Log {
case class Log10(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_log10"
- override protected def op(tile: Tile): Tile = fpTile(tile).localLog10()
+ protected def op(tile: Tile): Tile = fpTile(tile).localLog10()
override def dataType: DataType = child.dataType
}
@@ -84,11 +83,11 @@ object Log10 {
case class Log2(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_log2"
- override protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0)
+ protected def op(tile: Tile): Tile = fpTile(tile).localLog() / math.log(2.0)
override def dataType: DataType = child.dataType
}
-object Log2{
+object Log2 {
def apply(tile: Column): Column = new Column(Log2(tile.expr))
}
@@ -105,10 +104,10 @@ object Log2{
case class Log1p(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_log1p"
- override protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog()
+ protected def op(tile: Tile): Tile = fpTile(tile).localAdd(1.0).localLog()
override def dataType: DataType = child.dataType
}
-object Log1p{
+object Log1p {
def apply(tile: Column): Column = new Column(Log1p(tile.expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala
index ed92d329a..b68e49955 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Max.scala
@@ -44,9 +44,9 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
case class Max(left: Expression, right:Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName = "rf_local_max"
- override protected def op(left: Tile, right: Tile): Tile = left.localMax(right)
- override protected def op(left: Tile, right: Double): Tile = left.localMax(right)
- override protected def op(left: Tile, right: Int): Tile = left.localMax(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localMax(right)
+ protected def op(left: Tile, right: Double): Tile = left.localMax(right)
+ protected def op(left: Tile, right: Int): Tile = left.localMax(right)
}
object Max {
def apply(left: Column, right: Column): Column = new Column(Max(left.expr, right.expr))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala
index 769892709..0af8b3117 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Min.scala
@@ -44,9 +44,9 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
case class Min(left: Expression, right:Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName = "rf_local_min"
- override protected def op(left: Tile, right: Tile): Tile = left.localMin(right)
- override protected def op(left: Tile, right: Double): Tile = left.localMin(right)
- override protected def op(left: Tile, right: Int): Tile = left.localMin(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localMin(right)
+ protected def op(left: Tile, right: Double): Tile = left.localMin(right)
+ protected def op(left: Tile, right: Int): Tile = left.localMin(right)
}
object Min {
def apply(left: Column, right: Column): Column = new Column(Min(left.expr, right.expr))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala
index b6c397772..4dc7e8548 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Multiply.scala
@@ -43,13 +43,11 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Multiply(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_multiply"
- override protected def op(left: Tile, right: Tile): Tile = left.localMultiply(right)
- override protected def op(left: Tile, right: Double): Tile = left.localMultiply(right)
- override protected def op(left: Tile, right: Int): Tile = left.localMultiply(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localMultiply(right)
+ protected def op(left: Tile, right: Double): Tile = left.localMultiply(right)
+ protected def op(left: Tile, right: Int): Tile = left.localMultiply(right)
}
object Multiply {
- def apply(left: Column, right: Column): Column =
- new Column(Multiply(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Multiply(tile.expr, lit(value).expr))
+ def apply(left: Column, right: Column): Column = new Column(Multiply(left.expr, right.expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Multiply(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala
index e62ccfc37..f5a312296 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/NormalizedDifference.scala
@@ -31,7 +31,9 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@ExpressionDescription(
usage = "_FUNC_(left, right) - Computes the normalized difference '(left - right) / (left + right)' between two tile columns",
- note = "Common usage includes computing NDVI via red and NIR bands.",
+ note = """"
+ Common usage includes computing NDVI via red and NIR bands.
+ """,
arguments = """
Arguments:
* left - first tile argument
@@ -43,13 +45,12 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
)
case class NormalizedDifference(left: Expression, right: Expression) extends BinaryRasterOp with CodegenFallback {
override val nodeName: String = "rf_normalized_difference"
- override protected def op(left: Tile, right: Tile): Tile = {
+ protected def op(left: Tile, right: Tile): Tile = {
val diff = fpTile(left.localSubtract(right))
val sum = fpTile(left.localAdd(right))
diff.localDivide(sum)
}
}
object NormalizedDifference {
- def apply(left: Column, right: Column): TypedColumn[Any, Tile] =
- new Column(NormalizedDifference(left.expr, right.expr)).as[Tile]
+ def apply(left: Column, right: Column): TypedColumn[Any, Tile] = new Column(NormalizedDifference(left.expr, right.expr)).as[Tile]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala
index ebd6f0943..9bc0d829e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Resample.scala
@@ -23,7 +23,7 @@ package org.locationtech.rasterframes.expressions.localops
import geotrellis.raster.Tile
import geotrellis.raster.resample._
-import geotrellis.raster.resample.{ResampleMethod ⇒ GTResampleMethod, Max ⇒ RMax, Min ⇒ RMin}
+import geotrellis.raster.resample.{Max => RMax, Min => RMin, ResampleMethod => GTResampleMethod}
import org.apache.spark.sql.Column
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
@@ -31,28 +31,24 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, Literal, TernaryExpression}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.{DataType, StringType}
import org.apache.spark.unsafe.types.UTF8String
import org.locationtech.rasterframes.util.ResampleMethod
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.{fpTile, row}
+import org.locationtech.rasterframes.expressions.{RasterResult, fpTile, row}
import org.locationtech.rasterframes.expressions.DynamicExtractors._
-abstract class ResampleBase(left: Expression, right: Expression, method: Expression)
- extends TernaryExpression
- with CodegenFallback with Serializable {
+abstract class ResampleBase(left: Expression, right: Expression, method: Expression) extends TernaryExpression with RasterResult with CodegenFallback with Serializable {
override val nodeName: String = "rf_resample"
- override def dataType: DataType = left.dataType
- override def children: Seq[Expression] = Seq(left, right, method)
+ def dataType: DataType = left.dataType
+ def children: Seq[Expression] = Seq(left, right, method)
def targetFloatIfNeeded(t: Tile, method: GTResampleMethod): Tile =
- method match {
- case NearestNeighbor | Mode | RMax | RMin | Sum ⇒ t
- case _ ⇒ fpTile(t)
- }
+ method match {
+ case NearestNeighbor | Mode | RMax | RMin | Sum => t
+ case _ => fpTile(t)
+ }
// These methods define the core algorithms to be used.
def op(left: Tile, right: Tile, method: GTResampleMethod): Tile =
@@ -72,14 +68,13 @@ abstract class ResampleBase(left: Expression, right: Expression, method: Express
else if (!tileOrNumberExtractor.isDefinedAt(right.dataType)) {
TypeCheckFailure(s"Input type '${right.dataType}' does not conform to a compatible type.")
} else method.dataType match {
- case StringType ⇒ TypeCheckSuccess
- case _ ⇒ TypeCheckFailure(s"Cannot interpret value of type `${method.dataType.simpleString}` for resampling method; please provide a String method name.")
+ case StringType => TypeCheckSuccess
+ case _ => TypeCheckFailure(s"Cannot interpret value of type `${method.dataType.simpleString}` for resampling method; please provide a String method name.")
}
}
override def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
// more copypasta from BinaryLocalRasterOp
- implicit val tileSer = TileUDT.tileSerializer
val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
val methodString = input3.asInstanceOf[UTF8String].toString
@@ -91,16 +86,13 @@ abstract class ResampleBase(left: Expression, right: Expression, method: Express
val result: Tile = tileOrNumberExtractor(right.dataType)(input2) match {
// in this case we expect the left and right contexts to vary. no warnings raised.
- case TileArg(rightTile, _) ⇒ op(leftTile, rightTile, resamplingMethod)
- case DoubleArg(d) ⇒ op(leftTile, d, resamplingMethod)
- case IntegerArg(i) ⇒ op(leftTile, i.toDouble, resamplingMethod)
+ case TileArg(rightTile, _) => op(leftTile, rightTile, resamplingMethod)
+ case DoubleArg(d) => op(leftTile, d, resamplingMethod)
+ case IntegerArg(i) => op(leftTile, i.toDouble, resamplingMethod)
}
// reassemble the leftTile with its context. Note that this operation does not change Extent and CRS
- leftCtx match {
- case Some(ctx) ⇒ ctx.toProjectRasterTile(result).toInternalRow
- case None ⇒ result.toInternalRow
- }
+ toInternalRow(result, leftCtx)
}
override def eval(input: InternalRow): Any = {
@@ -135,8 +127,7 @@ Examples:
> SELECT _FUNC_(tile1, tile2, lit("cubic_spline"));
..."""
)
-case class Resample(left: Expression, factor: Expression, method: Expression)
- extends ResampleBase(left, factor, method)
+case class Resample(left: Expression, factor: Expression, method: Expression) extends ResampleBase(left, factor, method)
object Resample {
def apply(left: Column, right: Column, methodName: String): Column =
@@ -163,16 +154,13 @@ object Resample {
...
> SELECT _FUNC_(tile1, tile2);
...""")
-case class ResampleNearest(tile: Expression, target: Expression)
- extends ResampleBase(tile, target, Literal("nearest")) {
+case class ResampleNearest(tile: Expression, target: Expression) extends ResampleBase(tile, target, Literal("nearest")) {
override val nodeName: String = "rf_resample_nearest"
}
object ResampleNearest {
- def apply(tile: Column, target: Column): Column =
- new Column(ResampleNearest(tile.expr, target.expr))
+ def apply(tile: Column, target: Column): Column = new Column(ResampleNearest(tile.expr, target.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(ResampleNearest(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(ResampleNearest(tile.expr, lit(value).expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala
index 0d0cd036b..90bf4b508 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Round.scala
@@ -37,11 +37,10 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO
> SELECT _FUNC_(tile);
..."""
)
-case class Round(child: Expression) extends UnaryLocalRasterOp
- with NullToValue with CodegenFallback {
+case class Round(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_round"
- override def na: Any = null
- override protected def op(child: Tile): Tile = child.localRound()
+ def na: Any = null
+ protected def op(child: Tile): Tile = child.localRound()
}
object Round{
def apply(tile: Column): Column = new Column(Round(tile.expr))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala
index 3f5ea2d2e..d8e86fb34 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Sqrt.scala
@@ -42,7 +42,7 @@ import org.locationtech.rasterframes.expressions.{UnaryLocalRasterOp, fpTile}
)
case class Sqrt(child: Expression) extends UnaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_sqrt"
- override protected def op(tile: Tile): Tile = fpTile(tile).localPow(0.5)
+ protected def op(tile: Tile): Tile = fpTile(tile).localPow(0.5)
override def dataType: DataType = child.dataType
}
object Sqrt {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala
index bf52c1c9f..645049ce2 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Subtract.scala
@@ -43,14 +43,12 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
)
case class Subtract(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_subtract"
- override protected def op(left: Tile, right: Tile): Tile = left.localSubtract(right)
- override protected def op(left: Tile, right: Double): Tile = left.localSubtract(right)
- override protected def op(left: Tile, right: Int): Tile = left.localSubtract(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localSubtract(right)
+ protected def op(left: Tile, right: Double): Tile = left.localSubtract(right)
+ protected def op(left: Tile, right: Int): Tile = left.localSubtract(right)
}
object Subtract {
- def apply(left: Column, right: Column): Column =
- new Column(Subtract(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Subtract(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Subtract(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Subtract(tile.expr, lit(value).expr))
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala
index f91b54373..fb146451f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala
@@ -37,12 +37,11 @@ import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterO
> SELECT _FUNC_(tile);
..."""
)
-case class Undefined(child: Expression) extends UnaryLocalRasterOp
- with NullToValue with CodegenFallback {
+case class Undefined(child: Expression) extends UnaryLocalRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_local_no_data"
- override def na: Any = null
- override protected def op(child: Tile): Tile = child.localUndefined()
+ def na: Any = null
+ protected def op(child: Tile): Tile = child.localUndefined()
}
-object Undefined{
+object Undefined {
def apply(tile: Column): Column = new Column(Undefined(tile.expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala
index 3443cf35c..2cdc30292 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Unequal.scala
@@ -39,17 +39,15 @@ import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
> SELECT _FUNC_(tile1, tile2);
..."""
)
-case class Unequal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
+case class Unequal(left: Expression, right: Expression) extends BinaryLocalRasterOp with CodegenFallback {
override val nodeName: String = "rf_local_unequal"
- override protected def op(left: Tile, right: Tile): Tile = left.localUnequal(right)
- override protected def op(left: Tile, right: Double): Tile = left.localUnequal(right)
- override protected def op(left: Tile, right: Int): Tile = left.localUnequal(right)
+ protected def op(left: Tile, right: Tile): Tile = left.localUnequal(right)
+ protected def op(left: Tile, right: Double): Tile = left.localUnequal(right)
+ protected def op(left: Tile, right: Int): Tile = left.localUnequal(right)
}
object Unequal {
- def apply(left: Column, right: Column): Column =
- new Column(Unequal(left.expr, right.expr))
+ def apply(left: Column, right: Column): Column = new Column(Unequal(left.expr, right.expr))
- def apply[N: Numeric](tile: Column, value: N): Column =
- new Column(Unequal(tile.expr, lit(value).expr))
+ def apply[N: Numeric](tile: Column, value: N): Column = new Column(Unequal(tile.expr, lit(value).expr))
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Where.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Where.scala
index bdc13568d..9b0a605d9 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Where.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Where.scala
@@ -7,12 +7,10 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
import org.slf4j.LoggerFactory
@ExpressionDescription(
@@ -23,14 +21,13 @@ import org.slf4j.LoggerFactory
* x - tile with cell values to return if condition is true
* y - tile with cell values to return if condition is false"""
)
-case class Where(left: Expression, middle: Expression, right: Expression)
- extends TernaryExpression with CodegenFallback with Serializable {
+case class Where(left: Expression, middle: Expression, right: Expression) extends TernaryExpression with RasterResult with CodegenFallback with Serializable {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
- override def dataType: DataType = middle.dataType
+ def dataType: DataType = middle.dataType
- override def children: Seq[Expression] = Seq(left, middle, right)
+ def children: Seq[Expression] = Seq(left, middle, right)
override val nodeName = "rf_where"
@@ -46,7 +43,6 @@ case class Where(left: Expression, middle: Expression, right: Expression)
}
override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (conditionTile, conditionCtx) = tileExtractor(left.dataType)(row(input1))
val (xTile, xCtx) = tileExtractor(middle.dataType)(row(input2))
val (yTile, yCtx) = tileExtractor(right.dataType)(row(input3))
@@ -60,11 +56,7 @@ case class Where(left: Expression, middle: Expression, right: Expression)
logger.warn(s"Both '${middle}' and '${right}' provided an extent and CRS, but they are different. The former will be used.")
val result = op(conditionTile, xTile, yTile)
-
- xCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, xCtx)
}
def op(condition: Tile, x: Tile, y: Tile): Tile = {
@@ -76,15 +68,15 @@ case class Where(left: Expression, middle: Expression, right: Expression)
def getSet(c: Int, r: Int): Unit = {
(returnTile.cellType.isFloatingPoint, y.cellType.isFloatingPoint) match {
- case (true, true) ⇒ returnTile.setDouble(c, r, y.getDouble(c, r))
- case (true, false) ⇒ returnTile.setDouble(c, r, y.get(c, r))
- case (false, true) ⇒ returnTile.set(c, r, y.getDouble(c, r).toInt)
- case (false, false) ⇒ returnTile.set(c, r, y.get(c, r))
+ case (true, true) => returnTile.setDouble(c, r, y.getDouble(c, r))
+ case (true, false) => returnTile.setDouble(c, r, y.get(c, r))
+ case (false, true) => returnTile.set(c, r, y.getDouble(c, r).toInt)
+ case (false, false) => returnTile.set(c, r, y.get(c, r))
}
}
- cfor(0)(_ < x.rows, _ + 1) { r ⇒
- cfor(0)(_ < x.cols, _ + 1) { c ⇒
+ cfor(0)(_ < x.rows, _ + 1) { r =>
+ cfor(0)(_ < x.cols, _ + 1) { c =>
if(!isCellTrue(condition, c, r)) getSet(c, r)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala
index 8bfda70d6..b570b7b4a 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala
@@ -23,9 +23,11 @@ package org.locationtech.rasterframes
import geotrellis.raster.{DoubleConstantNoDataCellType, Tile}
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.{Expression, ScalaUDF}
import org.apache.spark.sql.catalyst.{InternalRow, ScalaReflection}
import org.apache.spark.sql.rf.VersionShims._
+import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{SQLContext, rf}
import org.locationtech.rasterframes.expressions.accessors._
import org.locationtech.rasterframes.expressions.aggregates.CellCountAggregate.DataCells
@@ -49,11 +51,14 @@ package object expressions {
private[expressions]
def fpTile(t: Tile) = if (t.cellType.isFloatingPoint) t else t.convert(DoubleConstantNoDataCellType)
- /** As opposed to `udf`, this constructs an unwrapped ScalaUDF Expression from a function. */
+ /**
+ * As opposed to `udf`, this constructs an unwrapped ScalaUDF Expression from a function.
+ * This ScalaUDF Expression expects the argument of type A1 to match the return type RT at runtime.
+ */
private[expressions]
- def udfexpr[RT: TypeTag, A1: TypeTag](name: String, f: A1 => RT): Expression => ScalaUDF = (child: Expression) => {
- val ScalaReflection.Schema(dataType, nullable) = ScalaReflection.schemaFor[RT]
- ScalaUDF(f, dataType, Seq(child), Seq(true), nullable = nullable, udfName = Some(name))
+ def udfiexpr[RT: TypeTag, A1: TypeTag](name: String, f: DataType => A1 => RT): Expression => ScalaUDF = (child: Expression) => {
+ val ScalaReflection.Schema(dataType, _) = ScalaReflection.schemaFor[RT]
+ ScalaUDF((row: A1) => f(child.dataType)(row), dataType, Seq(child), Seq(Option(ExpressionEncoder[RT]().resolveAndBind())), udfName = Some(name))
}
def register(sqlContext: SQLContext): Unit = {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala
index a18148db3..a27b78328 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/DataCells.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster._
import org.apache.spark.sql.{Column, TypedColumn}
@@ -39,25 +40,18 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
357"""
)
-case class DataCells(child: Expression) extends UnaryRasterOp
- with CodegenFallback with NullToValue {
+case class DataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue {
override def nodeName: String = "rf_data_cells"
- override def dataType: DataType = LongType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = DataCells.op(tile)
- override def na: Any = 0L
+ def dataType: DataType = LongType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = DataCells.op(tile)
+ def na: Any = 0L
}
object DataCells {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc
- def apply(tile: Column): TypedColumn[Any, Long] =
- new Column(DataCells(tile.expr)).as[Long]
+ def apply(tile: Column): TypedColumn[Any, Long] = new Column(DataCells(tile.expr)).as[Long]
- val op = (tile: Tile) => {
+ val op: Tile => Long = (tile: Tile) => {
var count: Long = 0
- tile.dualForeach(
- z ⇒ if(isData(z)) count = count + 1
- ) (
- z ⇒ if(isData(z)) count = count + 1
- )
+ tile.dualForeach(z => if(isData(z)) count = count + 1)(z => if(isData(z)) count = count + 1)
count
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala
index cd04b1467..1fa187409 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Exists.scala
@@ -5,6 +5,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.isCellTrue
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
@@ -25,19 +26,17 @@ import spire.syntax.cfor.cfor
)
case class Exists(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "exists"
- override def dataType: DataType = BooleanType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile)
+ def dataType: DataType = BooleanType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Exists.op(tile)
}
-object Exists{
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc
-
+object Exists {
def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(Exists(tile.expr)).as[Boolean]
def op(tile: Tile): Boolean = {
- cfor(0)(_ < tile.rows, _ + 1) { r ⇒
- cfor(0)(_ < tile.cols, _ + 1) { c ⇒
+ cfor(0)(_ < tile.rows, _ + 1) { r =>
+ cfor(0)(_ < tile.cols, _ + 1) { c =>
if(tile.cellType.isFloatingPoint) { if(isCellTrue(tile.getDouble(c, r))) return true }
else { if(isCellTrue(tile.get(c, r))) return true }
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala
index a912a8a0b..a49888845 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/ForAll.scala
@@ -5,7 +5,8 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.types._
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.isCellTrue
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
import spire.syntax.cfor.cfor
@@ -25,19 +26,16 @@ import spire.syntax.cfor.cfor
)
case class ForAll(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "for_all"
- override def dataType: DataType = BooleanType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile)
-
+ def dataType: DataType = BooleanType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = ForAll.op(tile)
}
object ForAll {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc
-
def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(ForAll(tile.expr)).as[Boolean]
def op(tile: Tile): Boolean = {
- cfor(0)(_ < tile.rows, _ + 1) { r ⇒
- cfor(0)(_ < tile.cols, _ + 1) { c ⇒
+ cfor(0)(_ < tile.rows, _ + 1) { r =>
+ cfor(0)(_ < tile.cols, _ + 1) { c =>
if (tile.cellType.isFloatingPoint) {
if (!isCellTrue(tile.getDouble(c, r))) return false
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala
index fd855cd39..f796e6019 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/IsNoDataTile.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster._
import org.apache.spark.sql.{Column, TypedColumn}
@@ -42,12 +43,10 @@ import org.locationtech.rasterframes.model.TileContext
case class IsNoDataTile(child: Expression) extends UnaryRasterOp
with CodegenFallback with NullToValue {
override def nodeName: String = "rf_is_no_data_tile"
- override def na: Any = true
- override def dataType: DataType = BooleanType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile.isNoDataTile
+ def na: Any = true
+ def dataType: DataType = BooleanType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = tile.isNoDataTile
}
object IsNoDataTile {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.boolEnc
- def apply(tile: Column): TypedColumn[Any, Boolean] =
- new Column(IsNoDataTile(tile.expr)).as[Boolean]
+ def apply(tile: Column): TypedColumn[Any, Boolean] = new Column(IsNoDataTile(tile.expr)).as[Boolean]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala
index cf47ba14e..2601bc4ae 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/NoDataCells.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster._
import org.apache.spark.sql.{Column, TypedColumn}
@@ -39,25 +40,19 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
12"""
)
-case class NoDataCells(child: Expression) extends UnaryRasterOp
- with CodegenFallback with NullToValue {
+case class NoDataCells(child: Expression) extends UnaryRasterOp with CodegenFallback with NullToValue {
override def nodeName: String = "rf_no_data_cells"
- override def dataType: DataType = LongType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = NoDataCells.op(tile)
- override def na: Any = 0L
+ def dataType: DataType = LongType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = NoDataCells.op(tile)
+ def na: Any = 0L
}
object NoDataCells {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.longEnc
def apply(tile: Column): TypedColumn[Any, Long] =
new Column(NoDataCells(tile.expr)).as[Long]
- val op = (tile: Tile) => {
+ val op: Tile => Long = (tile: Tile) => {
var count: Long = 0
- tile.dualForeach(
- z ⇒ if(isNoData(z)) count = count + 1
- ) (
- z ⇒ if(isNoData(z)) count = count + 1
- )
+ tile.dualForeach(z => if(isNoData(z)) count = count + 1)(z => if(isNoData(z)) count = count + 1)
count
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala
index 096acdab6..9e1861cda 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/Sum.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import geotrellis.raster._
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
@@ -41,18 +42,16 @@ import org.locationtech.rasterframes.model.TileContext
)
case class Sum(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "rf_tile_sum"
- override def dataType: DataType = DoubleType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile)
+ def dataType: DataType = DoubleType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = Sum.op(tile)
}
object Sum {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc
- def apply(tile: Column): TypedColumn[Any, Double] =
- new Column(Sum(tile.expr)).as[Double]
+ def apply(tile: Column): TypedColumn[Any, Double] = new Column(Sum(tile.expr)).as[Double]
- def op = (tile: Tile) => {
+ def op: Tile => Double = (tile: Tile) => {
var sum: Double = 0.0
- tile.foreachDouble(z ⇒ if(isData(z)) sum = sum + z)
+ tile.foreachDouble(z => if(isData(z)) sum = sum + z)
sum
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala
index 96e3d3dcc..567216ac5 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileHistogram.scala
@@ -21,7 +21,6 @@
package org.locationtech.rasterframes.expressions.tilestats
-import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.stats.CellHistogram
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.CatalystTypeConverters
@@ -42,20 +41,18 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
..."""
)
-case class TileHistogram(child: Expression) extends UnaryRasterOp
- with CodegenFallback {
+case class TileHistogram(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "rf_tile_histogram"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
TileHistogram.converter(TileHistogram.op(tile))
- override def dataType: DataType = CellHistogram.schema
+ def dataType: DataType = CellHistogram.schema
}
object TileHistogram {
- def apply(tile: Column): TypedColumn[Any, CellHistogram] =
- new Column(TileHistogram(tile.expr)).as[CellHistogram]
+ def apply(tile: Column): TypedColumn[Any, CellHistogram] = new Column(TileHistogram(tile.expr)).as[CellHistogram]
private lazy val converter = CatalystTypeConverters.createToCatalystConverter(CellHistogram.schema)
/** Single tile histogram. */
- val op = (t: Tile) ⇒ CellHistogram(t)
+ val op: Tile => CellHistogram = (t: Tile) => CellHistogram(t)
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala
index 3204f4aaf..8d3cd285a 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMax.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster.{Tile, isData}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -39,23 +40,20 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
1"""
)
-case class TileMax(child: Expression) extends UnaryRasterOp
- with NullToValue with CodegenFallback {
+case class TileMax(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_tile_max"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile)
- override def dataType: DataType = DoubleType
- override def na: Any = Double.MinValue
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMax.op(tile)
+ def dataType: DataType = DoubleType
+ def na: Any = Double.MinValue
}
-object TileMax {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc
- def apply(tile: Column): TypedColumn[Any, Double] =
- new Column(TileMax(tile.expr)).as[Double]
+object TileMax {
+ def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMax(tile.expr)).as[Double]
/** Find the maximum cell value. */
- val op = (tile: Tile) ⇒ {
+ val op: Tile => Double = (tile: Tile) => {
var max: Double = Double.MinValue
- tile.foreachDouble(z ⇒ if(isData(z)) max = math.max(max, z))
+ tile.foreachDouble(z => if(isData(z)) max = math.max(max, z))
if (max == Double.MinValue) Double.NaN
else max
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala
index 92c833f98..5fb7b1805 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMean.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster.{Tile, isData}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -39,28 +40,20 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
-1"""
)
-case class TileMean(child: Expression) extends UnaryRasterOp
- with NullToValue with CodegenFallback {
+case class TileMean(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_tile_mean"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile)
- override def dataType: DataType = DoubleType
- override def na: Any = Double.NaN
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMean.op(tile)
+ def dataType: DataType = DoubleType
+ def na: Any = Double.NaN
}
object TileMean {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc
-
- def apply(tile: Column): TypedColumn[Any, Double] =
- new Column(TileMean(tile.expr)).as[Double]
+ def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMean(tile.expr)).as[Double]
/** Single tile mean. */
- val op = (t: Tile) ⇒ {
+ val op: Tile => Double = (t: Tile) => {
var sum: Double = 0.0
var count: Long = 0
- t.dualForeach(
- z ⇒ if(isData(z)) { count = count + 1; sum = sum + z }
- ) (
- z ⇒ if(isData(z)) { count = count + 1; sum = sum + z }
- )
+ t.dualForeach(z => if(isData(z)) { count = count + 1; sum = sum + z }) (z => if(isData(z)) { count = count + 1; sum = sum + z })
sum/count
}
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala
index 71fa0194a..66698824e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileMin.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.tilestats
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryRasterOp}
import geotrellis.raster.{Tile, isData}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -39,23 +40,19 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
-1"""
)
-case class TileMin(child: Expression) extends UnaryRasterOp
- with NullToValue with CodegenFallback {
+case class TileMin(child: Expression) extends UnaryRasterOp with NullToValue with CodegenFallback {
override def nodeName: String = "rf_tile_min"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile)
- override def dataType: DataType = DoubleType
- override def na: Any = Double.MaxValue
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = TileMin.op(tile)
+ def dataType: DataType = DoubleType
+ def na: Any = Double.MaxValue
}
object TileMin {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.doubleEnc
-
- def apply(tile: Column): TypedColumn[Any, Double] =
- new Column(TileMin(tile.expr)).as[Double]
+ def apply(tile: Column): TypedColumn[Any, Double] = new Column(TileMin(tile.expr)).as[Double]
/** Find the minimum cell value. */
- val op = (tile: Tile) ⇒ {
+ val op: Tile => Double = (tile: Tile) => {
var min: Double = Double.MaxValue
- tile.foreachDouble(z ⇒ if(isData(z)) min = math.min(min, z))
+ tile.foreachDouble(z => if(isData(z)) min = math.min(min, z))
if (min == Double.MaxValue) Double.NaN
else min
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala
index fac6d330e..2ef501faa 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/tilestats/TileStats.scala
@@ -21,7 +21,6 @@
package org.locationtech.rasterframes.expressions.tilestats
-import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.stats.CellStatistics
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.CatalystTypeConverters
@@ -42,19 +41,17 @@ import org.locationtech.rasterframes.model.TileContext
> SELECT _FUNC_(tile);
..."""
)
-case class TileStats(child: Expression) extends UnaryRasterOp
- with CodegenFallback {
+case class TileStats(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "rf_tile_stats"
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
TileStats.converter(TileStats.op(tile).orNull)
- override def dataType: DataType = CellStatistics.schema
+ def dataType: DataType = CellStatistics.schema
}
object TileStats {
- def apply(tile: Column): TypedColumn[Any, CellStatistics] =
- new Column(TileStats(tile.expr)).as[CellStatistics]
+ def apply(tile: Column): TypedColumn[Any, CellStatistics] = new Column(TileStats(tile.expr)).as[CellStatistics]
private lazy val converter = CatalystTypeConverters.createToCatalystConverter(CellStatistics.schema)
/** Single tile statistics. */
- val op = (t: Tile) ⇒ CellStatistics(t)
+ val op: Tile => Option[CellStatistics] = (t: Tile) => CellStatistics(t)
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/CreateProjectedRaster.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/CreateProjectedRaster.scala
index fc5f639c7..759c14ebf 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/CreateProjectedRaster.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/CreateProjectedRaster.scala
@@ -21,18 +21,17 @@
package org.locationtech.rasterframes.expressions.transformers
-import geotrellis.proj4.CRS
-import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+import org.apache.spark.sql.rf.{CrsUDT, TileUDT}
+import org.locationtech.rasterframes.encoders._
@ExpressionDescription(
usage = "_FUNC_(extent, crs, tile) - Construct a `proj_raster` structure from individual CRS, Extent, and Tile columns",
@@ -42,30 +41,34 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile
* crs - crs component of `proj_raster`
* tile - tile component of `proj_raster`"""
)
-case class CreateProjectedRaster(tile: Expression, extent: Expression, crs: Expression) extends TernaryExpression with CodegenFallback {
+case class CreateProjectedRaster(tile: Expression, extent: Expression, crs: Expression) extends TernaryExpression with RasterResult with CodegenFallback {
override def nodeName: String = "rf_proj_raster"
- override def children: Seq[Expression] = Seq(tile, extent, crs)
+ def children: Seq[Expression] = Seq(tile, extent, crs)
- override def dataType: DataType = schemaOf[ProjectedRasterTile]
+ def dataType: DataType = ProjectedRasterTile.projectedRasterTileEncoder.schema
- override def checkInputDataTypes(): TypeCheckResult = {
+ override def checkInputDataTypes(): TypeCheckResult =
if (!tileExtractor.isDefinedAt(tile.dataType)) {
TypeCheckFailure(s"Column of type '${tile.dataType}' is not or does not have a Tile")
}
- else if (!extent.dataType.conformsTo[Extent]) {
+ else if (!extent.dataType.conformsToSchema(StandardEncoders.extentEncoder.schema)) {
TypeCheckFailure(s"Column of type '${extent.dataType}' is not an Extent")
}
- else if (!crs.dataType.conformsTo[CRS]) {
+ else if (!crs.dataType.isInstanceOf[CrsUDT]) {
TypeCheckFailure(s"Column of type '${crs.dataType}' is not a CRS")
}
else TypeCheckSuccess
- }
+
+ private lazy val extentDeser = StandardEncoders.extentEncoder.resolveAndBind().createDeserializer()
+ private lazy val crsUdt = new CrsUDT
+ private lazy val tileUdt = new TileUDT
override protected def nullSafeEval(tileInput: Any, extentInput: Any, crsInput: Any): Any = {
- val e = row(extentInput).to[Extent]
- val c = row(crsInput).to[CRS]
- val (t, _) = tileExtractor(tile.dataType)(row(tileInput))
- ProjectedRasterTile(t, e, c).toInternalRow
+ val e = extentDeser.apply(row(extentInput))
+ val c = crsUdt.deserialize(crsInput)
+ val t = tileUdt.deserialize(tileInput)
+ val prt = ProjectedRasterTile(t, e, c)
+ toInternalRow(prt)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala
index 54201152e..5f54506df 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala
@@ -28,16 +28,16 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescript
import org.apache.spark.sql.types.{DataType, StringType}
import org.apache.spark.sql.{Column, TypedColumn}
import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
import spire.syntax.cfor.cfor
-abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp
- with CodegenFallback with Serializable {
+abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp with CodegenFallback with Serializable {
import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix
- override def dataType: DataType = StringType
+ def dataType: DataType = StringType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
UTF8String.fromString(if (asciiArt)
s"\n${tile.renderAscii(AsciiArtEncoder.Palette.NARROW)}\n"
else
@@ -47,8 +47,6 @@ abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp
}
object DebugRender {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.stringEnc
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Coverts the contents of the given tile an ASCII art string rendering",
arguments = """
@@ -59,8 +57,7 @@ object DebugRender {
override def nodeName: String = "rf_render_ascii"
}
object RenderAscii {
- def apply(tile: Column): TypedColumn[Any, String] =
- new Column(RenderAscii(tile.expr)).as[String]
+ def apply(tile: Column): TypedColumn[Any, String] = new Column(RenderAscii(tile.expr)).as[String]
}
@ExpressionDescription(
@@ -73,8 +70,7 @@ object DebugRender {
override def nodeName: String = "rf_render_matrix"
}
object RenderMatrix {
- def apply(tile: Column): TypedColumn[Any, String] =
- new Column(RenderMatrix(tile.expr)).as[String]
+ def apply(tile: Column): TypedColumn[Any, String] = new Column(RenderMatrix(tile.expr)).as[String]
}
implicit class TileAsMatrix(val tile: Tile) extends AnyVal {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala
index 37b8a3c6c..e90c7046d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtentToGeometry.scala
@@ -21,10 +21,8 @@
package org.locationtech.rasterframes.expressions.transformers
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.{DynamicExtractors, row}
-import org.locationtech.jts.geom.{Envelope, Geometry}
-import geotrellis.vector.Extent
+import org.locationtech.jts.geom.Geometry
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
@@ -33,6 +31,7 @@ import org.apache.spark.sql.jts.JTSTypes
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
+import org.locationtech.rasterframes.encoders.StandardEncoders
/**
* Catalyst Expression for converting a bounding box structure into a JTS Geometry type.
@@ -40,14 +39,14 @@ import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
* @since 8/24/18
*/
case class ExtentToGeometry(child: Expression) extends UnaryExpression with CodegenFallback {
- override def nodeName: String = "st_geometry"
+ override def nodeName: String = "st_geometry"
- override def dataType: DataType = JTSTypes.GeometryTypeInstance
+ def dataType: DataType = JTSTypes.GeometryTypeInstance
override def checkInputDataTypes(): TypeCheckResult = {
if (!DynamicExtractors.extentExtractor.isDefinedAt(child.dataType)) {
TypeCheckFailure(
- s"Expected bounding box of form '${schemaOf[Envelope]}' or '${schemaOf[Extent]}' " +
+ s"Expected bounding box of form '${StandardEncoders.envelopeEncoder.schema}' or '${StandardEncoders.extentEncoder.schema}' " +
s"but received '${child.dataType.simpleString}'."
)
}
@@ -63,6 +62,5 @@ case class ExtentToGeometry(child: Expression) extends UnaryExpression with Code
}
object ExtentToGeometry extends SpatialEncoders {
- def apply(bounds: Column): TypedColumn[Any, Geometry] =
- new Column(new ExtentToGeometry(bounds.expr)).as[Geometry]
+ def apply(bounds: Column): TypedColumn[Any, Geometry] = new Column(new ExtentToGeometry(bounds.expr)).as[Geometry]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala
index 4ba658baa..661e3a087 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ExtractBits.scala
@@ -27,9 +27,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions._
@@ -46,12 +44,12 @@ import org.locationtech.rasterframes.expressions._
> SELECT _FUNC_(tile, lit(4), lit(2))
..."""
)
-case class ExtractBits(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with CodegenFallback with Serializable {
+case class ExtractBits(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with CodegenFallback with RasterResult with Serializable {
override val nodeName: String = "rf_local_extract_bits"
- override def children: Seq[Expression] = Seq(child1, child2, child3)
+ def children: Seq[Expression] = Seq(child1, child2, child3)
- override def dataType: DataType = child1.dataType
+ def dataType: DataType = child1.dataType
override def checkInputDataTypes(): TypeCheckResult =
if(!tileExtractor.isDefinedAt(child1.dataType)) {
@@ -64,21 +62,14 @@ case class ExtractBits(child1: Expression, child2: Expression, child3: Expressio
override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (childTile, childCtx) = tileExtractor(child1.dataType)(row(input1))
-
val startBits = intArgExtractor(child2.dataType)(input2).value
-
val numBits = intArgExtractor(child2.dataType)(input3).value
-
- childCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(op(childTile, startBits, numBits)).toInternalRow
- case None => op(childTile, startBits, numBits).toInternalRow
- }
+ val result = op(childTile, startBits, numBits)
+ toInternalRow(result,childCtx)
}
protected def op(tile: Tile, startBit: Int, numBits: Int): Tile = ExtractBits(tile, startBit, numBits)
-
}
object ExtractBits{
@@ -90,7 +81,7 @@ object ExtractBits{
// this is the last `numBits` positions of "111111111111111"
val widthMask = Int.MaxValue >> (63 - numBits)
// map preserving the nodata structure
- tile.mapIfSet(x ⇒ x >> startBit & widthMask)
+ tile.mapIfSet(x => x >> startBit & widthMask)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala
index adb52468b..410f9168c 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/GeometryToExtent.scala
@@ -21,7 +21,6 @@
package org.locationtech.rasterframes.expressions.transformers
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
@@ -30,6 +29,8 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression}
import org.apache.spark.sql.jts.{AbstractGeometryUDT, JTSTypes}
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.syntax._
/**
* Catalyst Expression for getting the extent of a geometry.
@@ -39,12 +40,12 @@ import org.apache.spark.sql.{Column, TypedColumn}
case class GeometryToExtent(child: Expression) extends UnaryExpression with CodegenFallback {
override def nodeName: String = "st_extent"
- override def dataType: DataType = schemaOf[Extent]
+ def dataType: DataType = extentEncoder.schema
override def checkInputDataTypes(): TypeCheckResult = {
child.dataType match {
- case _: AbstractGeometryUDT[_] ⇒ TypeCheckSuccess
- case o ⇒ TypeCheckFailure(
+ case _: AbstractGeometryUDT[_] => TypeCheckSuccess
+ case o => TypeCheckFailure(
s"Expected geometry but received '${o.simpleString}'."
)
}
@@ -52,14 +53,10 @@ case class GeometryToExtent(child: Expression) extends UnaryExpression with Code
override protected def nullSafeEval(input: Any): Any = {
val geom = JTSTypes.GeometryTypeInstance.deserialize(input)
- val extent = Extent(geom.getEnvelopeInternal)
- extent.toInternalRow
+ Extent(geom.getEnvelopeInternal).toInternalRow
}
}
object GeometryToExtent {
- import org.locationtech.rasterframes.encoders.StandardEncoders._
-
- def apply(bounds: Column): TypedColumn[Any, Extent] =
- new Column(new GeometryToExtent(bounds.expr)).as[Extent]
+ def apply(bounds: Column): TypedColumn[Any, Extent] = new Column(new GeometryToExtent(bounds.expr)).as[Extent]
}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/InterpretAs.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/InterpretAs.scala
index 169f84b33..678df26ab 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/InterpretAs.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/InterpretAs.scala
@@ -29,12 +29,13 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.{TileUDT}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.encoders.syntax._
+import org.locationtech.rasterframes.expressions.DynamicExtractors
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
@ExpressionDescription(
usage = "_FUNC_(tile, value) - Change the interpretation of the Tile's cell values according to specified CellType",
@@ -47,20 +48,19 @@ import org.locationtech.rasterframes.expressions.row
> SELECT _FUNC_(tile, 'int16ud0');
..."""
)
-case class InterpretAs(tile: Expression, cellType: Expression)
- extends BinaryExpression with CodegenFallback {
- def left = tile
- def right = cellType
+case class InterpretAs(tile: Expression, cellType: Expression) extends BinaryExpression with RasterResult with CodegenFallback {
+ def left: Expression = tile
+ def right: Expression = cellType
override def nodeName: String = "rf_interpret_cell_type_as"
- override def dataType: DataType = left.dataType
+ def dataType: DataType = left.dataType
override def checkInputDataTypes(): TypeCheckResult = {
- if (!tileExtractor.isDefinedAt(left.dataType))
+ if (!DynamicExtractors.tileExtractor.isDefinedAt(left.dataType))
TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.")
else
right.dataType match {
case StringType => TypeCheckSuccess
- case t if t.conformsTo[CellType] => TypeCheckSuccess
+ case t if t.conformsToSchema(cellTypeEncoder.schema) => TypeCheckSuccess
case _ =>
TypeCheckFailure(s"Expected CellType but received '${right.dataType.simpleString}'")
}
@@ -71,30 +71,20 @@ case class InterpretAs(tile: Expression, cellType: Expression)
case StringType =>
val text = datum.asInstanceOf[UTF8String].toString
CellType.fromName(text)
- case st if st.conformsTo[CellType] =>
- row(datum).to[CellType]
+ case st if st.conformsToSchema(cellTypeEncoder.schema) => row(datum).as[CellType]
}
}
override protected def nullSafeEval(tileInput: Any, ctInput: Any): InternalRow = {
- implicit val tileSer = TileUDT.tileSerializer
-
- val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput))
+ val (tile, ctx) = DynamicExtractors.tileExtractor(left.dataType)(row(tileInput))
val ct = toCellType(ctInput)
val result = tile.interpretAs(ct)
-
- ctx match {
- case Some(c) => c.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, ctx)
}
}
object InterpretAs{
- def apply(tile: Column, cellType: CellType): Column =
- new Column(new InterpretAs(tile.expr, lit(cellType.name).expr))
- def apply(tile: Column, cellType: String): Column =
- new Column(new InterpretAs(tile.expr, lit(cellType).expr))
- def apply(tile: Column, cellType: Column): Column =
- new Column(new InterpretAs(tile.expr, cellType.expr))
+ def apply(tile: Column, cellType: CellType): Column = new Column(new InterpretAs(tile.expr, lit(cellType.name).expr))
+ def apply(tile: Column, cellType: String): Column = new Column(new InterpretAs(tile.expr, lit(cellType).expr))
+ def apply(tile: Column, cellType: Column): Column = new Column(new InterpretAs(tile.expr, cellType.expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala
index 625183bdc..9f528cb92 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Mask.scala
@@ -24,18 +24,17 @@ package org.locationtech.rasterframes.expressions.transformers
import com.typesafe.scalalogging.Logger
import geotrellis.raster
import geotrellis.raster.{NoNoData, Tile}
-import geotrellis.raster.mapalgebra.local.{Undefined, InverseMask ⇒ gtInverseMask, Mask ⇒ gtMask}
+import geotrellis.raster.mapalgebra.local.{Undefined, InverseMask => gtInverseMask, Mask => gtMask}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, Literal, TernaryExpression}
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions.localops.IsIn
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
import org.slf4j.LoggerFactory
/** Convert cells in the `left` to NoData based on another tile's contents
@@ -47,17 +46,17 @@ import org.slf4j.LoggerFactory
* @param inverse if true, and defined is true, set `left` to NoData where `middle` is NOT nodata
*/
abstract class Mask(val left: Expression, val middle: Expression, val right: Expression, undefined: Boolean, inverse: Boolean)
- extends TernaryExpression with CodegenFallback with Serializable {
+ extends TernaryExpression with RasterResult with CodegenFallback with Serializable {
// aliases.
- def targetExp = left
- def maskExp = middle
- def maskValueExp = right
+ def targetExp: Expression = left
+ def maskExp: Expression = middle
+ def maskValueExp: Expression = right
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
- override def children: Seq[Expression] = Seq(left, middle, right)
+ def children: Seq[Expression] = Seq(left, middle, right)
- override def checkInputDataTypes(): TypeCheckResult = {
+ override def checkInputDataTypes(): TypeCheckResult =
if (!tileExtractor.isDefinedAt(targetExp.dataType)) {
TypeCheckFailure(s"Input type '${targetExp.dataType}' does not conform to a raster type.")
} else if (!tileExtractor.isDefinedAt(maskExp.dataType)) {
@@ -65,13 +64,12 @@ abstract class Mask(val left: Expression, val middle: Expression, val right: Exp
} else if (!intArgExtractor.isDefinedAt(maskValueExp.dataType)) {
TypeCheckFailure(s"Input type '${maskValueExp.dataType}' isn't an integral type.")
} else TypeCheckSuccess
- }
- override def dataType: DataType = left.dataType
+
+ def dataType: DataType = left.dataType
override def makeCopy(newArgs: Array[AnyRef]): Expression = super.makeCopy(newArgs)
override protected def nullSafeEval(targetInput: Any, maskInput: Any, maskValueInput: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (targetTile, targetCtx) = tileExtractor(targetExp.dataType)(row(targetInput))
require(! targetTile.cellType.isInstanceOf[NoNoData],
@@ -95,20 +93,12 @@ abstract class Mask(val left: Expression, val middle: Expression, val right: Exp
else maskTile.localEqual(maskValue.value) // Otherwise if `maskTile` locations equal `maskValue`, set location to ND
// apply the `masking` where values are 1 set to ND (possibly inverted!)
- val result = if (inverse)
- gtInverseMask(targetTile, masking, 1, raster.NODATA)
- else
- gtMask(targetTile, masking, 1, raster.NODATA)
-
- targetCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ val result = if (inverse) gtInverseMask(targetTile, masking, 1, raster.NODATA) else gtMask(targetTile, masking, 1, raster.NODATA)
+
+ toInternalRow(result, targetCtx)
}
}
object Mask {
- import org.locationtech.rasterframes.encoders.StandardEncoders.singlebandTileEncoder
-
@ExpressionDescription(
usage = "_FUNC_(target, mask) - Generate a tile with the values from the data tile, but where cells in the masking tile contain NODATA, replace the data value with NODATA.",
arguments = """
@@ -120,8 +110,7 @@ object Mask {
> SELECT _FUNC_(target, mask);
..."""
)
- case class MaskByDefined(target: Expression, mask: Expression)
- extends Mask(target, mask, Literal(0), true, false) {
+ case class MaskByDefined(target: Expression, mask: Expression) extends Mask(target, mask, Literal(0), true, false) {
override def nodeName: String = "rf_mask"
}
object MaskByDefined {
@@ -140,8 +129,7 @@ object Mask {
> SELECT _FUNC_(target, mask);
..."""
)
- case class InverseMaskByDefined(leftTile: Expression, rightTile: Expression)
- extends Mask(leftTile, rightTile, Literal(0), true, true) {
+ case class InverseMaskByDefined(leftTile: Expression, rightTile: Expression) extends Mask(leftTile, rightTile, Literal(0), true, true) {
override def nodeName: String = "rf_inverse_mask"
}
object InverseMaskByDefined {
@@ -160,8 +148,7 @@ object Mask {
> SELECT _FUNC_(target, mask, maskValue);
..."""
)
- case class MaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression)
- extends Mask(leftTile, rightTile, maskValue, false, false) {
+ case class MaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression) extends Mask(leftTile, rightTile, maskValue, false, false) {
override def nodeName: String = "rf_mask_by_value"
}
object MaskByValue {
@@ -182,8 +169,7 @@ object Mask {
> SELECT _FUNC_(target, mask, maskValue);
..."""
)
- case class InverseMaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression)
- extends Mask(leftTile, rightTile, maskValue, false, true) {
+ case class InverseMaskByValue(leftTile: Expression, rightTile: Expression, maskValue: Expression) extends Mask(leftTile, rightTile, maskValue, false, true) {
override def nodeName: String = "rf_inverse_mask_by_value"
}
object InverseMaskByValue {
@@ -204,8 +190,7 @@ object Mask {
> SELECT _FUNC_(data, mask, array(1, 2, 3))
..."""
)
- case class MaskByValues(dataTile: Expression, maskTile: Expression)
- extends Mask(dataTile, maskTile, Literal(1), false, false) {
+ case class MaskByValues(dataTile: Expression, maskTile: Expression) extends Mask(dataTile, maskTile, Literal(1), false, false) {
def this(dataTile: Expression, maskTile: Expression, maskValues: Expression) =
this(dataTile, IsIn(maskTile, maskValues))
override def nodeName: String = "rf_mask_by_values"
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RGBComposite.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RGBComposite.scala
index 5b266dd06..71a580b6f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RGBComposite.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RGBComposite.scala
@@ -27,12 +27,10 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
/**
* Expression to combine the given tile columns into an 32-bit RGB composite.
@@ -49,19 +47,17 @@ import org.locationtech.rasterframes.expressions.row
* green - tile column representing the green channel
* blue - tile column representing the blue channel"""
)
-case class RGBComposite(red: Expression, green: Expression, blue: Expression) extends TernaryExpression
- with CodegenFallback {
+case class RGBComposite(red: Expression, green: Expression, blue: Expression) extends TernaryExpression with RasterResult with CodegenFallback {
override def nodeName: String = "rf_rgb_composite"
- override def dataType: DataType = if(
+ def dataType: DataType = if(
tileExtractor.isDefinedAt(red.dataType) ||
tileExtractor.isDefinedAt(green.dataType) ||
tileExtractor.isDefinedAt(blue.dataType)
- ) red.dataType
- else TileType
+ ) red.dataType else tileUDT
- override def children: Seq[Expression] = Seq(red, green, blue)
+ def children: Seq[Expression] = Seq(red, green, blue)
override def checkInputDataTypes(): TypeCheckResult = {
if (!tileExtractor.isDefinedAt(red.dataType)) {
@@ -84,14 +80,11 @@ case class RGBComposite(red: Expression, green: Expression, blue: Expression) ex
// Pick the first available TileContext, if any, and reassociate with the result
val ctx = Seq(rc, gc, bc).flatten.headOption
val composite = ArrayMultibandTile(
- r.rescale(0, 255), g.rescale(0, 255), b.rescale(0, 255)
+ r.rescale(0, 255),
+ g.rescale(0, 255),
+ b.rescale(0, 255)
).color()
- ctx match {
- case Some(c) => c.toProjectRasterTile(composite).toInternalRow
- case None =>
- implicit val tileSer = TileUDT.tileSerializer
- composite.toInternalRow
- }
+ toInternalRow(composite, ctx)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala
index 3c699099a..7c0fb4ba2 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RasterRefToTile.scala
@@ -22,13 +22,12 @@
package org.locationtech.rasterframes.expressions.transformers
import com.typesafe.scalalogging.Logger
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression}
-import org.apache.spark.sql.rf._
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.ref.RasterRef
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import org.slf4j.LoggerFactory
@@ -45,14 +44,14 @@ case class RasterRefToTile(child: Expression) extends UnaryExpression
override def nodeName: String = "raster_ref_to_tile"
- override def inputTypes = Seq(schemaOf[RasterRef])
+ def inputTypes = Seq(RasterRef.rasterRefEncoder.schema)
- override def dataType: DataType = schemaOf[ProjectedRasterTile]
+ def dataType: DataType = ProjectedRasterTile.projectedRasterTileEncoder.schema
override protected def nullSafeEval(input: Any): Any = {
- implicit val ser = TileUDT.tileSerializer
- val ref = row(input).to[RasterRef]
- ref.tile.toInternalRow
+ // TODO: how is this different from RealizeTile expression, what work does it do for us? should it make tiles literal?
+ val ref = input.asInstanceOf[InternalRow].as[RasterRef]
+ ProjectedRasterTile(ref.tile, ref.extent, ref.crs).toInternalRow
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala
index 144a4abb6..a896a4342 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/RenderPNG.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.types.{BinaryType, DataType}
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
@@ -36,16 +37,14 @@ import org.locationtech.rasterframes.model.TileContext
* @param ramp color ramp to use for non-composite tiles.
*/
abstract class RenderPNG(child: Expression, ramp: Option[ColorRamp]) extends UnaryRasterOp with CodegenFallback with Serializable {
- override def dataType: DataType = BinaryType
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
+ def dataType: DataType = BinaryType
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
val png = ramp.map(tile.renderPng).getOrElse(tile.renderPng())
png.bytes
}
}
object RenderPNG {
- import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
-
@ExpressionDescription(
usage = "_FUNC_(tile) - Encode the given tile into a RGB composite PNG. Assumes the red, green, and " +
"blue channels are encoded as 8-bit channels within the 32-bit word.",
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala
index 4d8cb2a56..71c7800a4 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/ReprojectGeometry.scala
@@ -22,7 +22,6 @@
package org.locationtech.rasterframes.expressions.transformers
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.encoders.serialized_literal
import org.locationtech.jts.geom.Geometry
import geotrellis.proj4.CRS
@@ -50,13 +49,12 @@ import org.locationtech.rasterframes.model.LazyCRS
> SELECT _FUNC_(geom, srcCRS, dstCRS);
..."""
)
-case class ReprojectGeometry(geometry: Expression, srcCRS: Expression, dstCRS: Expression) extends Expression
- with CodegenFallback {
+case class ReprojectGeometry(geometry: Expression, srcCRS: Expression, dstCRS: Expression) extends Expression with CodegenFallback {
override def nodeName: String = "st_reproject"
- override def dataType: DataType = JTSTypes.GeometryTypeInstance
- override def nullable: Boolean = geometry.nullable || srcCRS.nullable || dstCRS.nullable
- override def children: Seq[Expression] = Seq(geometry, srcCRS, dstCRS)
+ def dataType: DataType = JTSTypes.GeometryTypeInstance
+ def nullable: Boolean = geometry.nullable || srcCRS.nullable || dstCRS.nullable
+ def children: Seq[Expression] = Seq(geometry, srcCRS, dstCRS)
override def checkInputDataTypes(): TypeCheckResult = {
if (!geometry.dataType.isInstanceOf[AbstractGeometryUDT[_]])
@@ -69,13 +67,13 @@ case class ReprojectGeometry(geometry: Expression, srcCRS: Expression, dstCRS: E
}
/** Reprojects a geometry column from one CRS to another. */
- val reproject: (Geometry, CRS, CRS) ⇒ Geometry =
- (sourceGeom, src, dst) ⇒ {
+ val reproject: (Geometry, CRS, CRS) => Geometry =
+ (sourceGeom, src, dst) => {
val trans = new ReprojectionTransformer(src, dst)
trans.transform(sourceGeom)
}
- override def eval(input: InternalRow): Any = {
+ def eval(input: InternalRow): Any = {
val src = DynamicExtractors.crsExtractor(srcCRS.dataType)(srcCRS.eval(input))
val dst = DynamicExtractors.crsExtractor(dstCRS.dataType)(dstCRS.eval(input))
(src, dst) match {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Rescale.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Rescale.scala
index 9ceb3bdd0..4261c7a36 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Rescale.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Rescale.scala
@@ -28,9 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions._
import org.locationtech.rasterframes.expressions.tilestats.TileStats
@@ -48,12 +46,12 @@ import org.locationtech.rasterframes.expressions.tilestats.TileStats
> SELECT _FUNC_(tile, lit(-2.2), lit(2.2))
..."""
)
-case class Rescale(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with CodegenFallback with Serializable {
+case class Rescale(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with RasterResult with CodegenFallback with Serializable {
override val nodeName: String = "rf_rescale"
- override def children: Seq[Expression] = Seq(child1, child2, child3)
+ def children: Seq[Expression] = Seq(child1, child2, child3)
- override def dataType: DataType = child1.dataType
+ def dataType: DataType = child1.dataType
override def checkInputDataTypes(): TypeCheckResult =
if(!tileExtractor.isDefinedAt(child1.dataType)) {
@@ -66,19 +64,11 @@ case class Rescale(child1: Expression, child2: Expression, child3: Expression) e
override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (childTile, childCtx) = tileExtractor(child1.dataType)(row(input1))
-
val min = doubleArgExtractor(child2.dataType)(input2).value
-
val max = doubleArgExtractor(child3.dataType)(input3).value
-
val result = op(childTile, min, max)
-
- childCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, childCtx)
}
protected def op(tile: Tile, min: Double, max: Double): Tile = {
@@ -108,5 +98,3 @@ object Rescale {
new Column(Rescale(tile.expr, min, max))
}
}
-
-
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala
index f7dcecf2a..32a329691 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala
@@ -29,12 +29,12 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.{TileUDT}
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.expressions.{DynamicExtractors, RasterResult, row}
+import org.locationtech.rasterframes.encoders._
+import org.locationtech.rasterframes.encoders.syntax._
/**
* Change the CellType of a Tile
@@ -53,54 +53,42 @@ import org.locationtech.rasterframes.expressions.row
> SELECT _FUNC_(tile, 'int16ud0');
..."""
)
-case class SetCellType(tile: Expression, cellType: Expression)
- extends BinaryExpression with CodegenFallback {
- def left = tile
- def right = cellType
+case class SetCellType(tile: Expression, cellType: Expression) extends BinaryExpression with RasterResult with CodegenFallback {
+ def left: Expression = tile
+ def right: Expression = cellType
override def nodeName: String = "rf_convert_cell_type"
- override def dataType: DataType = left.dataType
+ def dataType: DataType = left.dataType
- override def checkInputDataTypes(): TypeCheckResult = {
- if (!tileExtractor.isDefinedAt(left.dataType))
+ override def checkInputDataTypes(): TypeCheckResult =
+ if (!DynamicExtractors.tileExtractor.isDefinedAt(left.dataType))
TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.")
else
right.dataType match {
case StringType => TypeCheckSuccess
- case t if t.conformsTo[CellType] => TypeCheckSuccess
- case _ =>
- TypeCheckFailure(s"Expected CellType but received '${right.dataType.simpleString}'")
+ case t if t.conformsToSchema(cellTypeEncoder.schema) => TypeCheckSuccess
+ case _ => TypeCheckFailure(s"Expected CellType but received '${right.dataType.simpleString}'")
}
- }
private def toCellType(datum: Any): CellType = {
right.dataType match {
case StringType =>
val text = datum.asInstanceOf[UTF8String].toString
CellType.fromName(text)
- case st if st.conformsTo[CellType] =>
- row(datum).to[CellType]
+ case st if st.conformsToSchema(cellTypeEncoder.schema) =>
+ row(datum).as[CellType]
}
}
override protected def nullSafeEval(tileInput: Any, ctInput: Any): InternalRow = {
- implicit val tileSer = TileUDT.tileSerializer
-
- val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput))
+ val (tile, ctx) = DynamicExtractors.tileExtractor(left.dataType)(row(tileInput))
val ct = toCellType(ctInput)
val result = tile.convert(ct)
-
- ctx match {
- case Some(c) => c.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, ctx)
}
}
object SetCellType {
- def apply(tile: Column, cellType: CellType): Column =
- new Column(new SetCellType(tile.expr, lit(cellType.name).expr))
- def apply(tile: Column, cellType: String): Column =
- new Column(new SetCellType(tile.expr, lit(cellType).expr))
- def apply(tile: Column, cellType: Column): Column =
- new Column(new SetCellType(tile.expr, cellType.expr))
+ def apply(tile: Column, cellType: CellType): Column = new Column(new SetCellType(tile.expr, lit(cellType.name).expr))
+ def apply(tile: Column, cellType: String): Column = new Column(new SetCellType(tile.expr, lit(cellType).expr))
+ def apply(tile: Column, cellType: Column): Column = new Column(new SetCellType(tile.expr, cellType.expr))
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala
index eddca3508..2825d5334 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala
@@ -28,11 +28,9 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.expressions.row
+import org.locationtech.rasterframes.expressions.{RasterResult, row}
import org.slf4j.LoggerFactory
@ExpressionDescription(
@@ -46,11 +44,11 @@ import org.slf4j.LoggerFactory
> SELECT _FUNC_(tile, 1.5);
..."""
)
-case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExpression with CodegenFallback {
+case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExpression with RasterResult with CodegenFallback {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
override val nodeName: String = "rf_with_no_data"
- override def dataType: DataType = left.dataType
+ def dataType: DataType = left.dataType
override def checkInputDataTypes(): TypeCheckResult = {
if (!tileExtractor.isDefinedAt(left.dataType)) {
@@ -63,7 +61,6 @@ case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExp
}
override protected def nullSafeEval(input1: Any, input2: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
val result = numberArgExtractor(right.dataType)(input2) match {
@@ -71,10 +68,7 @@ case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExp
case IntegerArg(i) => leftTile.withNoData(Some(i.toDouble))
}
- leftCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
- case None => result.toInternalRow
- }
+ toInternalRow(result, leftCtx)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Standardize.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Standardize.scala
index e1d1aaa87..02a04e54c 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Standardize.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Standardize.scala
@@ -28,9 +28,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure,
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription, TernaryExpression}
import org.apache.spark.sql.functions.lit
-import org.apache.spark.sql.rf.TileUDT
import org.apache.spark.sql.types.DataType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions._
import org.locationtech.rasterframes.expressions.tilestats.TileStats
@@ -48,12 +46,12 @@ import org.locationtech.rasterframes.expressions.tilestats.TileStats
> SELECT _FUNC_(tile, lit(4.0), lit(2.2))
..."""
)
-case class Standardize(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with CodegenFallback with Serializable {
+case class Standardize(child1: Expression, child2: Expression, child3: Expression) extends TernaryExpression with RasterResult with CodegenFallback with Serializable {
override val nodeName: String = "rf_standardize"
- override def children: Seq[Expression] = Seq(child1, child2, child3)
+ def children: Seq[Expression] = Seq(child1, child2, child3)
- override def dataType: DataType = child1.dataType
+ def dataType: DataType = child1.dataType
override def checkInputDataTypes(): TypeCheckResult =
if(!tileExtractor.isDefinedAt(child1.dataType)) {
@@ -66,23 +64,20 @@ case class Standardize(child1: Expression, child2: Expression, child3: Expressio
override protected def nullSafeEval(input1: Any, input2: Any, input3: Any): Any = {
- implicit val tileSer = TileUDT.tileSerializer
val (childTile, childCtx) = tileExtractor(child1.dataType)(row(input1))
val mean = doubleArgExtractor(child2.dataType)(input2).value
-
val stdDev = doubleArgExtractor(child3.dataType)(input3).value
+ val result = op(childTile, mean, stdDev)
- childCtx match {
- case Some(ctx) => ctx.toProjectRasterTile(op(childTile, mean, stdDev)).toInternalRow
- case None => op(childTile, mean, stdDev).toInternalRow
- }
+ toInternalRow(result, childCtx)
}
protected def op(tile: Tile, mean: Double, stdDev: Double): Tile =
- tile.convert(FloatConstantNoDataCellType)
- .localSubtract(mean)
- .localDivide(stdDev)
+ tile
+ .convert(FloatConstantNoDataCellType)
+ .localSubtract(mean)
+ .localDivide(stdDev)
}
object Standardize {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala
index 5d7786f1c..6e52ed9ca 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayDouble.scala
@@ -21,6 +21,7 @@
package org.locationtech.rasterframes.expressions.transformers
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
@@ -38,13 +39,11 @@ import org.locationtech.rasterframes.model.TileContext
)
case class TileToArrayDouble(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "rf_tile_to_array_double"
- override def dataType: DataType = DataTypes.createArrayType(DoubleType, false)
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
+ def dataType: DataType = DataTypes.createArrayType(DoubleType, false)
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
ArrayData.toArrayData(tile.toArrayDouble())
- }
}
object TileToArrayDouble {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc
def apply(tile: Column): TypedColumn[Any, Array[Double]] =
new Column(TileToArrayDouble(tile.expr)).as[Array[Double]]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala
index c299d57c7..07b5dc58b 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/TileToArrayInt.scala
@@ -21,13 +21,13 @@
package org.locationtech.rasterframes.expressions.transformers
-import org.locationtech.rasterframes.expressions.UnaryRasterOp
import geotrellis.raster.Tile
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.types.{DataType, DataTypes, IntegerType}
import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.UnaryRasterOp
import org.locationtech.rasterframes.model.TileContext
@@ -39,13 +39,11 @@ import org.locationtech.rasterframes.model.TileContext
)
case class TileToArrayInt(child: Expression) extends UnaryRasterOp with CodegenFallback {
override def nodeName: String = "rf_tile_to_array_int"
- override def dataType: DataType = DataTypes.createArrayType(IntegerType, false)
- override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
+ def dataType: DataType = DataTypes.createArrayType(IntegerType, false)
+ protected def eval(tile: Tile, ctx: Option[TileContext]): Any =
ArrayData.toArrayData(tile.toArray())
- }
}
object TileToArrayInt {
- import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders.arrayEnc
def apply(tile: Column): TypedColumn[Any, Array[Int]] =
new Column(TileToArrayInt(tile.expr)).as[Array[Int]]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala
index 53f177daa..5356d7864 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala
@@ -21,40 +21,37 @@
package org.locationtech.rasterframes.expressions.transformers
-import java.net.URI
-
-import com.typesafe.scalalogging.Logger
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, UnaryExpression}
import org.apache.spark.sql.types.{DataType, StringType}
import org.apache.spark.sql.{Column, TypedColumn}
import org.apache.spark.unsafe.types.UTF8String
-import org.locationtech.rasterframes.RasterSourceType
+import org.locationtech.rasterframes._
import org.locationtech.rasterframes.ref.RFRasterSource
import org.slf4j.LoggerFactory
+import com.typesafe.scalalogging.Logger
+import java.net.URI
+
/**
* Catalyst generator to convert a geotiff download URL into a series of rows
* containing references to the internal tiles and associated extents.
*
* @since 5/4/18
*/
-case class URIToRasterSource(override val child: Expression)
- extends UnaryExpression with ExpectsInputTypes with CodegenFallback {
+case class URIToRasterSource(override val child: Expression) extends UnaryExpression with ExpectsInputTypes with CodegenFallback {
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
-
override def nodeName: String = "rf_uri_to_raster_source"
- override def dataType: DataType = RasterSourceType
-
- override def inputTypes = Seq(StringType)
+ def dataType: DataType = rasterSourceUDT
+ def inputTypes = Seq(StringType)
override protected def nullSafeEval(input: Any): Any = {
val uriString = input.asInstanceOf[UTF8String].toString
val uri = URI.create(uriString)
val ref = RFRasterSource(uri)
- RasterSourceType.serialize(ref)
+ rasterSourceUDT.serialize(ref)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala
index 58a86714f..dfca3d49d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression,
import org.apache.spark.sql.types.{DataType, LongType}
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.geomesa.curve.XZ2SFC
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions.accessors.GetCRS
import org.locationtech.rasterframes.jts.ReprojectionTransformer
@@ -52,12 +53,11 @@ import org.locationtech.rasterframes.jts.ReprojectionTransformer
* crs - the native CRS of the `geom` column
"""
)
-case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Short)
- extends BinaryExpression with CodegenFallback {
+case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Short) extends BinaryExpression with CodegenFallback {
override def nodeName: String = "rf_xz2_index"
- override def dataType: DataType = LongType
+ def dataType: DataType = LongType
override def checkInputDataTypes(): TypeCheckResult = {
if (!envelopeExtractor.isDefinedAt(left.dataType))
@@ -90,7 +90,6 @@ case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Shor
}
object XZ2Indexer {
- import org.locationtech.rasterframes.encoders.SparkBasicEncoders.longEnc
def apply(targetExtent: Column, targetCRS: Column, indexResolution: Short): TypedColumn[Any, Long] =
new Column(new XZ2Indexer(targetExtent.expr, targetCRS.expr, indexResolution)).as[Long]
def apply(targetExtent: Column, targetCRS: Column): TypedColumn[Any, Long] =
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Z2Indexer.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Z2Indexer.scala
index f9a081a19..d8f8a8ade 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Z2Indexer.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Z2Indexer.scala
@@ -29,6 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression,
import org.apache.spark.sql.types.{DataType, LongType}
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.geomesa.curve.Z2SFC
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.DynamicExtractors._
import org.locationtech.rasterframes.expressions.accessors.GetCRS
import org.locationtech.rasterframes.jts.ReprojectionTransformer
@@ -53,12 +54,11 @@ import org.locationtech.rasterframes.jts.ReprojectionTransformer
* crs - the native CRS of the `geom` column
"""
)
-case class Z2Indexer(left: Expression, right: Expression, indexResolution: Short)
- extends BinaryExpression with CodegenFallback {
+case class Z2Indexer(left: Expression, right: Expression, indexResolution: Short) extends BinaryExpression with CodegenFallback {
override def nodeName: String = "rf_z2_index"
- override def dataType: DataType = LongType
+ def dataType: DataType = LongType
override def checkInputDataTypes(): TypeCheckResult = {
if (!centroidExtractor.isDefinedAt(left.dataType))
@@ -80,12 +80,11 @@ case class Z2Indexer(left: Expression, right: Expression, indexResolution: Short
trans(coord)
}
- indexer.index(pt.getX, pt.getY, lenient = true).z
+ indexer.index(pt.getX, pt.getY, lenient = true)
}
}
object Z2Indexer {
- import org.locationtech.rasterframes.encoders.SparkBasicEncoders.longEnc
def apply(targetExtent: Column, targetCRS: Column, indexResolution: Short): TypedColumn[Any, Long] =
new Column(new Z2Indexer(targetExtent.expr, targetCRS.expr, indexResolution)).as[Long]
def apply(targetExtent: Column, targetCRS: Column): TypedColumn[Any, Long] =
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala
index 9929ea716..4bc1d3026 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ContextRDDMethods.scala
@@ -37,8 +37,7 @@ import org.locationtech.rasterframes.util._
* Extension method on `ContextRDD`-shaped RDDs with appropriate context bounds to create a RasterFrameLayer.
* @since 7/18/17
*/
-abstract class SpatialContextRDDMethods[T <: CellGrid[Int]](implicit spark: SparkSession)
- extends MethodExtensions[RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]] {
+abstract class SpatialContextRDDMethods[T <: CellGrid[Int]](implicit spark: SparkSession) extends MethodExtensions[RDD[(SpatialKey, T)] with Metadata[TileLayerMetadata[SpatialKey]]] {
import PairRDDConverter._
def toLayer(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = toLayer(TILE_COLUMN.columnName)
@@ -46,7 +45,7 @@ abstract class SpatialContextRDDMethods[T <: CellGrid[Int]](implicit spark: Spar
def toLayer(tileColumnName: String)(implicit converter: PairRDDConverter[SpatialKey, T]): RasterFrameLayer = {
val df = self.toDataFrame.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
val defName = TILE_COLUMN.columnName
- df.applyWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName))
+ df.applyWhen(_ => tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName))
.certify
}
}
@@ -55,9 +54,7 @@ abstract class SpatialContextRDDMethods[T <: CellGrid[Int]](implicit spark: Spar
* Extension method on `ContextRDD`-shaped `Tile` RDDs keyed with [[SpaceTimeKey]], with appropriate context bounds to create a RasterFrameLayer.
* @since 9/11/17
*/
-abstract class SpatioTemporalContextRDDMethods[T <: CellGrid[Int]](
- implicit spark: SparkSession)
- extends MethodExtensions[RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] {
+abstract class SpatioTemporalContextRDDMethods[T <: CellGrid[Int]](implicit spark: SparkSession) extends MethodExtensions[RDD[(SpaceTimeKey, T)] with Metadata[TileLayerMetadata[SpaceTimeKey]]] {
def toLayer(implicit converter: PairRDDConverter[SpaceTimeKey, T]): RasterFrameLayer = toLayer(TILE_COLUMN.columnName)
@@ -66,7 +63,6 @@ abstract class SpatioTemporalContextRDDMethods[T <: CellGrid[Int]](
.setSpatialColumnRole(SPATIAL_KEY_COLUMN, self.metadata)
.setTemporalColumnRole(TEMPORAL_KEY_COLUMN)
val defName = TILE_COLUMN.columnName
- df.applyWhen(_ ⇒ tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName))
- .certify
+ df.applyWhen(_ => tileColumnName != defName, _.withColumnRenamed(defName, tileColumnName)).certify
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala
index d02265c94..cfa4d3823 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/DataFrameMethods.scala
@@ -21,17 +21,15 @@
package org.locationtech.rasterframes.extensions
-import geotrellis.proj4.CRS
import geotrellis.layer._
import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod => GTResampleMethod}
import geotrellis.util.MethodExtensions
-import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.expressions.Attribute
+import org.apache.spark.sql.rf.CrsUDT
import org.apache.spark.sql.types.{MetadataBuilder, StructField}
import org.apache.spark.sql.{Column, DataFrame, TypedColumn}
-import org.locationtech.rasterframes.StandardColumns._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.StandardEncoders._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders._
import org.locationtech.rasterframes.expressions.DynamicExtractors
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import org.locationtech.rasterframes.util._
@@ -49,27 +47,27 @@ import scala.util.Try
trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with MetadataKeys {
import Implicits.{WithDataFrameMethods, WithMetadataBuilderMethods, WithMetadataMethods, WithRasterFrameLayerMethods}
- private def selector(column: Column) = (attr: Attribute) ⇒
+ private def selector(column: Column): Attribute => Boolean = (attr: Attribute) =>
attr.name == column.columnName || attr.semanticEquals(column.expr)
/** Map over the Attribute representation of Columns, modifying the one matching `column` with `op`. */
- private[rasterframes] def mapColumnAttribute(column: Column, op: Attribute ⇒ Attribute): DF = {
+ private[rasterframes] def mapColumnAttribute(column: Column, op: Attribute => Attribute): DF = {
val analyzed = self.queryExecution.analyzed.output
val selects = selector(column)
- val attrs = analyzed.map { attr ⇒
+ val attrs = analyzed.map { attr =>
if(selects(attr)) op(attr) else attr
}
- self.select(attrs.map(a ⇒ new Column(a)): _*).asInstanceOf[DF]
+ self.select(attrs.map(a => new Column(a)): _*).asInstanceOf[DF]
}
- private[rasterframes] def addColumnMetadata(column: Column, op: MetadataBuilder ⇒ MetadataBuilder): DF = {
- mapColumnAttribute(column, attr ⇒ {
+ private[rasterframes] def addColumnMetadata(column: Column, op: MetadataBuilder => MetadataBuilder): DF = {
+ mapColumnAttribute(column, attr => {
val md = new MetadataBuilder().withMetadata(attr.metadata)
attr.withMetadata(op(md).build)
})
}
- private[rasterframes] def fetchMetadataValue[D](column: Column, reader: (Attribute) ⇒ D): Option[D] = {
+ private[rasterframes] def fetchMetadataValue[D](column: Column, reader: Attribute => D): Option[D] = {
val analyzed = self.queryExecution.analyzed.output
analyzed.find(selector(column)).map(reader)
}
@@ -94,31 +92,31 @@ trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with Metada
def tileColumns: Seq[Column] =
self.schema.fields
.filter(f => DynamicExtractors.tileExtractor.isDefinedAt(f.dataType))
- .map(f ⇒ self.col(f.name))
+ .map(f => self.col(f.name))
/** Get the columns that look like `ProjectedRasterTile`s. */
def projRasterColumns: Seq[Column] =
self.schema.fields
- .filter(_.dataType.conformsTo[ProjectedRasterTile])
+ .filter(_.dataType.conformsToSchema(ProjectedRasterTile.projectedRasterTileEncoder.schema))
.map(f => self.col(f.name))
/** Get the columns that look like `Extent`s. */
def extentColumns: Seq[Column] =
self.schema.fields
- .filter(_.dataType.conformsTo[Extent])
+ .filter(_.dataType.conformsToSchema(extentEncoder.schema))
.map(f => self.col(f.name))
/** Get the columns that look like `CRS`s. */
def crsColumns: Seq[Column] =
self.schema.fields
- .filter(_.dataType.conformsTo[CRS])
+ .filter { f => f.dataType.conformsToDataType(crsExpressionEncoder.schema) || f.dataType.isInstanceOf[CrsUDT] }
.map(f => self.col(f.name))
/** Get the columns that are not of type `Tile` */
def notTileColumns: Seq[Column] =
self.schema.fields
.filter(f => !DynamicExtractors.tileExtractor.isDefinedAt(f.dataType))
- .map(f ⇒ self.col(f.name))
+ .map(f => self.col(f.name))
/** Get the spatial column. */
def spatialKeyColumn: Option[TypedColumn[Any, SpatialKey]] = {
@@ -137,7 +135,7 @@ trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with Metada
/** Find the field tagged with the requested `role` */
private[rasterframes] def findRoleField(role: String): Option[StructField] =
self.schema.fields.find(
- f ⇒
+ f =>
f.metadata.contains(SPATIAL_ROLE_KEY) &&
f.metadata.getString(SPATIAL_ROLE_KEY) == role
)
@@ -154,7 +152,7 @@ trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with Metada
* Useful for preparing dataframes for joins where duplicate names may arise.
*/
def withPrefixedColumnNames(prefix: String): DF =
- self.columns.foldLeft(self)((df, c) ⇒ df.withColumnRenamed(c, s"$prefix$c").asInstanceOf[DF])
+ self.columns.foldLeft(self)((df, c) => df.withColumnRenamed(c, s"$prefix$c").asInstanceOf[DF])
/**
* Performs a jeft join on the dataframe `right` to this one, reprojecting and merging tiles as necessary.
@@ -305,5 +303,5 @@ trait DataFrameMethods[DF <: DataFrame] extends MethodExtensions[DF] with Metada
/** Internal method for slapping the RasterFreameLayer seal of approval on a DataFrame.
* Only call if if you are sure it has a spatial key and tile columns and TileLayerMetadata. */
- private[rasterframes] def certify = certifyLayer(self)
+ private[rasterframes] def certify: RasterFrameLayer = certifyLayer(self)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala
index cedf3e06b..466c889eb 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/Implicits.scala
@@ -79,8 +79,7 @@ trait Implicits {
}
private[rasterframes]
- implicit class WithMetadataBuilderMethods(val self: MetadataBuilder)
- extends MetadataBuilderMethods
+ implicit class WithMetadataBuilderMethods(val self: MetadataBuilder) extends MetadataBuilderMethods
private[rasterframes]
implicit class TLMHasTotalCells(tlm: TileLayerMetadata[_]) {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala
index 7b291d7d6..aedd96c9e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/KryoMethods.scala
@@ -27,15 +27,15 @@ import org.apache.spark.sql.SparkSession
import org.locationtech.rasterframes.util.RFKryoRegistrator
object KryoMethods {
- val kryoProperties = Map("spark.serializer" -> classOf[KryoSerializer].getName,
+ val kryoProperties = Map(
+ "spark.serializer" -> classOf[KryoSerializer].getName,
"spark.kryo.registrator" -> classOf[RFKryoRegistrator].getName,
- "spark.kryoserializer.buffer.max" -> "500m")
+ "spark.kryoserializer.buffer.max" -> "500m"
+ )
trait BuilderKryoMethods extends MethodExtensions[SparkSession.Builder] {
def withKryoSerialization: SparkSession.Builder =
- kryoProperties.foldLeft(self) {
- case (bld, (key, value)) => bld.config(key, value)
- }
+ kryoProperties.foldLeft(self) { case (bld, (key, value)) => bld.config(key, value) }
}
trait SparkConfKryoMethods extends MethodExtensions[SparkConf] {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala
index 2cc58d5ac..f871c7904 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/LayerSpatialColumnMethods.scala
@@ -22,17 +22,16 @@
package org.locationtech.rasterframes.extensions
import org.locationtech.rasterframes.util._
-import org.locationtech.rasterframes.RasterFrameLayer
+import org.locationtech.rasterframes._
import org.locationtech.jts.geom.Point
import geotrellis.proj4.LatLng
-import geotrellis.layer._
+import geotrellis.layer.{MapKeyTransform, SpatialKey}
import geotrellis.util.MethodExtensions
import geotrellis.vector._
import org.apache.spark.sql.Row
import org.apache.spark.sql.functions.{asc, udf => sparkUdf}
import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
import org.locationtech.geomesa.curve.Z2SFC
-import org.locationtech.rasterframes.StandardColumns
import org.locationtech.rasterframes.encoders.serialized_literal
/**
@@ -47,15 +46,15 @@ trait LayerSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with
/** Returns the key-space to map-space coordinate transform. */
def mapTransform: MapKeyTransform = self.tileLayerMetadata.merge.mapTransform
- private def keyCol2Extent: Row ⇒ Extent = {
+ private def keyCol2Extent: Row => Extent = {
val transform = self.sparkSession.sparkContext.broadcast(mapTransform)
- r ⇒ transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1)))
+ r => transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1)))
}
- private def keyCol2LatLng: Row ⇒ (Double, Double) = {
+ private def keyCol2LatLng: Row => (Double, Double) = {
val transform = self.sparkSession.sparkContext.broadcast(mapTransform)
val crs = self.tileLayerMetadata.merge.crs
- r ⇒ {
+ r => {
val center = transform.value.keyToExtent(SpatialKey(r.getInt(0), r.getInt(1))).center.reproject(crs, LatLng)
(center.x, center.y)
}
@@ -121,10 +120,10 @@ trait LayerSpatialColumnMethods extends MethodExtensions[RasterFrameLayer] with
* @return RasterFrameLayer with index column.
*/
def withSpatialIndex(colName: String = SPATIAL_INDEX_COLUMN.columnName, applyOrdering: Boolean = true): RasterFrameLayer = {
- val zindex = sparkUdf(keyCol2LatLng andThen (p ⇒ Z2SFC.index(p._1, p._2).z))
+ val zindex = sparkUdf(keyCol2LatLng andThen (p => Z2SFC.index(p._1, p._2)))
self.withColumn(colName, zindex(self.spatialKeyColumn)) match {
- case rf if applyOrdering ⇒ rf.orderBy(asc(colName)).certify
- case rf ⇒ rf.certify
+ case rf if applyOrdering => rf.orderBy(asc(colName)).certify
+ case rf => rf.certify
}
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala
index fc2401bb5..2c33e6a35 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataBuilderMethods.scala
@@ -34,8 +34,8 @@ import org.locationtech.rasterframes.{MetadataKeys, StandardColumns}
*/
private[rasterframes]
abstract class MetadataBuilderMethods extends MethodExtensions[MetadataBuilder] with MetadataKeys with StandardColumns {
- def attachContext(md: Metadata) = self.putMetadata(CONTEXT_METADATA_KEY, md)
- def tagSpatialKey = self.putString(SPATIAL_ROLE_KEY, SPATIAL_KEY_COLUMN.columnName)
- def tagTemporalKey = self.putString(SPATIAL_ROLE_KEY, TEMPORAL_KEY_COLUMN.columnName)
- def tagSpatialIndex = self.putString(SPATIAL_ROLE_KEY, SPATIAL_INDEX_COLUMN.columnName)
+ def attachContext(md: Metadata): MetadataBuilder = self.putMetadata(CONTEXT_METADATA_KEY, md)
+ def tagSpatialKey: MetadataBuilder = self.putString(SPATIAL_ROLE_KEY, SPATIAL_KEY_COLUMN.columnName)
+ def tagTemporalKey: MetadataBuilder = self.putString(SPATIAL_ROLE_KEY, TEMPORAL_KEY_COLUMN.columnName)
+ def tagSpatialIndex: MetadataBuilder = self.putString(SPATIAL_ROLE_KEY, SPATIAL_INDEX_COLUMN.columnName)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala
index 5d96abdf4..efdd14189 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/MetadataMethods.scala
@@ -23,7 +23,7 @@ package org.locationtech.rasterframes.extensions
import geotrellis.util.MethodExtensions
import spray.json.{JsObject, JsonFormat}
-import org.apache.spark.sql.types.{Metadata ⇒ SQLMetadata}
+import org.apache.spark.sql.types.{Metadata => SQLMetadata}
/**
* Extension methods used for transforming the metadata in a ContextRDD.
@@ -34,8 +34,8 @@ abstract class MetadataMethods[M: JsonFormat] extends MethodExtensions[M] {
def asColumnMetadata: SQLMetadata = {
val fmt = implicitly[JsonFormat[M]]
fmt.write(self) match {
- case s: JsObject ⇒ SQLMetadata.fromJson(s.compactPrint)
- case _ ⇒ SQLMetadata.empty
+ case s: JsObject => SQLMetadata.fromJson(s.compactPrint)
+ case _ => SQLMetadata.empty
}
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/MultibandGeoTiffMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/MultibandGeoTiffMethods.scala
index 95ee8c1ce..98afe6f35 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/MultibandGeoTiffMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/MultibandGeoTiffMethods.scala
@@ -21,15 +21,14 @@
package org.locationtech.rasterframes.extensions
-import geotrellis.proj4.CRS
import geotrellis.raster.Dimensions
import geotrellis.raster.io.geotiff.MultibandGeoTiff
import geotrellis.util.MethodExtensions
-import geotrellis.vector.Extent
import org.apache.spark.sql.types.{StructField, StructType}
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.{NOMINAL_TILE_DIMS, TileType}
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.StandardEncoders
+import org.locationtech.rasterframes.encoders.syntax._
trait MultibandGeoTiffMethods extends MethodExtensions[MultibandGeoTiff] {
def toDF(dims: Dimensions[Int] = NOMINAL_TILE_DIMS)(implicit spark: SparkSession): DataFrame = {
@@ -41,20 +40,20 @@ trait MultibandGeoTiffMethods extends MethodExtensions[MultibandGeoTiff] {
val windows = segmentLayout.listWindows(dims.cols, dims.rows)
val subtiles = self.crop(windows)
- val rows = for {
- (gridbounds, tile) ← subtiles.toSeq
- } yield {
+ val rows = for { (gridbounds, tile) <- subtiles.toSeq } yield {
val extent = re.extentFor(gridbounds, false)
- Row(extent.toRow +: crs.toRow +: tile.bands: _*)
+ val extentRow = extent.toRow
+
+ Row(extentRow +: crs +: tile.bands: _*)
}
val schema =
- StructType(Seq(
- StructField("extent", schemaOf[Extent], false),
- StructField("crs", schemaOf[CRS], false)
- ) ++ (1 to bands).map { i =>
- StructField("b_" + i, TileType, false)
- })
+ StructType(
+ Seq(
+ StructField("extent", StandardEncoders.extentEncoder.schema, false),
+ StructField("crs", crsUDT, false)
+ ) ++ (1 to bands).map { i => StructField("b_" + i, tileUDT, false)}
+ )
spark.createDataFrame(spark.sparkContext.makeRDD(rows), schema)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala
index fe2867cc5..cac768925 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterFrameLayerMethods.scala
@@ -27,7 +27,7 @@ import com.typesafe.scalalogging.Logger
import geotrellis.proj4.CRS
import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod}
import geotrellis.raster.{MultibandTile, ProjectedRaster, Tile, TileLayout}
-import geotrellis.layer._
+import geotrellis.layer.{SpatialKey, SpaceTimeKey, TemporalKey, SpatialComponent, Boundable, Bounds, KeyBounds, TileLayerMetadata, LayoutDefinition}
import geotrellis.spark._
import geotrellis.spark.tiling.Tiler
import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD, TileLayerRDD}
@@ -38,8 +38,8 @@ import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.types.{Metadata, TimestampType}
import org.locationtech.rasterframes.{MetadataKeys, RasterFrameLayer}
-import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._
-import org.locationtech.rasterframes.encoders.StandardEncoders._
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.tiles.ShowableTile
import org.locationtech.rasterframes.util._
import org.locationtech.rasterframes.util.JsonCodecs._
@@ -53,8 +53,7 @@ import scala.reflect.runtime.universe._
*
* @since 7/18/17
*/
-trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
- with LayerSpatialColumnMethods with MetadataKeys {
+trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer] with LayerSpatialColumnMethods with MetadataKeys {
import Implicits.{WithDataFrameMethods, WithRasterFrameLayerMethods}
@transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
@@ -80,12 +79,10 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
def tileLayerMetadata: Either[TileLayerMetadata[SpatialKey], TileLayerMetadata[SpaceTimeKey]] = {
val spatialMD = self.findSpatialKeyField
.map(_.metadata)
- .getOrElse(throw new IllegalArgumentException(s"RasterFrameLayer operation requsted on non-RasterFrameLayer: $self"))
+ .getOrElse(throw new IllegalArgumentException(s"RasterFrameLayer operation requested on non-RasterFrameLayer: $self"))
- if (self.findTemporalKeyField.nonEmpty)
- Right(extract[TileLayerMetadata[SpaceTimeKey]](CONTEXT_METADATA_KEY)(spatialMD))
- else
- Left(extract[TileLayerMetadata[SpatialKey]](CONTEXT_METADATA_KEY)(spatialMD))
+ if (self.findTemporalKeyField.nonEmpty) Right(extract[TileLayerMetadata[SpaceTimeKey]](CONTEXT_METADATA_KEY)(spatialMD))
+ else Left(extract[TileLayerMetadata[SpatialKey]](CONTEXT_METADATA_KEY)(spatialMD))
}
/** Get the CRS covering the RasterFrameLayer. */
@@ -105,7 +102,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
// I wish there was a better way than this....
// can't do `lit(value)` because you get
// "Unsupported literal type class geotrellis.spark.TemporalKey" error
- val litKey = udf(() ⇒ value)
+ val litKey = udf(() => value)
val df = self.withColumn(TEMPORAL_KEY_COLUMN.columnName, litKey())
@@ -155,7 +152,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
prefix: String,
sk: TypedColumn[Any, SpatialKey],
tk: Option[TypedColumn[Any, TemporalKey]]) = {
- tk.combine(rf: DataFrame)((t, rf) ⇒ rf.withColumnRenamed(t.columnName, prefix + t.columnName))
+ tk.combine(rf: DataFrame)((t, rf) => rf.withColumnRenamed(t.columnName, prefix + t.columnName))
.withColumnRenamed(sk.columnName, prefix + sk.columnName)
.certify
}
@@ -169,9 +166,9 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
val rightTemporalKey = preppedRight.temporalKeyColumn
val spatialPred = leftSpatialKey === rightSpatialKey
- val temporalPred = leftTemporalKey.flatMap(l ⇒ rightTemporalKey.map(r ⇒ l === r))
+ val temporalPred = leftTemporalKey.flatMap(l => rightTemporalKey.map(r => l === r))
- val joinPred = temporalPred.map(t ⇒ spatialPred && t).getOrElse(spatialPred)
+ val joinPred = temporalPred.map(t => spatialPred && t).getOrElse(spatialPred)
val joined = preppedLeft.join(preppedRight, joinPred, joinType)
@@ -182,7 +179,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
.drop(rightSpatialKey.columnName)
left.temporalKeyColumn.tupleWith(leftTemporalKey).combine(spatialFix) {
- case ((orig, updated), rf) ⇒ rf
+ case ((orig, updated), rf) => rf
.withColumnRenamed(updated.columnName, orig.columnName)
.drop(rightTemporalKey.get.columnName)
}
@@ -199,12 +196,11 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
val layout = metadata.merge.layout
val trans = layout.mapTransform
- def updateBounds[T: SpatialComponent: Boundable: JsonFormat: TypeTag](tlm: TileLayerMetadata[T],
- keys: Dataset[T]): DataFrame = {
+ def updateBounds[T: SpatialComponent: Boundable: JsonFormat: TypeTag](tlm: TileLayerMetadata[T], keys: Dataset[T]): DataFrame = {
implicit val enc = Encoders.product[KeyBounds[T]]
val keyBounds = keys
- .map(k ⇒ KeyBounds(k, k))
- .reduce(_ combine _)
+ .map(k => KeyBounds(k, k))
+ .reduce{(_: KeyBounds[T]) combine (_: KeyBounds[T])}
val gridExtent = trans(keyBounds.toGridBounds())
val newExtent = gridExtent.intersection(extent).getOrElse(gridExtent)
@@ -212,13 +208,13 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
}
val df = metadata.fold(
- tlm ⇒ updateBounds(tlm, self.select(self.spatialKeyColumn)),
- tlm ⇒ {
+ tlm => updateBounds(tlm, self.select(self.spatialKeyColumn)),
+ tlm => {
updateBounds(
tlm,
self
.select(self.spatialKeyColumn, self.temporalKeyColumn.get)
- .map { case (s, t) ⇒ SpaceTimeKey(s, t) }
+ .map { case (s, t) => SpaceTimeKey(s, t) }
)
}
)
@@ -232,7 +228,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
*/
def toTileLayerRDD(tileCol: Column): Either[TileLayerRDD[SpatialKey], TileLayerRDD[SpaceTimeKey]] =
tileLayerMetadata.fold(
- tlm ⇒ {
+ tlm => {
val rdd = self.select(self.spatialKeyColumn, tileCol.as[Tile])
.rdd
.map {
@@ -243,12 +239,12 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
Left(ContextRDD(rdd, tlm))
},
- tlm ⇒ {
+ tlm => {
val rdd = self
.select(self.spatialKeyColumn, self.temporalKeyColumn.get, tileCol.as[Tile])
.rdd
.map {
- case (sk, tk, v) ⇒
+ case (sk, tk, v) =>
val tile = v match {
// Wrapped tiles can break GeoTrellis Avro code.
case wrapped: ShowableTile => wrapped.delegate
@@ -268,36 +264,38 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
/** Convert the specified tile columns in a Rasterrame to a GeoTrellis [[MultibandTileLayerRDD]] */
def toMultibandTileLayerRDD(tileCols: Column*): Either[MultibandTileLayerRDD[SpatialKey], MultibandTileLayerRDD[SpaceTimeKey]] =
tileLayerMetadata.fold(
- tlm ⇒ {
+ tlm => {
implicit val genEnc = expressionEncoder[(SpatialKey, Array[Tile])]
val rdd = self
.select(self.spatialKeyColumn, array(tileCols: _*)).as[(SpatialKey, Array[Tile])]
.rdd
- .map { case (sk, tiles) ⇒
+ .map { case (sk, tiles) =>
(sk, MultibandTile(tiles))
}
Left(ContextRDD(rdd, tlm))
},
- tlm ⇒ {
+ tlm => {
implicit val genEnc = expressionEncoder[(SpatialKey, TemporalKey, Array[Tile])]
val rdd = self
.select(self.spatialKeyColumn, self.temporalKeyColumn.get, array(tileCols: _*)).as[(SpatialKey, TemporalKey, Array[Tile])]
.rdd
- .map { case (sk, tk, tiles) ⇒ (SpaceTimeKey(sk, tk), MultibandTile(tiles)) }
+ .map { case (sk, tk, tiles) => (SpaceTimeKey(sk, tk), MultibandTile(tiles)) }
Right(ContextRDD(rdd, tlm))
}
)
/** Extract metadata value. */
- private[rasterframes] def extract[M: JsonFormat](metadataKey: String)(md: Metadata) =
+ private[rasterframes] def extract[M: JsonFormat](metadataKey: String)(md: Metadata): M =
md.getMetadata(metadataKey).json.parseJson.convertTo[M]
/** Convert the tiles in the RasterFrameLayer into a single raster. For RasterFrames keyed with temporal keys, they
* will be merge undeterministically. */
- def toRaster(tileCol: Column,
- rasterCols: Int,
- rasterRows: Int,
- resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[Tile] = {
+ def toRaster(
+ tileCol: Column,
+ rasterCols: Int,
+ rasterRows: Int,
+ resampler: ResampleMethod = NearestNeighbor
+ ): ProjectedRaster[Tile] = {
val clipped = clipLayerExtent
@@ -306,19 +304,17 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
val newLayout = LayoutDefinition(md.extent, TileLayout(1, 1, rasterCols, rasterRows))
val rdd = clipped.toTileLayerRDD(tileCol)
- .fold(identity, _.map{ case(stk, t) ⇒ (stk.spatialKey, t) }) // <-- Drops the temporal key outright
+ .fold(identity, _.map{ case(stk, t) => (stk.spatialKey, t) }) // <-- Drops the temporal key outright
val cellType = rdd.first()._2.cellType
val newLayerMetadata =
md.copy(layout = newLayout, bounds = Bounds(SpatialKey(0, 0), SpatialKey(0, 0)), cellType = cellType)
- val newLayer = rdd
- .map {
- case (key, tile) ⇒
- (ProjectedExtent(trans(key), md.crs), tile)
- }
- .tileToLayout(newLayerMetadata, Tiler.Options(resampler))
+ val newLayer =
+ rdd
+ .map { case (key, tile) => (ProjectedExtent(trans(key), md.crs), tile) }
+ .tileToLayout(newLayerMetadata, Tiler.Options(resampler))
val stitchedTile = newLayer.stitch()
@@ -333,7 +329,8 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
tileCols: Seq[Column],
rasterCols: Int,
rasterRows: Int,
- resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[MultibandTile] = {
+ resampler: ResampleMethod = NearestNeighbor
+ ): ProjectedRaster[MultibandTile] = {
val clipped = clipLayerExtent
@@ -342,7 +339,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
val newLayout = LayoutDefinition(md.extent, TileLayout(1, 1, rasterCols, rasterRows))
val rdd = clipped.toMultibandTileLayerRDD(tileCols: _*)
- .fold(identity, _.map{ case(stk, t) ⇒ (stk.spatialKey, t)}) // <-- Drops the temporal key outright
+ .fold(identity, _.map{ case(stk, t) => (stk.spatialKey, t)}) // <-- Drops the temporal key outright
val cellType = rdd.first()._2.cellType
@@ -351,7 +348,7 @@ trait RasterFrameLayerMethods extends MethodExtensions[RasterFrameLayer]
val newLayer = rdd
.map {
- case (key, tile) ⇒
+ case (key, tile) =>
(ProjectedExtent(trans(key), md.crs), tile)
}
.tileToLayout(newLayerMetadata, Tiler.Options(resampler))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala
index 89455355e..ca7a027b6 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/RasterJoin.scala
@@ -69,10 +69,10 @@ object RasterJoin {
// Convert resolved column into a symbolic one.
def unresolved(c: Column): Column = col(c.columnName)
-// checkType(leftExtent, "Extent", DynamicExtractors.extentExtractor)
-// checkType(leftCRS, "CRS", DynamicExtractors.crsExtractor)
-// checkType(rightExtent, "Extent", DynamicExtractors.extentExtractor)
-// checkType(rightCRS, "CRS", DynamicExtractors.crsExtractor)
+ // checkType(leftExtent, "Extent", DynamicExtractors.extentExtractor)
+ // checkType(leftCRS, "CRS", DynamicExtractors.crsExtractor)
+ // checkType(rightExtent, "Extent", DynamicExtractors.extentExtractor)
+ // checkType(rightCRS, "CRS", DynamicExtractors.crsExtractor)
// Unique id for temporary columns
val id = Random.alphanumeric.take(5).mkString("_", "", "_")
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala
index ac799c35e..22de68b81 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/ReprojectToLayer.scala
@@ -26,7 +26,6 @@ import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod => GTResample
import org.apache.spark.sql._
import org.apache.spark.sql.functions.broadcast
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.StandardEncoders.crsSparkEncoder
import org.locationtech.rasterframes.util._
/** Algorithm for projecting an arbitrary RasterFrame into a layer with consistent CRS and gridding. */
@@ -38,7 +37,7 @@ object ReprojectToLayer {
val crs = tlm.crs
import df.sparkSession.implicits._
- implicit val enc = Encoders.tuple(spatialKeyEncoder, extentEncoder, crsSparkEncoder)
+ implicit val enc = Encoders.tuple(spatialKeyEncoder, extentEncoder, crsExpressionEncoder)
val gridItems = for {
(col, row) <- gb.coordsIter
diff --git a/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala b/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala
index 3f2c90683..7955880a0 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/extensions/SinglebandGeoTiffMethods.scala
@@ -26,15 +26,13 @@ import geotrellis.raster.Dimensions
import geotrellis.raster.io.geotiff.SinglebandGeoTiff
import geotrellis.util.MethodExtensions
import geotrellis.vector.Extent
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.apache.spark.sql.{DataFrame, SparkSession}
import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
+import geotrellis.raster.Tile
trait SinglebandGeoTiffMethods extends MethodExtensions[SinglebandGeoTiff] {
def toDF(dims: Dimensions[Int] = NOMINAL_TILE_DIMS)(implicit spark: SparkSession): DataFrame = {
-
val segmentLayout = self.imageData.segmentLayout
val re = self.rasterExtent
val crs = self.crs
@@ -42,21 +40,13 @@ trait SinglebandGeoTiffMethods extends MethodExtensions[SinglebandGeoTiff] {
val windows = segmentLayout.listWindows(dims.cols, dims.rows)
val subtiles = self.crop(windows)
- val rows = for {
- (gridbounds, tile) ← subtiles.toSeq
- } yield {
+ val rows = for { (gridbounds, tile) <- subtiles.toSeq } yield {
val extent = re.extentFor(gridbounds, false)
- Row(extent.toRow, crs.toRow, tile)
+ (extent, crs, tile)
}
- val schema = StructType(Seq(
- StructField("extent", schemaOf[Extent], false),
- StructField("crs", schemaOf[CRS], false),
- StructField("tile", TileType, false)
- ))
-
- spark.createDataFrame(spark.sparkContext.makeRDD(rows), schema)
+ spark.createDataset(rows)(typedExpressionEncoder[(Extent, CRS, Tile)]).toDF("extent", "crs", "tile")
}
- def toProjectedRasterTile: ProjectedRasterTile = ProjectedRasterTile(self.projectedRaster)
+ def toProjectedRasterTile: ProjectedRasterTile = ProjectedRasterTile(self.tile, self.extent, self.crs)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/TileFunctions.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/TileFunctions.scala
index ab86436d5..f671f59d8 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/functions/TileFunctions.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/functions/TileFunctions.scala
@@ -20,9 +20,10 @@
*/
package org.locationtech.rasterframes.functions
+
import geotrellis.raster.render.ColorRamp
import geotrellis.raster.{CellType, Tile}
-import org.apache.spark.sql.functions.{lit, udf}
+import org.apache.spark.sql.functions.{lit, typedLit, udf}
import org.apache.spark.sql.{Column, TypedColumn}
import org.locationtech.jts.geom.Geometry
import org.locationtech.rasterframes.expressions.TileAssembler
@@ -35,7 +36,7 @@ import org.locationtech.rasterframes.expressions.transformers._
import org.locationtech.rasterframes.stats._
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
import org.locationtech.rasterframes.util.{ColorRampNames, withTypedAlias, _}
-import org.locationtech.rasterframes.{encoders, singlebandTileEncoder, functions ⇒ F}
+import org.locationtech.rasterframes.{tileEncoder, functions => F}
/** Functions associated with creating and transforming tiles, including tile-wise statistics and rendering. */
trait TileFunctions {
@@ -66,7 +67,7 @@ trait TileFunctions {
ct: CellType): TypedColumn[Any, Tile] =
rf_convert_cell_type(TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows)), ct)
.as(cellData.columnName)
- .as[Tile](singlebandTileEncoder)
+ .as[Tile](tileEncoder)
/** Create a Tile from a column of cell data with location indexes and perform cell conversion. */
def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int): TypedColumn[Any, Tile] =
@@ -145,8 +146,7 @@ trait TileFunctions {
/** Create a column constant tiles of zero */
def rf_make_zeros_tile(cols: Int, rows: Int, cellTypeName: String): TypedColumn[Any, Tile] = {
- import org.apache.spark.sql.rf.TileUDT.tileSerializer
- val constTile = encoders.serialized_literal(F.tileZeros(cols, rows, cellTypeName))
+ val constTile = typedLit(F.tileZeros(cols, rows, cellTypeName))
withTypedAlias(s"rf_make_zeros_tile($cols, $rows, $cellTypeName)")(constTile)
}
@@ -156,8 +156,7 @@ trait TileFunctions {
/** Creates a column of tiles containing all ones */
def rf_make_ones_tile(cols: Int, rows: Int, cellTypeName: String): TypedColumn[Any, Tile] = {
- import org.apache.spark.sql.rf.TileUDT.tileSerializer
- val constTile = encoders.serialized_literal(F.tileOnes(cols, rows, cellTypeName))
+ val constTile = typedLit(F.tileOnes(cols, rows, cellTypeName))
withTypedAlias(s"rf_make_ones_tile($cols, $rows, $cellTypeName)")(constTile)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala
index 8103e73c1..6545da41a 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala
@@ -19,14 +19,15 @@
*
*/
package org.locationtech.rasterframes
+
import geotrellis.proj4.CRS
import geotrellis.raster.reproject.Reproject
-import geotrellis.raster.{Tile, _}
+import geotrellis.raster._
import geotrellis.vector.Extent
import org.apache.spark.sql.functions.udf
import org.apache.spark.sql.{Row, SQLContext}
import org.locationtech.jts.geom.Geometry
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.syntax._
import org.locationtech.rasterframes.util.ResampleMethod
/**
@@ -37,80 +38,80 @@ import org.locationtech.rasterframes.util.ResampleMethod
package object functions {
@inline
- private[rasterframes] def safeBinaryOp[T <: AnyRef, R >: T](op: (T, T) ⇒ R): ((T, T) ⇒ R) =
- (o1: T, o2: T) ⇒ {
+ private[rasterframes] def safeBinaryOp[T <: AnyRef, R >: T](op: (T, T) => R): (T, T) => R =
+ (o1: T, o2: T) => {
if (o1 == null) o2
else if (o2 == null) o1
else op(o1, o2)
}
@inline
- private[rasterframes] def safeEval[P, R <: AnyRef](f: P ⇒ R): P ⇒ R =
- (p) ⇒ if (p == null) null.asInstanceOf[R] else f(p)
+ private[rasterframes] def safeEval[P, R <: AnyRef](f: P => R): P => R =
+ p => if (p == null) null.asInstanceOf[R] else f(p)
@inline
- private[rasterframes] def safeEval[P](f: P ⇒ Double)(implicit d: DummyImplicit): P ⇒ Double =
- (p) ⇒ if (p == null) Double.NaN else f(p)
+ private[rasterframes] def safeEval[P](f: P => Double)(implicit d: DummyImplicit): P => Double =
+ p => if (p == null) Double.NaN else f(p)
@inline
- private[rasterframes] def safeEval[P](f: P ⇒ Long)(implicit d1: DummyImplicit, d2: DummyImplicit): P ⇒ Long =
- (p) ⇒ if (p == null) 0l else f(p)
+ private[rasterframes] def safeEval[P](f: P => Long)(implicit d1: DummyImplicit, d2: DummyImplicit): P => Long =
+ p => if (p == null) 0l else f(p)
@inline
- private[rasterframes] def safeEval[P1, P2, R](f: (P1, P2) ⇒ R): (P1, P2) ⇒ R =
- (p1, p2) ⇒ if (p1 == null || p2 == null) null.asInstanceOf[R] else f(p1, p2)
+ private[rasterframes] def safeEval[P1, P2, R](f: (P1, P2) => R): (P1, P2) => R =
+ (p1, p2) => if (p1 == null || p2 == null) null.asInstanceOf[R] else f(p1, p2)
/** Converts an array into a tile. */
private[rasterframes] def arrayToTile(cols: Int, rows: Int) = {
safeEval[AnyRef, Tile]{
- case s: Seq[_] ⇒ s.headOption match {
- case Some(_: Int) ⇒ RawArrayTile(s.asInstanceOf[Seq[Int]].toArray[Int], cols, rows)
- case Some(_: Double) ⇒ RawArrayTile(s.asInstanceOf[Seq[Double]].toArray[Double], cols, rows)
- case Some(_: Byte) ⇒ RawArrayTile(s.asInstanceOf[Seq[Byte]].toArray[Byte], cols, rows)
- case Some(_: Short) ⇒ RawArrayTile(s.asInstanceOf[Seq[Short]].toArray[Short], cols, rows)
- case Some(_: Float) ⇒ RawArrayTile(s.asInstanceOf[Seq[Float]].toArray[Float], cols, rows)
- case Some(o @ _) ⇒ throw new MatchError(o)
- case None ⇒ null
+ case s: Seq[_] => s.headOption match {
+ case Some(_: Int) => RawArrayTile(s.asInstanceOf[Seq[Int]].toArray[Int], cols, rows)
+ case Some(_: Double) => RawArrayTile(s.asInstanceOf[Seq[Double]].toArray[Double], cols, rows)
+ case Some(_: Byte) => RawArrayTile(s.asInstanceOf[Seq[Byte]].toArray[Byte], cols, rows)
+ case Some(_: Short) => RawArrayTile(s.asInstanceOf[Seq[Short]].toArray[Short], cols, rows)
+ case Some(_: Float) => RawArrayTile(s.asInstanceOf[Seq[Float]].toArray[Float], cols, rows)
+ case Some(o @ _) => throw new MatchError(o)
+ case None => null
}
}
}
- private[rasterframes] val arrayToTile: (Array[_], Int, Int) ⇒ Tile = (a, cols, rows) ⇒ {
+ private[rasterframes] val arrayToTileFunc3: (Array[Double], Int, Int) => Tile = (a, cols, rows) => {
arrayToTile(cols, rows).apply(a)
}
/** Constructor for constant tiles */
- private[rasterframes] val makeConstantTile: (Number, Int, Int, String) ⇒ Tile = (value, cols, rows, cellTypeName) ⇒ {
+ private[rasterframes] val makeConstantTile: (Number, Int, Int, String) => Tile = (value, cols, rows, cellTypeName) => {
val cellType = CellType.fromName(cellTypeName)
cellType match {
- case BitCellType ⇒ BitConstantTile(if (value.intValue() == 0) false else true, cols, rows)
- case ct: ByteCells ⇒ ByteConstantTile(value.byteValue(), cols, rows, ct)
- case ct: UByteCells ⇒ UByteConstantTile(value.byteValue(), cols, rows, ct)
- case ct: ShortCells ⇒ ShortConstantTile(value.shortValue(), cols, rows, ct)
- case ct: UShortCells ⇒ UShortConstantTile(value.shortValue(), cols, rows, ct)
- case ct: IntCells ⇒ IntConstantTile(value.intValue(), cols, rows, ct)
- case ct: FloatCells ⇒ FloatConstantTile(value.floatValue(), cols, rows, ct)
- case ct: DoubleCells ⇒ DoubleConstantTile(value.doubleValue(), cols, rows, ct)
+ case BitCellType => BitConstantTile(if (value.intValue() == 0) false else true, cols, rows)
+ case ct: ByteCells => ByteConstantTile(value.byteValue(), cols, rows, ct)
+ case ct: UByteCells => UByteConstantTile(value.byteValue(), cols, rows, ct)
+ case ct: ShortCells => ShortConstantTile(value.shortValue(), cols, rows, ct)
+ case ct: UShortCells => UShortConstantTile(value.shortValue(), cols, rows, ct)
+ case ct: IntCells => IntConstantTile(value.intValue(), cols, rows, ct)
+ case ct: FloatCells => FloatConstantTile(value.floatValue(), cols, rows, ct)
+ case ct: DoubleCells => DoubleConstantTile(value.doubleValue(), cols, rows, ct)
}
}
/** Alias for constant tiles of zero */
- private[rasterframes] val tileZeros: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒
+ private[rasterframes] val tileZeros: (Int, Int, String) => Tile = (cols, rows, cellTypeName) =>
makeConstantTile(0, cols, rows, cellTypeName)
/** Alias for constant tiles of one */
- private[rasterframes] val tileOnes: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒
+ private[rasterframes] val tileOnes: (Int, Int, String) => Tile = (cols, rows, cellTypeName) =>
makeConstantTile(1, cols, rows, cellTypeName)
- val reproject_and_merge_f: (Row, Row, Seq[Tile], Seq[Row], Seq[Row], Row, String) => Tile = (leftExtentEnc: Row, leftCRSEnc: Row, tiles: Seq[Tile], rightExtentEnc: Seq[Row], rightCRSEnc: Seq[Row], leftDimsEnc: Row, resampleMethod: String) => {
+ val reproject_and_merge_f: (Row, CRS, Seq[Tile], Seq[Row], Seq[CRS], Row, String) => Tile = (leftExtentEnc: Row, leftCRSEnc: CRS, tiles: Seq[Tile], rightExtentEnc: Seq[Row], rightCRSEnc: Seq[CRS], leftDimsEnc: Row, resampleMethod: String) => {
if (tiles.isEmpty) null
else {
require(tiles.length == rightExtentEnc.length && tiles.length == rightCRSEnc.length, "size mismatch")
- val leftExtent = leftExtentEnc.to[Extent]
- val leftDims = leftDimsEnc.to[Dimensions[Int]]
- val leftCRS = leftCRSEnc.to[CRS]
- lazy val rightExtents = rightExtentEnc.map(_.to[Extent])
- lazy val rightCRSs = rightCRSEnc.map(_.to[CRS])
+ val leftExtent: Extent = leftExtentEnc.as[Extent]
+ val leftDims: Dimensions[Int] = leftDimsEnc.as[Dimensions[Int]]
+ val leftCRS: CRS = leftCRSEnc
+ lazy val rightExtents: Seq[Extent] = rightExtentEnc.map(_.as[Extent])
+ lazy val rightCRSs: Seq[CRS] = rightCRSEnc
lazy val resample = resampleMethod match {
- case ResampleMethod(mm) ⇒ mm
- case _ ⇒ throw new IllegalArgumentException(s"Unable to parse ResampleMethod for ${resampleMethod}.")
+ case ResampleMethod(mm) => mm
+ case _ => throw new IllegalArgumentException(s"Unable to parse ResampleMethod for ${resampleMethod}.")
}
if (leftExtent == null || leftDims == null || leftCRS == null) null
@@ -133,11 +134,10 @@ package object functions {
}
// NB: Don't be tempted to make this a `val`. Spark will barf if `withRasterFrames` hasn't been called first.
- def reproject_and_merge = udf(reproject_and_merge_f)
- .withName("reproject_and_merge")
+ def reproject_and_merge = udf(reproject_and_merge_f).withName("reproject_and_merge")
- private[rasterframes] val cellTypes: () ⇒ Seq[String] = () ⇒
+ private[rasterframes] val cellTypes: () => Seq[String] = () =>
Seq(
BitCellType,
ByteCellType,
@@ -159,8 +159,8 @@ package object functions {
/**
* Rasterize geometry into tiles.
*/
- private[rasterframes] val rasterize: (Geometry, Geometry, Int, Int, Int) ⇒ Tile = {
- (geom, bounds, value, cols, rows) ⇒ {
+ private[rasterframes] val rasterize: (Geometry, Geometry, Int, Int, Int) => Tile = {
+ (geom, bounds, value, cols, rows) => {
// We have to do this because (as of spark 2.2.x) Encoder-only types
// can't be used as UDF inputs. Only Spark-native types and UDTs.
val extent = Extent(bounds.getEnvelopeInternal)
@@ -174,6 +174,6 @@ package object functions {
sqlContext.udf.register("rf_make_ones_tile", tileOnes)
sqlContext.udf.register("rf_cell_types", cellTypes)
sqlContext.udf.register("rf_rasterize", rasterize)
- sqlContext.udf.register("rf_array_to_tile", arrayToTile)
+ sqlContext.udf.register("rf_array_to_tile", arrayToTileFunc3)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala b/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala
index 92527abb2..190a65cac 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/jts/Implicits.scala
@@ -24,22 +24,21 @@ package org.locationtech.rasterframes.jts
import java.sql.{Date, Timestamp}
import java.time.{LocalDate, ZonedDateTime}
+import org.locationtech.rasterframes.encoders.SparkBasicEncoders._
import org.locationtech.rasterframes.expressions.SpatialRelation.{Contains, Intersects}
import org.locationtech.jts.geom._
import geotrellis.util.MethodExtensions
-import geotrellis.vector.{Point ⇒ gtPoint}
+import geotrellis.vector.{Point => gtPoint}
import org.apache.spark.sql.{Column, TypedColumn}
import org.apache.spark.sql.functions._
import org.locationtech.geomesa.spark.jts.DataFrameFunctions.SpatialConstructors
-import org.locationtech.rasterframes.encoders.StandardEncoders.PrimitiveEncoders._
/**
* Extension methods on typed columns allowing for DSL-like queries over JTS types.
* @since 1/10/18
*/
trait Implicits extends SpatialConstructors {
- implicit class ExtentColumnMethods[T <: Geometry](val self: TypedColumn[Any, T])
- extends MethodExtensions[TypedColumn[Any, T]] {
+ implicit class ExtentColumnMethods[T <: Geometry](val self: TypedColumn[Any, T]) extends MethodExtensions[TypedColumn[Any, T]] {
def intersects(geom: Geometry): TypedColumn[Any, Boolean] =
new Column(Intersects(self.expr, geomLit(geom).expr)).as[Boolean]
@@ -59,8 +58,7 @@ trait Implicits extends SpatialConstructors {
new Column(Intersects(self.expr, geomLit(geom).expr)).as[Boolean]
}
- implicit class TimestampColumnMethods(val self: TypedColumn[Any, Timestamp])
- extends MethodExtensions[TypedColumn[Any, Timestamp]] {
+ implicit class TimestampColumnMethods(val self: TypedColumn[Any, Timestamp]) extends MethodExtensions[TypedColumn[Any, Timestamp]] {
import scala.language.implicitConversions
private implicit def zdt2ts(time: ZonedDateTime): Timestamp =
@@ -78,8 +76,7 @@ trait Implicits extends SpatialConstructors {
betweenTimes(start: Timestamp, end: Timestamp)
}
- implicit class DateColumnMethods(val self: TypedColumn[Any, Date])
- extends MethodExtensions[TypedColumn[Any, Date]] {
+ implicit class DateColumnMethods(val self: TypedColumn[Any, Date]) extends MethodExtensions[TypedColumn[Any, Date]] {
import scala.language.implicitConversions
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala
index 5cd9e780e..0b75e215d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ml/NoDataFilter.scala
@@ -31,7 +31,7 @@ import java.util.ArrayList
import org.locationtech.rasterframes.ml.Parameters.HasInputCols
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
/**
* Transformer filtering out rows containing NoData/NA values in
@@ -45,7 +45,7 @@ class NoDataFilter (override val uid: String) extends Transformer
def this() = this(Identifiable.randomUID("nodata-filter"))
final def setInputCols(value: Array[String]): NoDataFilter = set(inputCols, value)
final def setInputCols(values: ArrayList[String]): this.type = {
- val valueArr = Array[String](values:_*)
+ val valueArr = Array[String](values.asScala:_*)
set(inputCols, valueArr)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala
index 4d273a7f9..dc2e5725f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ml/Parameters.scala
@@ -29,7 +29,7 @@ import org.apache.spark.ml.param.{Params, StringArrayParam}
* @since 9/21/17
*/
object Parameters {
- trait HasInputCols { self: Params ⇒
+ trait HasInputCols { self: Params =>
final val inputCols = new StringArrayParam(this, "inputCols", "array of input column names")
final def getInputCols: Array[String] = $(inputCols)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala b/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala
index 38f978231..b5c438a2f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ml/TileExploder.scala
@@ -25,7 +25,7 @@ import org.locationtech.rasterframes._
import org.apache.spark.ml.Transformer
import org.apache.spark.ml.param.ParamMap
import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable}
-import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.{DataFrame, Dataset}
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.types._
import org.locationtech.rasterframes.util._
@@ -44,7 +44,7 @@ class TileExploder(override val uid: String) extends Transformer
override def copy(extra: ParamMap): TileExploder = defaultCopy(extra)
/** Checks the incoming schema and determines what the output schema will be. */
- def transformSchema(schema: StructType) = {
+ def transformSchema(schema: StructType): StructType = {
val (tiles, nonTiles) = selectTileAndNonTileFields(schema)
val cells = tiles.map(_.copy(dataType = DoubleType, nullable = false))
val indexes = Seq(
@@ -54,10 +54,10 @@ class TileExploder(override val uid: String) extends Transformer
StructType(nonTiles ++ indexes ++ cells)
}
- def transform(dataset: Dataset[_]) = {
+ def transform(dataset: Dataset[_]): DataFrame = {
val (tiles, nonTiles) = selectTileAndNonTileFields(dataset.schema)
- val tileCols = tiles.map(f ⇒ col(f.name))
- val nonTileCols = nonTiles.map(f ⇒ col(f.name))
+ val tileCols = tiles.map(f => col(f.name))
+ val nonTileCols = nonTiles.map(f => col(f.name))
val exploder = rf_explode_tiles(tileCols: _*)
dataset.select(nonTileCols :+ exploder: _*)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala
index dfc083774..66731e5a1 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/model/CellContext.scala
@@ -21,32 +21,4 @@
package org.locationtech.rasterframes.model
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.types.{ShortType, StructField, StructType}
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
-import CatalystSerializer._
-
case class CellContext(tileContext: TileContext, tileDataContext: TileDataContext, colIndex: Short, rowIndex: Short)
-object CellContext {
- implicit val serializer: CatalystSerializer[CellContext] = new CatalystSerializer[CellContext] {
- override val schema: StructType = StructType(Seq(
- StructField("tileContext", schemaOf[TileContext], false),
- StructField("tileDataContext", schemaOf[TileDataContext], false),
- StructField("colIndex", ShortType, false),
- StructField("rowIndex", ShortType, false)
- ))
- override protected def to[R](t: CellContext, io: CatalystSerializer.CatalystIO[R]): R = io.create(
- io.to(t.tileContext),
- io.to(t.tileDataContext),
- t.colIndex,
- t.rowIndex
- )
- override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): CellContext = CellContext(
- io.get[TileContext](t, 0),
- io.get[TileDataContext](t, 1),
- io.getShort(t, 2),
- io.getShort(t, 3)
- )
- }
- implicit def encoder: ExpressionEncoder[CellContext] = CatalystSerializerEncoder[CellContext]()
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala b/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala
deleted file mode 100644
index 0993e39dc..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/model/Cells.scala
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2019 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.model
-
-import geotrellis.raster.{ArrayTile, ConstantTile, Tile}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.types.{BinaryType, StructField, StructType}
-import org.locationtech.rasterframes
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
-import org.locationtech.rasterframes.ref.RasterRef
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
-import org.locationtech.rasterframes.tiles.ProjectedRasterTile.ConcreteProjectedRasterTile
-import org.locationtech.rasterframes.tiles.ShowableTile
-
-/** Represents the union of binary cell datas or a reference to the data.*/
-case class Cells(data: Either[Array[Byte], RasterRef]) {
- def isRef: Boolean = data.isRight
-
- /** Convert cells into either a RasterRefTile or an ArrayTile. */
- def toTile(ctx: TileDataContext): Tile = {
- data.fold(
- bytes => {
- val t = ArrayTile.fromBytes(bytes, ctx.cellType, ctx.dimensions.cols, ctx.dimensions.rows)
- if (Cells.showableTiles) new ShowableTile(t)
- else t
- },
- ref => RasterRefTile(ref)
- )
- }
-}
-
-object Cells {
- private val showableTiles = rasterframes.rfConfig.getBoolean("showable-tiles")
- /** Extracts the Cells from a Tile. */
- def apply(t: Tile): Cells = {
- t match {
- case prt: ConcreteProjectedRasterTile =>
- apply(prt.t)
- case ref: RasterRefTile =>
- Cells(Right(ref.rr))
- case const: ConstantTile =>
- // Need to expand constant tiles so they can be interpreted properly in catalyst and Python.
- // If we don't, the serialization breaks.
- Cells(Left(const.toArrayTile().toBytes))
- case o =>
- Cells(Left(o.toBytes))
- }
- }
-
- implicit def cellsSerializer: CatalystSerializer[Cells] = new CatalystSerializer[Cells] {
- override val schema: StructType =
- StructType(
- Seq(
- StructField("cells", BinaryType, true),
- StructField("ref", RasterRef.embeddedSchema, true)
- ))
- override protected def to[R](t: Cells, io: CatalystSerializer.CatalystIO[R]): R = io.create(
- t.data.left.getOrElse(null),
- t.data.right.map(rr => io.to(rr)).right.getOrElse(null)
- )
- override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): Cells = {
- if (!io.isNullAt(t, 0))
- Cells(Left(io.getByteArray(t, 0)))
- else if (!io.isNullAt(t, 1))
- Cells(Right(io.get[RasterRef](t, 1)))
- else throw new IllegalArgumentException("must be eithe cell data or a ref, but not null")
- }
- }
-
- implicit def encoder: ExpressionEncoder[Cells] = CatalystSerializerEncoder[Cells]()
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala b/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala
index d99255973..ca31d5405 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala
@@ -24,7 +24,6 @@ package org.locationtech.rasterframes.model
import com.github.blemale.scaffeine.Scaffeine
import geotrellis.proj4.CRS
import org.locationtech.proj4j.CoordinateReferenceSystem
-import org.locationtech.rasterframes.encoders.CatalystSerializer
import org.locationtech.rasterframes.model.LazyCRS.EncodedCRS
class LazyCRS(val encoded: EncodedCRS) extends CRS {
@@ -54,7 +53,7 @@ object LazyCRS {
private object WKTCRS {
def unapply(src: String): Option[CRS] =
- if (wktKeywords.exists { prefix ⇒ src.toUpperCase().startsWith(prefix)})
+ if (wktKeywords.exists { prefix => src.toUpperCase().startsWith(prefix)})
CRS.fromWKT(src)
else None
}
@@ -78,6 +77,4 @@ object LazyCRS {
else throw new IllegalArgumentException(
s"CRS string must be either EPSG code, +proj string, or OGC WKT (WKT1). Argument value was ${if (value.length > 50) value.substring(0, 50) + "..." else value} ")
}
-
- implicit val crsSererializer: CatalystSerializer[LazyCRS] = CatalystSerializer.crsSerializer.asInstanceOf[CatalystSerializer[LazyCRS]]
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/LongExtent.scala b/core/src/main/scala/org/locationtech/rasterframes/model/LongExtent.scala
index f18ea88af..34c5df32b 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/model/LongExtent.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/model/LongExtent.scala
@@ -22,30 +22,7 @@
package org.locationtech.rasterframes.model
import geotrellis.vector.Extent
-import org.apache.spark.sql.types.{LongType, StructField, StructType}
-import org.locationtech.rasterframes.encoders.CatalystSerializer
-import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO
case class LongExtent(xmin: Long, ymin: Long, xmax: Long, ymax: Long) {
def toExtent: Extent = Extent(xmin.toDouble, ymin.toDouble, xmax.toDouble, ymax.toDouble)
}
-
-object LongExtent {
- implicit val bigIntExtentSerializer: CatalystSerializer[LongExtent] = new CatalystSerializer[LongExtent] {
- override val schema: StructType = StructType(Seq(
- StructField("xmin", LongType, false),
- StructField("ymin", LongType, false),
- StructField("xmax", LongType, false),
- StructField("ymax", LongType, false)
- ))
- override def to[R](t: LongExtent, io: CatalystIO[R]): R = io.create(
- t.xmin, t.ymin, t.xmax, t.ymax
- )
- override def from[R](row: R, io: CatalystIO[R]): LongExtent = LongExtent(
- io.getLong(row, 0),
- io.getLong(row, 1),
- io.getLong(row, 2),
- io.getLong(row, 3)
- )
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala
index 436b46982..9b75b2dbd 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileContext.scala
@@ -24,10 +24,6 @@ package org.locationtech.rasterframes.model
import geotrellis.proj4.CRS
import geotrellis.raster.Tile
import geotrellis.vector.{Extent, ProjectedExtent}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
case class TileContext(extent: Extent, crs: CRS) {
@@ -40,19 +36,4 @@ object TileContext {
case prt: ProjectedRasterTile => Some((prt.extent, prt.crs))
case _ => None
}
- implicit val serializer: CatalystSerializer[TileContext] = new CatalystSerializer[TileContext] {
- override val schema: StructType = StructType(Seq(
- StructField("extent", schemaOf[Extent], false),
- StructField("crs", schemaOf[CRS], false)
- ))
- override protected def to[R](t: TileContext, io: CatalystSerializer.CatalystIO[R]): R = io.create(
- io.to(t.extent),
- io.to(t.crs)
- )
- override protected def from[R](t: R, io: CatalystSerializer.CatalystIO[R]): TileContext = TileContext(
- io.get[Extent](t, 0),
- io.get[CRS](t, 1)
- )
- }
- implicit def encoder: ExpressionEncoder[TileContext] = CatalystSerializerEncoder[TileContext]()
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala
index 9d0d5f387..c628eebfb 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/model/TileDataContext.scala
@@ -21,11 +21,7 @@
package org.locationtech.rasterframes.model
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
import geotrellis.raster.{CellType, Dimensions, Tile}
-import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
/** Encapsulates all information about a tile aside from actual cell values. */
case class TileDataContext(cellType: CellType, dimensions: Dimensions[Int])
@@ -35,26 +31,6 @@ object TileDataContext {
def apply(t: Tile): TileDataContext = {
require(t.cols <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.cols}")
require(t.rows <= Short.MaxValue, s"RasterFrames doesn't support tiles of size ${t.rows}")
- TileDataContext(
- t.cellType, t.dimensions
- )
+ TileDataContext(t.cellType, t.dimensions)
}
-
- implicit val serializer: CatalystSerializer[TileDataContext] = new CatalystSerializer[TileDataContext] {
- override val schema: StructType = StructType(Seq(
- StructField("cellType", schemaOf[CellType], false),
- StructField("dimensions", schemaOf[Dimensions[Int]], false)
- ))
-
- override protected def to[R](t: TileDataContext, io: CatalystIO[R]): R = io.create(
- io.to(t.cellType),
- io.to(t.dimensions)
- )
- override protected def from[R](t: R, io: CatalystIO[R]): TileDataContext = TileDataContext(
- io.get[CellType](t, 0),
- io.get[Dimensions[Int]](t, 1)
- )
- }
-
- implicit def encoder: ExpressionEncoder[TileDataContext] = CatalystSerializerEncoder[TileDataContext]()
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/package.scala b/core/src/main/scala/org/locationtech/rasterframes/package.scala
index 4b4800eef..eb4cd492b 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/package.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/package.scala
@@ -20,14 +20,13 @@
*/
package org.locationtech
-import com.typesafe.config.ConfigFactory
+
+import com.typesafe.config.{Config, ConfigFactory}
import com.typesafe.scalalogging.Logger
import geotrellis.raster.{Dimensions, Tile, TileFeature, isData}
-import geotrellis.raster.resample._
import geotrellis.layer._
import geotrellis.spark.ContextRDD
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT}
import org.apache.spark.sql.{DataFrame, SQLContext, rf}
import org.locationtech.geomesa.spark.jts.DataFrameFunctions
import org.locationtech.rasterframes.encoders.StandardEncoders
@@ -47,8 +46,7 @@ package object rasterframes extends StandardColumns
// Don't make this a `lazy val`... breaks Spark assemblies for some reason.
protected def logger: Logger = Logger(LoggerFactory.getLogger(getClass.getName))
- private[rasterframes]
- def rfConfig = ConfigFactory.load().getConfig("rasterframes")
+ def rfConfig: Config = ConfigFactory.load().getConfig("rasterframes")
/** The generally expected tile size, as defined by configuration property `rasterframes.nominal-tile-size`.*/
@transient
@@ -84,12 +82,6 @@ package object rasterframes extends StandardColumns
rasterframes.rules.register(sqlContext)
}
- /** TileUDT type reference. */
- def TileType = new TileUDT()
-
- /** RasterSourceUDT type reference. */
- def RasterSourceType = new RasterSourceUDT()
-
/**
* A RasterFrameLayer is just a DataFrame with certain invariants, enforced via the methods that create and transform them:
* 1. One column is a `SpatialKey` or `SpaceTimeKey``
@@ -146,9 +138,4 @@ package object rasterframes extends StandardColumns
def isCellTrue(t: Tile, col: Int, row: Int): Boolean =
if (t.cellType.isFloatingPoint) isCellTrue(t.getDouble(col, row))
else isCellTrue(t.get(col, row))
-
-
-
-
-
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala
index 1b845d6e4..6364aba48 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala
@@ -66,15 +66,15 @@ abstract class DelegatingRasterSource(source: URI, delegateBuilder: () => GTRast
retryableRead(rs => SimpleRasterInfo(rs))
)
- override def cols: Int = info.cols.toInt
- override def rows: Int = info.rows.toInt
- override def crs: CRS = info.crs
- override def extent: Extent = info.extent
- override def cellType: CellType = info.cellType
- override def bandCount: Int = info.bandCount
- override def tags: Tags = info.tags
+ def cols: Int = info.cols.toInt
+ def rows: Int = info.rows.toInt
+ def crs: CRS = info.crs
+ def extent: Extent = info.extent
+ def cellType: CellType = info.cellType
+ def bandCount: Int = info.bandCount
+ def tags: Tags = info.tags
- override def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
+ def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
retryableRead(_.readBounds(bounds.map(_.toGridType[Long]), bands))
override def read(bounds: GridBounds[Int], bands: Seq[Int]): Raster[MultibandTile] =
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala
index 47c7037f5..8c01ec269 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala
@@ -52,27 +52,26 @@ case class GDALRasterSource(source: URI) extends RFRasterSource with URIRasterSo
protected def tiffInfo = SimpleRasterInfo(source.toASCIIString, _ => SimpleRasterInfo(gdal))
- override def crs: CRS = tiffInfo.crs
+ def crs: CRS = tiffInfo.crs
- override def extent: Extent = tiffInfo.extent
+ def extent: Extent = tiffInfo.extent
- private def metadata = Map.empty[String, String]
+ def metadata = Map.empty[String, String]
- override def cellType: CellType = tiffInfo.cellType
+ def cellType: CellType = tiffInfo.cellType
- override def bandCount: Int = tiffInfo.bandCount
+ def bandCount: Int = tiffInfo.bandCount
- override def cols: Int = tiffInfo.cols.toInt
+ def cols: Int = tiffInfo.cols.toInt
- override def rows: Int = tiffInfo.rows.toInt
+ def rows: Int = tiffInfo.rows.toInt
- override def tags: Tags = Tags(metadata, List.empty)
+ def tags: Tags = Tags(metadata, List.empty)
- override def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
+ def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
try {
gdal.readBounds(bounds.map(_.toGridType[Long]), bands)
- }
- catch {
+ } catch {
case e: Exception => throw new IOException(s"Error reading '$source'", e)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala
index a222485b8..35e7dd614 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/HadoopGeoTiffRasterSource.scala
@@ -28,8 +28,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.locationtech.rasterframes.ref.RFRasterSource.{URIRasterSource, URIRasterSourceDebugString}
-case class HadoopGeoTiffRasterSource(source: URI, config: () => Configuration)
- extends RangeReaderRasterSource with URIRasterSource with URIRasterSourceDebugString { self =>
+case class HadoopGeoTiffRasterSource(source: URI, config: () => Configuration) extends RangeReaderRasterSource with URIRasterSource with URIRasterSourceDebugString { self =>
@transient
protected lazy val rangeReader = HdfsRangeReader(new Path(source.getPath), config())
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala
index fb53f3b63..1ca82f6de 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/InMemoryRasterSource.scala
@@ -31,19 +31,19 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile
case class InMemoryRasterSource(tile: Tile, extent: Extent, crs: CRS) extends RFRasterSource {
def this(prt: ProjectedRasterTile) = this(prt, prt.extent, prt.crs)
- override def rows: Int = tile.rows
+ def rows: Int = tile.rows
- override def cols: Int = tile.cols
+ def cols: Int = tile.cols
- override def cellType: CellType = tile.cellType
+ def cellType: CellType = tile.cellType
- override def bandCount: Int = 1
+ def bandCount: Int = 1
- override def tags: Tags = EMPTY_TAGS
+ def tags: Tags = EMPTY_TAGS
- override def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
+ def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
bounds
- .map(b => {
+ .map({ b =>
val subext = rasterExtent.extentFor(b)
Raster(MultibandTile(tile.crop(b)), subext)
})
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala
index 57b8c883d..4d5594282 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/JVMGeoTiffRasterSource.scala
@@ -25,5 +25,4 @@ import java.net.URI
import geotrellis.raster.geotiff.GeoTiffRasterSource
-
case class JVMGeoTiffRasterSource(source: URI) extends DelegatingRasterSource(source, () => GeoTiffRasterSource(source.toASCIIString))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RFRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RFRasterSource.scala
index e3ac69c66..31a8ac5dc 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/RFRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RFRasterSource.scala
@@ -22,7 +22,6 @@
package org.locationtech.rasterframes.ref
import java.net.URI
-
import com.github.blemale.scaffeine.Scaffeine
import com.typesafe.scalalogging.LazyLogging
import geotrellis.proj4.CRS
@@ -36,7 +35,7 @@ import org.apache.spark.sql.rf.RasterSourceUDT
import org.locationtech.rasterframes.model.TileContext
import org.locationtech.rasterframes.{NOMINAL_TILE_DIMS, rfConfig}
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, FiniteDuration}
/**
* Abstraction over fetching geospatial raster data.
@@ -92,7 +91,7 @@ object RFRasterSource extends LazyLogging {
final val SINGLEBAND = Seq(0)
final val EMPTY_TAGS = Tags(Map.empty, List.empty)
- val cacheTimeout: Duration = Duration.fromNanos(rfConfig.getDuration("raster-source-cache-timeout").toNanos)
+ val cacheTimeout: FiniteDuration = Duration.fromNanos(rfConfig.getDuration("raster-source-cache-timeout").toNanos)
private[ref] val rsCache = Scaffeine()
.recordStats()
@@ -149,7 +148,7 @@ object RFRasterSource extends LazyLogging {
object IsDefaultGeoTiff {
def unapply(source: URI): Boolean = source.getScheme match {
case "file" | "http" | "https" | "s3" => true
- case null | "" ⇒ true
+ case null | "" => true
case _ => false
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala
index 28854ee7d..cd7ff3448 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RangeReaderRasterSource.scala
@@ -45,23 +45,22 @@ trait RangeReaderRasterSource extends RFRasterSource with GeoTiffInfoSupport {
def extent: Extent = tiffInfo.extent
- override def cols: Int = tiffInfo.rasterExtent.cols
+ def cols: Int = tiffInfo.rasterExtent.cols
- override def rows: Int = tiffInfo.rasterExtent.rows
+ def rows: Int = tiffInfo.rasterExtent.rows
def cellType: CellType = tiffInfo.cellType
def bandCount: Int = tiffInfo.bandCount
- override def tags: Tags = tiffInfo.tags
+ def tags: Tags = tiffInfo.tags
- override def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
+ def readBounds(bounds: Traversable[GridBounds[Int]], bands: Seq[Int]): Iterator[Raster[MultibandTile]] = {
val info = realInfo
val geoTiffTile = GeoTiffReader.geoTiffMultibandTile(info)
val intersectingBounds = bounds.flatMap(_.intersection(this.gridBounds)).toSeq
geoTiffTile.crop(intersectingBounds, bands.toArray).map {
- case (gb, tile) =>
- Raster(tile, rasterExtent.extentFor(gb, clamp = true))
+ case (gb, tile) => Raster(tile, rasterExtent.extentFor(gb, clamp = true))
}
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala
index 8c03c8427..04497f489 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala
@@ -22,16 +22,11 @@
package org.locationtech.rasterframes.ref
import com.typesafe.scalalogging.LazyLogging
+import frameless.TypedExpressionEncoder
import geotrellis.proj4.CRS
-import geotrellis.raster.{CellGrid, CellType, GridBounds, Tile}
-import geotrellis.vector.{Extent, ProjectedExtent}
+import geotrellis.raster.{CellType, GridBounds, Tile}
+import geotrellis.vector.Extent
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.rf.RasterSourceUDT
-import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
-import org.locationtech.rasterframes.RasterSourceType
-import org.locationtech.rasterframes.encoders.CatalystSerializer.{CatalystIO, _}
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
/**
@@ -39,74 +34,36 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile
*
* @since 8/21/18
*/
-case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[GridBounds[Int]])
- extends CellGrid[Int] with ProjectedRasterLike {
- def crs: CRS = source.crs
+case class RasterRef(source: RFRasterSource, bandIndex: Int, subextent: Option[Extent], subgrid: Option[Subgrid]) extends ProjectedRasterTile {
+ def tile: Tile = this
def extent: Extent = subextent.getOrElse(source.extent)
- def projectedExtent: ProjectedExtent = ProjectedExtent(extent, crs)
- def cols: Int = grid.width
- def rows: Int = grid.height
- def cellType: CellType = source.cellType
- def tile: ProjectedRasterTile = RasterRefTile(this)
+ def crs: CRS = source.crs
+ def delegate = realizedTile
+
+ override def cols: Int = grid.width
+ override def rows: Int = grid.height
+ override def cellType: CellType = source.cellType
protected lazy val grid: GridBounds[Int] =
- subgrid.getOrElse(source.rasterExtent.gridBoundsFor(extent, true))
+ subgrid.map(_.toGridBounds).getOrElse(source.rasterExtent.gridBoundsFor(extent, true))
- protected lazy val realizedTile: Tile = {
+ lazy val realizedTile: Tile = {
RasterRef.log.trace(s"Fetching $extent ($grid) from band $bandIndex of $source")
source.read(grid, Seq(bandIndex)).tile.band(0)
}
+
+ override def toString: String = s"RasterRef($source,$bandIndex,$cellType)"
}
object RasterRef extends LazyLogging {
private val log = logger
- case class RasterRefTile(rr: RasterRef) extends ProjectedRasterTile {
- def extent: Extent = rr.extent
- def crs: CRS = rr.crs
- override def cellType = rr.cellType
-
- override def cols: Int = rr.cols
- override def rows: Int = rr.rows
-
- protected def delegate: Tile = rr.realizedTile
- // NB: This saves us from stack overflow exception
- override def convert(ct: CellType): ProjectedRasterTile =
- ProjectedRasterTile(rr.realizedTile.convert(ct), extent, crs)
- override def toString: String = s"$productPrefix($rr)"
- }
-
- val embeddedSchema: StructType = StructType(Seq(
- StructField("source", RasterSourceType.sqlType, true),
- StructField("bandIndex", IntegerType, false),
- StructField("subextent", schemaOf[Extent], true),
- StructField("subgrid", schemaOf[GridBounds[Int]], true)
- ))
-
- implicit val rasterRefSerializer: CatalystSerializer[RasterRef] = new CatalystSerializer[RasterRef] {
- override val schema: StructType = StructType(Seq(
- StructField("source", RasterSourceType, true),
- StructField("bandIndex", IntegerType, false),
- StructField("subextent", schemaOf[Extent], true),
- StructField("subgrid", schemaOf[GridBounds[Int]], true)
- ))
+ def apply(source: RFRasterSource, bandIndex: Int): RasterRef =
+ RasterRef(source, bandIndex, None, None)
- override def to[R](t: RasterRef, io: CatalystIO[R]): R = io.create(
- io.to(t.source)(RasterSourceUDT.rasterSourceSerializer),
- t.bandIndex,
- t.subextent.map(io.to[Extent]).orNull,
- t.subgrid.map(io.to[GridBounds[Int]]).orNull
- )
+ def apply(source: RFRasterSource, bandIndex: Int, subextent: Extent, subgrid: GridBounds[Int]): RasterRef =
+ RasterRef(source, bandIndex, Some(subextent), Some(Subgrid(subgrid)))
- override def from[R](row: R, io: CatalystIO[R]): RasterRef = RasterRef(
- io.get[RFRasterSource](row, 0)(RasterSourceUDT.rasterSourceSerializer),
- io.getInt(row, 1),
- if (io.isNullAt(row, 2)) None
- else Option(io.get[Extent](row, 2)),
- if (io.isNullAt(row, 3)) None
- else Option(io.get[GridBounds[Int]](row, 3))
- )
- }
-
- implicit def rrEncoder: ExpressionEncoder[RasterRef] = CatalystSerializerEncoder[RasterRef](true)
-}
+ implicit val rasterRefEncoder: ExpressionEncoder[RasterRef] =
+ TypedExpressionEncoder[RasterRef].asInstanceOf[ExpressionEncoder[RasterRef]]
+}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala
index 501e17639..a474dba9f 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala
@@ -80,9 +80,10 @@ object SimpleRasterInfo {
)
}
- private lazy val cache = Scaffeine()
- .recordStats()
- .build[String, SimpleRasterInfo]
+ private lazy val cache =
+ Scaffeine()
+ .recordStats()
+ .build[String, SimpleRasterInfo]
def cacheStats = cache.stats()
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/Subgrid.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/Subgrid.scala
new file mode 100644
index 000000000..811b191f7
--- /dev/null
+++ b/core/src/main/scala/org/locationtech/rasterframes/ref/Subgrid.scala
@@ -0,0 +1,13 @@
+package org.locationtech.rasterframes.ref
+
+import geotrellis.raster.GridBounds
+
+case class Subgrid(colMin: Int, rowMin: Int, colMax: Int, rowMax: Int) {
+ def toGridBounds: GridBounds[Int] =
+ GridBounds(colMin, rowMin, colMax, rowMax)
+}
+
+object Subgrid {
+ def apply(grid: GridBounds[Int]): Subgrid =
+ Subgrid(grid.colMin, grid.rowMin, grid.colMax, grid.rowMax)
+}
\ No newline at end of file
diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala
index 3b3e54d6f..a4be9d849 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilterPushdownRules.scala
@@ -33,14 +33,14 @@ import org.apache.spark.sql.rf.{FilterTranslator, VersionShims}
object SpatialFilterPushdownRules extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = {
plan.transformUp {
- case f @ Filter(condition, lr @ SpatialRelationReceiver(sr: SpatialRelationReceiver[_] @unchecked)) ⇒
+ case f @ Filter(condition, lr @ SpatialRelationReceiver(sr: SpatialRelationReceiver[_] @unchecked)) =>
val preds = FilterTranslator.translateFilter(condition)
def foldIt[T <: SpatialRelationReceiver[T]](rel: T): T =
- preds.foldLeft(rel)((r, f) ⇒ r.withFilter(f))
+ preds.foldLeft(rel)((r, f) => r.withFilter(f))
- preds.filterNot(sr.hasFilter).map(p ⇒ {
+ preds.filterNot(sr.hasFilter).map(p => {
val newRec = foldIt(sr)
Filter(condition, VersionShims.updateRelation(lr, newRec.asBaseRelation))
}).getOrElse(f)
diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala
index cf731b658..5bbe5148e 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialFilters.scala
@@ -22,7 +22,6 @@
package org.locationtech.rasterframes.rules
import org.locationtech.jts.geom.Geometry
-import org.apache.spark.sql.sources.Filter
/**
* New filter types captured and rewritten for use in spatiotemporal data sources that can handle them.
@@ -30,11 +29,11 @@ import org.apache.spark.sql.sources.Filter
* @since 1/11/18
*/
object SpatialFilters {
- case class Intersects(attribute: String, value: Geometry) extends Filter {
+ case class Intersects(attribute: String, value: Geometry) {
def references: Array[String] = Array(attribute)
}
- case class Contains(attribute: String, value: Geometry) extends Filter {
+ case class Contains(attribute: String, value: Geometry) {
def references: Array[String] = Array(attribute)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala
index 403d122ea..a0b81e127 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/rules/SpatialRelationReceiver.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.sources.{BaseRelation, Filter}
*
* @since 7/16/18
*/
-trait SpatialRelationReceiver[+T <: SpatialRelationReceiver[T]] { self: BaseRelation ⇒
+trait SpatialRelationReceiver[+T <: SpatialRelationReceiver[T]] { self: BaseRelation =>
/** Create new relation with the give filter added. */
def withFilter(filter: Filter): T
/** Check to see if relation already exists in this. */
@@ -40,7 +40,7 @@ trait SpatialRelationReceiver[+T <: SpatialRelationReceiver[T]] { self: BaseRela
object SpatialRelationReceiver {
def unapply[T <: SpatialRelationReceiver[T]](lr: LogicalRelation): Option[SpatialRelationReceiver[T]] = lr.relation match {
- case t: SpatialRelationReceiver[T] @unchecked ⇒ Some(t)
- case _ ⇒ None
+ case t: SpatialRelationReceiver[T] @unchecked => Some(t)
+ case _ => None
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala
index 5315b63b7..57836749c 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/rules/TemporalFilters.scala
@@ -23,7 +23,6 @@ package org.locationtech.rasterframes.rules
import java.sql.{Date, Timestamp}
-import org.apache.spark.sql.sources.Filter
/**
* New filter types captured and rewritten for use in spatiotemporal data sources that can handle them.
@@ -32,11 +31,11 @@ import org.apache.spark.sql.sources.Filter
*/
object TemporalFilters {
- case class BetweenTimes(attribute: String, start: Timestamp, end: Timestamp) extends Filter {
+ case class BetweenTimes(attribute: String, start: Timestamp, end: Timestamp) {
def references: Array[String] = Array(attribute)
}
- case class BetweenDates(attribute: String, start: Date, end: Date) extends Filter {
+ case class BetweenDates(attribute: String, start: Date, end: Date) {
def references: Array[String] = Array(attribute)
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala b/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala
index 0f028e14e..9c0d7f1bd 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/rules/package.scala
@@ -38,9 +38,10 @@ package object rules {
}
def register(sqlContext: SQLContext): Unit = {
- //org.locationtech.geomesa.spark.jts.rules.registerOptimizations(sqlContext)
+ org.locationtech.geomesa.spark.jts.rules.registerOptimizations(sqlContext)
registerOptimization(sqlContext, SpatialUDFSubstitutionRules)
- registerOptimization(sqlContext, SpatialFilterPushdownRules)
+ // TODO: implement [[FilterTranslator]]
+ // registerOptimization(sqlContext, SpatialFilterPushdownRules)
}
/** Separate And conditions into separate filters. */
diff --git a/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala b/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala
index be3d547a3..fa988e00d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/stats/CellHistogram.scala
@@ -36,17 +36,15 @@ import scala.collection.mutable.{ListBuffer => MutableListBuffer}
case class CellHistogram(bins: Seq[CellHistogram.Bin]) {
lazy val labels: Seq[Double] = bins.map(_.value)
lazy val totalCount = bins.foldLeft(0L)(_ + _.count)
- def asciiHistogram(width: Int = 80)= {
+ def asciiHistogram(width: Int = 80) = {
val counts = bins.map(_.count)
val maxCount = counts.max.toFloat
val maxLabelLen = labels.map(_.toString.length).max
- val maxCountLen = counts.map(c ⇒ f"$c%,d".length).max
+ val maxCountLen = counts.map(c => f"$c%,d".length).max
val fmt = s"%${maxLabelLen}s: %,${maxCountLen}d | %s"
val barlen = width - fmt.format(0, 0, "").length
- val lines = for {
- (l, c) ← labels.zip(counts)
- } yield {
+ val lines = for { (l, c) <- labels.zip(counts) } yield {
val width = (barlen * (c/maxCount)).round
val bar = "*" * width
fmt.format(l, c, bar)
@@ -83,7 +81,7 @@ case class CellHistogram(bins: Seq[CellHistogram.Bin]) {
val cdf = pdf.scanLeft(0.0)(_ + _)
val data = ds.zip(cdf).sliding(2)
- data.map({ ab => (ab.head, ab.tail.head) })
+ data.map { ab => (ab.head, ab.tail.head) }
}
}
@@ -91,7 +89,7 @@ case class CellHistogram(bins: Seq[CellHistogram.Bin]) {
private def percentileBreaks(qs: Seq[Double]): Seq[Double] = {
if(bins.size == 1) {
- qs.map(z => bins.head.value)
+ qs.map(_ => bins.head.value)
} else {
val data = cdfIntervals
if(!data.hasNext) {
@@ -158,20 +156,20 @@ object CellHistogram {
def apply(tile: Tile): CellHistogram = {
val bins = if (tile.cellType.isFloatingPoint) {
val h = tile.histogramDouble
- h.binCounts().map(p ⇒ Bin(p._1, p._2))
+ h.binCounts().map(p => Bin(p._1, p._2))
}
else {
val h = tile.histogram
- h.binCounts().map(p ⇒ Bin(p._1.toDouble, p._2))
+ h.binCounts().map(p => Bin(p._1.toDouble, p._2))
}
CellHistogram(bins)
}
def apply(hist: GTHistogram[Int]): CellHistogram = {
- CellHistogram(hist.binCounts().map(p ⇒ Bin(p._1.toDouble, p._2)))
+ CellHistogram(hist.binCounts().map(p => Bin(p._1.toDouble, p._2)))
}
def apply(hist: GTHistogram[Double])(implicit ev: DummyImplicit): CellHistogram = {
- CellHistogram(hist.binCounts().map(p ⇒ Bin(p._1, p._2)))
+ CellHistogram(hist.binCounts().map(p => Bin(p._1, p._2)))
}
lazy val schema: StructType = StandardEncoders.cellHistEncoder.schema
diff --git a/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala b/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala
index ea371666d..f16e0c669 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/stats/CellStatistics.scala
@@ -32,7 +32,7 @@ import org.locationtech.rasterframes.encoders.StandardEncoders
*/
case class CellStatistics(data_cells: Long, no_data_cells: Long, min: Double, max: Double, mean: Double, variance: Double) {
def stddev: Double = math.sqrt(variance)
- def asciiStats = Seq(
+ def asciiStats: String = Seq(
"data_cells: " + data_cells,
"no_data_cells: " + no_data_cells,
"min: " + min,
@@ -46,7 +46,7 @@ case class CellStatistics(data_cells: Long, no_data_cells: Long, min: Double, ma
val fields = Seq("data_cells", "no_data_cells", "min", "max", "mean", "variance")
fields.iterator
.zip(productIterator)
- .map(p ⇒ p._1 + "=" + p._2)
+ .map(p => p._1 + "=" + p._2)
.mkString(productPrefix + "(", ",", ")")
}
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala
deleted file mode 100644
index 169166ce0..000000000
--- a/core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * This software is licensed under the Apache 2 license, quoted below.
- *
- * Copyright 2018 Astraea, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * [http://www.apache.org/licenses/LICENSE-2.0]
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- * SPDX-License-Identifier: Apache-2.0
- *
- */
-
-package org.locationtech.rasterframes.tiles
-
-import java.nio.ByteBuffer
-
-import geotrellis.raster._
-import org.apache.spark.sql.catalyst.InternalRow
-import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO
-import org.locationtech.rasterframes.model.{Cells, TileDataContext}
-
-/**
- * Wrapper around a `Tile` encoded in a Catalyst `InternalRow`, for the purpose
- * of providing compatible semantics over common operations.
- *
- * @since 11/29/17
- */
-class InternalRowTile(val mem: InternalRow) extends DelegatingTile {
- import InternalRowTile._
-
- override def toArrayTile(): ArrayTile = realizedTile.toArrayTile()
-
- // TODO: We want to reimplement relevant delegated methods so that they read directly from tungsten storage
- lazy val realizedTile: Tile = cells.toTile(cellContext)
-
- protected override def delegate: Tile = realizedTile
-
- private def cellContext: TileDataContext =
- CatalystIO[InternalRow].get[TileDataContext](mem, 0)
-
- private def cells: Cells = CatalystIO[InternalRow].get[Cells](mem, 1)
-
- /** Retrieve the cell type from the internal encoding. */
- override def cellType: CellType = cellContext.cellType
-
- /** Retrieve the number of columns from the internal encoding. */
- override def cols: Int = cellContext.dimensions.cols
-
- /** Retrieve the number of rows from the internal encoding. */
- override def rows: Int = cellContext.dimensions.rows
-
- /** Get the internally encoded tile data cells. */
- override lazy val toBytes: Array[Byte] = {
- cells.data.left
- .getOrElse(throw new IllegalStateException(
- "Expected tile cell bytes, but received RasterRef instead: " + cells.data.right.get)
- )
- }
-
- private lazy val toByteBuffer: ByteBuffer = {
- val data = toBytes
- if(data.length < cols * rows && cellType.name != "bool") {
- // Handling constant tiles like this is inefficient and ugly. All the edge
- // cases associated with them create too much undue complexity for
- // something that's unlikely to be
- // used much in production to warrant handling them specially.
- // If a more efficient handling is necessary, consider a flag in
- // the UDT struct.
- ByteBuffer.wrap(toArrayTile().toBytes())
- } else ByteBuffer.wrap(data)
- }
-
- /** Reads the cell value at the given index as an Int. */
- def apply(i: Int): Int = cellReader.apply(i)
-
- /** Reads the cell value at the given index as a Double. */
- def applyDouble(i: Int): Double = cellReader.applyDouble(i)
-
- def copy = new InternalRowTile(mem.copy)
-
- private lazy val cellReader: CellReader = {
- cellType match {
- case ct: ByteUserDefinedNoDataCellType ⇒
- ByteUDNDCellReader(this, ct.noDataValue)
- case ct: UByteUserDefinedNoDataCellType ⇒
- UByteUDNDCellReader(this, ct.noDataValue)
- case ct: ShortUserDefinedNoDataCellType ⇒
- ShortUDNDCellReader(this, ct.noDataValue)
- case ct: UShortUserDefinedNoDataCellType ⇒
- UShortUDNDCellReader(this, ct.noDataValue)
- case ct: IntUserDefinedNoDataCellType ⇒
- IntUDNDCellReader(this, ct.noDataValue)
- case ct: FloatUserDefinedNoDataCellType ⇒
- FloatUDNDCellReader(this, ct.noDataValue)
- case ct: DoubleUserDefinedNoDataCellType ⇒
- DoubleUDNDCellReader(this, ct.noDataValue)
- case _: BitCells ⇒ BitCellReader(this)
- case _: ByteCells ⇒ ByteCellReader(this)
- case _: UByteCells ⇒ UByteCellReader(this)
- case _: ShortCells ⇒ ShortCellReader(this)
- case _: UShortCells ⇒ UShortCellReader(this)
- case _: IntCells ⇒ IntCellReader(this)
- case _: FloatCells ⇒ FloatCellReader(this)
- case _: DoubleCells ⇒ DoubleCellReader(this)
- }
- }
-
- override def toString: String = ShowableTile.show(this)
-}
-
-object InternalRowTile {
- sealed trait CellReader {
- def apply(index: Int): Int
- def applyDouble(index: Int): Double
- }
-
- case class BitCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int =
- (t.toByteBuffer.get(i >> 3) >> (i & 7)) & 1 // See BitArrayTile.apply
- def applyDouble(i: Int): Double = apply(i).toDouble
- }
-
- case class ByteCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = b2i(t.toByteBuffer.get(i))
- def applyDouble(i: Int): Double = b2d(t.toByteBuffer.get(i))
- }
-
- case class ByteUDNDCellReader(t: InternalRowTile, userDefinedByteNoDataValue: Byte)
- extends CellReader with UserDefinedByteNoDataConversions {
- def apply(i: Int): Int = udb2i(t.toByteBuffer.get(i))
- def applyDouble(i: Int): Double = udb2d(t.toByteBuffer.get(i))
- }
-
- case class UByteCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = ub2i(t.toByteBuffer.get(i))
- def applyDouble(i: Int): Double = ub2d(t.toByteBuffer.get(i))
- }
-
- case class UByteUDNDCellReader(t: InternalRowTile, userDefinedByteNoDataValue: Byte)
- extends CellReader with UserDefinedByteNoDataConversions {
- def apply(i: Int): Int = udub2i(t.toByteBuffer.get(i))
- def applyDouble(i: Int): Double = udub2d(t.toByteBuffer.get(i))
- }
-
- case class ShortCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = s2i(t.toByteBuffer.asShortBuffer().get(i))
- def applyDouble(i: Int): Double = s2d(t.toByteBuffer.asShortBuffer().get(i))
- }
-
- case class ShortUDNDCellReader(t: InternalRowTile, userDefinedShortNoDataValue: Short)
- extends CellReader with UserDefinedShortNoDataConversions {
- def apply(i: Int): Int = uds2i(t.toByteBuffer.asShortBuffer().get(i))
- def applyDouble(i: Int): Double = uds2d(t.toByteBuffer.asShortBuffer().get(i))
- }
-
- case class UShortCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = us2i(t.toByteBuffer.asShortBuffer().get(i))
- def applyDouble(i: Int): Double = us2d(t.toByteBuffer.asShortBuffer().get(i))
- }
-
- case class UShortUDNDCellReader(t: InternalRowTile, userDefinedShortNoDataValue: Short)
- extends CellReader with UserDefinedShortNoDataConversions {
- def apply(i: Int): Int = udus2i(t.toByteBuffer.asShortBuffer().get(i))
- def applyDouble(i: Int): Double = udus2d(t.toByteBuffer.asShortBuffer().get(i))
- }
-
- case class IntCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = t.toByteBuffer.asIntBuffer().get(i)
- def applyDouble(i: Int): Double = i2d(t.toByteBuffer.asIntBuffer().get(i))
- }
-
- case class IntUDNDCellReader(t: InternalRowTile, userDefinedIntNoDataValue: Int)
- extends CellReader with UserDefinedIntNoDataConversions {
- def apply(i: Int): Int = udi2i(t.toByteBuffer.asIntBuffer().get(i))
- def applyDouble(i: Int): Double = udi2d(t.toByteBuffer.asIntBuffer().get(i))
- }
-
- case class FloatCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = f2i(t.toByteBuffer.asFloatBuffer().get(i))
- def applyDouble(i: Int): Double = f2d(t.toByteBuffer.asFloatBuffer().get(i))
- }
-
- case class FloatUDNDCellReader(t: InternalRowTile, userDefinedFloatNoDataValue: Float)
- extends CellReader with UserDefinedFloatNoDataConversions{
- def apply(i: Int): Int = udf2i(t.toByteBuffer.asFloatBuffer().get(i))
- def applyDouble(i: Int): Double = udf2d(t.toByteBuffer.asFloatBuffer().get(i))
- }
-
- case class DoubleCellReader(t: InternalRowTile) extends CellReader {
- def apply(i: Int): Int = d2i(t.toByteBuffer.asDoubleBuffer().get(i))
- def applyDouble(i: Int): Double = t.toByteBuffer.asDoubleBuffer().get(i)
- }
-
- case class DoubleUDNDCellReader(t: InternalRowTile, userDefinedDoubleNoDataValue: Double)
- extends CellReader with UserDefinedDoubleNoDataConversions{
- def apply(i: Int): Int = udd2i(t.toByteBuffer.asDoubleBuffer().get(i))
- def applyDouble(i: Int): Double = udd2d(t.toByteBuffer.asDoubleBuffer().get(i))
- }
-}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala
index b5701b095..564664211 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala
@@ -22,25 +22,19 @@
package org.locationtech.rasterframes.tiles
import geotrellis.proj4.CRS
-import geotrellis.raster.io.geotiff.SinglebandGeoTiff
-import geotrellis.raster.{ArrayTile, CellType, DelegatingTile, ProjectedRaster, Tile}
+import geotrellis.raster.{DelegatingTile, ProjectedRaster, Tile}
import geotrellis.vector.{Extent, ProjectedExtent}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.rf.TileUDT
-import org.apache.spark.sql.types.{StructField, StructType}
-import org.locationtech.rasterframes.TileType
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.{CatalystSerializer, CatalystSerializerEncoder}
-import org.locationtech.rasterframes.model.TileContext
import org.locationtech.rasterframes.ref.ProjectedRasterLike
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
+import org.apache.spark.sql.catalyst.DefinedByConstructorParams
/**
* A Tile that's also like a ProjectedRaster, with delayed evaluation support.
*
* @since 9/5/18
*/
-abstract class ProjectedRasterTile extends DelegatingTile with ProjectedRasterLike {
+trait ProjectedRasterTile extends DelegatingTile with ProjectedRasterLike with DefinedByConstructorParams {
+ def tile: Tile
def extent: Extent
def crs: CRS
def projectedExtent: ProjectedExtent = ProjectedExtent(extent, crs)
@@ -49,69 +43,20 @@ abstract class ProjectedRasterTile extends DelegatingTile with ProjectedRasterLi
}
object ProjectedRasterTile {
- def apply(t: Tile, extent: Extent, crs: CRS): ProjectedRasterTile =
- ConcreteProjectedRasterTile(t, extent, crs)
- def apply(pr: ProjectedRaster[Tile]): ProjectedRasterTile =
- ConcreteProjectedRasterTile(pr.tile, pr.extent, pr.crs)
- def apply(tiff: SinglebandGeoTiff): ProjectedRasterTile =
- ConcreteProjectedRasterTile(tiff.tile, tiff.extent, tiff.crs)
-
- case class ConcreteProjectedRasterTile(t: Tile, extent: Extent, crs: CRS)
- extends ProjectedRasterTile {
- def delegate: Tile = t
-
- // NB: Don't be tempted to move this into the parent trait. Will get stack overflow.
- override def convert(cellType: CellType): Tile =
- ConcreteProjectedRasterTile(t.convert(cellType), extent, crs)
-
- override def toString: String = {
- val e = s"(${extent.xmin}, ${extent.ymin}, ${extent.xmax}, ${extent.ymax})"
- val c = crs.toProj4String
- s"[${ShowableTile.show(t)}, $e, $c]"
- }
-
- // Not sure why the following are still needed with this being closed:
- // https://github.com/locationtech/geotrellis/issues/3153
- // Without them, TileFunctionsSpec.`conditional cell values`.`should evaluate rf_where` fails
- override def combine(r2: Tile)(f: (Int, Int) ⇒ Int): Tile = (delegate, r2) match {
- case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combine(r2.toArrayTile())(f)
- case _ ⇒ delegate.combine(r2)(f)
- }
-
- override def combineDouble(r2: Tile)(f: (Double, Double) ⇒ Double): Tile = (delegate, r2) match {
- case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combineDouble(r2.toArrayTile())(f)
- case _ ⇒ delegate.combineDouble(r2)(f)
- }
-
- }
- implicit val serializer: CatalystSerializer[ProjectedRasterTile] = new CatalystSerializer[ProjectedRasterTile] {
- override val schema: StructType = StructType(Seq(
- StructField("tile_context", schemaOf[TileContext], true),
- StructField("tile", TileType, false))
- )
-
- override protected def to[R](t: ProjectedRasterTile, io: CatalystIO[R]): R = io.create(
- t match {
- case _: RasterRefTile => null
- case o => io.to(TileContext(o.extent, o.crs))
- },
- io.to[Tile](t)(TileUDT.tileSerializer)
- )
-
- override protected def from[R](t: R, io: CatalystIO[R]): ProjectedRasterTile = {
- val tile = io.get[Tile](t, 1)(TileUDT.tileSerializer)
- tile match {
- case r: RasterRefTile => r
- case _ =>
- val ctx = io.get[TileContext](t, 0)
- val resolved = tile match {
- case i: InternalRowTile => i.toArrayTile()
- case o => o
- }
- ProjectedRasterTile(resolved, ctx.extent, ctx.crs)
+ def apply(tile: Tile, extent: Extent, crs: CRS): ProjectedRasterTile = {
+ val tileArg = tile
+ val extentArg = extent
+ val crsArg = crs
+ new ProjectedRasterTile {
+ def tile: Tile = tileArg
+ def delegate: Tile = tileArg
+ def extent: Extent = extentArg
+ def crs: CRS = crsArg
}
- }
}
- implicit val prtEncoder: ExpressionEncoder[ProjectedRasterTile] = CatalystSerializerEncoder[ProjectedRasterTile](true)
+ def unapply(prt: ProjectedRasterTile): Option[(Tile, Extent, CRS)] =
+ Some((prt.tile, prt.extent, prt.crs))
+
+ implicit lazy val projectedRasterTileEncoder: ExpressionEncoder[ProjectedRasterTile] = ExpressionEncoder()
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala
index ba241b914..5cfebe493 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala
@@ -20,6 +20,7 @@
*/
package org.locationtech.rasterframes.tiles
+
import geotrellis.raster.{DelegatingTile, Tile, isNoData}
import org.locationtech.rasterframes._
@@ -38,24 +39,24 @@ object ShowableTile {
val ct = tile.cellType
val dims = tile.dimensions
- val data = if (tile.cellType.isFloatingPoint)
- tile.toArrayDouble().map {
- case c if isNoData(c) => "--"
- case c => c.toString
- }
- else tile.toArray().map {
- case c if isNoData(c) => "--"
- case c => c.toString
- }
+ val data =
+ if (tile.cellType.isFloatingPoint)
+ tile.toArrayDouble().map {
+ case c if isNoData(c) => "--"
+ case c => c.toString
+ } else tile.toArray().map {
+ case c if isNoData(c) => "--"
+ case c => c.toString
+ }
- val cells = if(tile.size <= maxCells) {
- data.mkString("[", ",", "]")
- }
- else {
- val front = data.take(maxCells/2).mkString("[", ",", "")
- val back = data.takeRight(maxCells/2).mkString("", ",", "]")
- front + ",...," + back
+ val cells =
+ if(tile.size <= maxCells) {
+ data.mkString("[", ",", "]")
+ } else {
+ val front = data.take(maxCells / 2).mkString("[", ",", "")
+ val back = data.takeRight(maxCells / 2).mkString("", ",", "]")
+ front + ",...," + back
+ }
+ s"[${ct.name}, $dims, $cells]"
}
- s"[${ct.name}, $dims, $cells]"
- }
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala b/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala
index b286fde0f..02eb9709d 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/util/DataBiasedOp.scala
@@ -32,18 +32,18 @@ import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
*/
object DataBiasedOp {
object BiasedMin extends DataBiasedOp {
- def op(z1: Int, z2: Int) = math.min(z1, z2)
- def op(z1: Double, z2: Double) = math.min(z1, z2)
+ def op(z1: Int, z2: Int): Int = math.min(z1, z2)
+ def op(z1: Double, z2: Double): Double = math.min(z1, z2)
}
object BiasedMax extends DataBiasedOp {
- def op(z1: Int, z2: Int) = math.max(z1, z2)
- def op(z1: Double, z2: Double) = math.max(z1, z2)
+ def op(z1: Int, z2: Int): Int = math.max(z1, z2)
+ def op(z1: Double, z2: Double): Double = math.max(z1, z2)
}
object BiasedAdd extends DataBiasedOp {
- def op(z1: Int, z2: Int) = z1 + z2
- def op(z1: Double, z2: Double) = z1 + z2
+ def op(z1: Int, z2: Int): Int = z1 + z2
+ def op(z1: Double, z2: Double): Double = z1 + z2
}
}
trait DataBiasedOp extends LocalTileBinaryOp {
@@ -52,19 +52,13 @@ trait DataBiasedOp extends LocalTileBinaryOp {
def combine(z1: Int, z2: Int): Int =
if (isNoData(z1) && isNoData(z2)) raster.NODATA
- else if (isNoData(z1))
- z2
- else if (isNoData(z2))
- z1
- else
- op(z1, z2)
+ else if (isNoData(z1)) z2
+ else if (isNoData(z2)) z1
+ else op(z1, z2)
def combine(z1: Double, z2: Double): Double =
if (isNoData(z1) && isNoData(z2)) raster.doubleNODATA
- else if (isNoData(z1))
- z2
- else if (isNoData(z2))
- z1
- else
- op(z1, z2)
+ else if (isNoData(z1)) z2
+ else if (isNoData(z2)) z1
+ else op(z1, z2)
}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala b/core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala
index 6c16975fe..57e1ac9a8 100644
--- a/core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala
+++ b/core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala
@@ -22,10 +22,10 @@
package org.locationtech.rasterframes.util
import geotrellis.raster.render.ColorRamps
-import org.apache.spark.sql.Dataset
+import org.apache.spark.sql.{Column, Dataset}
import org.apache.spark.sql.functions.{base64, concat, concat_ws, length, lit, substring, when}
import org.apache.spark.sql.jts.JTSTypes
-import org.apache.spark.sql.types.{StringType, StructField, BinaryType}
+import org.apache.spark.sql.types.{BinaryType, StringType, StructField}
import org.locationtech.rasterframes.expressions.DynamicExtractors
import org.locationtech.rasterframes.{rfConfig, rf_render_png, rf_resample}
import org.apache.spark.sql.rf.WithTypeConformity
@@ -37,8 +37,7 @@ trait DataFrameRenderers {
private val truncateWidth = rfConfig.getInt("max-truncate-row-element-length")
implicit class DFWithPrettyPrint(val df: Dataset[_]) {
-
- private def stringifyRowElements(cols: Seq[StructField], truncate: Boolean, renderTiles: Boolean) = {
+ private def stringifyRowElements(cols: Seq[StructField], truncate: Boolean, renderTiles: Boolean): Seq[Column] = {
cols
.map(c => {
val resolved = df.col(s"`${c.name}`")
@@ -73,16 +72,16 @@ trait DataFrameRenderers {
val header = cols.map(_.name).mkString("| ", " | ", " |") + "\n" + ("|---" * cols.length) + "|\n"
val stringifiers = stringifyRowElements(cols, truncate, renderTiles)
val cat = concat_ws(" | ", stringifiers: _*)
- val rows = df
- .select(cat)
- .limit(numRows)
- .as[String]
- .collect()
- .map(_.replaceAll("\\[", "\\\\["))
- .map(_.replace('\n', '↩'))
+ val rows =
+ df
+ .select(cat)
+ .limit(numRows)
+ .as[String]
+ .collect()
+ .map(_.replaceAll("\\[", "\\\\["))
+ .map(_.replace('\n', '↩'))
- val body = rows
- .mkString("| ", " |\n| ", " |")
+ val body = rows.mkString("| ", " |\n| ", " |")
val caption = if (rows.length >= numRows) s"\n_Showing only top $numRows rows_.\n\n" else ""
caption + header + body
@@ -94,13 +93,14 @@ trait DataFrameRenderers {
val header = "\n" + cols.map(_.name).mkString(" \n") + "\n"
val stringifiers = stringifyRowElements(cols, truncate, renderTiles)
val cat = concat_ws("", " ", "