From 675f51f34c3f87cc81d7d9982c4ddcfb310e7390 Mon Sep 17 00:00:00 2001 From: "Jason T. Brown" Date: Wed, 31 Jul 2019 13:11:38 -0400 Subject: [PATCH 1/6] Python test_sql refactor all comparisons to inside SQL Signed-off-by: Jason T. Brown --- .../main/python/tests/PyRasterFramesTests.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py index f93383555..9b1add5f0 100644 --- a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py +++ b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py @@ -234,22 +234,34 @@ def test_sql(self): rf_local_add(tile, 1) AS and_one, rf_local_subtract(tile, 1) AS less_one, rf_local_multiply(tile, 2) AS times_two, - rf_local_divide(tile, 2) AS over_two + rf_local_divide(rf_convert_cell_type(tile, "float32"), 2) AS over_two FROM rf_test_sql""").createOrReplaceTempView('rf_test_sql_1') - statsRow = self.spark.sql(""" + stats = self.spark.sql(""" SELECT rf_tile_mean(tile) as base, rf_tile_mean(and_one) as plus_one, rf_tile_mean(less_one) as minus_one, rf_tile_mean(times_two) as double, rf_tile_mean(over_two) as half FROM rf_test_sql_1 - """).first() - - self.assertTrue(self.rounded_compare(statsRow.base, statsRow.plus_one - 1)) - self.assertTrue(self.rounded_compare(statsRow.base, statsRow.minus_one + 1)) - self.assertTrue(self.rounded_compare(statsRow.base, statsRow.double / 2)) - self.assertTrue(self.rounded_compare(statsRow.base, statsRow.half * 2)) + """) + stats.createOrReplaceTempView('rf_test_sql_stats') + + compare = self.spark.sql(""" + SELECT + plus_one - 1.0 = base as add, + minus_one + 1.0 = base as subtract, + double / 2.0 = base as multiply, + half * 2.0 = base as divide + FROM rf_test_sql_stats + """) + + expect_row = compare.first() + + self.assertTrue(expect_row.add) + self.assertTrue(expect_row.subtract) + self.assertTrue(expect_row.multiply) + self.assertTrue(expect_row.divide) def test_explode(self): import pyspark.sql.functions as F From 09be7a4298542fe37ae96ba7f19804b6ddbf99d4 Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Thu, 1 Aug 2019 11:36:40 -0400 Subject: [PATCH 2/6] Added dataframe output for post error analysis. --- .../src/main/python/tests/PyRasterFramesTests.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py index 9b1add5f0..2931d86e6 100644 --- a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py +++ b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py @@ -230,21 +230,24 @@ def test_aggregations(self): def test_sql(self): self.rf.createOrReplaceTempView("rf_test_sql") - self.spark.sql("""SELECT tile, - rf_local_add(tile, 1) AS and_one, + arith = self.spark.sql("""SELECT tile, + rf_local_add(tile, 1) AS add_one, rf_local_subtract(tile, 1) AS less_one, rf_local_multiply(tile, 2) AS times_two, rf_local_divide(rf_convert_cell_type(tile, "float32"), 2) AS over_two - FROM rf_test_sql""").createOrReplaceTempView('rf_test_sql_1') + FROM rf_test_sql""")\ + arith.createOrReplaceTempView('rf_test_sql_1') + arith.show(truncate=False) stats = self.spark.sql(""" SELECT rf_tile_mean(tile) as base, - rf_tile_mean(and_one) as plus_one, + rf_tile_mean(add_one) as plus_one, rf_tile_mean(less_one) as minus_one, rf_tile_mean(times_two) as double, rf_tile_mean(over_two) as half FROM rf_test_sql_1 """) + stats.show(truncate=False) stats.createOrReplaceTempView('rf_test_sql_stats') compare = self.spark.sql(""" From 2a380dc242ef497d211920d905cb4ba82fd7be91 Mon Sep 17 00:00:00 2001 From: "Simeon H.K. Fitch" Date: Thu, 1 Aug 2019 12:04:24 -0400 Subject: [PATCH 3/6] Changed ordering of tests to see if others pass. --- pyrasterframes/src/main/python/tests/PyRasterFramesTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py index 2931d86e6..2daaafd5d 100644 --- a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py +++ b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py @@ -261,10 +261,10 @@ def test_sql(self): expect_row = compare.first() - self.assertTrue(expect_row.add) self.assertTrue(expect_row.subtract) self.assertTrue(expect_row.multiply) self.assertTrue(expect_row.divide) + self.assertTrue(expect_row.add) def test_explode(self): import pyspark.sql.functions as F From 76df01c921b5be2462ed5ea777f9179a4777e592 Mon Sep 17 00:00:00 2001 From: "Jason T. Brown" Date: Fri, 2 Aug 2019 12:06:33 -0400 Subject: [PATCH 4/6] Pin python test to pyrsistent 0.15.3 Signed-off-by: Jason T. Brown --- pyrasterframes/src/main/python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyrasterframes/src/main/python/setup.py b/pyrasterframes/src/main/python/setup.py index 91d5cd188..f4bc3f449 100644 --- a/pyrasterframes/src/main/python/setup.py +++ b/pyrasterframes/src/main/python/setup.py @@ -128,7 +128,8 @@ def doctype(self): 'setuptools>=0.8', 'ipython==6.2.1', "ipykernel==4.8.0", - 'Pweave==0.30.3' + 'Pweave==0.30.3', + 'pyrsistent==0.15.3', ], tests_require=[ 'pytest==3.4.2', From 85121ecc3f1958fc4ec7e4f193ea1223c4c1904a Mon Sep 17 00:00:00 2001 From: "Jason T. Brown" Date: Fri, 2 Aug 2019 13:02:30 -0400 Subject: [PATCH 5/6] Add deterministic failing conditions to python test_sql Signed-off-by: Jason T. Brown --- .../main/python/tests/PyRasterFramesTests.py | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py index 2daaafd5d..3468e7e67 100644 --- a/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py +++ b/pyrasterframes/src/main/python/tests/PyRasterFramesTests.py @@ -244,27 +244,48 @@ def test_sql(self): rf_tile_mean(add_one) as plus_one, rf_tile_mean(less_one) as minus_one, rf_tile_mean(times_two) as double, - rf_tile_mean(over_two) as half + rf_tile_mean(over_two) as half, + rf_no_data_cells(tile) as nd + FROM rf_test_sql_1 + ORDER BY rf_no_data_cells(tile) """) stats.show(truncate=False) stats.createOrReplaceTempView('rf_test_sql_stats') - compare = self.spark.sql(""" + compare1 = self.spark.sql(""" SELECT plus_one - 1.0 = base as add, minus_one + 1.0 = base as subtract, double / 2.0 = base as multiply, half * 2.0 = base as divide FROM rf_test_sql_stats + ORDER BY nd """) - expect_row = compare.first() + expect_row1 = compare1.first() - self.assertTrue(expect_row.subtract) - self.assertTrue(expect_row.multiply) - self.assertTrue(expect_row.divide) - self.assertTrue(expect_row.add) + self.assertTrue(expect_row1.subtract) + self.assertTrue(expect_row1.multiply) + self.assertTrue(expect_row1.divide) + self.assertTrue(expect_row1.add) + + compare2 = self.spark.sql(""" + SELECT + plus_one - 1.0 = base as add, + minus_one + 1.0 = base as subtract, + double / 2.0 = base as multiply, + half * 2.0 = base as divide + FROM rf_test_sql_stats + ORDER BY -nd + """) + + expect_row2 = compare2.first() + + self.assertTrue(expect_row2.subtract) + self.assertTrue(expect_row2.multiply) + self.assertTrue(expect_row2.divide) + self.assertTrue(expect_row2.add) # <-- fails due to NoData handling. ND + 1 = 1 def test_explode(self): import pyspark.sql.functions as F From 83ea6dcfe53236bb836057f85e489a9609268d37 Mon Sep 17 00:00:00 2001 From: "Jason T. Brown" Date: Fri, 2 Aug 2019 13:14:24 -0400 Subject: [PATCH 6/6] No need to pin pyrsistent version Signed-off-by: Jason T. Brown --- pyrasterframes/src/main/python/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyrasterframes/src/main/python/setup.py b/pyrasterframes/src/main/python/setup.py index f4bc3f449..720e07b53 100644 --- a/pyrasterframes/src/main/python/setup.py +++ b/pyrasterframes/src/main/python/setup.py @@ -129,7 +129,6 @@ def doctype(self): 'ipython==6.2.1', "ipykernel==4.8.0", 'Pweave==0.30.3', - 'pyrsistent==0.15.3', ], tests_require=[ 'pytest==3.4.2',