Change the assert to require for handling TIMESTAMP_MILLIS in isDateT…

…imeRebaseNeeded (NVIDIA#1147) * Change datatype assert to require for TIMESTAMP_MILLIS Signed-off-by: Thomas Graves <tgraves@nvidia.com> * update test to ignore order * orc ignore order
tgravescs · Nov 18, 2020 · 0ff7c56 · 0ff7c56
1 parent 09ce5c7
commit 0ff7c56
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/integration_tests/src/main/python/orc_write_test.py b/integration_tests/src/main/python/orc_write_test.py
@@ -109,6 +109,7 @@ def test_orc_write_compression_fallback(spark_tmp_path, codec, spark_tmp_table_f
             'DataWritingCommandExec',
             conf=all_confs)
 
+@ignore_order
 @allow_non_gpu('DataWritingCommandExec')
 def test_buckets_write_fallback(spark_tmp_path, spark_tmp_table_factory):
     data_path = spark_tmp_path + '/ORC_DATA'

diff --git a/integration_tests/src/main/python/parquet_write_test.py b/integration_tests/src/main/python/parquet_write_test.py
@@ -229,6 +229,7 @@ def test_parquet_writeLegacyFormat_fallback(spark_tmp_path, spark_tmp_table_fact
             'DataWritingCommandExec',
             conf=all_confs)
 
+@ignore_order
 @allow_non_gpu('DataWritingCommandExec')
 def test_buckets_write_fallback(spark_tmp_path, spark_tmp_table_factory):
     data_path = spark_tmp_path + '/PARQUET_DATA'

diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/RebaseHelper.scala b/sql-plugin/src/main/scala/com/nvidia/spark/RebaseHelper.scala
@@ -39,7 +39,9 @@ object RebaseHelper extends Arm {
         }
       }
     } else if (dtype.isTimestampType) {
-      assert(dtype == DType.TIMESTAMP_MICROSECONDS)
+      // TODO - https://github.com/NVIDIA/spark-rapids/issues/1130 to properly handle
+      // TIMESTAMP_MILLIS, for use require so we fail if that happens
+      require(dtype == DType.TIMESTAMP_MICROSECONDS)
       withResource(
         Scalar.timestampFromLong(DType.TIMESTAMP_MICROSECONDS, startTs)) { minGood =>
         withResource(column.lessThan(minGood)) { hasBad =>