Skip to content

Commit

Permalink
address the comment
Browse files Browse the repository at this point in the history
Signed-off-by: Firestarman <firestarmanllc@gmail.com>
  • Loading branch information
firestarman committed Mar 15, 2022
1 parent 5e81386 commit 6ccf551
Showing 1 changed file with 5 additions and 11 deletions.
16 changes: 5 additions & 11 deletions integration_tests/src/main/python/cache_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,29 +297,23 @@ def test_cache_daytimeinterval_input_row():

@pytest.mark.skipif(is_before_spark_330(), reason='DayTimeInterval is not supported before Spark3.3.0')
@ignore_order(local=True)
@allow_non_gpu("FileSourceScanExec", "ColumnarToRowExec")
@pytest.mark.parametrize('alongside_gen', [int_gen, ArrayGen(int_gen)], ids=idfn)
@pytest.mark.parametrize('with_rapids_memoryscan', ['true', 'false'],
ids=["rapids_memoryscan_on", "rapids_memoryscan_off"])
@pytest.mark.parametrize('with_rapids_reader', ['true', 'false'],
ids=["rapids_reader_on", "rapids_reader_off"])
def test_cache_daytimeinterval_input_columnar(spark_tmp_path, alongside_gen,
with_rapids_memoryscan, with_rapids_reader):
@pytest.mark.parametrize('enable_vectorized', ['true', 'false'], ids=idfn)
def test_cache_daytimeinterval_input_columnar(spark_tmp_path, enable_vectorized):
tmp_data_path = spark_tmp_path + '/PARQUET_DATA'
def test_func(spark):
two_col_df(spark,
DayTimeIntervalGen(), alongside_gen).write.mode("overwrite").parquet(tmp_data_path)
DayTimeIntervalGen(), int_gen).write.mode("overwrite").parquet(tmp_data_path)
df = spark.read.parquet(tmp_data_path)
df.cache().count()
return df.selectExpr('b', 'a')

test_conf = {
# Copied the configs from `test_cache_columnar`
# rapids-spark doesn't support LEGACY read for parquet, also set the int96 rebase mode
# values because LEGACY in databricks which will preclude this op from running on GPU.
'spark.sql.parquet.datetimeRebaseModeInWrite': 'CORRECTED',
'spark.sql.parquet.datetimeRebaseModeInRead' : 'CORRECTED',
'spark.sql.parquet.int96RebaseModeInWrite': 'CORRECTED',
'spark.sql.parquet.int96RebaseModeInRead' : 'CORRECTED',
'spark.rapids.sql.format.parquet.read.enabled': with_rapids_reader,
'spark.rapids.sql.exec.InMemoryTableScanExec': with_rapids_memoryscan}
'spark.sql.inMemoryColumnarStorage.enableVectorizedReader' : enable_vectorized}
assert_gpu_and_cpu_are_equal_collect(test_func, test_conf)

0 comments on commit 6ccf551

Please sign in to comment.