Skip to content

Commit

Permalink
Remove the ORC encryption tests [databricks] (#5737)
Browse files Browse the repository at this point in the history
* Write Scala test for ORC encrypted write

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

* add a defensive piece of code to make sure we don't run this code for 3.1.x

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

* park

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

* remove the redundant test

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

Co-authored-by: Raza Jafri <rjafri@nvidia.com>
  • Loading branch information
razajafri and razajafri authored Jun 7, 2022
1 parent 6b44c21 commit b7649ec
Showing 1 changed file with 0 additions and 27 deletions.
27 changes: 0 additions & 27 deletions integration_tests/src/main/python/orc_write_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,30 +167,3 @@ def create_empty_df(spark, path):
lambda spark, path: spark.read.orc(path),
data_path,
conf={'spark.rapids.sql.format.orc.write.enabled': True})

@allow_non_gpu('DataWritingCommandExec')
@pytest.mark.parametrize("path", ["", "kms://http@localhost:9600/kms"])
@pytest.mark.parametrize("provider", ["", "hadoop"])
@pytest.mark.parametrize("encrypt", ["", "pii:a"])
@pytest.mark.parametrize("mask", ["", "sha256:a"])
@pytest.mark.skipif(is_databricks104_or_later(), reason="The test will fail on Databricks10.4 because `HadoopShimsPre2_3$NullKeyProvider` is loaded")
def test_orc_write_encryption_fallback(spark_tmp_path, spark_tmp_table_factory, path, provider, encrypt, mask):
def write_func(spark, write_path):
writer = unary_op_df(spark, gen).coalesce(1).write
if path != "":
writer.option("hadoop.security.key.provider.path", path)
if provider != "":
writer.option("orc.key.provider", provider)
if encrypt != "":
writer.option("orc.encrypt", encrypt)
if mask != "":
writer.option("orc.mask", mask)
writer.format("orc").mode('overwrite').option("path", write_path).saveAsTable(spark_tmp_table_factory.get())
if path == "" and provider == "" and encrypt == "" and mask == "":
pytest.skip("Skip this test when none of the encryption confs are set")
gen = IntegerGen()
data_path = spark_tmp_path + '/ORC_DATA'
assert_gpu_fallback_write(write_func,
lambda spark, path: spark.read.orc(path),
data_path,
'DataWritingCommandExec')

0 comments on commit b7649ec

Please sign in to comment.