Override the seed for test_map_scalars_supported_key_types for ver…

…sion of Spark before 3.4.0 [Databricks] (#9949)

* Override the seed for version of Spark before 3.4.0

* Signing off

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

* addressed review comments

* made seed param mandatory

---------

Signed-off-by: Raza Jafri <rjafri@nvidia.com>

integration_tests/src/main/python/conftest.py

@@ -154,7 +154,14 @@ def pytest_runtest_setup(item):
     _inject_oom = item.get_closest_marker('inject_oom')
     datagen_overrides = item.get_closest_marker('datagen_overrides')
     if datagen_overrides:
-        _test_datagen_random_seed = datagen_overrides.kwargs.get('seed', _test_datagen_random_seed)
+        try:
+            seed = datagen_overrides.kwargs["seed"]
+        except KeyError:
+            raise Exception("datagen_overrides requires an override seed value")
+
+        override_seed = datagen_overrides.kwargs.get('condition', True)
+        if override_seed:
+            _test_datagen_random_seed = seed
 
     order = item.get_closest_marker('ignore_order')
     if order:

integration_tests/src/main/python/map_test.py

@@ -192,7 +192,9 @@ def query_map_scalar(spark):
 
 
 @allow_non_gpu('WindowLocalExec')
-@datagen_overrides(seed=0, reason='https://github.com/NVIDIA/spark-rapids/issues/9683')
+@datagen_overrides(seed=0, condition=is_before_spark_314()
+                             or (not is_before_spark_320() and is_before_spark_323())
+                             or (not is_before_spark_330() and is_before_spark_331()), reason="https://issues.apache.org/jira/browse/SPARK-40089")
 @pytest.mark.parametrize('data_gen', supported_key_map_gens, ids=idfn)
 @pytest.mark.xfail(condition = is_not_utc(), reason = 'xfail non-UTC time zone tests because of https://github.com/NVIDIA/spark-rapids/issues/9653')
 def test_map_scalars_supported_key_types(data_gen):

integration_tests/src/main/python/spark_session.py

@@ -128,12 +128,18 @@ def is_before_spark_312():
 def is_before_spark_313():
     return spark_version() < "3.1.3"
 
+def is_before_spark_314():
+    return spark_version() < "3.1.4"
+
 def is_before_spark_320():
     return spark_version() < "3.2.0"
 
 def is_before_spark_322():
     return spark_version() < "3.2.2"
 
+def is_before_spark_323():
+    return spark_version() < "3.2.3"
+
 def is_before_spark_330():
     return spark_version() < "3.3.0"