Stop using copyWithBooleanColumnAsValidity [databricks] (#11418)

* stop using copyWithBooleanColumnAsValidity Signed-off-by: Chong Gao <res_life@163.com> * Refactor: rename variable --------- Signed-off-by: Chong Gao <res_life@163.com> Co-authored-by: Chong Gao <res_life@163.com>
NVIDIA · Sep 3, 2024 · 6d4c4ef · 6d4c4ef
1 parent ee2049a
commit 6d4c4ef
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 7 deletions.
diff --git a/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOrcScan.scala b/sql-plugin/src/main/scala/com/nvidia/spark/rapids/GpuOrcScan.scala
@@ -202,8 +202,8 @@ object GpuOrcScan {
       withResource(overflowFlags) { _ =>
         // This is an integer type so we don't have to worry about
         // nested DTypes here.
-        withResource(Scalar.fromNull(toType)) { NULL =>
-          overflowFlags.ifElse(casted, NULL)
+        withResource(Scalar.fromNull(toType)) { nullVal =>
+          overflowFlags.ifElse(casted, nullVal)
         }
       }
     }
@@ -338,8 +338,8 @@ object GpuOrcScan {
         //   next convert to long,
         //   then down cast long to the target integral type.
         val longDoubles = withResource(doubleCanFitInLong(col)) { fitLongs =>
-          withResource(Scalar.fromNull(fromDt)) { NULL =>
-            fitLongs.ifElse(col, NULL)
+          withResource(Scalar.fromNull(fromDt)) { nullVal =>
+            fitLongs.ifElse(col, nullVal)
           }
         }
         withResource(longDoubles) { _ =>
@@ -394,7 +394,9 @@ object GpuOrcScan {
               withResource(doubleMillis.add(half)) { doubleMillisPlusHalf =>
                 withResource(doubleMillisPlusHalf.floor()) { millis =>
                   withResource(getOverflowFlags(doubleMillis, millis)) { overflowFlags =>
-                    millis.copyWithBooleanColumnAsValidity(overflowFlags)
+                    withResource(Scalar.fromNull(millis.getType)) { nullVal =>
+                      overflowFlags.ifElse(millis, nullVal)
+                    }
                   }
                 }
               }

diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/rapids/complexTypeExtractors.scala
@@ -298,8 +298,8 @@ case class GpuArrayContains(left: Expression, right: Expression)
       containsResult.or(_)
     }
     withResource(containsKeyOrNotContainsNull) { lcnn =>
-      withResource(Scalar.fromNull(DType.BOOL8)) { NULL =>
-        lcnn.ifElse(containsResult, NULL)
+      withResource(Scalar.fromNull(DType.BOOL8)) { nullVal =>
+        lcnn.ifElse(containsResult, nullVal)
       }
     }
   }