Fallback to CPU when encoding is not supported for JSON reader (#4622)

* fallback to CPU when encoding is not supported for JSON reader Signed-off-by: Bobby Wang <wbo4958@gmail.com> * change the log
NVIDIA · Jan 26, 2022 · 871a1df · 871a1df
1 parent 8e7a8c9
commit 871a1df
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala b/sql-plugin/src/main/scala/org/apache/spark/sql/catalyst/json/rapids/GpuJsonScan.scala
@@ -16,6 +16,8 @@
 
 package org.apache.spark.sql.catalyst.json.rapids
 
+import java.nio.charset.StandardCharsets
+
 import scala.collection.JavaConverters._
 
 import ai.rapids.cudf
@@ -138,6 +140,11 @@ object GpuJsonScan {
       meta.willNotWorkOnGpu("GpuJsonScan only supports \"\\n\" as a line separator")
     }
 
+    parsedOptions.encoding.foreach(enc =>
+      if (enc != StandardCharsets.UTF_8.name() && enc != StandardCharsets.US_ASCII.name()) {
+      meta.willNotWorkOnGpu("GpuJsonScan only supports UTF8 or US-ASCII encoded data")
+    })
+
     if (readSchema.map(_.dataType).contains(DateType)) {
       ShimLoader.getSparkShims.dateFormatInRead(parsedOptions).foreach { dateFormat =>
         if (!supportedDateFormats.contains(dateFormat)) {