Support automatic Binary/LargeBinary --> Utf8 coercion

apache · Oct 16, 2023 · dd8f41e · dd8f41e
1 parent 2a4e7f7
commit dd8f41e
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 8 deletions.
diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs
@@ -1558,10 +1558,9 @@ macro_rules! make_utf8_to_return_type {
         }
     };
 }
-
-/// `utf8_to_str_type` returns either a Utf8 or LargeUtf8 based on the input type size.
+// `utf8_to_str_type` returns either a Utf8 or LargeUtf8 based on the input type size.
 make_utf8_to_return_type!(utf8_to_str_type, DataType::LargeUtf8, DataType::Utf8);
-/// `utf8_to_str_type` returns either a Int32 or Int64 based on the input type size.
+// `utf8_to_str_type` returns either a Int32 or Int64 based on the input type size.
 make_utf8_to_return_type!(utf8_to_int_type, DataType::Int64, DataType::Int32);
 
 fn utf8_or_binary_to_binary_type(arg_type: &DataType, name: &str) -> Result<DataType> {

diff --git a/datafusion/expr/src/type_coercion/binary.rs b/datafusion/expr/src/type_coercion/binary.rs
@@ -650,13 +650,17 @@ fn string_concat_internal_coercion(
 
 /// Coercion rules for Strings: the type that both lhs and rhs can be
 /// casted to for the purpose of a string computation
+///
+/// Note this also permits coercing `Binary` and `LargeBinary` types to
+/// `Utf8` and `LargeUtf`, which will actually generate an error at runtime
+/// if the binary field holds invalid Utf8 data
 fn string_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option<DataType> {
     use arrow::datatypes::DataType::*;
     match (lhs_type, rhs_type) {
-        (Utf8, Utf8) => Some(Utf8),
-        (LargeUtf8, Utf8) => Some(LargeUtf8),
-        (Utf8, LargeUtf8) => Some(LargeUtf8),
-        (LargeUtf8, LargeUtf8) => Some(LargeUtf8),
+        (Utf8, Utf8 | Binary) => Some(Utf8),
+        (Utf8 | Binary, LargeUtf8) => Some(LargeUtf8),
+        (LargeUtf8 | LargeBinary, Utf8 | Binary) => Some(LargeUtf8),
+        (LargeUtf8 | LargeBinary, LargeUtf8 | LargeBinary) => Some(LargeUtf8),
         // TODO: cast between array elements (#6558)
         (List(_), List(_)) => Some(lhs_type.clone()),
         (List(_), _) => Some(lhs_type.clone()),

diff --git a/datafusion/sqllogictest/test_files/binary.slt b/datafusion/sqllogictest/test_files/binary.slt
@@ -221,8 +221,9 @@ NULL
 query error DataFusion error: type_coercion
 SELECT binary FROM t where binary LIKE '%F';
 
-query error DataFusion error: type_coercion
+query ?
 SELECT largebinary FROM t where largebinary LIKE '%F';
+----
 
 
 # character_length function
@@ -239,6 +240,15 @@ NULL NULL NULL NULL
 Bar 3 Bar 3
 FooBar 6 FooBar 6
 
+query I
+SELECT character_length(X'20');
+----
+1
+
+# still errors on values that can not be coerced to utf8
+query error Encountered non UTF\-8 data: invalid utf\-8 sequence of 1 bytes from index 0
+SELECT character_length(X'c328');
+
 # regexp_replace
 query TTTT
 SELECT