DRILL-7887: JDBC plugin returns incorrect precision for SUM(DECIMAL) …

…queries
logicdrop · Mar 30, 2021 · 82fc42e · 82fc42e
1 parent 4aefcef
commit 82fc42e
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 4 deletions.
diff --git a/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcRecordReader.java b/contrib/storage-jdbc/src/main/java/org/apache/drill/exec/store/jdbc/JdbcRecordReader.java
@@ -64,6 +64,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.drill.exec.planner.types.DrillRelDataTypeSystem.DRILL_REL_DATATYPE_SYSTEM;
+
 class JdbcRecordReader extends AbstractRecordReader {
 
   private static final Logger logger = LoggerFactory.getLogger(JdbcRecordReader.class);
@@ -210,8 +212,8 @@ public void setup(OperatorContext operatorContext, OutputMutator output) {
         String name = columns.get(i - 1).getRootSegmentPath();
         // column index in ResultSetMetaData starts from 1
         int jdbcType = meta.getColumnType(i);
-        int width = meta.getPrecision(i);
-        int scale = meta.getScale(i);
+        int width = Math.min(meta.getPrecision(i), DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision());
+        int scale = Math.min(meta.getScale(i), DRILL_REL_DATATYPE_SYSTEM.getMaxNumericScale());
         MinorType minorType = JDBC_TYPE_MAPPINGS.get(jdbcType);
         if (minorType == null) {
           logger.warn("Ignoring column that is unsupported.", UserException
@@ -381,6 +383,14 @@ private static class DecimalCopier extends Copier<NullableVarDecimalVector.Mutat
     void copy(int index) throws SQLException {
       BigDecimal decimal = result.getBigDecimal(columnIndex);
       if (decimal != null) {
+        if (decimal.precision() > DRILL_REL_DATATYPE_SYSTEM.getMaxNumericPrecision()
+            || decimal.scale() > DRILL_REL_DATATYPE_SYSTEM.getMaxNumericScale()) {
+          throw UserException.unsupportedError()
+              .message("Drill doesn't support reading values with precision or scale larger than 38.\n" +
+                  "Please use round() UDF to obtain results with supported precision")
+              .addContext("Column index", index)
+              .build(logger);
+        }
         mutator.setSafe(index, decimal);
       }
     }

diff --git a/...torage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java b/...torage-jdbc/src/test/java/org/apache/drill/exec/store/jdbc/TestJdbcPluginWithMySQLIT.java
@@ -23,10 +23,16 @@
 import com.wix.mysql.config.SchemaConfig;
 import com.wix.mysql.distribution.Version;
 import org.apache.drill.categories.JdbcStorageTest;
+import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.expr.fn.impl.DateUtility;
+import org.apache.drill.exec.physical.rowSet.DirectRowSet;
+import org.apache.drill.exec.physical.rowSet.RowSet;
+import org.apache.drill.exec.record.metadata.SchemaBuilder;
+import org.apache.drill.exec.record.metadata.TupleMetadata;
 import org.apache.drill.test.ClusterFixture;
 import org.apache.drill.test.ClusterTest;
 import org.apache.drill.test.QueryTestUtil;
+import org.apache.drill.test.rowSet.RowSetUtilities;
 import org.joda.time.DateTimeZone;
 import org.junit.AfterClass;
 import org.junit.Assume;
@@ -197,6 +203,24 @@ public void pushDownJoinAndFilterPushDown() throws Exception {
         .match();
   }
 
+  @Test
+  public void pushDownAggWithDecimal() throws Exception {
+    String query = "SELECT sum(decimal_field * smallint_field) AS `order_total`\n" +
+        "FROM mysql.`drill_mysql_test`.person e";
+
+    DirectRowSet results = queryBuilder().sql(query).rowSet();
+
+    TupleMetadata expectedSchema = new SchemaBuilder()
+        .addNullable("order_total", TypeProtos.MinorType.VARDECIMAL, 38, 2)
+        .buildSchema();
+
+    RowSet expected = client.rowSetBuilder(expectedSchema)
+        .addRow(123.32)
+        .build();
+
+    RowSetUtilities.verify(expected, results);
+  }
+
   @Test
   public void testPhysicalPlanSubmission() throws Exception {
     String query = "select * from mysql.`drill_mysql_test`.person";

diff --git a/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql b/contrib/storage-jdbc/src/test/resources/mysql-test-data.sql
@@ -38,6 +38,7 @@ create table person (
   long_text_field LONGTEXT,
   blob_field      BLOB,
   bit_field       BIT,
+  decimal_field   DECIMAL(15, 2),
 
   enum_field      ENUM('XXX', 'YYY', 'ZZZ') NOT NULL
 );
@@ -47,7 +48,7 @@ insert into person (first_name, last_name, address, city, state, zip, bigint_fie
                     time_field, timestamp_field, date_field, datetime_field, year_field,
                     json,
                     text_field, tiny_text_field, medium_text_field, long_text_field, blob_field, bit_field,
-                    enum_field)
+                    decimal_field, enum_field)
     values ('first_name_1', 'last_name_1', '1401 John F Kennedy Blvd', 'Philadelphia', 'PA', 19107, 123456789, 1, 10.01,
             false, 1.0, 1.1, 1.2,
             '13:00:01', '2012-02-29 13:00:01', '2012-02-29', '2012-02-29 13:00:01', 2015,
@@ -56,7 +57,7 @@ insert into person (first_name, last_name, address, city, state, zip, bigint_fie
             'a medium piece of text', 'a longer piece of text this is going on.....',
             'this is a test',
             1,
-            'XXX');
+            123.321, 'XXX');
 
 insert into person (first_name, last_name, address, city, state, zip, bigint_field, smallint_field, numeric_field,
                     boolean_field, double_field, float_field, real_field,