executor: using ToHashKey to check equality of decimal when count(dis…

…tinct) (#9901) (#9931)
pingcap · Mar 28, 2019 · 2033c23 · 2033c23
1 parent 0f69856
commit 2033c23
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 22 deletions.
diff --git a/executor/aggfuncs/func_count.go b/executor/aggfuncs/func_count.go
@@ -280,54 +280,61 @@ func (e *countOriginalWithDistinct) UpdatePartialResult(sctx sessionctx.Context,
 func (e *countOriginalWithDistinct) evalAndEncode(
 	sctx sessionctx.Context, arg expression.Expression,
 	row chunk.Row, buf, encodedBytes []byte,
-) ([]byte, bool, error) {
+) (_ []byte, isNull bool, err error) {
 	switch tp := arg.GetType().EvalType(); tp {
 	case types.ETInt:
-		val, isNull, err := arg.EvalInt(sctx, row)
+		var val int64
+		val, isNull, err = arg.EvalInt(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendInt64(encodedBytes, buf, val)
 	case types.ETReal:
-		val, isNull, err := arg.EvalReal(sctx, row)
+		var val float64
+		val, isNull, err = arg.EvalReal(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendFloat64(encodedBytes, buf, val)
 	case types.ETDecimal:
-		val, isNull, err := arg.EvalDecimal(sctx, row)
+		var val *types.MyDecimal
+		val, isNull, err = arg.EvalDecimal(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
-		encodedBytes = appendDecimal(encodedBytes, buf, val)
+		encodedBytes, err = appendDecimal(encodedBytes, val)
 	case types.ETTimestamp, types.ETDatetime:
-		val, isNull, err := arg.EvalTime(sctx, row)
+		var val types.Time
+		val, isNull, err = arg.EvalTime(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendTime(encodedBytes, buf, val)
 	case types.ETDuration:
-		val, isNull, err := arg.EvalDuration(sctx, row)
+		var val types.Duration
+		val, isNull, err = arg.EvalDuration(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendDuration(encodedBytes, buf, val)
 	case types.ETJson:
-		val, isNull, err := arg.EvalJSON(sctx, row)
+		var val json.BinaryJSON
+		val, isNull, err = arg.EvalJSON(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendJSON(encodedBytes, buf, val)
 	case types.ETString:
-		val, isNull, err := arg.EvalString(sctx, row)
+		var val string
+		val, isNull, err = arg.EvalString(sctx, row)
 		if err != nil || isNull {
-			return encodedBytes, isNull, errors.Trace(err)
+			break
 		}
 		encodedBytes = appendString(encodedBytes, buf, val)
 	default:
 		return nil, false, errors.Errorf("unsupported column type for encode %d", tp)
 	}
-	return encodedBytes, false, nil
+	return encodedBytes, isNull, err
 }
 
 func appendInt64(encodedBytes, buf []byte, val int64) []byte {
@@ -344,11 +351,10 @@ func appendFloat64(encodedBytes, buf []byte, val float64) []byte {
 	return encodedBytes
 }
 
-func appendDecimal(encodedBytes, buf []byte, val *types.MyDecimal) []byte {
-	*(*types.MyDecimal)(unsafe.Pointer(&buf[0])) = *val
-	buf = buf[:types.MyDecimalStructSize]
-	encodedBytes = append(encodedBytes, buf...)
-	return encodedBytes
+func appendDecimal(encodedBytes []byte, val *types.MyDecimal) ([]byte, error) {
+	hash, err := val.ToHashKey()
+	encodedBytes = append(encodedBytes, hash...)
+	return encodedBytes, err
 }
 
 func writeTime(buf []byte, t types.Time) {

diff --git a/executor/executor_test.go b/executor/executor_test.go
@@ -1116,6 +1116,12 @@ func (s *testSuite) TestUnion(c *C) {
 	tk.MustExec("analyze table t2")
 	_, err = tk.Exec("(select a,b from t1 limit 2) union all (select a,b from t2 order by a limit 1) order by t1.b")
 	c.Assert(err.Error(), Equals, "[planner:1250]Table 't1' from one of the SELECTs cannot be used in global ORDER clause")
+
+	// #issue 9900
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b decimal(6, 3))")
+	tk.MustExec("insert into t values(1, 1.000)")
+	tk.MustQuery("select count(distinct a) from (select a from t union select b from t) tmp;").Check(testkit.Rows("1"))
 }
 
 func (s *testSuite) TestNeighbouringProj(c *C) {