util, types: fix overflow & adjustment for the year type in index ran…

…ger (#20338) (#20907)

Signed-off-by: ti-srebot <ti-srebot@pingcap.com>

executor/write_test.go

@@ -1470,7 +1470,7 @@ func (s *testSuite8) TestUpdate(c *C) {
 	tk.MustExec("drop table t")
 	tk.MustExec("CREATE TABLE `t` (	`c1` year DEFAULT NULL, `c2` year DEFAULT NULL, `c3` date DEFAULT NULL, `c4` datetime DEFAULT NULL,	KEY `idx` (`c1`,`c2`))")
 	_, err = tk.Exec("UPDATE t SET c2=16777215 WHERE c1>= -8388608 AND c1 < -9 ORDER BY c1 LIMIT 2")
-	c.Assert(err.Error(), Equals, "[types:1690]DECIMAL value is out of range in '(4, 0)'")
+	c.Assert(err, IsNil)
 
 	tk.MustExec("update (select * from t) t set c1 = 1111111")
 

types/convert_test.go

@@ -673,20 +673,20 @@ func (s *testTypeConvertSuite) TestConvert(c *C) {
 	signedAccept(c, mysql.TypeDouble, "1e+1", "10")
 
 	// year
-	signedDeny(c, mysql.TypeYear, 123, "0")
-	signedDeny(c, mysql.TypeYear, 3000, "0")
+	signedDeny(c, mysql.TypeYear, 123, "1901")
+	signedDeny(c, mysql.TypeYear, 3000, "2155")
 	signedAccept(c, mysql.TypeYear, "2000", "2000")
 	signedAccept(c, mysql.TypeYear, "abc", "0")
 	signedAccept(c, mysql.TypeYear, "00abc", "2000")
 	signedAccept(c, mysql.TypeYear, "0019", "2019")
 	signedAccept(c, mysql.TypeYear, 2155, "2155")
 	signedAccept(c, mysql.TypeYear, 2155.123, "2155")
-	signedDeny(c, mysql.TypeYear, 2156, "0")
-	signedDeny(c, mysql.TypeYear, 123.123, "0")
-	signedDeny(c, mysql.TypeYear, 1900, "0")
+	signedDeny(c, mysql.TypeYear, 2156, "2155")
+	signedDeny(c, mysql.TypeYear, 123.123, "1901")
+	signedDeny(c, mysql.TypeYear, 1900, "1901")
 	signedAccept(c, mysql.TypeYear, 1901, "1901")
 	signedAccept(c, mysql.TypeYear, 1900.567, "1901")
-	signedDeny(c, mysql.TypeYear, 1900.456, "0")
+	signedDeny(c, mysql.TypeYear, 1900.456, "1901")
 	signedAccept(c, mysql.TypeYear, 0, "0")
 	signedAccept(c, mysql.TypeYear, "0", "2000")
 	signedAccept(c, mysql.TypeYear, "00", "2000")
@@ -707,7 +707,7 @@ func (s *testTypeConvertSuite) TestConvert(c *C) {
 	signedAccept(c, mysql.TypeYear, "70", "1970")
 	signedAccept(c, mysql.TypeYear, 99, "1999")
 	signedAccept(c, mysql.TypeYear, "99", "1999")
-	signedDeny(c, mysql.TypeYear, 100, "0")
+	signedDeny(c, mysql.TypeYear, 100, "1901")
 	signedDeny(c, mysql.TypeYear, "99999999999999999999999999999999999", "0")
 
 	// time from string

types/datum.go

@@ -1370,6 +1370,50 @@ func (d *Datum) convertToMysqlYear(sc *stmtctx.StatementContext, target *FieldTy
 	return ret, err
 }
 
+// ConvertDatumToFloatYear converts datum into MySQL year with float type
+func ConvertDatumToFloatYear(sc *stmtctx.StatementContext, d Datum) (Datum, error) {
+	return d.convertToMysqlFloatYear(sc, types.NewFieldType(mysql.TypeYear))
+}
+
+func (d *Datum) convertToMysqlFloatYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) {
+	var (
+		ret    Datum
+		y      float64
+		err    error
+		adjust bool
+	)
+	switch d.k {
+	case KindString, KindBytes:
+		s := d.GetString()
+		trimS := strings.TrimSpace(s)
+		y, err = StrToFloat(sc, trimS, false)
+		if err != nil {
+			ret.SetFloat64(0)
+			return ret, errors.Trace(err)
+		}
+		// condition:
+		// parsed to 0, not a string of length 4, the first valid char is a 0 digit
+		if len(s) != 4 && y == 0 && strings.HasPrefix(trimS, "0") {
+			adjust = true
+		}
+	case KindMysqlTime:
+		y = float64(d.GetMysqlTime().Year())
+	case KindMysqlDuration:
+		y = float64(time.Now().Year())
+	default:
+		ret, err = d.convertToFloat(sc, NewFieldType(mysql.TypeDouble))
+		if err != nil {
+			_, err = invalidConv(d, target.Tp)
+			ret.SetFloat64(0)
+			return ret, err
+		}
+		y = ret.GetFloat64()
+	}
+	y = adjustYearForFloat(y, adjust)
+	ret.SetFloat64(y)
+	return ret, err
+}
+
 func (d *Datum) convertToMysqlBit(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) {
 	var ret Datum
 	var uintValue uint64

types/time.go

@@ -1207,13 +1207,31 @@ func AdjustYear(y int64, shouldAdjust bool) (int64, error) {
 		return y, nil
 	}
 	y = int64(adjustYear(int(y)))
-	if y < int64(MinYear) || y > int64(MaxYear) {
+	if y < 0 {
 		return 0, errors.Trace(ErrInvalidYear)
 	}
+	if y < int64(MinYear) {
+		return int64(MinYear), errors.Trace(ErrInvalidYear)
+	}
+	if y > int64(MaxYear) {
+		return int64(MaxYear), errors.Trace(ErrInvalidYear)
+	}
 
 	return y, nil
 }
 
+func adjustYearForFloat(y float64, shouldAdjust bool) float64 {
+	if y == 0 && !shouldAdjust {
+		return y
+	}
+	if y >= 0 && y <= 69 {
+		y = 2000 + y
+	} else if y >= 70 && y <= 99 {
+		y = 1900 + y
+	}
+	return y
+}
+
 // NewDuration construct duration with time.
 func NewDuration(hour, minute, second, microsecond int, fsp int8) Duration {
 	return Duration{

util/ranger/points.go

@@ -210,19 +210,28 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []point {
 		ft    *types.FieldType
 	)
 
-	// refineValue refines the constant datum for string type since we may eval the constant to another collation instead of its own collation.
-	refineValue := func(col *expression.Column, value *types.Datum) {
+	// refineValue refines the constant datum:
+	// 1. for string type since we may eval the constant to another collation instead of its own collation.
+	// 2. for year type since 2-digit year value need adjustment, see https://dev.mysql.com/doc/refman/5.6/en/year.html
+	refineValue := func(col *expression.Column, value *types.Datum) (err error) {
 		if col.RetType.EvalType() == types.ETString && value.Kind() == types.KindString {
 			value.SetString(value.GetString(), col.RetType.Collate)
 		}
+		if col.GetType().Tp == mysql.TypeYear {
+			*value, err = types.ConvertDatumToFloatYear(r.sc, *value)
+		}
+		return
 	}
 	if col, ok := expr.GetArgs()[0].(*expression.Column); ok {
 		ft = col.RetType
 		value, err = expr.GetArgs()[1].Eval(chunk.Row{})
 		if err != nil {
 			return nil
 		}
-		refineValue(col, &value)
+		err = refineValue(col, &value)
+		if err != nil {
+			return nil
+		}
 		op = expr.FuncName.L
 	} else {
 		col, ok := expr.GetArgs()[1].(*expression.Column)
@@ -234,7 +243,10 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []point {
 		if err != nil {
 			return nil
 		}
-		refineValue(col, &value)
+		err = refineValue(col, &value)
+		if err != nil {
+			return nil
+		}
 
 		switch expr.FuncName.L {
 		case ast.GE:
@@ -383,6 +395,13 @@ func (r *builder) buildFromIn(expr *expression.ScalarFunction) ([]point, bool) {
 		if dt.Kind() == types.KindString {
 			dt.SetString(dt.GetString(), colCollate)
 		}
+		if expr.GetArgs()[0].GetType().Tp == mysql.TypeYear {
+			dt, err = types.ConvertDatumToFloatYear(r.sc, dt)
+			if err != nil {
+				r.err = ErrUnsupportedType.GenWithStack("expr:%v is not converted to year", e)
+				return fullRange, hasNull
+			}
+		}
 		var startValue, endValue types.Datum
 		dt.Copy(&startValue)
 		dt.Copy(&endValue)

util/ranger/ranger.go

@@ -23,6 +23,7 @@ import (
 	"github.com/pingcap/parser/ast"
 	"github.com/pingcap/parser/charset"
 	"github.com/pingcap/parser/mysql"
+	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/expression"
 	"github.com/pingcap/tidb/kv"
 	"github.com/pingcap/tidb/sessionctx"
@@ -92,7 +93,10 @@ func convertPoint(sc *stmtctx.StatementContext, point point, tp *types.FieldType
 	}
 	casted, err := point.value.ConvertTo(sc, tp)
 	if err != nil {
-		return point, errors.Trace(err)
+		// see issue #20101: overflow when converting integer to year
+		if tp.Tp != mysql.TypeYear || !terror.ErrorEqual(err, types.ErrOverflow) {
+			return point, errors.Trace(err)
+		}
 	}
 	valCmpCasted, err := point.value.CompareDatum(sc, &casted)
 	if err != nil {

util/ranger/ranger_test.go

@@ -1199,3 +1199,154 @@ func (s *testRangerSuite) TestIndexStringIsTrueRange(c *C) {
 		testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...))
 	}
 }
+
+func (s *testRangerSuite) TestIndexRangeForYear(c *C) {
+	defer testleak.AfterTest(c)()
+	dom, store, err := newDomainStoreWithBootstrap(c)
+	defer func() {
+		dom.Close()
+		store.Close()
+	}()
+	c.Assert(err, IsNil)
+	testKit := testkit.NewTestKit(c, store)
+
+	// for issue #20101: overflow when converting integer to year
+	testKit.MustExec("use test")
+	testKit.MustExec("DROP TABLE IF EXISTS `table_30_utf8_undef`")
+	testKit.MustExec("CREATE TABLE `table_30_utf8_undef` (\n  `pk` int(11) NOT NULL\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin")
+	testKit.MustExec("INSERT INTO `table_30_utf8_undef` VALUES (29)")
+
+	testKit.MustExec("DROP TABLE IF EXISTS `table_40_utf8_4`")
+	testKit.MustExec("CREATE TABLE `table_40_utf8_4`(\n  `pk` int(11) NOT NULL,\n  `col_int_key_unsigned` int(10) unsigned DEFAULT NULL,\n  `col_year_key_signed` year(4) DEFAULT NULL,\n" +
+		"PRIMARY KEY (`pk`),\n  KEY `col_int_key_unsigned` (`col_int_key_unsigned`),\n  KEY `col_year_key_signed` (`col_year_key_signed`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin")
+
+	testKit.MustExec("INSERT INTO `table_40_utf8_4` VALUES (36, 10 ,1)")
+
+	testKit.MustQuery("SELECT sum(tmp.val) AS val FROM (" +
+		"SELECT count(1) AS val FROM table_40_utf8_4 JOIN table_30_utf8_undef\n" +
+		"WHERE table_40_utf8_4.col_year_key_signed!=table_40_utf8_4.col_int_key_unsigned\n" +
+		"AND table_40_utf8_4.col_int_key_unsigned=\"15698\") AS tmp").
+		Check(testkit.Rows("0"))
+
+	// test index range
+	testKit.MustExec("DROP TABLE IF EXISTS t")
+	testKit.MustExec("CREATE TABLE t (a year(4), key(a))")
+	testKit.MustExec("INSERT INTO t VALUES (1), (70), (99), (0), ('0')")
+	testKit.MustQuery("SELECT * FROM t WHERE a < 15698").Check(testkit.Rows("0", "1970", "1999", "2000", "2001"))
+	testKit.MustQuery("SELECT * FROM t WHERE a <= 0").Check(testkit.Rows("0"))
+	testKit.MustQuery("SELECT * FROM t WHERE a < 2000").Check(testkit.Rows("0", "1970", "1999"))
+	testKit.MustQuery("SELECT * FROM t WHERE a > -1").Check(testkit.Rows("0", "1970", "1999", "2000", "2001"))
+
+	tests := []struct {
+		indexPos    int
+		exprStr     string
+		accessConds string
+		filterConds string
+		resultStr   string
+	}{
+		{
+			indexPos:    0,
+			exprStr:     `a not in (0, 1, 2)`,
+			accessConds: "[not(in(test.t.a, 0, 1, 2))]",
+			filterConds: "[]",
+			resultStr:   `[(NULL,0) (0,2001) (2002,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a not in (-1, 1, 2)`,
+			accessConds: "[not(in(test.t.a, -1, 1, 2))]",
+			filterConds: "[]",
+			resultStr:   `[(NULL,0) [0,2001) (2002,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a not in (1, 2, 70)`,
+			accessConds: "[not(in(test.t.a, 1, 2, 70))]",
+			filterConds: "[]",
+			resultStr:   `[(NULL,1970) (1970,2001) (2002,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a not in (99)`,
+			accessConds: "[ne(test.t.a, 99)]",
+			filterConds: "[]",
+			resultStr:   `[[-inf,1999) (1999,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a not in (1, 2, 15698)`,
+			accessConds: "[not(in(test.t.a, 1, 2, 15698))]",
+			filterConds: "[]",
+			resultStr:   `[(NULL,2001) (2002,2155] (2155,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a >= -1000`,
+			accessConds: "[ge(test.t.a, -1000)]",
+			filterConds: "[]",
+			resultStr:   `[[0,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a > -1000`,
+			accessConds: "[gt(test.t.a, -1000)]",
+			filterConds: "[]",
+			resultStr:   `[[0,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a != 1`,
+			accessConds: "[ne(test.t.a, 1)]",
+			filterConds: "[]",
+			resultStr:   `[[-inf,2001) (2001,+inf]]`,
+		},
+		{
+			indexPos:    0,
+			exprStr:     `a != 2156`,
+			accessConds: "[ne(test.t.a, 2156)]",
+			filterConds: "[]",
+			resultStr:   `[[-inf,2155] (2155,+inf]]`,
+		},
+		{
+			exprStr:     "a < 99 or a > 01",
+			accessConds: "[or(lt(test.t.a, 99), gt(test.t.a, 1))]",
+			filterConds: "[]",
+			resultStr:   "[[-inf,1999) (2001,+inf]]",
+		},
+		{
+			exprStr:     "a >= 70 and a <= 69",
+			accessConds: "[ge(test.t.a, 70) le(test.t.a, 69)]",
+			filterConds: "[]",
+			resultStr:   "[[1970,2069]]",
+		},
+	}
+
+	ctx := context.Background()
+	for _, tt := range tests {
+		sql := "select * from t where " + tt.exprStr
+		sctx := testKit.Se.(sessionctx.Context)
+		stmts, err := session.Parse(sctx, sql)
+		c.Assert(err, IsNil, Commentf("error %v, for expr %s", err, tt.exprStr))
+		c.Assert(stmts, HasLen, 1)
+		is := domain.GetDomain(sctx).InfoSchema()
+		err = plannercore.Preprocess(sctx, stmts[0], is)
+		c.Assert(err, IsNil, Commentf("error %v, for resolve name, expr %s", err, tt.exprStr))
+		p, _, err := plannercore.BuildLogicalPlan(ctx, sctx, stmts[0], is)
+		c.Assert(err, IsNil, Commentf("error %v, for build plan, expr %s", err, tt.exprStr))
+		selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection)
+		tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo()
+		c.Assert(selection, NotNil, Commentf("expr:%v", tt.exprStr))
+		conds := make([]expression.Expression, len(selection.Conditions))
+		for i, cond := range selection.Conditions {
+			conds[i] = expression.PushDownNot(sctx, cond)
+		}
+		cols, lengths := expression.IndexInfo2PrefixCols(tbl.Columns, selection.Schema().Columns, tbl.Indices[tt.indexPos])
+		c.Assert(cols, NotNil)
+		res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths)
+		c.Assert(err, IsNil)
+		c.Assert(fmt.Sprintf("%s", res.AccessConds), Equals, tt.accessConds, Commentf("wrong access conditions for expr: %s", tt.exprStr))
+		c.Assert(fmt.Sprintf("%s", res.RemainedConds), Equals, tt.filterConds, Commentf("wrong filter conditions for expr: %s", tt.exprStr))
+		got := fmt.Sprintf("%v", res.Ranges)
+		c.Assert(got, Equals, tt.resultStr, Commentf("different for expr %s", tt.exprStr))
+	}
+}