From 161beeeb067f00c1aee743a862c67d8f6735034a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=99=8E?= Date: Tue, 24 Nov 2020 18:06:16 +0800 Subject: [PATCH 1/2] cherry pick #20987 to release-4.0 Signed-off-by: ti-srebot --- executor/insert_test.go | 37 +++++++++++++++++++++++++++++++++++++ types/datum.go | 26 ++++++++++++++++++++++---- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/executor/insert_test.go b/executor/insert_test.go index 561def8a7c00e..19a68871d4322 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -1268,3 +1268,40 @@ func (s *testSuite9) TestIssue20768(c *C) { tk.MustQuery("select /*+ inl_merge_join(t1) */ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows("0 0")) tk.MustQuery("select /*+ merge_join(t1) */ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows("0 0")) } +<<<<<<< HEAD +======= + +func (s *testSuite9) TestIssue10402(c *C) { + tk := testkit.NewTestKit(c, s.store) + tk.MustExec("use test") + tk.MustExec("create table vctt (v varchar(4), c char(4))") + tk.MustExec("insert into vctt values ('ab ', 'ab ')") + tk.MustQuery("select * from vctt").Check(testkit.Rows("ab ab")) + tk.MustExec("delete from vctt") + tk.Se.GetSessionVars().StmtCtx.SetWarnings(nil) + tk.MustExec("insert into vctt values ('ab\\n\\n\\n', 'ab\\n\\n\\n'), ('ab\\t\\t\\t', 'ab\\t\\t\\t'), ('ab ', 'ab '), ('ab\\r\\r\\r', 'ab\\r\\r\\r')") + c.Check(tk.Se.GetSessionVars().StmtCtx.WarningCount(), Equals, uint16(4)) + warns := tk.Se.GetSessionVars().StmtCtx.GetWarnings() + c.Check(fmt.Sprintf("%v", warns), Equals, "[{Warning [types:1265]Data truncated, field len 4, data len 5} {Warning [types:1265]Data truncated, field len 4, data len 5} {Warning [types:1265]Data truncated, field len 4, data len 6} {Warning [types:1265]Data truncated, field len 4, data len 5}]") + tk.MustQuery("select * from vctt").Check(testkit.Rows("ab\n\n ab\n\n", "ab\t\t ab\t\t", "ab ab", "ab\r\r ab\r\r")) + tk.MustQuery("select length(v), length(c) from vctt").Check(testkit.Rows("4 4", "4 4", "4 2", "4 4")) +} + +func combination(items []string) func() []string { + current := 1 + buf := make([]string, len(items)) + return func() []string { + if current >= int(math.Pow(2, float64(len(items)))) { + return nil + } + buf = buf[:0] + for i, e := range items { + if (1<>>>>>> 83c165263... expression, executor: allow insert strings with overflowed trailing spaces (#20987) diff --git a/types/datum.go b/types/datum.go index dd7ffd96de3ed..eefc0fb79e8ef 100644 --- a/types/datum.go +++ b/types/datum.go @@ -952,10 +952,14 @@ func (d *Datum) convertToString(sc *stmtctx.StatementContext, target *FieldType) func ProduceStrWithSpecifiedTp(s string, tp *FieldType, sc *stmtctx.StatementContext, padZero bool) (_ string, err error) { flen, chs := tp.Flen, tp.Charset if flen >= 0 { + // overflowed stores the part of the string that is out of the length contraint, it is later checked to see if the + // overflowed part is all whitespaces + var overflowed string + var characterLen int // Flen is the rune length, not binary length, for UTF8 charset, we need to calculate the // rune count and truncate to Flen runes if it is too long. if chs == charset.CharsetUTF8 || chs == charset.CharsetUTF8MB4 { - characterLen := utf8.RuneCountInString(s) + characterLen = utf8.RuneCountInString(s) if characterLen > flen { // 1. If len(s) is 0 and flen is 0, truncateLen will be 0, don't truncate s. // CREATE TABLE t (a char(0)); @@ -972,13 +976,27 @@ func ProduceStrWithSpecifiedTp(s string, tp *FieldType, sc *stmtctx.StatementCon } runeCount++ } - err = ErrDataTooLong.GenWithStack("Data Too Long, field len %d, data len %d", flen, characterLen) + overflowed = s[truncateLen:] s = truncateStr(s, truncateLen) } } else if len(s) > flen { - err = ErrDataTooLong.GenWithStack("Data Too Long, field len %d, data len %d", flen, len(s)) + characterLen = len(s) + overflowed = s[flen:] s = truncateStr(s, flen) - } else if tp.Tp == mysql.TypeString && IsBinaryStr(tp) && len(s) < flen && padZero { + } + + if len(overflowed) != 0 { + trimed := strings.TrimRight(overflowed, " \t\n\r") + if len(trimed) == 0 && !IsBinaryStr(tp) && IsTypeChar(tp.Tp) { + if tp.Tp == mysql.TypeVarchar { + sc.AppendWarning(ErrTruncated.GenWithStack("Data truncated, field len %d, data len %d", flen, characterLen)) + } + } else { + err = ErrDataTooLong.GenWithStack("Data Too Long, field len %d, data len %d", flen, characterLen) + } + } + + if tp.Tp == mysql.TypeString && IsBinaryStr(tp) && len(s) < flen && padZero { padding := make([]byte, flen-len(s)) s = string(append([]byte(s), padding...)) } From 85270993bbfa5cb89265836be8f1ad0365190eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=99=8E?= Date: Wed, 25 Nov 2020 16:18:59 +0800 Subject: [PATCH 2/2] Update insert_test.go --- executor/insert_test.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/executor/insert_test.go b/executor/insert_test.go index 19a68871d4322..d8671e6e25e45 100644 --- a/executor/insert_test.go +++ b/executor/insert_test.go @@ -1268,8 +1268,6 @@ func (s *testSuite9) TestIssue20768(c *C) { tk.MustQuery("select /*+ inl_merge_join(t1) */ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows("0 0")) tk.MustQuery("select /*+ merge_join(t1) */ * from t1 join t2 on t1.a = t2.a").Check(testkit.Rows("0 0")) } -<<<<<<< HEAD -======= func (s *testSuite9) TestIssue10402(c *C) { tk := testkit.NewTestKit(c, s.store) @@ -1286,22 +1284,3 @@ func (s *testSuite9) TestIssue10402(c *C) { tk.MustQuery("select * from vctt").Check(testkit.Rows("ab\n\n ab\n\n", "ab\t\t ab\t\t", "ab ab", "ab\r\r ab\r\r")) tk.MustQuery("select length(v), length(c) from vctt").Check(testkit.Rows("4 4", "4 4", "4 2", "4 4")) } - -func combination(items []string) func() []string { - current := 1 - buf := make([]string, len(items)) - return func() []string { - if current >= int(math.Pow(2, float64(len(items)))) { - return nil - } - buf = buf[:0] - for i, e := range items { - if (1<>>>>>> 83c165263... expression, executor: allow insert strings with overflowed trailing spaces (#20987)