expression,types: Adjusts UNIX_TIMESTAMP() for non-existing DST values (#28739)

expression/builtin_time.go

@@ -4856,7 +4856,7 @@ func (b *builtinUnixTimestampIntSig) evalIntWithCtx(ctx sessionctx.Context, row
 	}
 
 	tz := ctx.GetSessionVars().Location()
-	t, err := val.GoTime(tz)
+	t, err := val.AdjustedGoTime(tz)
 	if err != nil {
 		return 0, false, nil
 	}

expression/builtin_time_vec.go

@@ -2390,7 +2390,7 @@ func (b *builtinUnixTimestampIntSig) vecEvalInt(input *chunk.Chunk, result *chun
 				continue
 			}
 
-			t, err := buf.GetTime(i).GoTime(getTimeZone(b.ctx))
+			t, err := buf.GetTime(i).AdjustedGoTime(getTimeZone(b.ctx))
 			if err != nil {
 				i64s[i] = 0
 				continue

expression/integration_test.go

@@ -9367,6 +9367,25 @@ func (s *testIntegrationSuite) TestIssue30101(c *C) {
 	tk.MustQuery("select greatest(c1, c2) from t1;").Sort().Check(testkit.Rows("9223372036854775809"))
 }
 
+func (s *testIntegrationSuite) TestIssue28739(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec(`USE test`)
+	tk.MustExec("SET time_zone = 'Europe/Vilnius'")
+	tk.MustQuery("SELECT UNIX_TIMESTAMP('2020-03-29 03:45:00')").Check(testkit.Rows("1585443600"))
+	tk.MustQuery("SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('2020-03-29 03:45:00'))").Check(testkit.Rows("2020-03-29 04:00:00"))
+	tk.MustExec(`DROP TABLE IF EXISTS t`)
+	tk.MustExec(`CREATE TABLE t (dt DATETIME NULL)`)
+	defer tk.MustExec(`DROP TABLE t`)
+	// Test the vector implememtation
+	tk.MustExec(`INSERT INTO t VALUES ('2021-10-31 02:30:00'), ('2021-03-28 02:30:00'), ('2020-10-04 02:15:00'), ('2020-03-29 03:45:00'), (NULL)`)
+	tk.MustQuery(`SELECT dt, UNIX_TIMESTAMP(dt) FROM t`).Sort().Check(testkit.Rows(
+		"2020-03-29 03:45:00 1585443600",
+		"2020-10-04 02:15:00 1601766900",
+		"2021-03-28 02:30:00 1616891400",
+		"2021-10-31 02:30:00 1635636600",
+		"<nil> <nil>"))
+}
+
 func (s *testIntegrationSuite) TestIssue30326(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")

types/core_time.go

@@ -184,6 +184,59 @@ func (t CoreTime) GoTime(loc *gotime.Location) (gotime.Time, error) {
 	return tm, nil
 }
 
+// FindZoneTransition check for one Time Zone transition within +/- 4h
+// Currently the needed functions are not exported, if gotime.Location.lookup would be exported
+// then it would be easy to use that directly
+func FindZoneTransition(tIn gotime.Time) (gotime.Time, error) {
+	// Check most common case first, DST transition on full hour.
+	// round truncates away from zero!
+	t2 := tIn.Round(gotime.Hour).Add(-1 * gotime.Hour)
+	t1 := t2.Add(-1 * gotime.Second)
+	_, offset1 := t1.Zone()
+	_, offset2 := t2.Zone()
+	if offset1 != offset2 {
+		return t2, nil
+	}
+
+	// Check if any offset change?
+	t1 = tIn.Add(-4 * gotime.Hour)
+	t2 = tIn.Add(4 * gotime.Hour)
+	_, offset1 = t1.Zone()
+	_, offset2 = t2.Zone()
+	if offset1 == offset2 {
+		return tIn, errors.Trace(ErrWrongValue.GenWithStackByArgs(TimeStr, tIn))
+	}
+
+	// Check generic case, like for 'Australia/Lord_Howe'
+	for t2.After(t1.Add(gotime.Second)) {
+		t := t1.Add(t2.Sub(t1) / 2).Round(gotime.Second)
+		_, offset := t.Zone()
+		if offset == offset1 {
+			t1 = t
+		} else {
+			t2 = t
+		}
+	}
+	return t2, nil
+}
+
+// AdjustedGoTime converts Time to GoTime and adjust for invalid DST times
+// like during the DST change with increased offset,
+// normally moving to Daylight Saving Time.
+// see https://github.com/pingcap/tidb/issues/28739
+func (t CoreTime) AdjustedGoTime(loc *gotime.Location) (gotime.Time, error) {
+	tm, err := t.GoTime(loc)
+	if err == nil {
+		return tm, nil
+	}
+
+	tAdj, err2 := FindZoneTransition(tm)
+	if err2 == nil {
+		return tAdj, nil
+	}
+	return tm, err
+}
+
 // IsLeapYear returns if it's leap year.
 func (t CoreTime) IsLeapYear() bool {
 	return isLeapYear(t.getYear())

types/core_time_test.go

@@ -316,3 +316,81 @@ func TestWeekday(t *testing.T) {
 		require.Equal(t, tt.Expect, weekday.String())
 	}
 }
+
+func TestFindZoneTransition(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		TZ      string
+		dt      string
+		Expect  string
+		Success bool
+	}{
+		{"Australia/Lord_Howe", "2020-06-29 03:45:00", "", false},
+		{"Australia/Lord_Howe", "2020-10-04 02:15:00", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", "2020-10-04 02:29:59", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", "2020-10-04 02:29:59.99", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", "2020-10-04 02:30:00.0001", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", "2020-10-04 02:30:00", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", "2020-10-04 02:30:01", "2020-10-04 02:30:00 +11 +1100", true},
+		{"Europe/Vilnius", "2020-03-29 03:45:00", "2020-03-29 04:00:00 EEST +0300", true},
+		{"Europe/Vilnius", "2020-10-25 03:45:00", "2020-10-25 03:00:00 EET +0200", true},
+		{"Europe/Vilnius", "2020-06-29 03:45:00", "", false},
+		{"Europe/Amsterdam", "2020-03-29 02:45:00", "2020-03-29 03:00:00 CEST +0200", true},
+		{"Europe/Amsterdam", "2020-10-25 02:35:00", "2020-10-25 02:00:00 CET +0100", true},
+		{"Europe/Amsterdam", "2020-03-29 02:59:59", "2020-03-29 03:00:00 CEST +0200", true},
+		{"Europe/Amsterdam", "2020-03-29 02:59:59.999999999", "2020-03-29 03:00:00 CEST +0200", true},
+		{"Europe/Amsterdam", "2020-03-29 03:00:00.000000001", "2020-03-29 03:00:00 CEST +0200", true},
+	}
+
+	for _, tt := range tests {
+		loc, err := time.LoadLocation(tt.TZ)
+		require.NoError(t, err)
+		tm, err := time.ParseInLocation("2006-01-02 15:04:05", tt.dt, loc)
+		require.NoError(t, err)
+		tp, err := FindZoneTransition(tm)
+		if !tt.Success {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, tt.Expect, tp.Format("2006-01-02 15:04:05.999999999 MST -0700"))
+		}
+	}
+}
+
+func TestAdjustedGoTime(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		TZ      string
+		dt      CoreTime
+		Expect  string
+		Success bool
+	}{
+		{"Australia/Lord_Howe", FromDate(2020, 10, 04, 01, 59, 59, 997), "2020-10-04 01:59:59.000997 +1030 +1030", true},
+		{"Australia/Lord_Howe", FromDate(2020, 10, 04, 02, 00, 00, 0), "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", FromDate(2020, 10, 04, 02, 15, 00, 0), "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", FromDate(2020, 10, 04, 02, 29, 59, 999999), "2020-10-04 02:30:00 +11 +1100", true},
+		{"Australia/Lord_Howe", FromDate(2020, 10, 04, 02, 30, 00, 1), "2020-10-04 02:30:00.000001 +11 +1100", true},
+		{"Australia/Lord_Howe", FromDate(2020, 06, 29, 03, 45, 00, 0), "2020-06-29 03:45:00 +1030 +1030", true},
+		{"Australia/Lord_Howe", FromDate(2020, 04, 04, 01, 45, 00, 0), "2020-04-04 01:45:00 +11 +1100", true},
+		{"Europe/Vilnius", FromDate(2020, 03, 29, 03, 45, 00, 0), "2020-03-29 04:00:00 EEST +0300", true},
+		{"Europe/Vilnius", FromDate(2020, 03, 29, 03, 59, 59, 456789), "2020-03-29 04:00:00 EEST +0300", true},
+		{"Europe/Vilnius", FromDate(2020, 03, 29, 04, 00, 01, 130000), "2020-03-29 04:00:01.13 EEST +0300", true},
+		{"Europe/Vilnius", FromDate(2020, 10, 25, 03, 45, 00, 0), "2020-10-25 03:45:00 EET +0200", true},
+		{"Europe/Vilnius", FromDate(2020, 06, 29, 03, 45, 00, 0), "2020-06-29 03:45:00 EEST +0300", true},
+		{"Europe/Amsterdam", FromDate(2020, 03, 29, 02, 45, 00, 0), "2020-03-29 03:00:00 CEST +0200", true},
+		{"Europe/Amsterdam", FromDate(2020, 10, 25, 02, 35, 00, 0), "2020-10-25 02:35:00 CET +0100", true},
+		{"UTC", FromDate(2020, 2, 31, 02, 35, 00, 0), "", false},
+	}
+
+	for _, tt := range tests {
+		loc, err := time.LoadLocation(tt.TZ)
+		require.NoError(t, err)
+		tp, err := tt.dt.AdjustedGoTime(loc)
+		if !tt.Success {
+			require.Error(t, err)
+		} else {
+			require.NoError(t, err)
+			require.Equal(t, tt.Expect, tp.Format("2006-01-02 15:04:05.999999999 MST -0700"))
+		}
+	}
+}