Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

domain: make ru_stats duration compatible with DST #52408

Merged
merged 4 commits into from
Apr 8, 2024
Merged

domain: make ru_stats duration compatible with DST #52408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5542edc
make ru_stats duration compatible with DST
glorv Apr 8, 2024
4c21553
correct comments
glorv Apr 8, 2024
5972f02
fix lint
glorv Apr 8, 2024
546ecb9
fix nil
glorv Apr 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions pkg/domain/ru_stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,29 @@ func (do *Domain) requestUnitsWriterLoop() {
}

// GetLastExpectedTime return the last written ru time.
// NOTE:
// - due to DST(daylight saving time), the actual duration for a specific
// time may be shorter or longer than the interval when DST happens.
// - All the tidb-server should be deployed in the same timezone to ensure
// the duration is calculated correctly.
// - The interval must not be longer than 24h.
func GetLastExpectedTime(now time.Time, interval time.Duration) time.Time {
nowTs := now.Unix()
intervalSecs := int64(interval / time.Second)
tzOffset := time.Date(1971, 1, 1, 0, 0, 0, 0, time.Local).Unix()
targetTs := nowTs - (nowTs+intervalSecs-tzOffset)%intervalSecs
return time.Unix(targetTs, 0)
return GetLastExpectedTimeTZ(now, interval, time.Local)
}

// GetLastExpectedTimeTZ return the last written ru time under specifical timezone.
// make it public only for test.
func GetLastExpectedTimeTZ(now time.Time, interval time.Duration, tz *time.Location) time.Time {
if tz == nil {
tz = time.Local
}
year, month, day := now.Date()
start := time.Date(year, month, day, 0, 0, 0, 0, tz)
// cast to int64 to bypass the durationcheck lint.
count := int64(now.Sub(start) / interval)
targetDur := time.Duration(count) * interval
// use UTC timezone to calculate target time so it can be compatible with DST.
return start.In(time.UTC).Add(targetDur).In(tz)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can tz be nil here?

Copy link
Contributor Author

@glorv glorv Apr 8, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, this caller should always pass a valid tz. Anyway, I added a check to set time.Local for nil tz.

}

// DoWriteRUStatistics write ru historical data into mysql.request_unit_by_group.
Expand Down
58 changes: 40 additions & 18 deletions pkg/domain/ru_stats_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,18 @@ import (
)

func TestWriteRUStatistics(t *testing.T) {
tz, _ := time.LoadLocation("Asia/Shanghai")
testWriteRUStatisticsTz(t, tz)

// test with DST timezone.
tz, _ = time.LoadLocation("Australia/Lord_Howe")
testWriteRUStatisticsTz(t, tz)

testWriteRUStatisticsTz(t, time.Local)
testWriteRUStatisticsTz(t, time.UTC)
}

func testWriteRUStatisticsTz(t *testing.T, tz *time.Location) {
store, dom := testkit.CreateMockStoreAndDomain(t)
tk := newTestKit(t, store)

Expand Down Expand Up @@ -70,22 +82,22 @@ func TestWriteRUStatistics(t *testing.T) {

tk.MustQuery("SELECT count(*) from mysql.request_unit_by_group").Check(testkit.Rows("0"))

testRUWriter.StartTime = time.Date(2023, 12, 26, 0, 0, 1, 0, time.Local)
testRUWriter.StartTime = time.Date(2023, 12, 26, 0, 0, 1, 0, tz)
require.NoError(t, testRUWriter.DoWriteRUStatistics(context.Background()))
tk.MustQuery("SELECT resource_group, total_ru from mysql.request_unit_by_group").Check(testkit.Rows("default 350", "test 150"))

// after 1 day, only 1 group has delta ru.
testRMClient.groups[1].RUStats.RRU = 500
testRUWriter.StartTime = time.Date(2023, 12, 27, 0, 0, 1, 0, time.Local)
testRUWriter.StartTime = time.Date(2023, 12, 27, 0, 0, 1, 0, tz)
require.NoError(t, testRUWriter.DoWriteRUStatistics(context.Background()))
tk.MustQuery("SELECT resource_group, total_ru from mysql.request_unit_by_group where end_time = '2023-12-27'").Check(testkit.Rows("test 400"))

// test after 1 day with 0 delta ru, no data inserted.
testRUWriter.StartTime = time.Date(2023, 12, 28, 0, 0, 1, 0, time.Local)
testRUWriter.StartTime = time.Date(2023, 12, 28, 0, 0, 1, 0, tz)
require.NoError(t, testRUWriter.DoWriteRUStatistics(context.Background()))
tk.MustQuery("SELECT count(*) from mysql.request_unit_by_group where end_time = '2023-12-28'").Check(testkit.Rows("0"))

testRUWriter.StartTime = time.Date(2023, 12, 29, 0, 0, 0, 0, time.Local)
testRUWriter.StartTime = time.Date(2023, 12, 29, 0, 0, 0, 0, tz)
testRMClient.groups[0].RUStats.WRU = 200
require.NoError(t, testRUWriter.DoWriteRUStatistics(context.Background()))
tk.MustQuery("SELECT resource_group, total_ru from mysql.request_unit_by_group where end_time = '2023-12-29'").Check(testkit.Rows("default 50"))
Expand All @@ -94,12 +106,12 @@ func TestWriteRUStatistics(t *testing.T) {
// This is to test after restart, no unexpected data are inserted.
testRMClient.groups[0].RUStats.RRU = 1000
testRMClient.groups[1].RUStats.WRU = 2000
testRUWriter.StartTime = time.Date(2023, 12, 29, 1, 0, 0, 0, time.Local)
testRUWriter.StartTime = time.Date(2023, 12, 29, 1, 0, 0, 0, tz)
require.NoError(t, testRUWriter.DoWriteRUStatistics(context.Background()))
tk.MustQuery("SELECT resource_group, total_ru from mysql.request_unit_by_group where end_time = '2023-12-29'").Check(testkit.Rows("default 50"))

// after 61 days, old record should be GCed.
testRUWriter.StartTime = time.Date(2023, 12, 26, 0, 0, 0, 0, time.Local).Add(92 * 24 * time.Hour)
testRUWriter.StartTime = time.Date(2023, 12, 26, 0, 0, 0, 0, tz).Add(92 * 24 * time.Hour)
tk.MustQuery("SELECT count(*) from mysql.request_unit_by_group where end_time = '2023-12-26'").Check(testkit.Rows("2"))
require.NoError(t, testRUWriter.GCOutdatedRecords(testRUWriter.StartTime))
tk.MustQuery("SELECT count(*) from mysql.request_unit_by_group where end_time = '2023-12-26'").Check(testkit.Rows("0"))
Expand Down Expand Up @@ -130,20 +142,30 @@ func (is *testInfoschema) SchemaMetaVersion() int64 {
}

func TestGetLastExpectedTime(t *testing.T) {
tz, _ := time.LoadLocation("Asia/Shanghai")
testGetLastExpectedTimeTz(t, tz)

// test with DST affected timezone.
tz, _ = time.LoadLocation("Australia/Lord_Howe")
testGetLastExpectedTimeTz(t, tz)
testGetLastExpectedTimeTz(t, time.Local)
}

func testGetLastExpectedTimeTz(t *testing.T, tz *time.Location) {
// 2023-12-28 10:46:23.000
now := time.Date(2023, 12, 28, 10, 46, 23, 0, time.Local)
now := time.Date(2023, 12, 28, 10, 46, 23, 0, tz)
newTime := func(hour, minute int) time.Time {
return time.Date(2023, 12, 28, hour, minute, 0, 0, time.Local)
return time.Date(2023, 12, 28, hour, minute, 0, 0, tz)
}

require.Equal(t, domain.GetLastExpectedTime(now, 5*time.Minute), newTime(10, 45))
require.Equal(t, domain.GetLastExpectedTime(time.Date(2023, 12, 28, 10, 45, 0, 0, time.Local), 5*time.Minute), newTime(10, 45))
require.Equal(t, domain.GetLastExpectedTime(now, 10*time.Minute), newTime(10, 40))
require.Equal(t, domain.GetLastExpectedTime(now, 30*time.Minute), newTime(10, 30))
require.Equal(t, domain.GetLastExpectedTime(now, time.Hour), newTime(10, 0))
require.Equal(t, domain.GetLastExpectedTime(now, 3*time.Hour), newTime(9, 0))
require.Equal(t, domain.GetLastExpectedTime(now, 4*time.Hour), newTime(8, 0))
require.Equal(t, domain.GetLastExpectedTime(now, 12*time.Hour), newTime(0, 0))
require.Equal(t, domain.GetLastExpectedTime(now, 24*time.Hour), newTime(0, 0))
require.Equal(t, domain.GetLastExpectedTime(time.Date(2023, 12, 28, 0, 0, 0, 0, time.Local), 24*time.Hour), newTime(0, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 5*time.Minute, tz), newTime(10, 45))
require.Equal(t, domain.GetLastExpectedTimeTZ(time.Date(2023, 12, 28, 10, 45, 0, 0, tz), 5*time.Minute, tz), newTime(10, 45))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 10*time.Minute, tz), newTime(10, 40))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 30*time.Minute, tz), newTime(10, 30))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, time.Hour, tz), newTime(10, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 3*time.Hour, tz), newTime(9, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 4*time.Hour, tz), newTime(8, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 12*time.Hour, tz), newTime(0, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(now, 24*time.Hour, tz), newTime(0, 0))
require.Equal(t, domain.GetLastExpectedTimeTZ(time.Date(2023, 12, 28, 0, 0, 0, 0, tz), 24*time.Hour, tz), newTime(0, 0))
}