gctuner: fix goleak in the memoryLimitTuner.tuning (#41523)

close #41524
pingcap · Feb 16, 2023 · 218ca1c · 218ca1c
1 parent cd51608
commit 218ca1c
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 0 deletions.
diff --git a/domain/BUILD.bazel b/domain/BUILD.bazel
@@ -58,6 +58,7 @@ go_library(
         "//util/etcd",
         "//util/execdetails",
         "//util/expensivequery",
+        "//util/gctuner",
         "//util/intest",
         "//util/logutil",
         "//util/memory",

diff --git a/domain/domain.go b/domain/domain.go
@@ -69,6 +69,7 @@ import (
 	"github.com/pingcap/tidb/util/engine"
 	"github.com/pingcap/tidb/util/etcd"
 	"github.com/pingcap/tidb/util/expensivequery"
+	"github.com/pingcap/tidb/util/gctuner"
 	"github.com/pingcap/tidb/util/intest"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/memory"
@@ -938,6 +939,7 @@ func (do *Domain) Close() {
 	if do.onClose != nil {
 		do.onClose()
 	}
+	gctuner.WaitMemoryLimitTunerExitInTest()
 	logutil.BgLogger().Info("domain closed", zap.Duration("take time", time.Since(startTime)))
 }
 

diff --git a/util/gctuner/BUILD.bazel b/util/gctuner/BUILD.bazel
@@ -12,6 +12,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//util",
+        "//util/intest",
         "//util/memory",
         "@com_github_pingcap_failpoint//:failpoint",
         "@org_uber_go_atomic//:atomic",

diff --git a/util/gctuner/memory_limit_tuner.go b/util/gctuner/memory_limit_tuner.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/pingcap/failpoint"
 	"github.com/pingcap/tidb/util"
+	"github.com/pingcap/tidb/util/intest"
 	"github.com/pingcap/tidb/util/memory"
 	atomicutil "go.uber.org/atomic"
 )
@@ -41,6 +42,17 @@ type memoryLimitTuner struct {
 // fallbackPercentage indicates the fallback memory limit percentage when turning.
 const fallbackPercentage float64 = 1.1
 
+var memoryGoroutineCntInTest = *atomicutil.NewInt64(0)
+
+// WaitMemoryLimitTunerExitInTest is used to wait memory limit tuner exit in test.
+func WaitMemoryLimitTunerExitInTest() {
+	if intest.InTest {
+		for memoryGoroutineCntInTest.Load() > 0 {
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+}
+
 // tuning check the memory nextGC and judge whether this GC is trigger by memory limit.
 // Go runtime ensure that it will be called serially.
 func (t *memoryLimitTuner) tuning() {
@@ -62,10 +74,17 @@ func (t *memoryLimitTuner) tuning() {
 	if float64(r.HeapInuse)*ratio > float64(debug.SetMemoryLimit(-1)) {
 		if t.nextGCTriggeredByMemoryLimit.Load() && t.waitingReset.CompareAndSwap(false, true) {
 			go func() {
+				if intest.InTest {
+					memoryGoroutineCntInTest.Inc()
+					defer memoryGoroutineCntInTest.Dec()
+				}
 				memory.MemoryLimitGCLast.Store(time.Now())
 				memory.MemoryLimitGCTotal.Add(1)
 				debug.SetMemoryLimit(t.calcMemoryLimit(fallbackPercentage))
 				resetInterval := 1 * time.Minute // Wait 1 minute and set back, to avoid frequent GC
+				if intest.InTest {
+					resetInterval = 3 * time.Second
+				}
 				failpoint.Inject("testMemoryLimitTuner", func(val failpoint.Value) {
 					if val, ok := val.(bool); val && ok {
 						resetInterval = 1 * time.Second