util: add gctuner to avoid gc frequently

build/nogo_config.json

@@ -181,6 +181,7 @@
       "util/tracing/": "util/tracing/ code",
       "util/trxevents/": "util/trxevents/ code",
       "util/watcher/": "util/watcher/ code",
+      "util/gctuner": "util/gctuner",
       "store/mockstore/unistore/util": "store/mockstore/unistore/util code",
       "ddl/util/": "ddl/util code"
     }
@@ -197,6 +198,7 @@
       ".*_generated\\.go$": "ignore generated code"
     },
     "only_files": {
+      "util/gctuner": "util/gctuner",
       "br/pkg/lightning/mydump/": "br/pkg/lightning/mydump/",
       "br/pkg/lightning/restore/opts": "br/pkg/lightning/restore/opts",
       "executor/aggregate.go": "executor/aggregate.go",
@@ -743,6 +745,7 @@
       ".*_generated\\.go$": "ignore generated code"
     },
     "only_files": {
+      "util/gctuner": "util/gctuner",
       "br/pkg/lightning/mydump/": "br/pkg/lightning/mydump/",
       "br/pkg/lightning/restore/opts": "br/pkg/lightning/restore/opts",
       "executor/aggregate.go": "executor/aggregate.go",

sessionctx/variable/BUILD.bazel

@@ -41,6 +41,7 @@ go_library(
         "//util/collate",
         "//util/dbterror",
         "//util/execdetails",
+        "//util/gctuner",
         "//util/kvcache",
         "//util/logutil",
         "//util/mathutil",

sessionctx/variable/sysvar.go

@@ -36,6 +36,7 @@ import (
 	"github.com/pingcap/tidb/types"
 	_ "github.com/pingcap/tidb/types/parser_driver" // for parser driver
 	"github.com/pingcap/tidb/util/collate"
+	"github.com/pingcap/tidb/util/gctuner"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/mathutil"
 	"github.com/pingcap/tidb/util/memory"
@@ -619,6 +620,14 @@ var defaultSysVars = []*SysVar{
 		VarTiDBSuperReadOnly.Store(TiDBOptOn(val))
 		return nil
 	}},
+	{Scope: ScopeGlobal, Name: TiDBEnableGOGCTuner, Value: BoolToOnOff(DefTiDBEnableGOGCTuner), Type: TypeBool, SetGlobal: func(s *SessionVars, val string) error {
+		on := TiDBOptOn(val)
+		gctuner.EnableGOGCTuner.Store(on)
+		if !on {
+			gctuner.SetDefaultGOGC()
+		}
+		return nil
+	}},
 	{Scope: ScopeGlobal, Name: TiDBEnableTelemetry, Value: BoolToOnOff(DefTiDBEnableTelemetry), Type: TypeBool},
 	{Scope: ScopeGlobal, Name: TiDBEnableHistoricalStats, Value: Off, Type: TypeBool},
 	/* tikv gc metrics */

sessionctx/variable/tidb_vars.go

@@ -822,6 +822,8 @@ const (
 	TiDBServerMemoryLimit = "tidb_server_memory_limit"
 	// TiDBServerMemoryLimitSessMinSize indicates the minimal memory used of a session, that becomes a candidate for session kill.
 	TiDBServerMemoryLimitSessMinSize = "tidb_server_memory_limit_sess_min_size"
+	// TiDBEnableGOGCTuner is to enable GOGC tuner. it can tuner GOGC
+	TiDBEnableGOGCTuner = "tidb_enable_gogc_tuner"
 )
 
 // TiDB intentional limits
@@ -1048,6 +1050,7 @@ const (
 	DefTiDBOptRangeMaxSize                          = 0
 	DefTiDBCostModelVer                             = 1
 	DefTiDBServerMemoryLimitSessMinSize             = 128 << 20
+	DefTiDBEnableGOGCTuner                          = true
 )
 
 // Process global variables.

tidb-server/BUILD.bazel

@@ -36,6 +36,7 @@ go_library(
         "//util/deadlockhistory",
         "//util/disk",
         "//util/domainutil",
+        "//util/gctuner",
         "//util/kvcache",
         "//util/logutil",
         "//util/memory",

tidb-server/main.go

@@ -61,6 +61,7 @@ import (
 	"github.com/pingcap/tidb/util/deadlockhistory"
 	"github.com/pingcap/tidb/util/disk"
 	"github.com/pingcap/tidb/util/domainutil"
+	"github.com/pingcap/tidb/util/gctuner"
 	"github.com/pingcap/tidb/util/kvcache"
 	"github.com/pingcap/tidb/util/logutil"
 	"github.com/pingcap/tidb/util/memory"
@@ -204,7 +205,7 @@ func main() {
 	printInfo()
 	setupBinlogClient()
 	setupMetrics()
-
+	setupGCTuner()
 	storage, dom := createStoreAndDomain()
 	svr := createServer(storage, dom)
 
@@ -783,6 +784,15 @@ func setupTracing() {
 	opentracing.SetGlobalTracer(tracer)
 }
 
+func setupGCTuner() {
+	limit, err := memory.MemTotal()
+	if err != nil {
+		log.Fatal("setupGCTuner failed", zap.Error(err))
+	}
+	threshold := limit * 7 / 10
+	gctuner.Tuning(threshold)
+}
+
 func closeDomainAndStorage(storage kv.Storage, dom *domain.Domain) {
 	tikv.StoreShuttingDown(1)
 	dom.Close()

util/gctuner/BUILD.bazel

@@ -0,0 +1,31 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "gctuner",
+    srcs = [
+        "finalizer.go",
+        "mem.go",
+        "tuner.go",
+    ],
+    importpath = "github.com/pingcap/tidb/util/gctuner",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//util",
+        "@org_uber_go_atomic//:atomic",
+    ],
+)
+
+go_test(
+    name = "gctuner_test",
+    srcs = [
+        "finalizer_test.go",
+        "mem_test.go",
+        "tuner_test.go",
+    ],
+    embed = [":gctuner"],
+    flaky = True,
+    deps = [
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+    ],
+)

util/gctuner/finalizer.go

@@ -0,0 +1,58 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gctuner
+
+import (
+	"runtime"
+
+	"go.uber.org/atomic"
+)
+
+type finalizerCallback func()
+
+type finalizer struct {
+	ref      *finalizerRef
+	callback finalizerCallback
+	stopped  atomic.Int32
+}
+
+type finalizerRef struct {
+	parent *finalizer
+}
+
+func finalizerHandler(f *finalizerRef) {
+	// stop calling callback
+	if f.parent.stopped.Load() > 0 {
+		return
+	}
+	f.parent.callback()
+	runtime.SetFinalizer(f, finalizerHandler)
+}
+
+// newFinalizer return a finalizer object and caller should save it to make sure it will not be gc.
+// the go runtime promise the callback function should be called every gc time.
+func newFinalizer(callback finalizerCallback) *finalizer {
+	f := &finalizer{
+		callback: callback,
+	}
+	f.ref = &finalizerRef{parent: f}
+	runtime.SetFinalizer(f.ref, finalizerHandler)
+	f.ref = nil // trigger gc
+	return f
+}
+
+func (f *finalizer) stop() {
+	f.stopped.Store(1)
+}

util/gctuner/finalizer_test.go

@@ -0,0 +1,50 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gctuner
+
+import (
+	"runtime"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type testState struct {
+	count int32
+}
+
+func TestFinalizer(t *testing.T) {
+	maxCount := int32(16)
+	state := &testState{}
+	f := newFinalizer(func() {
+		n := atomic.AddInt32(&state.count, 1)
+		if n > maxCount {
+			t.Fatalf("cannot exec finalizer callback after f has been gc")
+		}
+	})
+	for i := int32(1); i <= maxCount; i++ {
+		runtime.GC()
+		require.Equal(t, i, atomic.LoadInt32(&state.count))
+	}
+	require.Nil(t, f.ref)
+
+	f.stop()
+	require.Equal(t, maxCount, atomic.LoadInt32(&state.count))
+	runtime.GC()
+	require.Equal(t, maxCount, atomic.LoadInt32(&state.count))
+	runtime.GC()
+	require.Equal(t, maxCount, atomic.LoadInt32(&state.count))
+}

util/gctuner/mem.go

@@ -0,0 +1,26 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gctuner
+
+import (
+	"runtime"
+)
+
+var memStats runtime.MemStats
+
+func readMemoryInuse() uint64 {
+	runtime.ReadMemStats(&memStats)
+	return memStats.HeapInuse
+}

util/gctuner/mem_test.go

@@ -0,0 +1,31 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gctuner
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMem(t *testing.T) {
+	const mb = 1024 * 1024
+
+	heap := make([]byte, 100*mb+1)
+	inuse := readMemoryInuse()
+	t.Logf("mem inuse: %d MB", inuse/mb)
+	require.GreaterOrEqual(t, inuse, uint64(100*mb))
+	heap[0] = 0
+}