diff --git a/DEPS.bzl b/DEPS.bzl index fa7de8864d94c..108f79343c64e 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -2316,8 +2316,8 @@ def go_deps(): name = "com_github_mattn_go_runewidth", build_file_proto_mode = "disable", importpath = "github.com/mattn/go-runewidth", - sum = "h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=", - version = "v0.0.13", + sum = "h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=", + version = "v0.0.14", ) go_repository( name = "com_github_mattn_goveralls", @@ -2375,8 +2375,8 @@ def go_deps(): name = "com_github_mgechev_revive", build_file_proto_mode = "disable", importpath = "github.com/mgechev/revive", - sum = "h1:a+itKsYpxka50MyaWQW1XCZ1vwfgjzVy/OzZ7DC/4+U=", - version = "v1.2.4-0.20220827111817-553604eaced5", + sum = "h1:+2Hd/S8oO2H0Ikq2+egtNwQsVhAeELHjxjIUFX5ajLI=", + version = "v1.2.4", ) go_repository( @@ -2993,8 +2993,8 @@ def go_deps(): patches = [ "//build/patches:com_github_rivo_uniseg.patch", ], - sum = "h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw=", - version = "v0.3.4", + sum = "h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8=", + version = "v0.4.2", ) go_repository( name = "com_github_rlmcpherson_s3gof3r", @@ -3422,8 +3422,8 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sum = "h1:GJNu6XDT8W2Oahh+w/fhb37PNhFr4EZjdehIklZnhU4=", - version = "v2.0.1-0.20220921101651-ce9203ef66e9", + sum = "h1:TxDSQAmtGdE34BvOaYF35mRrAXePeZEq8quvuAwrKsI=", + version = "v2.0.1-0.20220923061703-33efe476e022", ) go_repository( name = "com_github_tikv_pd_client", @@ -4321,8 +4321,8 @@ def go_deps(): name = "org_golang_x_sys", build_file_proto_mode = "disable_global", importpath = "golang.org/x/sys", - sum = "h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho=", - version = "v0.0.0-20220909162455-aba9fc2a8ff2", + sum = "h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=", + version = "v0.0.0-20220928140112-f11e5e49a4ec", ) go_repository( name = "org_golang_x_term", @@ -4454,8 +4454,8 @@ def go_deps(): name = "org_uber_go_atomic", build_file_proto_mode = "disable_global", importpath = "go.uber.org/atomic", - sum = "h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=", - version = "v1.9.0", + sum = "h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=", + version = "v1.10.0", ) go_repository( name = "org_uber_go_automaxprocs", diff --git a/Makefile b/Makefile index 917963a5cc858..bc350cea3a255 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ dev: checklist check explaintest gogenerate br_unit_test test_part_parser_dev ut # Install the check tools. check-setup:tools/bin/revive -check: parser_yacc check-parallel lint tidy testSuite errdoc bazel_all_build +check: parser_yacc check-parallel lint tidy testSuite errdoc fmt: @echo "gofmt (simplify)" @@ -409,15 +409,10 @@ bazel_coverage_test: failpoint-enable bazel_ci_prepare -- //... -//cmd/... -//tests/graceshutdown/... \ -//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/... -bazel_all_build: bazel_ci_prepare - mkdir -p bin - bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \ - //... --//build:with_nogo_flag=true - bazel_build: bazel_ci_prepare mkdir -p bin bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \ - //cmd/importer:importer //tidb-server:tidb-server //tidb-server:tidb-server-check --//build:with_nogo_flag=true + //... --//build:with_nogo_flag=true cp bazel-out/k8-fastbuild/bin/tidb-server/tidb-server_/tidb-server ./bin cp bazel-out/k8-fastbuild/bin/cmd/importer/importer_/importer ./bin cp bazel-out/k8-fastbuild/bin/tidb-server/tidb-server-check_/tidb-server-check ./bin diff --git a/WORKSPACE b/WORKSPACE index d40e3d83dbdae..35ae55b7388a3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,18 +21,15 @@ http_archive( load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") load("//:DEPS.bzl", "go_deps") -load("//build:lint.bzl", "nogo_deps") # gazelle:repository_macro DEPS.bzl%go_deps go_deps() -nogo_deps() - go_rules_dependencies() go_register_toolchains( nogo = "@//build:tidb_nogo", - version = "1.19.1", + version = "1.19.2", ) gazelle_dependencies() diff --git a/br/pkg/lightning/backend/importer/BUILD.bazel b/br/pkg/lightning/backend/importer/BUILD.bazel deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/br/pkg/lightning/backend/kv/BUILD.bazel b/br/pkg/lightning/backend/kv/BUILD.bazel index fe398c2fd65d2..ea2cfefc2440e 100644 --- a/br/pkg/lightning/backend/kv/BUILD.bazel +++ b/br/pkg/lightning/backend/kv/BUILD.bazel @@ -50,6 +50,7 @@ go_test( "sql2kv_test.go", ], flaky = True, + race = "on", deps = [ ":kv", "//br/pkg/lightning/common", diff --git a/br/pkg/lightning/checkpoints/BUILD.bazel b/br/pkg/lightning/checkpoints/BUILD.bazel index 5e51011339197..35312b44a4998 100644 --- a/br/pkg/lightning/checkpoints/BUILD.bazel +++ b/br/pkg/lightning/checkpoints/BUILD.bazel @@ -42,6 +42,7 @@ go_test( ], embed = [":checkpoints"], flaky = True, + race = "on", deps = [ "//br/pkg/lightning/checkpoints/checkpointspb", "//br/pkg/lightning/config", diff --git a/br/pkg/rtree/BUILD.bazel b/br/pkg/rtree/BUILD.bazel index 78a6dd6403d77..5f837e021dd34 100644 --- a/br/pkg/rtree/BUILD.bazel +++ b/br/pkg/rtree/BUILD.bazel @@ -28,6 +28,7 @@ go_test( "rtree_test.go", ], flaky = True, + race = "on", deps = [ ":rtree", "//testkit/testsetup", diff --git a/build/lint.bzl b/build/lint.bzl deleted file mode 100644 index fc7103a750e2e..0000000000000 --- a/build/lint.bzl +++ /dev/null @@ -1,21 +0,0 @@ -load("@bazel_gazelle//:deps.bzl", "go_repository") - -def nogo_deps(): - go_repository( - name = "com_github_gostaticanalysis_analysisutil", - importpath = "github.com/gostaticanalysis/analysisutil", - sum = "h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk=", - version = "v0.7.1", - ) - go_repository( - name = "com_github_gostaticanalysis_comment", - importpath = "github.com/gostaticanalysis/comment", - sum = "h1:hlnx5+S2fY9Zo9ePo4AhgYsYHbM2+eAv8m/s1JiCd6Q=", - version = "v1.4.2", - ) - go_repository( - name = "com_github_timakin_bodyclose", - importpath = "github.com/timakin/bodyclose", - sum = "h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=", - version = "v0.0.0-20210704033933-f49887972144", - ) diff --git a/build/patches/com_github_rivo_uniseg.patch b/build/patches/com_github_rivo_uniseg.patch index 80f00263463bd..43c2c40933b11 100644 --- a/build/patches/com_github_rivo_uniseg.patch +++ b/build/patches/com_github_rivo_uniseg.patch @@ -1,15 +1,15 @@ -From 5097c14ae5e40af636fc3bd353edd64ca1f91713 Mon Sep 17 00:00:00 2001 +From 1492043a155839cb863210d4f564be3fa640c0d9 Mon Sep 17 00:00:00 2001 From: Weizhen Wang -Date: Mon, 29 Aug 2022 18:15:12 +0800 -Subject: [PATCH] *: update dep +Date: Sat, 8 Oct 2022 11:41:06 +0800 +Subject: [PATCH] update Signed-off-by: Weizhen Wang --- - BUILD.bazel | 26 +++++ + BUILD.bazel | 27 +++++ WORKSPACE | 2 + - gen_breaktest.go | 213 ---------------------------------------- - gen_properties.go | 240 ---------------------------------------------- - 4 files changed, 28 insertions(+), 453 deletions(-) + gen_breaktest.go | 213 -------------------------------------- + gen_properties.go | 256 ---------------------------------------------- + 4 files changed, 29 insertions(+), 469 deletions(-) create mode 100644 BUILD.bazel create mode 100644 WORKSPACE delete mode 100644 gen_breaktest.go @@ -17,10 +17,10 @@ Signed-off-by: Weizhen Wang diff --git a/BUILD.bazel b/BUILD.bazel new file mode 100644 -index 0000000..c6423e8 +index 0000000..a1e5c89 --- /dev/null +++ b/BUILD.bazel -@@ -0,0 +1,26 @@ +@@ -0,0 +1,27 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( @@ -28,6 +28,7 @@ index 0000000..c6423e8 + srcs = [ + "doc.go", + "eastasianwidth.go", ++ "emojipresentation.go", + "grapheme.go", + "graphemeproperties.go", + "graphemerules.go", @@ -39,6 +40,7 @@ index 0000000..c6423e8 + "sentenceproperties.go", + "sentencerules.go", + "step.go", ++ "width.go", + "word.go", + "wordproperties.go", + "wordrules.go", @@ -46,7 +48,6 @@ index 0000000..c6423e8 + importpath = "github.com/rivo/uniseg", + visibility = ["//visibility:public"], +) -+ diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..d596273 @@ -276,28 +277,31 @@ index e613c4c..0000000 -} diff --git a/gen_properties.go b/gen_properties.go deleted file mode 100644 -index 6451270..0000000 +index 999d5ef..0000000 --- a/gen_properties.go +++ /dev/null -@@ -1,240 +0,0 @@ +@@ -1,256 +0,0 @@ -//go:build generate - -// This program generates a property file in Go file from Unicode Character -// Database auxiliary data files. The command line arguments are as follows: -// --// 1. The name of the Unicode data file (just the filename, without extension). --// 2. The name of the locally generated Go file. --// 3. The name of the slice mapping code points to properties. --// 4. The name of the generator, for logging purposes. --// 5. (Optional) Flags, comma-separated. The following flags are available: --// - "emojis": include emoji properties (Extended Pictographic only). --// - "gencat": include general category properties. +-// 1. The name of the Unicode data file (just the filename, without extension). +-// Can be "-" (to skip) if the emoji flag is included. +-// 2. The name of the locally generated Go file. +-// 3. The name of the slice mapping code points to properties. +-// 4. The name of the generator, for logging purposes. +-// 5. (Optional) Flags, comma-separated. The following flags are available: +-// - "emojis=": include the specified emoji properties (e.g. +-// "Extended_Pictographic"). +-// - "gencat": include general category properties. -// --//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis --//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis +-//go:generate go run gen_properties.go auxiliary/GraphemeBreakProperty graphemeproperties.go graphemeCodePoints graphemes emojis=Extended_Pictographic +-//go:generate go run gen_properties.go auxiliary/WordBreakProperty wordproperties.go workBreakCodePoints words emojis=Extended_Pictographic -//go:generate go run gen_properties.go auxiliary/SentenceBreakProperty sentenceproperties.go sentenceBreakCodePoints sentences -//go:generate go run gen_properties.go LineBreak lineproperties.go lineBreakCodePoints lines gencat -//go:generate go run gen_properties.go EastAsianWidth eastasianwidth.go eastAsianWidth eastasianwidth +-//go:generate go run gen_properties.go - emojipresentation.go emojiPresentation emojipresentation emojis=Emoji_Presentation -package main - -import ( @@ -320,8 +324,8 @@ index 6451270..0000000 -// We want to test against a specific version rather than the latest. When the -// package is upgraded to a new version, change these to generate new tests. -const ( -- gbpURL = `https://www.unicode.org/Public/14.0.0/ucd/%s.txt` -- emojiURL = `https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt` +- propertyURL = `https://www.unicode.org/Public/14.0.0/ucd/%s.txt` +- emojiURL = `https://unicode.org/Public/14.0.0/ucd/emoji/emoji-data.txt` -) - -// The regular expression for a line containing a code point range property. @@ -337,20 +341,25 @@ index 6451270..0000000 - log.SetFlags(0) - - // Parse flags. -- flags := make(map[string]struct{}) +- flags := make(map[string]string) - if len(os.Args) >= 6 { - for _, flag := range strings.Split(os.Args[5], ",") { -- flags[flag] = struct{}{} +- flagFields := strings.Split(flag, "=") +- if len(flagFields) == 1 { +- flags[flagFields[0]] = "yes" +- } else { +- flags[flagFields[0]] = flagFields[1] +- } - } - } - - // Parse the text file and generate Go source code from it. -- var emojis string -- if _, ok := flags["emojis"]; ok { -- emojis = emojiURL -- } - _, includeGeneralCategory := flags["gencat"] -- src, err := parse(fmt.Sprintf(gbpURL, os.Args[1]), emojis, includeGeneralCategory) +- var mainURL string +- if os.Args[1] != "-" { +- mainURL = fmt.Sprintf(propertyURL, os.Args[1]) +- } +- src, err := parse(mainURL, flags["emojis"], includeGeneralCategory) - if err != nil { - log.Fatal(err) - } @@ -370,49 +379,57 @@ index 6451270..0000000 - -// parse parses the Unicode Properties text files located at the given URLs and -// returns their equivalent Go source code to be used in the uniseg package. If --// "emojiURL" is an empty string, no emoji code points will be included. If +-// "emojiProperty" is not an empty string, emoji code points for that emoji +-// property (e.g. "Extended_Pictographic") will be included. In those cases, you +-// may pass an empty "propertyURL" to skip parsing the main properties file. If -// "includeGeneralCategory" is true, the Unicode General Category property will -// be extracted from the comments and included in the output. --func parse(gbpURL, emojiURL string, includeGeneralCategory bool) (string, error) { +-func parse(propertyURL, emojiProperty string, includeGeneralCategory bool) (string, error) { +- if propertyURL == "" && emojiProperty == "" { +- return "", errors.New("no properties to parse") +- } +- - // Temporary buffer to hold properties. - var properties [][4]string - - // Open the first URL. -- log.Printf("Parsing %s", gbpURL) -- res, err := http.Get(gbpURL) -- if err != nil { -- return "", err -- } -- in1 := res.Body -- defer in1.Close() +- if propertyURL != "" { +- log.Printf("Parsing %s", propertyURL) +- res, err := http.Get(propertyURL) +- if err != nil { +- return "", err +- } +- in1 := res.Body +- defer in1.Close() - -- // Parse it. -- scanner := bufio.NewScanner(in1) -- num := 0 -- for scanner.Scan() { -- num++ -- line := strings.TrimSpace(scanner.Text()) +- // Parse it. +- scanner := bufio.NewScanner(in1) +- num := 0 +- for scanner.Scan() { +- num++ +- line := strings.TrimSpace(scanner.Text()) - -- // Skip comments and empty lines. -- if strings.HasPrefix(line, "#") || line == "" { -- continue -- } +- // Skip comments and empty lines. +- if strings.HasPrefix(line, "#") || line == "" { +- continue +- } - -- // Everything else must be a code point range, a property and a comment. -- from, to, property, comment, err := parseProperty(line) -- if err != nil { -- return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err) +- // Everything else must be a code point range, a property and a comment. +- from, to, property, comment, err := parseProperty(line) +- if err != nil { +- return "", fmt.Errorf("%s line %d: %v", os.Args[4], num, err) +- } +- properties = append(properties, [4]string{from, to, property, comment}) +- } +- if err := scanner.Err(); err != nil { +- return "", err - } -- properties = append(properties, [4]string{from, to, property, comment}) -- } -- if err := scanner.Err(); err != nil { -- return "", err - } - - // Open the second URL. -- if emojiURL != "" { +- if emojiProperty != "" { - log.Printf("Parsing %s", emojiURL) -- res, err = http.Get(emojiURL) +- res, err := http.Get(emojiURL) - if err != nil { - return "", err - } @@ -420,15 +437,15 @@ index 6451270..0000000 - defer in2.Close() - - // Parse it. -- scanner = bufio.NewScanner(in2) -- num = 0 +- scanner := bufio.NewScanner(in2) +- num := 0 - for scanner.Scan() { - num++ - line := scanner.Text() - - // Skip comments, empty lines, and everything not containing - // "Extended_Pictographic". -- if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, "Extended_Pictographic") { +- if strings.HasPrefix(line, "#") || line == "" || !strings.Contains(line, emojiProperty) { - continue - } - @@ -471,7 +488,7 @@ index 6451270..0000000 -// Code generated via go generate from gen_properties.go. DO NOT EDIT. - -// ` + os.Args[3] + ` are taken from --// ` + gbpURL + emojiComment + ` +-// ` + propertyURL + emojiComment + ` -// on ` + time.Now().Format("January 2, 2006") + `. See https://www.unicode.org/license.html for the Unicode -// license agreement. -var ` + os.Args[3] + ` = [][` + strconv.Itoa(columns) + `]int{ @@ -520,6 +537,6 @@ index 6451270..0000000 -func translateProperty(prefix, property string) string { - return prefix + strings.ReplaceAll(property, "_", "") -} --- -2.37.2 +-- +2.38.0 diff --git a/ddl/reorg.go b/ddl/reorg.go index 8d5da96f1dd86..2c7508d24b38f 100644 --- a/ddl/reorg.go +++ b/ddl/reorg.go @@ -199,12 +199,6 @@ func (w *worker) runReorgJob(rh *reorgHandler, reorgInfo *reorgInfo, tblInfo *mo rc := w.getReorgCtx(job) if rc == nil { - // Since reorg job will be interrupted for polling the cancel action outside. we don't need to wait for 2.5s - // for the later entrances. - // lease = 0 means it's in an integration test. In this case we don't delay so the test won't run too slowly. - if lease > 0 { - delayForAsyncCommit() - } // This job is cancelling, we should return ErrCancelledDDLJob directly. // Q: Is there any possibility that the job is cancelling and has no reorgCtx? // A: Yes, consider the case that we cancel the job when backfilling the last batch of data, the cancel txn is commit first, @@ -608,7 +602,9 @@ func getReorgInfo(ctx *JobContext, d *ddlCtx, rh *reorgHandler, job *model.Job, }) info.first = true - // get the current version for reorganization if we don't have + if d.lease > 0 { // Only delay when it's not in test. + delayForAsyncCommit() + } ver, err := getValidCurrentVersion(d.store) if err != nil { return nil, errors.Trace(err) @@ -696,7 +692,9 @@ func getReorgInfoFromPartitions(ctx *JobContext, d *ddlCtx, rh *reorgHandler, jo ) if job.SnapshotVer == 0 { info.first = true - // get the current version for reorganization if we don't have + if d.lease > 0 { // Only delay when it's not in test. + delayForAsyncCommit() + } ver, err := getValidCurrentVersion(d.store) if err != nil { return nil, errors.Trace(err) diff --git a/domain/domain.go b/domain/domain.go index f27e746c1b2d0..9259e5c1e8b52 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -91,7 +91,7 @@ type Domain struct { store kv.Storage infoCache *infoschema.InfoCache privHandle *privileges.Handle - bindHandle *bindinfo.BindHandle + bindHandle atomic.Pointer[bindinfo.BindHandle] statsHandle unsafe.Pointer statsLease time.Duration ddl ddl.DDL @@ -855,7 +855,7 @@ func (do *Domain) Close() { } do.wg.Wait() do.sysSessionPool.Close() - variable.UnregisterStatistics(do.bindHandle) + variable.UnregisterStatistics(do.bindHandle.Load()) if do.onClose != nil { do.onClose() } @@ -1372,7 +1372,7 @@ func (do *Domain) PrivilegeHandle() *privileges.Handle { // BindHandle returns domain's bindHandle. func (do *Domain) BindHandle() *bindinfo.BindHandle { - return do.bindHandle + return do.bindHandle.Load() } // LoadBindInfoLoop create a goroutine loads BindInfo in a loop, it should @@ -1380,13 +1380,11 @@ func (do *Domain) BindHandle() *bindinfo.BindHandle { func (do *Domain) LoadBindInfoLoop(ctxForHandle sessionctx.Context, ctxForEvolve sessionctx.Context) error { ctxForHandle.GetSessionVars().InRestrictedSQL = true ctxForEvolve.GetSessionVars().InRestrictedSQL = true - if do.bindHandle == nil { - do.bindHandle = bindinfo.NewBindHandle(ctxForHandle) - } else { - do.bindHandle.Reset(ctxForHandle) + if !do.bindHandle.CompareAndSwap(nil, bindinfo.NewBindHandle(ctxForHandle)) { + do.bindHandle.Load().Reset(ctxForHandle) } - err := do.bindHandle.Update(true) + err := do.bindHandle.Load().Update(true) if err != nil || bindinfo.Lease == 0 { return err } @@ -1416,22 +1414,23 @@ func (do *Domain) globalBindHandleWorkerLoop(owner owner.Manager) { case <-do.exit: return case <-bindWorkerTicker.C: - err := do.bindHandle.Update(false) + bindHandle := do.bindHandle.Load() + err := bindHandle.Update(false) if err != nil { logutil.BgLogger().Error("update bindinfo failed", zap.Error(err)) } - do.bindHandle.DropInvalidBindRecord() + bindHandle.DropInvalidBindRecord() // Get Global optVal, err := do.GetGlobalVar(variable.TiDBCapturePlanBaseline) if err == nil && variable.TiDBOptOn(optVal) { - do.bindHandle.CaptureBaselines() + bindHandle.CaptureBaselines() } - do.bindHandle.SaveEvolveTasksToStore() + bindHandle.SaveEvolveTasksToStore() case <-gcBindTicker.C: if !owner.IsOwner() { continue } - err := do.bindHandle.GCBindRecord() + err := do.bindHandle.Load().GCBindRecord() if err != nil { logutil.BgLogger().Error("GC bind record failed", zap.Error(err)) } @@ -1456,7 +1455,7 @@ func (do *Domain) handleEvolvePlanTasksLoop(ctx sessionctx.Context, owner owner. case <-time.After(bindinfo.Lease): } if owner.IsOwner() { - err := do.bindHandle.HandleEvolvePlanTask(ctx, false) + err := do.bindHandle.Load().HandleEvolvePlanTask(ctx, false) if err != nil { logutil.BgLogger().Info("evolve plan failed", zap.Error(err)) } diff --git a/executor/infoschema_reader.go b/executor/infoschema_reader.go index e157ac72f93aa..a1f57da26e411 100644 --- a/executor/infoschema_reader.go +++ b/executor/infoschema_reader.go @@ -395,23 +395,6 @@ func hasPriv(ctx sessionctx.Context, priv mysql.PrivilegeType) bool { return pm.RequestVerification(ctx.GetSessionVars().ActiveRoles, "", "", "", priv) } -func scopeStr(sv *variable.SysVar) string { - var scopes []string - if sv.HasNoneScope() { - return "NONE" - } - if sv.HasSessionScope() { - scopes = append(scopes, "SESSION") - } - if sv.HasGlobalScope() { - scopes = append(scopes, "GLOBAL") - } - if sv.HasInstanceScope() { - scopes = append(scopes, "INSTANCE") - } - return strings.Join(scopes, ",") -} - func (e *memtableRetriever) setDataForVariablesInfo(ctx sessionctx.Context) error { sysVars := variable.GetSysVars() rows := make([][]types.Datum, 0, len(sysVars)) @@ -428,14 +411,14 @@ func (e *memtableRetriever) setDataForVariablesInfo(ctx sessionctx.Context) erro isNoop = "YES" } row := types.MakeDatums( - sv.Name, // VARIABLE_NAME - scopeStr(sv), // VARIABLE_SCOPE - sv.Value, // DEFAULT_VALUE - currentVal, // CURRENT_VALUE - sv.MinValue, // MIN_VALUE - sv.MaxValue, // MAX_VALUE - nil, // POSSIBLE_VALUES - isNoop, // IS_NOOP + sv.Name, // VARIABLE_NAME + sv.Scope.String(), // VARIABLE_SCOPE + sv.Value, // DEFAULT_VALUE + currentVal, // CURRENT_VALUE + sv.MinValue, // MIN_VALUE + sv.MaxValue, // MAX_VALUE + nil, // POSSIBLE_VALUES + isNoop, // IS_NOOP ) // min and max value is only supported for numeric types if !(sv.Type == variable.TypeUnsigned || sv.Type == variable.TypeInt || sv.Type == variable.TypeFloat) { diff --git a/expression/BUILD.bazel b/expression/BUILD.bazel index 38f1cc9ca3b34..c98dcdb1ec483 100644 --- a/expression/BUILD.bazel +++ b/expression/BUILD.bazel @@ -110,6 +110,7 @@ go_library( "@com_github_tikv_client_go_v2//oracle", "@org_golang_x_exp//slices", "@org_golang_x_tools//container/intsets", + "@org_uber_go_atomic//:atomic", "@org_uber_go_zap//:zap", ], ) diff --git a/expression/collation.go b/expression/collation.go index 11731087cbd4c..eebab0aa5bc1f 100644 --- a/expression/collation.go +++ b/expression/collation.go @@ -15,6 +15,8 @@ package expression import ( + goatomic "sync/atomic" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/mysql" @@ -49,12 +51,12 @@ func (c *collationInfo) HasCoercibility() bool { } func (c *collationInfo) Coercibility() Coercibility { - return c.coer + return Coercibility(goatomic.LoadInt32((*int32)(&c.coer))) } // SetCoercibility implements CollationInfo SetCoercibility interface. func (c *collationInfo) SetCoercibility(val Coercibility) { - c.coer = val + goatomic.StoreInt32((*int32)(&c.coer), int32(val)) c.coerInit.Store(true) } @@ -100,7 +102,7 @@ type CollationInfo interface { // Coercibility values are used to check whether the collation of one item can be coerced to // the collation of other. See https://dev.mysql.com/doc/refman/8.0/en/charset-collation-coercibility.html -type Coercibility int +type Coercibility int32 const ( // CoercibilityExplicit is derived from an explicit COLLATE clause. diff --git a/go.mod b/go.mod index caf35784115a0..9e778b9b3aacb 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/kisielk/errcheck v1.6.2 github.com/klauspost/compress v1.15.1 github.com/kyoh86/exportloopref v0.1.8 - github.com/mgechev/revive v1.2.4-0.20220827111817-553604eaced5 + github.com/mgechev/revive v1.2.4 github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 github.com/nishanths/predeclared v0.2.2 github.com/opentracing/basictracer-go v1.0.0 @@ -99,7 +99,7 @@ require ( go.etcd.io/etcd/server/v3 v3.5.2 go.etcd.io/etcd/tests/v3 v3.5.2 go.opencensus.io v0.23.0 - go.uber.org/atomic v1.9.0 + go.uber.org/atomic v1.10.0 go.uber.org/automaxprocs v1.4.0 go.uber.org/goleak v1.2.0 go.uber.org/multierr v1.8.0 @@ -108,7 +108,7 @@ require ( golang.org/x/net v0.0.0-20220722155237-a158d28d115b golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 + golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 @@ -182,7 +182,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -200,7 +200,7 @@ require ( github.com/prometheus/tsdb v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect - github.com/rivo/uniseg v0.3.4 // indirect + github.com/rivo/uniseg v0.4.2 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/shurcooL/vfsgen v0.0.0-20180711163814-62bca832be04 // indirect diff --git a/go.sum b/go.sum index 036326aff8a42..4717bbaf91361 100644 --- a/go.sum +++ b/go.sum @@ -647,16 +647,16 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/maxatome/go-testdeep v1.11.0 h1:Tgh5efyCYyJFGUYiT0qxBSIDeXw0F5zSoatlou685kk= github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/mgechev/revive v1.2.4-0.20220827111817-553604eaced5 h1:a+itKsYpxka50MyaWQW1XCZ1vwfgjzVy/OzZ7DC/4+U= -github.com/mgechev/revive v1.2.4-0.20220827111817-553604eaced5/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8GQMX/Q= +github.com/mgechev/revive v1.2.4 h1:+2Hd/S8oO2H0Ikq2+egtNwQsVhAeELHjxjIUFX5ajLI= +github.com/mgechev/revive v1.2.4/go.mod h1:iAWlQishqCuj4yhV24FTnKSXGpbAA+0SckXB8GQMX/Q= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.10/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -821,8 +821,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6O github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw= -github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= +github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rlmcpherson/s3gof3r v0.5.0/go.mod h1:s7vv7SMDPInkitQMuZzH615G7yWHdrU2r/Go7Bo71Rs= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1009,8 +1009,9 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -1280,8 +1281,8 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 h1:wM1k/lXfpc5HdkJJyW9GELpd8ERGdnh8sMGL6Gzq3Ho= -golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/planner/core/BUILD.bazel b/planner/core/BUILD.bazel index fb76503f81bb1..4bcc24f0430a7 100644 --- a/planner/core/BUILD.bazel +++ b/planner/core/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "plan_cacheable_checker.go", "plan_cost.go", "plan_cost_detail.go", + "plan_cost_ver2.go", "plan_stats.go", "plan_to_pb.go", "planbuilder.go", @@ -186,6 +187,7 @@ go_test( "plan_cacheable_checker_test.go", "plan_cost_detail_test.go", "plan_cost_test.go", + "plan_cost_ver2_test.go", "plan_stats_test.go", "plan_test.go", "plan_to_pb_test.go", diff --git a/planner/core/common_plans.go b/planner/core/common_plans.go index 412664bb935f7..a814089ecd2bb 100644 --- a/planner/core/common_plans.go +++ b/planner/core/common_plans.go @@ -599,7 +599,7 @@ func (e *Explain) RenderResult() error { if e.Analyze && strings.ToLower(e.Format) == types.ExplainFormatTrueCardCost { pp, ok := e.TargetPlan.(PhysicalPlan) if ok { - if _, err := pp.GetPlanCost(property.RootTaskType, + if _, err := getPlanCost(pp, property.RootTaskType, NewDefaultPlanCostOption().WithCostFlag(CostFlagRecalculate|CostFlagUseTrueCardinality)); err != nil { return err } @@ -769,7 +769,7 @@ func (e *Explain) getOperatorInfo(p Plan, id string) (string, string, string, st } estCost := "N/A" if pp, ok := p.(PhysicalPlan); ok { - planCost, _ := pp.GetPlanCost(property.RootTaskType, NewDefaultPlanCostOption()) + planCost, _ := getPlanCost(pp, property.RootTaskType, NewDefaultPlanCostOption()) estCost = strconv.FormatFloat(planCost, 'f', 2, 64) } var accessObject, operatorInfo string @@ -875,7 +875,7 @@ func binaryOpFromFlatOp(explainCtx sessionctx.Context, op *FlatOperator, out *ti } if op.IsPhysicalPlan { p := op.Origin.(PhysicalPlan) - out.Cost, _ = p.GetPlanCost(property.RootTaskType, NewDefaultPlanCostOption()) + out.Cost, _ = getPlanCost(p, property.RootTaskType, NewDefaultPlanCostOption()) } if rootStats != nil { basic, groups := rootStats.MergeStats() diff --git a/planner/core/find_best_task.go b/planner/core/find_best_task.go index ad66f366b9842..c73ce9f3c086d 100644 --- a/planner/core/find_best_task.go +++ b/planner/core/find_best_task.go @@ -327,7 +327,7 @@ func getTaskPlanCost(t task, op *physicalOptimizeOp) (float64, bool, error) { default: return 0, false, errors.New("unknown task type") } - cost, err := t.plan().GetPlanCost(taskType, NewDefaultPlanCostOption().WithOptimizeTracer(op)) + cost, err := getPlanCost(t.plan(), taskType, NewDefaultPlanCostOption().WithOptimizeTracer(op)) return cost, false, err } diff --git a/planner/core/logical_plans.go b/planner/core/logical_plans.go index 66ce98cc97d59..e3c24d2375134 100644 --- a/planner/core/logical_plans.go +++ b/planner/core/logical_plans.go @@ -1890,6 +1890,19 @@ type ShowContents struct { Limit *ast.Limit // Used for limit Result Set row number. } +const emptyShowContentsSize = int64(unsafe.Sizeof(ShowContents{})) + +// MemoryUsage return the memory usage of ShowContents +func (s *ShowContents) MemoryUsage() (sum int64) { + if s == nil { + return + } + + sum = emptyShowContentsSize + int64(len(s.DBName)) + s.Partition.MemoryUsage() + s.IndexName.MemoryUsage() + + int64(cap(s.Roles))*size.SizeOfPointer + return +} + // LogicalShow represents a show plan. type LogicalShow struct { logicalSchemaProducer diff --git a/planner/core/optimizer.go b/planner/core/optimizer.go index ee802a4bb775d..09c370dd11f60 100644 --- a/planner/core/optimizer.go +++ b/planner/core/optimizer.go @@ -659,7 +659,7 @@ func physicalOptimize(logic LogicalPlan, planCounter *PlanCounterTp) (plan Physi if err = t.plan().ResolveIndices(); err != nil { return nil, 0, err } - cost, err = t.plan().GetPlanCost(property.RootTaskType, NewDefaultPlanCostOption()) + cost, err = getPlanCost(t.plan(), property.RootTaskType, NewDefaultPlanCostOption()) return t.plan(), cost, err } diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index a4419b2ab5caf..54180e2ce58fe 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -1963,6 +1963,15 @@ func (p *PhysicalMaxOneRow) Clone() (PhysicalPlan, error) { return cloned, nil } +// MemoryUsage return the memory usage of PhysicalMaxOneRow +func (p *PhysicalMaxOneRow) MemoryUsage() (sum int64) { + if p == nil { + return + } + + return p.basePhysicalPlan.MemoryUsage() +} + // PhysicalTableDual is the physical operator of dual. type PhysicalTableDual struct { physicalSchemaProducer @@ -1984,6 +1993,19 @@ func (p *PhysicalTableDual) SetOutputNames(names types.NameSlice) { p.names = names } +// MemoryUsage return the memory usage of PhysicalTableDual +func (p *PhysicalTableDual) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = p.physicalSchemaProducer.MemoryUsage() + size.SizeOfInt + size.SizeOfSlice + int64(cap(p.names))*size.SizeOfPointer + for _, name := range p.names { + sum += name.MemoryUsage() + } + return +} + // PhysicalWindow is the physical operator of window function. type PhysicalWindow struct { physicalSchemaProducer @@ -2048,6 +2070,27 @@ func (p *PhysicalWindow) Clone() (PhysicalPlan, error) { return cloned, nil } +// MemoryUsage return the memory usage of PhysicalWindow +func (p *PhysicalWindow) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = p.physicalSchemaProducer.MemoryUsage() + size.SizeOfSlice*3 + int64(cap(p.WindowFuncDescs))*size.SizeOfPointer + + size.SizeOfUint8 + + for _, windowFunc := range p.WindowFuncDescs { + sum += windowFunc.MemoryUsage() + } + for _, item := range p.PartitionBy { + sum += item.MemoryUsage() + } + for _, item := range p.OrderBy { + sum += item.MemoryUsage() + } + return +} + // PhysicalShuffle represents a shuffle plan. // `Tails` and `DataSources` are the last plan within and the first plan following the "shuffle", respectively, // @@ -2070,6 +2113,30 @@ type PhysicalShuffle struct { ByItemArrays [][]expression.Expression } +// MemoryUsage return the memory usage of PhysicalShuffle +func (p *PhysicalShuffle) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = p.basePhysicalPlan.MemoryUsage() + size.SizeOfInt*2 + size.SizeOfSlice*(3+int64(cap(p.ByItemArrays))) + + int64(cap(p.Tails)+cap(p.DataSources))*size.SizeOfInterface + + for _, plan := range p.Tails { + sum += plan.MemoryUsage() + } + for _, plan := range p.DataSources { + sum += plan.MemoryUsage() + } + for _, exprs := range p.ByItemArrays { + sum += int64(cap(exprs)) * size.SizeOfInterface + for _, expr := range exprs { + sum += expr.MemoryUsage() + } + } + return +} + // PartitionSplitterType is the type of `Shuffle` executor splitter, which splits data source into partitions. type PartitionSplitterType int @@ -2091,6 +2158,16 @@ type PhysicalShuffleReceiverStub struct { DataSource PhysicalPlan } +// MemoryUsage return the memory usage of PhysicalShuffleReceiverStub +func (p *PhysicalShuffleReceiverStub) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = p.physicalSchemaProducer.MemoryUsage() + size.SizeOfPointer + size.SizeOfInterface + p.DataSource.MemoryUsage() + return +} + // CollectPlanStatsVersion uses to collect the statistics version of the plan. func CollectPlanStatsVersion(plan PhysicalPlan, statsInfos map[string]uint64) map[string]uint64 { for _, child := range plan.Children() { @@ -2123,6 +2200,16 @@ type PhysicalShow struct { Extractor ShowPredicateExtractor } +// MemoryUsage return the memory usage of PhysicalShow +func (p *PhysicalShow) MemoryUsage() (sum int64) { + if p == nil { + return + } + + sum = p.physicalSchemaProducer.MemoryUsage() + p.ShowContents.MemoryUsage() + size.SizeOfInterface + return +} + // PhysicalShowDDLJobs is for showing DDL job list. type PhysicalShowDDLJobs struct { physicalSchemaProducer @@ -2130,6 +2217,14 @@ type PhysicalShowDDLJobs struct { JobNumber int64 } +// MemoryUsage return the memory usage of PhysicalShowDDLJobs +func (p *PhysicalShowDDLJobs) MemoryUsage() (sum int64) { + if p == nil { + return + } + return p.physicalSchemaProducer.MemoryUsage() + size.SizeOfInt64 +} + // BuildMergeJoinPlan builds a PhysicalMergeJoin from the given fields. Currently, it is only used for test purpose. func BuildMergeJoinPlan(ctx sessionctx.Context, joinType JoinType, leftKeys, rightKeys []*expression.Column) *PhysicalMergeJoin { baseJoin := basePhysicalJoin{ diff --git a/planner/core/plan.go b/planner/core/plan.go index 7ba79bfd41e7b..dbdad6c11e863 100644 --- a/planner/core/plan.go +++ b/planner/core/plan.go @@ -330,8 +330,11 @@ type LogicalPlan interface { type PhysicalPlan interface { Plan - // GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. - GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) + // getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost on model ver1. + getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) + + // getPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost on model ver2. + getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) // attach2Task makes the current physical plan as the father of task's physicalPlan and updates the cost of // current task. If the child's task is cop task, some operator may close this task and return a new rootTask. diff --git a/planner/core/plan_cache_lru.go b/planner/core/plan_cache_lru.go index 16d2a9db786e2..ed29a1274e97f 100644 --- a/planner/core/plan_cache_lru.go +++ b/planner/core/plan_cache_lru.go @@ -70,12 +70,20 @@ func NewLRUPlanCache(capacity uint, guard float64, quota uint64, } } +// strHashKey control deep or Shallow copy of string +func strHashKey(key kvcache.Key, deepCopy bool) string { + if deepCopy { + return string(key.Hash()) + } + return string(hack.String(key.Hash())) +} + // Get tries to find the corresponding value according to the given key. func (l *LRUPlanCache) Get(key kvcache.Key, paramTypes []*types.FieldType) (value kvcache.Value, ok bool) { l.lock.Lock() defer l.lock.Unlock() - bucket, bucketExist := l.buckets[string(hack.String(key.Hash()))] + bucket, bucketExist := l.buckets[strHashKey(key, false)] if bucketExist { if element, exist := l.pickFromBucket(bucket, paramTypes); exist { l.lruList.MoveToFront(element) @@ -90,7 +98,7 @@ func (l *LRUPlanCache) Put(key kvcache.Key, value kvcache.Value, paramTypes []*t l.lock.Lock() defer l.lock.Unlock() - hash := string(key.Hash()) + hash := strHashKey(key, true) bucket, bucketExist := l.buckets[hash] if bucketExist { if element, exist := l.pickFromBucket(bucket, paramTypes); exist { @@ -120,14 +128,14 @@ func (l *LRUPlanCache) Delete(key kvcache.Key) { l.lock.Lock() defer l.lock.Unlock() - hash := hack.String(key.Hash()) - bucket, bucketExist := l.buckets[string(hash)] + hash := strHashKey(key, false) + bucket, bucketExist := l.buckets[hash] if bucketExist { for element := range bucket { l.lruList.Remove(element) l.size-- } - delete(l.buckets, string(hash)) + delete(l.buckets, hash) } } @@ -183,8 +191,12 @@ func (l *LRUPlanCache) removeOldest() { // removeFromBucket remove element from bucket func (l *LRUPlanCache) removeFromBucket(element *list.Element) { - bucket := l.buckets[string(hack.String(element.Value.(*planCacheEntry).PlanKey.Hash()))] + hash := strHashKey(element.Value.(*planCacheEntry).PlanKey, false) + bucket := l.buckets[hash] delete(bucket, element) + if len(bucket) == 0 { + delete(l.buckets, hash) + } } // memoryControl control the memory by quota and guard diff --git a/planner/core/plan_cache_lru_test.go b/planner/core/plan_cache_lru_test.go index 09a4e10a28035..0ec108e35b838 100644 --- a/planner/core/plan_cache_lru_test.go +++ b/planner/core/plan_cache_lru_test.go @@ -334,3 +334,30 @@ func TestIssue37914(t *testing.T) { lru.Put(key, val, pTypes) }) } + +func TestIssue38244(t *testing.T) { + lru := NewLRUPlanCache(3, 0, 0, pickFromBucket) + require.Equal(t, uint(3), lru.capacity) + + keys := make([]*mockCacheKey, 5) + vals := make([]*fakePlan, 5) + pTypes := [][]*types.FieldType{{types.NewFieldType(mysql.TypeFloat), types.NewFieldType(mysql.TypeDouble)}, + {types.NewFieldType(mysql.TypeFloat), types.NewFieldType(mysql.TypeEnum)}, + {types.NewFieldType(mysql.TypeFloat), types.NewFieldType(mysql.TypeDate)}, + {types.NewFieldType(mysql.TypeFloat), types.NewFieldType(mysql.TypeLong)}, + {types.NewFieldType(mysql.TypeFloat), types.NewFieldType(mysql.TypeInt24)}, + } + + // one key corresponding to multi values + for i := 0; i < 5; i++ { + keys[i] = newMockHashKey(int64(i)) + vals[i] = &fakePlan{ + plan: int64(i), + tps: pTypes[i], + } + lru.Put(keys[i], vals[i], pTypes[i]) + } + require.Equal(t, lru.size, lru.capacity) + require.Equal(t, uint(3), lru.size) + require.Equal(t, len(lru.buckets), 3) +} diff --git a/planner/core/plan_cost.go b/planner/core/plan_cost_ver1.go similarity index 78% rename from planner/core/plan_cost.go rename to planner/core/plan_cost_ver1.go index 3bfe1fddb1a97..49a4cd32618c5 100644 --- a/planner/core/plan_cost.go +++ b/planner/core/plan_cost_ver1.go @@ -43,8 +43,8 @@ func hasCostFlag(costFlag, flag uint64) bool { return (costFlag & flag) > 0 } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *basePhysicalPlan) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *basePhysicalPlan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { // just calculate the cost once and always reuse it @@ -52,7 +52,7 @@ func (p *basePhysicalPlan) GetPlanCost(taskType property.TaskType, option *PlanC } p.planCost = 0 // the default implementation, the operator have no cost for _, child := range p.children { - childCost, err := child.GetPlanCost(taskType, option) + childCost, err := child.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -62,34 +62,29 @@ func (p *basePhysicalPlan) GetPlanCost(taskType property.TaskType, option *PlanC return p.planCost, nil } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalSelection) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalSelection) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } var selfCost float64 - switch p.ctx.GetSessionVars().CostModelVersion { - case modelVer1: // selection cost: rows * cpu-factor - var cpuFactor float64 - switch taskType { - case property.RootTaskType, property.MppTaskType: - cpuFactor = p.ctx.GetSessionVars().GetCPUFactor() - case property.CopSingleReadTaskType, property.CopDoubleReadTaskType: - cpuFactor = p.ctx.GetSessionVars().GetCopCPUFactor() - default: - return 0, errors.Errorf("unknown task type %v", taskType) - } - selfCost = getCardinality(p.children[0], costFlag) * cpuFactor - if p.fromDataSource { - selfCost = 0 // for compatibility, see https://github.com/pingcap/tidb/issues/36243 - } - case modelVer2: - return p.getPlanCostVer2(taskType, option) + var cpuFactor float64 + switch taskType { + case property.RootTaskType, property.MppTaskType: + cpuFactor = p.ctx.GetSessionVars().GetCPUFactor() + case property.CopSingleReadTaskType, property.CopDoubleReadTaskType: + cpuFactor = p.ctx.GetSessionVars().GetCopCPUFactor() + default: + return 0, errors.Errorf("unknown task type %v", taskType) + } + selfCost = getCardinality(p.children[0], costFlag) * cpuFactor + if p.fromDataSource { + selfCost = 0 // for compatibility, see https://github.com/pingcap/tidb/issues/36243 } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -111,16 +106,13 @@ func (p *PhysicalProjection) GetCost(count float64) float64 { return cpuCost + concurrencyCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalProjection) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalProjection) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -176,20 +168,17 @@ func (p *PhysicalIndexLookUpReader) GetCost(costFlag uint64) (cost float64) { return } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexLookUpReader) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexLookUpReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = 0 // child's cost for _, child := range []PhysicalPlan{p.indexPlan, p.tablePlan} { - childCost, err := child.GetPlanCost(property.CopDoubleReadTaskType, option) + childCost, err := child.getPlanCostVer1(property.CopDoubleReadTaskType, option) if err != nil { return 0, err } @@ -202,14 +191,12 @@ func (p *PhysicalIndexLookUpReader) GetPlanCost(taskType property.TaskType, opti for tmp = p.tablePlan; len(tmp.Children()) > 0; tmp = tmp.Children()[0] { } ts := tmp.(*PhysicalTableScan) - if p.ctx.GetSessionVars().CostModelVersion == modelVer1 { - tblCost, err := ts.GetPlanCost(property.CopDoubleReadTaskType, option) - if err != nil { - return 0, err - } - p.planCost -= tblCost - p.planCost += getCardinality(p.indexPlan, costFlag) * ts.getScanRowSize() * p.SCtx().GetSessionVars().GetScanFactor(ts.Table) + tblCost, err := ts.getPlanCostVer1(property.CopDoubleReadTaskType, option) + if err != nil { + return 0, err } + p.planCost -= tblCost + p.planCost += getCardinality(p.indexPlan, costFlag) * ts.getScanRowSize() * p.SCtx().GetSessionVars().GetScanFactor(ts.Table) // index-side net I/O cost: rows * row-size * net-factor netFactor := getTableNetFactor(p.tablePlan) @@ -235,20 +222,17 @@ func (p *PhysicalIndexLookUpReader) GetPlanCost(taskType property.TaskType, opti return p.planCost, nil } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexReader) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } var rowCount, rowSize, netFactor, indexPlanCost, netSeekCost float64 sqlScanConcurrency := p.ctx.GetSessionVars().DistSQLScanConcurrency() // child's cost - childCost, err := p.indexPlan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := p.indexPlan.getPlanCostVer1(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -280,15 +264,12 @@ func (p *PhysicalIndexReader) GetNetDataSize() float64 { return p.indexPlan.StatsCount() * rowSize } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableReader) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalTableReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = 0 netFactor := getTableNetFactor(p.tablePlan) @@ -298,7 +279,7 @@ func (p *PhysicalTableReader) GetPlanCost(taskType property.TaskType, option *Pl switch storeType { case kv.TiKV: // child's cost - childCost, err := p.tablePlan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := p.tablePlan.getPlanCostVer1(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -321,7 +302,7 @@ func (p *PhysicalTableReader) GetPlanCost(taskType property.TaskType, option *Pl concurrency = p.ctx.GetSessionVars().CopTiFlashConcurrencyFactor rowSize = collectRowSizeFromMPPPlan(p.tablePlan) seekCost = accumulateNetSeekCost4MPP(p.tablePlan) - childCost, err := p.tablePlan.GetPlanCost(property.MppTaskType, option) + childCost, err := p.tablePlan.getPlanCostVer1(property.MppTaskType, option) if err != nil { return 0, err } @@ -331,12 +312,8 @@ func (p *PhysicalTableReader) GetPlanCost(taskType property.TaskType, option *Pl concurrency = float64(p.ctx.GetSessionVars().DistSQLScanConcurrency()) rowSize = getTblStats(p.tablePlan).GetAvgRowSize(p.ctx, p.tablePlan.Schema().Columns, false, false) seekCost = estimateNetSeekCost(p.tablePlan) - tType := property.MppTaskType - if p.ctx.GetSessionVars().CostModelVersion == modelVer1 { - // regard the underlying tasks as cop-task on modelVer1 for compatibility - tType = property.CopSingleReadTaskType - } - childCost, err := p.tablePlan.GetPlanCost(tType, option) + tType := property.CopSingleReadTaskType + childCost, err := p.tablePlan.getPlanCostVer1(tType, option) if err != nil { return 0, err } @@ -370,19 +347,16 @@ func (p *PhysicalTableReader) GetNetDataSize() float64 { return p.tablePlan.StatsCount() * rowSize } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexMergeReader) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexMergeReader) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = 0 if tblScan := p.tablePlan; tblScan != nil { - childCost, err := tblScan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := tblScan.getPlanCostVer1(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -393,7 +367,7 @@ func (p *PhysicalIndexMergeReader) GetPlanCost(taskType property.TaskType, optio p.planCost += getCardinality(tblScan, costFlag) * rowSize * netFactor // net I/O cost } for _, partialScan := range p.partialPlans { - childCost, err := partialScan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := partialScan.getPlanCostVer1(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -427,8 +401,8 @@ func (p *PhysicalIndexMergeReader) GetPartialReaderNetDataSize(plan PhysicalPlan return plan.StatsCount() * getTblStats(plan).GetAvgRowSize(p.ctx, plan.Schema().Columns, isIdxScan, false) } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTableScan) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalTableScan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -437,18 +411,13 @@ func (p *PhysicalTableScan) GetPlanCost(taskType property.TaskType, option *Plan var selfCost float64 var rowCount, rowSize, scanFactor float64 costModelVersion := p.ctx.GetSessionVars().CostModelVersion - switch costModelVersion { - case modelVer1: // scan cost: rows * row-size * scan-factor - scanFactor = p.ctx.GetSessionVars().GetScanFactor(p.Table) - if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold { - scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table) - } - rowCount = getCardinality(p, costFlag) - rowSize = p.getScanRowSize() - selfCost = rowCount * rowSize * scanFactor - case modelVer2: - return p.getPlanCostVer2(taskType, option) + scanFactor = p.ctx.GetSessionVars().GetScanFactor(p.Table) + if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold { + scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table) } + rowCount = getCardinality(p, costFlag) + rowSize = p.getScanRowSize() + selfCost = rowCount * rowSize * scanFactor if option.tracer != nil { setPhysicalTableOrIndexScanCostDetail(p, option.tracer, rowCount, rowSize, scanFactor, costModelVersion) } @@ -457,8 +426,8 @@ func (p *PhysicalTableScan) GetPlanCost(taskType property.TaskType, option *Plan return p.planCost, nil } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexScan) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexScan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil @@ -467,18 +436,13 @@ func (p *PhysicalIndexScan) GetPlanCost(taskType property.TaskType, option *Plan var selfCost float64 var rowCount, rowSize, scanFactor float64 costModelVersion := p.ctx.GetSessionVars().CostModelVersion - switch costModelVersion { - case modelVer1: // scan cost: rows * row-size * scan-factor - scanFactor = p.ctx.GetSessionVars().GetScanFactor(p.Table) - if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold { - scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table) - } - rowCount = getCardinality(p, costFlag) - rowSize = p.getScanRowSize() - selfCost = rowCount * rowSize * scanFactor - case modelVer2: - return p.getPlanCostVer2(taskType, option) + scanFactor = p.ctx.GetSessionVars().GetScanFactor(p.Table) + if p.Desc && p.prop != nil && p.prop.ExpectedCnt >= smallScanThreshold { + scanFactor = p.ctx.GetSessionVars().GetDescScanFactor(p.Table) } + rowCount = getCardinality(p, costFlag) + rowSize = p.getScanRowSize() + selfCost = rowCount * rowSize * scanFactor if option.tracer != nil { setPhysicalTableOrIndexScanCostDetail(p, option.tracer, rowCount, rowSize, scanFactor, costModelVersion) } @@ -544,40 +508,21 @@ func (p *PhysicalIndexJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost flo batchRatio := 30.0 innerPlanCost /= batchRatio } - return outerCost + innerPlanCost + cpuCost + memoryCost + p.estDoubleReadCost(outerCnt) -} - -func (p *PhysicalIndexJoin) estDoubleReadCost(doubleReadRows float64) float64 { - if p.ctx.GetSessionVars().CostModelVersion == modelVer1 { - // only consider double-read cost on modelVer2 - return 0 - } - // estimate the double read cost for IndexJoin: (double-read-tasks * seek-factor) / concurrency - seekFactor := p.ctx.GetSessionVars().GetSeekFactor(nil) - batchSize := math.Max(1.0, float64(p.ctx.GetSessionVars().IndexJoinBatchSize)) - concurrency := math.Max(1.0, float64(p.ctx.GetSessionVars().IndexLookupJoinConcurrency())) - // distRatio indicates how many requests corresponding to a batch, current value is from experiments. - // TODO: estimate it by using index correlation or make it configurable. - distRatio := 40.0 - numDoubleReadTasks := (doubleReadRows / batchSize) * distRatio - return (numDoubleReadTasks * seekFactor) / concurrency + return outerCost + innerPlanCost + cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexJoin) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexJoin) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] - outerCost, err := outerChild.GetPlanCost(taskType, option) + outerCost, err := outerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } - innerCost, err := innerChild.GetPlanCost(taskType, option) + innerCost, err := innerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -652,24 +597,21 @@ func (p *PhysicalIndexHashJoin) GetCost(outerCnt, innerCnt, outerCost, innerCost memoryCost := concurrency * (batchSize * distinctFactor) * innerCnt * sessVars.GetMemoryFactor() // Cost of inner child plan, i.e, mainly I/O and network cost. innerPlanCost := outerCnt * innerCost - return outerCost + innerPlanCost + cpuCost + memoryCost + p.estDoubleReadCost(outerCnt) + return outerCost + innerPlanCost + cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexHashJoin) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexHashJoin) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] - outerCost, err := outerChild.GetPlanCost(taskType, option) + outerCost, err := outerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } - innerCost, err := innerChild.GetPlanCost(taskType, option) + innerCost, err := innerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -746,24 +688,21 @@ func (p *PhysicalIndexMergeJoin) GetCost(outerCnt, innerCnt, outerCost, innerCos memoryCost := innerConcurrency * (batchSize * avgProbeCnt) * sessVars.GetMemoryFactor() innerPlanCost := outerCnt * innerCost - return outerCost + innerPlanCost + cpuCost + memoryCost + p.estDoubleReadCost(outerCnt) + return outerCost + innerPlanCost + cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalIndexMergeJoin) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalIndexMergeJoin) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] - outerCost, err := outerChild.GetPlanCost(taskType, option) + outerCost, err := outerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } - innerCost, err := innerChild.GetPlanCost(taskType, option) + innerCost, err := innerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -805,21 +744,18 @@ func (p *PhysicalApply) GetCost(lCount, rCount, lCost, rCost float64) float64 { return cpuCost + lCost + lCount*rCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalApply) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalApply) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } outerChild, innerChild := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] - outerCost, err := outerChild.GetPlanCost(taskType, option) + outerCost, err := outerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } - innerCost, err := innerChild.GetPlanCost(taskType, option) + innerCost, err := innerChild.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -882,18 +818,15 @@ func (p *PhysicalMergeJoin) GetCost(lCnt, rCnt float64, costFlag uint64) float64 return cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalMergeJoin) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalMergeJoin) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = 0 for _, child := range p.children { - childCost, err := child.GetPlanCost(taskType, option) + childCost, err := child.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -920,9 +853,6 @@ func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, isMPP bool, costFlag uint spill := oomUseTmpStorage && memQuota > 0 && rowSize*buildCnt > float64(memQuota) && p.storeTp != kv.TiFlash // Cost of building hash table. cpuFactor := sessVars.GetCPUFactor() - if isMPP && p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - cpuFactor = sessVars.GetTiFlashCPUFactor() // use the dedicated TiFlash CPU Factor on modelVer2 - } diskFactor := sessVars.GetDiskFactor() memoryFactor := sessVars.GetMemoryFactor() concurrencyFactor := sessVars.GetConcurrencyFactor() @@ -1003,18 +933,15 @@ func (p *PhysicalHashJoin) GetCost(lCnt, rCnt float64, isMPP bool, costFlag uint return cpuCost + memoryCost + diskCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalHashJoin) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalHashJoin) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = 0 for _, child := range p.children { - childCost, err := child.GetPlanCost(taskType, option) + childCost, err := child.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1033,13 +960,6 @@ func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot, isMPP bool, costF sessVars := p.ctx.GetSessionVars() if isRoot { cpuCost = inputRows * sessVars.GetCPUFactor() * aggFuncFactor - } else if isMPP { - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - // use the dedicated CPU factor for TiFlash on modelVer2 - cpuCost = inputRows * sessVars.GetTiFlashCPUFactor() * aggFuncFactor - } else { - cpuCost = inputRows * sessVars.GetCopCPUFactor() * aggFuncFactor - } } else { cpuCost = inputRows * sessVars.GetCopCPUFactor() * aggFuncFactor } @@ -1048,16 +968,13 @@ func (p *PhysicalStreamAgg) GetCost(inputRows float64, isRoot, isMPP bool, costF return cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalStreamAgg) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalStreamAgg) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1082,13 +999,6 @@ func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot, isMPP bool, costFla // Cost of additional goroutines. cpuCost += (con + 1) * sessVars.GetConcurrencyFactor() } - } else if isMPP { - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - // use the dedicated CPU factor for TiFlash on modelVer2 - cpuCost = inputRows * sessVars.GetTiFlashCPUFactor() * aggFuncFactor - } else { - cpuCost = inputRows * sessVars.GetCopCPUFactor() * aggFuncFactor - } } else { cpuCost = inputRows * sessVars.GetCopCPUFactor() * aggFuncFactor } @@ -1099,16 +1009,13 @@ func (p *PhysicalHashAgg) GetCost(inputRows float64, isRoot, isMPP bool, costFla return cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalHashAgg) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalHashAgg) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1150,16 +1057,13 @@ func (p *PhysicalSort) GetCost(count float64, schema *expression.Schema) float64 return cpuCost + memoryCost + diskCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalSort) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalSort) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1192,16 +1096,13 @@ func (p *PhysicalTopN) GetCost(count float64, isRoot bool) float64 { return cpuCost + memoryCost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalTopN) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalTopN) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1239,15 +1140,12 @@ func (p *BatchPointGetPlan) GetCost(opt *physicalOptimizeOp) float64 { return cost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *BatchPointGetPlan) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *BatchPointGetPlan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx != nil && p.ctx.GetSessionVars() != nil && p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = p.GetCost(option.tracer) p.planCostInit = true return p.planCost, nil @@ -1290,15 +1188,12 @@ func (p *PointGetPlan) GetCost(opt *physicalOptimizeOp) float64 { return cost } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PointGetPlan) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PointGetPlan) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx != nil && p.ctx.GetSessionVars() != nil && p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } p.planCost = p.GetCost(option.tracer) p.planCostInit = true return p.planCost, nil @@ -1316,15 +1211,15 @@ func (p *PointGetPlan) GetAvgRowSize() float64 { return p.stats.HistColl.GetIndexAvgRowSize(p.ctx, cols, p.IndexInfo.Unique) } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalUnionAll) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalUnionAll) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } var childMaxCost float64 for _, child := range p.children { - childCost, err := child.GetPlanCost(taskType, option) + childCost, err := child.getPlanCostVer1(taskType, option) if err != nil { return 0, err } @@ -1335,16 +1230,13 @@ func (p *PhysicalUnionAll) GetPlanCost(taskType property.TaskType, option *PlanC return p.planCost, nil } -// GetPlanCost calculates the cost of the plan if it has not been calculated yet and returns the cost. -func (p *PhysicalExchangeReceiver) GetPlanCost(taskType property.TaskType, option *PlanCostOption) (float64, error) { +// getPlanCostVer1 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *PhysicalExchangeReceiver) getPlanCostVer1(taskType property.TaskType, option *PlanCostOption) (float64, error) { costFlag := option.CostFlag if p.planCostInit && !hasCostFlag(costFlag, CostFlagRecalculate) { return p.planCost, nil } - if p.ctx.GetSessionVars().CostModelVersion == modelVer2 { - return p.getPlanCostVer2(taskType, option) - } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer1(taskType, option) if err != nil { return 0, err } diff --git a/planner/core/plan_cost_test.go b/planner/core/plan_cost_ver1_test.go similarity index 100% rename from planner/core/plan_cost_test.go rename to planner/core/plan_cost_ver1_test.go diff --git a/planner/core/plan_cost_ver2.go b/planner/core/plan_cost_ver2.go index 47856d4293006..b7f09c32bdce2 100644 --- a/planner/core/plan_cost_ver2.go +++ b/planner/core/plan_cost_ver2.go @@ -25,15 +25,43 @@ import ( "github.com/pingcap/tidb/sessionctx/variable" ) +func getPlanCost(p PhysicalPlan, taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.SCtx().GetSessionVars().CostModelVersion == modelVer1 { + return p.getPlanCostVer1(taskType, option) + } + return p.getPlanCostVer2(taskType, option) +} + +// getPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost. +func (p *basePhysicalPlan) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + p.planCost = 0 // the default implementation, the operator have no cost + for _, child := range p.children { + childCost, err := child.getPlanCostVer2(taskType, option) + if err != nil { + return 0, err + } + p.planCost += childCost + } + p.planCostInit = true + return p.planCost, nil +} + // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + filter-cost func (p *PhysicalSelection) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + inputRows := getCardinality(p.children[0], option.CostFlag) - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) filterCost := filterCostVer2(inputRows, p.Conditions, cpuFactor) - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -47,13 +75,17 @@ func (p *PhysicalSelection) getPlanCostVer2(taskType property.TaskType, option * // plan-cost = child-cost + proj-cost / concurrency // proj-cost = input-rows * len(expressions) * cpu-factor func (p *PhysicalProjection) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + inputRows := getCardinality(p.children[0], option.CostFlag) - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) concurrency := float64(p.ctx.GetSessionVars().ProjectionConcurrency()) - projCost := inputRows * float64(len(p.Exprs)) * cpuFactor + projCost := inputRows * float64(len(p.Exprs)) * cpuFactor.Value - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -67,9 +99,13 @@ func (p *PhysicalProjection) getPlanCostVer2(taskType property.TaskType, option // plan-cost = rows * log2(row-size) * scan-factor // log2(row-size) is from experiments. func (p *PhysicalIndexScan) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p, option.CostFlag) rowSize := math.Max(p.getScanRowSize(), 2.0) - scanFactor := getTaskScanFactor(p, taskType) + scanFactor := getTaskScanFactorVer2(p, taskType) p.planCost = scanCostVer2(rows, rowSize, scanFactor) p.planCostInit = true @@ -80,9 +116,13 @@ func (p *PhysicalIndexScan) getPlanCostVer2(taskType property.TaskType, option * // plan-cost = rows * log2(row-size) * scan-factor // log2(row-size) is from experiments. func (p *PhysicalTableScan) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p, option.CostFlag) rowSize := math.Max(p.getScanRowSize(), 2.0) - scanFactor := getTaskScanFactor(p, taskType) + scanFactor := getTaskScanFactorVer2(p, taskType) p.planCost = scanCostVer2(rows, rowSize, scanFactor) @@ -100,15 +140,19 @@ func (p *PhysicalTableScan) getPlanCostVer2(taskType property.TaskType, option * // net-cost = rows * row-size * net-factor // seek-cost = num-tasks * seek-factor func (p *PhysicalIndexReader) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p.indexPlan, option.CostFlag) rowSize := getAvgRowSize(p.indexPlan.Stats(), p.indexPlan.Schema()) - netFactor := getTaskNetFactor(p, taskType) + netFactor := getTaskNetFactorVer2(p, taskType) concurrency := float64(p.ctx.GetSessionVars().DistSQLScanConcurrency()) netCost := netCostVer2(rows, rowSize, netFactor) seekCost := estimateNetSeekCost(p.indexPlan) - childCost, err := p.indexPlan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := p.indexPlan.getPlanCostVer2(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -123,15 +167,19 @@ func (p *PhysicalIndexReader) getPlanCostVer2(taskType property.TaskType, option // net-cost = rows * row-size * net-factor // seek-cost = num-tasks * seek-factor func (p *PhysicalTableReader) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p.tablePlan, option.CostFlag) rowSize := getAvgRowSize(p.tablePlan.Stats(), p.tablePlan.Schema()) - netFactor := getTaskNetFactor(p, taskType) + netFactor := getTaskNetFactorVer2(p, taskType) concurrency := float64(p.ctx.GetSessionVars().DistSQLScanConcurrency()) netCost := netCostVer2(rows, rowSize, netFactor) seekCost := estimateNetSeekCost(p.tablePlan) - childCost, err := p.tablePlan.GetPlanCost(property.CopSingleReadTaskType, option) + childCost, err := p.tablePlan.getPlanCostVer2(property.CopSingleReadTaskType, option) if err != nil { return 0, err } @@ -157,20 +205,24 @@ func (p *PhysicalTableReader) getPlanCostVer2(taskType property.TaskType, option // double-read-cpu-cost = index-rows * cpu-factor // double-read-tasks = index-rows / batch-size * task-per-batch # task-per-batch is a magic number now func (p *PhysicalIndexLookUpReader) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + indexRows := getCardinality(p.indexPlan, option.CostFlag) tableRows := getCardinality(p.indexPlan, option.CostFlag) indexRowSize := getTblStats(p.indexPlan).GetAvgRowSize(p.ctx, p.indexPlan.Schema().Columns, true, false) tableRowSize := getTblStats(p.tablePlan).GetAvgRowSize(p.ctx, p.tablePlan.Schema().Columns, false, false) - cpuFactor := getTaskCPUFactor(p, taskType) - netFactor := getTaskNetFactor(p, taskType) - seekFactor := getTaskSeekFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) + netFactor := getTaskNetFactorVer2(p, taskType) + seekFactor := getTaskSeekFactorVer2(p, taskType) distConcurrency := float64(p.ctx.GetSessionVars().DistSQLScanConcurrency()) doubleReadConcurrency := float64(p.ctx.GetSessionVars().IndexLookupConcurrency()) // index-side indexNetCost := netCostVer2(indexRows, indexRowSize, netFactor) indexSeekCost := estimateNetSeekCost(p.indexPlan) - indexChildCost, err := p.indexPlan.GetPlanCost(property.CopDoubleReadTaskType, option) + indexChildCost, err := p.indexPlan.getPlanCostVer2(property.CopDoubleReadTaskType, option) if err != nil { return 0, err } @@ -179,18 +231,18 @@ func (p *PhysicalIndexLookUpReader) getPlanCostVer2(taskType property.TaskType, // table-side tableNetCost := netCostVer2(tableRows, tableRowSize, netFactor) tableSeekCost := estimateNetSeekCost(p.tablePlan) - tableChildCost, err := p.tablePlan.GetPlanCost(property.CopDoubleReadTaskType, option) + tableChildCost, err := p.tablePlan.getPlanCostVer2(property.CopDoubleReadTaskType, option) if err != nil { return 0, err } tableSideCost := (tableNetCost + tableSeekCost + tableChildCost) / distConcurrency // double-read - doubleReadCPUCost := indexRows * cpuFactor + doubleReadCPUCost := indexRows * cpuFactor.Value batchSize := float64(p.ctx.GetSessionVars().IndexLookupSize) taskPerBatch := 40.0 // TODO: remove this magic number doubleReadTasks := indexRows / batchSize * taskPerBatch - doubleReadSeekCost := doubleReadTasks * seekFactor + doubleReadSeekCost := doubleReadTasks * seekFactor.Value doubleReadCost := doubleReadCPUCost + doubleReadSeekCost p.planCost = indexSideCost + (tableSideCost+doubleReadCost)/doubleReadConcurrency @@ -203,7 +255,11 @@ func (p *PhysicalIndexLookUpReader) getPlanCostVer2(taskType property.TaskType, // index-side-cost = (index-child-cost + index-net-cost + index-seek-cost) / dist-concurrency # same with IndexReader // table-side-cost = (table-child-cost + table-net-cost + table-seek-cost) / dist-concurrency # same with TableReader func (p *PhysicalIndexMergeReader) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { - netFactor := getTaskNetFactor(p, taskType) + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + + netFactor := getTaskNetFactorVer2(p, taskType) distConcurrency := float64(p.ctx.GetSessionVars().DistSQLScanConcurrency()) var tableSideCost float64 @@ -213,7 +269,7 @@ func (p *PhysicalIndexMergeReader) getPlanCostVer2(taskType property.TaskType, o tableNetCost := netCostVer2(rows, rowSize, netFactor) tableSeekCost := estimateNetSeekCost(tablePath) - tableChildCost, err := tablePath.GetPlanCost(taskType, option) + tableChildCost, err := tablePath.getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -227,7 +283,7 @@ func (p *PhysicalIndexMergeReader) getPlanCostVer2(taskType property.TaskType, o indexNetCost := netCostVer2(rows, rowSize, netFactor) indexSeekCost := estimateNetSeekCost(indexPath) - indexChildCost, err := indexPath.GetPlanCost(taskType, option) + indexChildCost, err := indexPath.getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -249,11 +305,15 @@ func (p *PhysicalIndexMergeReader) getPlanCostVer2(taskType property.TaskType, o // 1. sort-mem-cost = mem-quota * mem-factor // 2. sort-disk-cost = rows * row-size * disk-factor func (p *PhysicalSort) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := math.Max(getCardinality(p.children[0], option.CostFlag), 1) rowSize := getAvgRowSize(p.statsInfo(), p.Schema()) - cpuFactor := getTaskCPUFactor(p, taskType) - memFactor := getTaskMemFactor(p, taskType) - diskFactor := p.ctx.GetSessionVars().GetDiskFactor() + cpuFactor := getTaskCPUFactorVer2(p, taskType) + memFactor := getTaskMemFactorVer2(p, taskType) + diskFactor := defaultVer2Factors.TiDBDisk oomUseTmpStorage := variable.EnableTmpStorageOnOOM.Load() memQuota := p.ctx.GetSessionVars().StmtCtx.MemTracker.GetBytesLimit() spill := taskType == property.RootTaskType && // only TiDB can spill @@ -261,18 +321,18 @@ func (p *PhysicalSort) getPlanCostVer2(taskType property.TaskType, option *PlanC memQuota > 0 && // mem-quota is set rowSize*rows > float64(memQuota) // exceed the mem-quota - sortCPUCost := rows * math.Log2(rows) * float64(len(p.ByItems)) * cpuFactor + sortCPUCost := rows * math.Log2(rows) * float64(len(p.ByItems)) * cpuFactor.Value var sortMemCost, sortDiskCost float64 if !spill { - sortMemCost = rows * rowSize * memFactor + sortMemCost = rows * rowSize * memFactor.Value sortDiskCost = 0 } else { - sortMemCost = float64(memQuota) * memFactor - sortDiskCost = rows * rowSize * diskFactor + sortMemCost = float64(memQuota) * memFactor.Value + sortDiskCost = rows * rowSize * diskFactor.Value } - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -287,16 +347,20 @@ func (p *PhysicalSort) getPlanCostVer2(taskType property.TaskType, option *PlanC // topn-cpu-cost = rows * log2(N) * len(sort-items) * cpu-factor // topn-mem-cost = N * row-size * mem-factor func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p.children[0], option.CostFlag) N := math.Max(1, float64(p.Count+p.Offset)) rowSize := getAvgRowSize(p.statsInfo(), p.Schema()) - cpuFactor := getTaskCPUFactor(p, taskType) - memFactor := getTaskMemFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) + memFactor := getTaskMemFactorVer2(p, taskType) - topNCPUCost := rows * math.Log2(N) * float64(len(p.ByItems)) * cpuFactor - topNMemCost := N * rowSize * memFactor + topNCPUCost := rows * math.Log2(N) * float64(len(p.ByItems)) * cpuFactor.Value + topNMemCost := N * rowSize * memFactor.Value - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -309,13 +373,17 @@ func (p *PhysicalTopN) getPlanCostVer2(taskType property.TaskType, option *PlanC // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + agg-cost + group-cost func (p *PhysicalStreamAgg) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p.children[0], option.CostFlag) - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) aggCost := aggCostVer2(rows, p.AggFuncs, cpuFactor) groupCost := groupCostVer2(rows, p.GroupByItems, cpuFactor) - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -328,11 +396,15 @@ func (p *PhysicalStreamAgg) getPlanCostVer2(taskType property.TaskType, option * // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + (agg-cost + group-cost + hash-build-cost + hash-probe-cost) / concurrency func (p *PhysicalHashAgg) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + inputRows := getCardinality(p.children[0], option.CostFlag) outputRows := getCardinality(p, option.CostFlag) outputRowSize := getAvgRowSize(p.Stats(), p.Schema()) - cpuFactor := getTaskCPUFactor(p, taskType) - memFactor := getTaskMemFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) + memFactor := getTaskMemFactorVer2(p, taskType) concurrency := p.ctx.GetSessionVars().GetConcurrencyFactor() aggCost := aggCostVer2(inputRows, p.AggFuncs, cpuFactor) @@ -340,7 +412,7 @@ func (p *PhysicalHashAgg) getPlanCostVer2(taskType property.TaskType, option *Pl hashBuildCost := hashBuildCostVer2(outputRows, outputRowSize, p.GroupByItems, cpuFactor, memFactor) hashProbeCost := hashProbeCostVer2(inputRows, p.GroupByItems, cpuFactor) - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -353,20 +425,24 @@ func (p *PhysicalHashAgg) getPlanCostVer2(taskType property.TaskType, option *Pl // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = left-child-cost + right-child-cost + filter-cost + group-cost func (p *PhysicalMergeJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + leftRows := getCardinality(p.children[0], option.CostFlag) rightRows := getCardinality(p.children[1], option.CostFlag) - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) filterCost := filterCostVer2(leftRows, p.LeftConditions, cpuFactor) + filterCostVer2(rightRows, p.RightConditions, cpuFactor) groupCost := groupCostVer2(leftRows, cols2Exprs(p.LeftJoinKeys), cpuFactor) + groupCostVer2(rightRows, cols2Exprs(p.LeftJoinKeys), cpuFactor) - leftChildCost, err := p.children[0].GetPlanCost(taskType, option) + leftChildCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } - rightChildCost, err := p.children[1].GetPlanCost(taskType, option) + rightChildCost, err := p.children[1].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -381,6 +457,10 @@ func (p *PhysicalMergeJoin) getPlanCostVer2(taskType property.TaskType, option * // build-hash-cost + build-filter-cost + // (probe-filter-cost + probe-hash-cost) / concurrency func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + build, probe := p.children[0], p.children[1] buildFilters, probeFilters := p.LeftConditions, p.RightConditions buildKeys, probeKeys := p.LeftJoinKeys, p.RightJoinKeys @@ -392,8 +472,8 @@ func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *P probeRows := getCardinality(probe, option.CostFlag) buildRowSize := getAvgRowSize(build.Stats(), build.Schema()) concurrency := float64(p.Concurrency) - cpuFactor := getTaskCPUFactor(p, taskType) - memFactor := getTaskMemFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) + memFactor := getTaskMemFactorVer2(p, taskType) buildFilterCost := filterCostVer2(buildRows, buildFilters, cpuFactor) buildHashCost := hashBuildCostVer2(buildRows, buildRowSize, cols2Exprs(buildKeys), cpuFactor, memFactor) @@ -401,11 +481,11 @@ func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *P probeFilterCost := filterCostVer2(probeRows, probeFilters, cpuFactor) probeHashCost := hashProbeCostVer2(probeRows, cols2Exprs(probeKeys), cpuFactor) - buildChildCost, err := build.GetPlanCost(taskType, option) + buildChildCost, err := build.getPlanCostVer2(taskType, option) if err != nil { return 0, err } - probeChildCost, err := probe.GetPlanCost(taskType, option) + probeChildCost, err := probe.getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -421,22 +501,26 @@ func (p *PhysicalHashJoin) getPlanCostVer2(taskType property.TaskType, option *P // (probe-cost + probe-filter-cost) / concurrency // probe-cost = probe-child-cost * build-rows / batchRatio func (p *PhysicalIndexJoin) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + build, probe := p.children[1-p.InnerChildIdx], p.children[p.InnerChildIdx] buildRows := getCardinality(build, option.CostFlag) probeRowsOne := getCardinality(probe, option.CostFlag) probeRowsTot := probeRowsOne * buildRows buildFilters, probeFilters := p.LeftConditions, p.RightConditions probeConcurrency := float64(p.ctx.GetSessionVars().IndexLookupJoinConcurrency()) - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) buildFilterCost := filterCostVer2(buildRows, buildFilters, cpuFactor) - buildChildCost, err := build.GetPlanCost(taskType, option) + buildChildCost, err := build.getPlanCostVer2(taskType, option) if err != nil { return 0, err } probeFilterCost := filterCostVer2(probeRowsTot, probeFilters, cpuFactor) - probeChildCost, err := probe.GetPlanCost(taskType, option) + probeChildCost, err := probe.getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -466,19 +550,23 @@ func (p *PhysicalIndexMergeJoin) getPlanCostVer2(taskType property.TaskType, opt // plan-cost = build-child-cost + build-filter-cost + probe-cost + probe-filter-cost // probe-cost = probe-child-cost * build-rows func (p *PhysicalApply) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + buildRows := getCardinality(p.children[0], option.CostFlag) probeRowsOne := getCardinality(p.children[1], option.CostFlag) probeRowsTot := buildRows * probeRowsOne - cpuFactor := getTaskCPUFactor(p, taskType) + cpuFactor := getTaskCPUFactorVer2(p, taskType) buildFilterCost := filterCostVer2(buildRows, p.LeftConditions, cpuFactor) - buildChildCost, err := p.children[0].GetPlanCost(taskType, option) + buildChildCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } probeFilterCost := filterCostVer2(probeRowsTot, p.RightConditions, cpuFactor) - probeChildCost, err := p.children[1].GetPlanCost(taskType, option) + probeChildCost, err := p.children[1].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -489,15 +577,40 @@ func (p *PhysicalApply) getPlanCostVer2(taskType property.TaskType, option *Plan return p.planCost, nil } +// getPlanCostVer2 calculates the cost of the plan if it has not been calculated yet and returns the cost. +// plan-cost = sum(child-cost) / concurrency +func (p *PhysicalUnionAll) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + + concurrency := p.ctx.GetSessionVars().GetConcurrencyFactor() + var sumChildCost float64 + for _, child := range p.children { + childCost, err := child.getPlanCostVer2(taskType, option) + if err != nil { + return 0, err + } + sumChildCost += childCost + } + p.planCost = sumChildCost / concurrency + p.planCostInit = true + return p.planCost, nil +} + // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = child-cost + net-cost func (p *PhysicalExchangeReceiver) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + rows := getCardinality(p, option.CostFlag) rowSize := getAvgRowSize(p.stats, p.Schema()) - netFactor := getTableNetFactor(p) + netFactor := getTaskNetFactorVer2(p, taskType) netCost := netCostVer2(rows, rowSize, netFactor) - childCost, err := p.children[0].GetPlanCost(taskType, option) + childCost, err := p.children[0].getPlanCostVer2(taskType, option) if err != nil { return 0, err } @@ -509,18 +622,22 @@ func (p *PhysicalExchangeReceiver) getPlanCostVer2(taskType property.TaskType, o // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = seek-cost + net-cost -func (p *PointGetPlan) getPlanCostVer2(taskType property.TaskType, _ *PlanCostOption) (float64, error) { +func (p *PointGetPlan) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + if p.accessCols == nil { // from fast plan code path p.planCost = 0 p.planCostInit = true return 0, nil } rowSize := getAvgRowSize(p.stats, p.schema) - netFactor := getTaskNetFactor(p, taskType) - seekFactor := getTaskSeekFactor(p, taskType) + netFactor := getTaskNetFactorVer2(p, taskType) + seekFactor := getTaskSeekFactorVer2(p, taskType) netCost := netCostVer2(1, rowSize, netFactor) - seekCost := 1 * seekFactor / 20 // 20 times faster than general request + seekCost := 1 * seekFactor.Value / 20 // 20 times faster than general request p.planCost = netCost + seekCost p.planCostInit = true @@ -530,6 +647,10 @@ func (p *PointGetPlan) getPlanCostVer2(taskType property.TaskType, _ *PlanCostOp // getPlanCostVer2 returns the plan-cost of this sub-plan, which is: // plan-cost = seek-cost + net-cost func (p *BatchPointGetPlan) getPlanCostVer2(taskType property.TaskType, option *PlanCostOption) (float64, error) { + if p.planCostInit && !hasCostFlag(option.CostFlag, CostFlagRecalculate) { + return p.planCost, nil + } + if p.accessCols == nil { // from fast plan code path p.planCost = 0 p.planCostInit = true @@ -537,100 +658,165 @@ func (p *BatchPointGetPlan) getPlanCostVer2(taskType property.TaskType, option * } rows := getCardinality(p, option.CostFlag) rowSize := getAvgRowSize(p.stats, p.schema) - netFactor := getTaskNetFactor(p, taskType) - seekFactor := getTaskSeekFactor(p, taskType) + netFactor := getTaskNetFactorVer2(p, taskType) + seekFactor := getTaskSeekFactorVer2(p, taskType) netCost := netCostVer2(rows, rowSize, netFactor) - seekCost := 1 * seekFactor / 20 // in one batch + seekCost := 1 * seekFactor.Value / 20 // in one batch p.planCost = netCost + seekCost p.planCostInit = true return p.planCost, nil } -func scanCostVer2(rows, rowSize, scanFactor float64) float64 { +func scanCostVer2(rows, rowSize float64, scanFactor costVer2Factor) float64 { // log2 from experiments - return rows * math.Log2(math.Max(1, rowSize)) * scanFactor + return rows * math.Log2(math.Max(1, rowSize)) * scanFactor.Value } -func netCostVer2(rows, rowSize, netFactor float64) float64 { - return rows * rowSize * netFactor +func netCostVer2(rows, rowSize float64, netFactor costVer2Factor) float64 { + return rows * rowSize * netFactor.Value } -func filterCostVer2(rows float64, filters []expression.Expression, cpuFactor float64) float64 { +func filterCostVer2(rows float64, filters []expression.Expression, cpuFactor costVer2Factor) float64 { // TODO: consider types of filters - return rows * float64(len(filters)) * cpuFactor + return rows * float64(len(filters)) * cpuFactor.Value } -func aggCostVer2(rows float64, aggFuncs []*aggregation.AggFuncDesc, cpuFactor float64) float64 { +func aggCostVer2(rows float64, aggFuncs []*aggregation.AggFuncDesc, cpuFactor costVer2Factor) float64 { // TODO: consider types of agg-funcs - return rows * float64(len(aggFuncs)) * cpuFactor + return rows * float64(len(aggFuncs)) * cpuFactor.Value } -func groupCostVer2(rows float64, groupItems []expression.Expression, cpuFactor float64) float64 { - return rows * float64(len(groupItems)) * cpuFactor +func groupCostVer2(rows float64, groupItems []expression.Expression, cpuFactor costVer2Factor) float64 { + return rows * float64(len(groupItems)) * cpuFactor.Value } -func hashBuildCostVer2(buildRows, buildRowSize float64, keys []expression.Expression, cpuFactor, memFactor float64) float64 { +func hashBuildCostVer2(buildRows, buildRowSize float64, keys []expression.Expression, cpuFactor, memFactor costVer2Factor) float64 { // TODO: 1) consider types of keys, 2) dedicated factor for build-probe hash table - hashKeyCost := buildRows * float64(len(keys)) * cpuFactor - hashMemCost := buildRows * buildRowSize * memFactor - hashBuildCost := buildRows * float64(len(keys)) * cpuFactor + hashKeyCost := buildRows * float64(len(keys)) * cpuFactor.Value + hashMemCost := buildRows * buildRowSize * memFactor.Value + hashBuildCost := buildRows * float64(len(keys)) * cpuFactor.Value return hashKeyCost + hashMemCost + hashBuildCost } -func hashProbeCostVer2(probeRows float64, keys []expression.Expression, cpuFactor float64) float64 { +func hashProbeCostVer2(probeRows float64, keys []expression.Expression, cpuFactor costVer2Factor) float64 { // TODO: 1) consider types of keys, 2) dedicated factor for build-probe hash table - hashKeyCost := probeRows * float64(len(keys)) * cpuFactor - hashProbeCost := probeRows * float64(len(keys)) * cpuFactor + hashKeyCost := probeRows * float64(len(keys)) * cpuFactor.Value + hashProbeCost := probeRows * float64(len(keys)) * cpuFactor.Value return hashKeyCost + hashProbeCost } -func getTaskCPUFactor(p PhysicalPlan, taskType property.TaskType) float64 { +type costVer2Factor struct { + Name string + Value float64 +} + +// In Cost Ver2, we hide cost factors from users and deprecate SQL variables like `tidb_opt_scan_factor`. +type costVer2Factors struct { + TiDBTemp costVer2Factor // operations on TiDB temporary table + TiKVScan costVer2Factor // per byte + TiKVDescScan costVer2Factor // per byte + TiFlashScan costVer2Factor // per byte + TiDBCPU costVer2Factor // per column or expression + TiKVCPU costVer2Factor // per column or expression + TiFlashCPU costVer2Factor // per column or expression + TiDB2KVNet costVer2Factor // per byte + TiDB2FlashNet costVer2Factor // per byte + TiFlashMPPNet costVer2Factor // per byte + TiDBMem costVer2Factor // per byte + TiKVMem costVer2Factor // per byte + TiFlashMem costVer2Factor // per byte + TiDBDisk costVer2Factor // per byte + TiDBRequest costVer2Factor // per net request +} + +var defaultVer2Factors = costVer2Factors{ + TiDBTemp: costVer2Factor{"tidb_temp_table_factor", 0}, + TiKVScan: costVer2Factor{"tikv_scan_factor", 100}, + TiKVDescScan: costVer2Factor{"tikv_desc_scan_factor", 150}, + TiFlashScan: costVer2Factor{"tiflash_scan_factor", 5}, + TiDBCPU: costVer2Factor{"tidb_cpu_factor", 30}, + TiKVCPU: costVer2Factor{"tikv_cpu_factor", 30}, + TiFlashCPU: costVer2Factor{"tiflash_cpu_factor", 5}, + TiDB2KVNet: costVer2Factor{"tidb_kv_net_factor", 8}, + TiDB2FlashNet: costVer2Factor{"tidb_flash_net_factor", 4}, + TiFlashMPPNet: costVer2Factor{"tiflash_mpp_net_factor", 4}, + TiDBMem: costVer2Factor{"tidb_mem_factor", 1}, + TiKVMem: costVer2Factor{"tikv_mem_factor", 1}, + TiFlashMem: costVer2Factor{"tiflash_mem_factor", 1}, + TiDBDisk: costVer2Factor{"tidb_disk_factor", 1000}, + TiDBRequest: costVer2Factor{"tidb_request_factor", 9500000}, +} + +func getTaskCPUFactorVer2(p PhysicalPlan, taskType property.TaskType) costVer2Factor { switch taskType { case property.RootTaskType: // TiDB - return p.SCtx().GetSessionVars().GetCPUFactor() + return defaultVer2Factors.TiDBCPU case property.MppTaskType: // TiFlash - return p.SCtx().GetSessionVars().GetTiFlashCPUFactor() + return defaultVer2Factors.TiFlashCPU default: // TiKV - return p.SCtx().GetSessionVars().GetCopCPUFactor() + return defaultVer2Factors.TiKVCPU } } -func getTaskMemFactor(p PhysicalPlan, taskType property.TaskType) float64 { - // TODO: introduce a dedicated mem factor for TiFlash - return p.SCtx().GetSessionVars().GetMemoryFactor() +func getTaskMemFactorVer2(p PhysicalPlan, taskType property.TaskType) costVer2Factor { + switch taskType { + case property.RootTaskType: // TiDB + return defaultVer2Factors.TiDBMem + case property.MppTaskType: // TiFlash + return defaultVer2Factors.TiFlashMem + default: // TiKV + return defaultVer2Factors.TiKVMem + } } -func getTaskScanFactor(p PhysicalPlan, taskType property.TaskType) float64 { +func getTaskScanFactorVer2(p PhysicalPlan, taskType property.TaskType) costVer2Factor { + if isTemporaryTable(getTableInfo(p)) { + return defaultVer2Factors.TiDBTemp + } switch taskType { case property.MppTaskType: // TiFlash - return p.SCtx().GetSessionVars().GetTiFlashScanFactor() + return defaultVer2Factors.TiFlashScan default: // TiKV var desc bool - var tbl *model.TableInfo if indexScan, ok := p.(*PhysicalIndexScan); ok { desc = indexScan.Desc - tbl = indexScan.Table } if tableScan, ok := p.(*PhysicalTableScan); ok { desc = tableScan.Desc - tbl = tableScan.Table } if desc { - return p.SCtx().GetSessionVars().GetDescScanFactor(tbl) + return defaultVer2Factors.TiKVDescScan } - return p.SCtx().GetSessionVars().GetScanFactor(tbl) + return defaultVer2Factors.TiKVScan } } -func getTaskNetFactor(p PhysicalPlan, _ property.TaskType) float64 { - // TODO: introduce a dedicated net factor for TiFlash - return p.SCtx().GetSessionVars().GetNetworkFactor(getTableInfo(p)) +func getTaskNetFactorVer2(p PhysicalPlan, _ property.TaskType) costVer2Factor { + if isTemporaryTable(getTableInfo(p)) { + return defaultVer2Factors.TiDBTemp + } + if _, ok := p.(*PhysicalExchangeReceiver); ok { // TiFlash MPP + return defaultVer2Factors.TiFlashMPPNet + } + if tblReader, ok := p.(*PhysicalTableReader); ok { + if _, isMPP := tblReader.tablePlan.(*PhysicalExchangeSender); isMPP { // TiDB to TiFlash with mpp protocol + return defaultVer2Factors.TiDB2FlashNet + } + } + return defaultVer2Factors.TiDB2KVNet } -func getTaskSeekFactor(p PhysicalPlan, _ property.TaskType) float64 { - return p.SCtx().GetSessionVars().GetSeekFactor(getTableInfo(p)) +func getTaskSeekFactorVer2(p PhysicalPlan, _ property.TaskType) costVer2Factor { + if isTemporaryTable(getTableInfo(p)) { + return defaultVer2Factors.TiDBTemp + } + return defaultVer2Factors.TiDBRequest +} + +func isTemporaryTable(tbl *model.TableInfo) bool { + return tbl != nil && tbl.TempTableType != model.TempTableNone } func getTableInfo(p PhysicalPlan) *model.TableInfo { @@ -641,6 +827,11 @@ func getTableInfo(p PhysicalPlan) *model.TableInfo { return getTableInfo(x.tablePlan) case *PhysicalIndexLookUpReader: return getTableInfo(x.tablePlan) + case *PhysicalIndexMergeReader: + if x.tablePlan != nil { + return getTableInfo(x.tablePlan) + } + return getTableInfo(x.partialPlans[0]) case *PhysicalTableScan: return x.Table case *PhysicalIndexScan: diff --git a/server/BUILD.bazel b/server/BUILD.bazel index 0b37bbb27a27c..9045b932c7f61 100644 --- a/server/BUILD.bazel +++ b/server/BUILD.bazel @@ -140,6 +140,7 @@ go_test( ], embed = [":server"], flaky = True, + race = "on", shard_count = 50, deps = [ "//config", diff --git a/sessionctx/variable/BUILD.bazel b/sessionctx/variable/BUILD.bazel index 6b41b0b8f4529..3a9eb1439ce2c 100644 --- a/sessionctx/variable/BUILD.bazel +++ b/sessionctx/variable/BUILD.bazel @@ -92,6 +92,7 @@ go_test( "//parser/terror", "//planner/core", "//sessionctx/stmtctx", + "//testkit", "//testkit/testsetup", "//types", "//util/execdetails", diff --git a/sessionctx/variable/noop.go b/sessionctx/variable/noop.go index b720490d03423..398ea09f3ec92 100644 --- a/sessionctx/variable/noop.go +++ b/sessionctx/variable/noop.go @@ -295,7 +295,8 @@ var noopSysVars = []*SysVar{ {Scope: ScopeGlobal, Name: SlowQueryLog, Value: "0"}, {Scope: ScopeSession, Name: "debug_sync", Value: ""}, {Scope: ScopeGlobal, Name: InnodbStatsAutoRecalc, Value: "1"}, - {Scope: ScopeGlobal | ScopeSession, Name: "lc_messages", Value: "en_US", ReadOnly: true}, + // lc_messages cannot be read_only, see https://github.com/pingcap/tidb/issues/38231. + {Scope: ScopeGlobal | ScopeSession, Name: "lc_messages", Value: "en_US"}, {Scope: ScopeGlobal | ScopeSession, Name: "bulk_insert_buffer_size", Value: "8388608", IsHintUpdatable: true}, {Scope: ScopeGlobal | ScopeSession, Name: BinlogDirectNonTransactionalUpdates, Value: Off, Type: TypeBool}, {Scope: ScopeGlobal, Name: "innodb_change_buffering", Value: "all"}, diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index c4dc136ea3dec..4d3d16f351965 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -827,29 +827,6 @@ type SessionVars struct { // concurrencyFactor is the CPU cost of additional one goroutine. concurrencyFactor float64 - // factors for cost model v2 - // cpuFactorV2 is the CPU factor for the Cost Model Ver2. - cpuFactorV2 float64 - // copCPUFactorV2 is the cop-cpu factor for the Cost Model Ver2. - copCPUFactorV2 float64 - // tiflashCPUFactorV2 is the cop-cpu factor for the Cost Model Ver2. - tiflashCPUFactorV2 float64 - // networkFactorV2 is the network factor for the Cost Model Ver2. - networkFactorV2 float64 - // scanFactorV2 is the scan factor for the Cost Model Ver2. - scanFactorV2 float64 - // descScanFactorV2 is the desc-scan factor for the Cost Model Ver2. - descScanFactorV2 float64 - // tiflashScanFactorV2 is the tiflash-scan factor for the Cost Model Ver2. - tiflashScanFactorV2 float64 - // seekFactorV2 is the seek factor for the Cost Model Ver2. - seekFactorV2 float64 - // memoryFactorV2 is the memory factor for the Cost Model Ver2. - memoryFactorV2 float64 - // diskFactorV2 is the disk factor for the Cost Model Ver2. - diskFactorV2 float64 - // concurrencyFactorV2 is the concurrency factor for the Cost Model Ver2. - concurrencyFactorV2 float64 // enableForceInlineCTE is used to enable/disable force inline CTE. enableForceInlineCTE bool @@ -2934,46 +2911,26 @@ func (s *SessionVars) CleanupTxnReadTSIfUsed() { // GetCPUFactor returns the session variable cpuFactor func (s *SessionVars) GetCPUFactor() float64 { - if s.CostModelVersion == 2 { - return s.cpuFactorV2 - } return s.cpuFactor } // GetCopCPUFactor returns the session variable copCPUFactor func (s *SessionVars) GetCopCPUFactor() float64 { - if s.CostModelVersion == 2 { - return s.copCPUFactorV2 - } return s.copCPUFactor } -// GetTiFlashCPUFactor returns the session -func (s *SessionVars) GetTiFlashCPUFactor() float64 { - return s.tiflashCPUFactorV2 -} - // GetMemoryFactor returns the session variable memoryFactor func (s *SessionVars) GetMemoryFactor() float64 { - if s.CostModelVersion == 2 { - return s.memoryFactorV2 - } return s.memoryFactor } // GetDiskFactor returns the session variable diskFactor func (s *SessionVars) GetDiskFactor() float64 { - if s.CostModelVersion == 2 { - return s.diskFactorV2 - } return s.diskFactor } // GetConcurrencyFactor returns the session variable concurrencyFactor func (s *SessionVars) GetConcurrencyFactor() float64 { - if s.CostModelVersion == 2 { - return s.concurrencyFactorV2 - } return s.concurrencyFactor } @@ -2985,9 +2942,6 @@ func (s *SessionVars) GetNetworkFactor(tbl *model.TableInfo) float64 { return 0 } } - if s.CostModelVersion == 2 { - return s.networkFactorV2 - } return s.networkFactor } @@ -2999,9 +2953,6 @@ func (s *SessionVars) GetScanFactor(tbl *model.TableInfo) float64 { return 0 } } - if s.CostModelVersion == 2 { - return s.scanFactorV2 - } return s.scanFactor } @@ -3013,17 +2964,9 @@ func (s *SessionVars) GetDescScanFactor(tbl *model.TableInfo) float64 { return 0 } } - if s.CostModelVersion == 2 { - return s.descScanFactorV2 - } return s.descScanFactor } -// GetTiFlashScanFactor returns the session variable tiflashScanFactorV2 -func (s *SessionVars) GetTiFlashScanFactor() float64 { - return s.tiflashScanFactorV2 -} - // GetSeekFactor returns the session variable seekFactor // returns 0 when tbl is a temporary table. func (s *SessionVars) GetSeekFactor(tbl *model.TableInfo) float64 { @@ -3032,9 +2975,6 @@ func (s *SessionVars) GetSeekFactor(tbl *model.TableInfo) float64 { return 0 } } - if s.CostModelVersion == 2 { - return s.seekFactorV2 - } return s.seekFactor } diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index b2bcce7dad275..93b581744874e 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1302,50 +1302,6 @@ var defaultSysVars = []*SysVar{ s.concurrencyFactor = tidbOptFloat64(val, DefOptConcurrencyFactor) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCPUFactorV2, Value: strconv.FormatFloat(DefOptCPUFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.cpuFactorV2 = tidbOptFloat64(val, DefOptCPUFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptCopCPUFactorV2, Value: strconv.FormatFloat(DefOptCopCPUFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.copCPUFactorV2 = tidbOptFloat64(val, DefOptCopCPUFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptTiFlashCPUFactorV2, Value: strconv.FormatFloat(DefOptTiFlashCPUFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.tiflashCPUFactorV2 = tidbOptFloat64(val, DefOptTiFlashCPUFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptNetworkFactorV2, Value: strconv.FormatFloat(DefOptNetworkFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.networkFactorV2 = tidbOptFloat64(val, DefOptNetworkFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptScanFactorV2, Value: strconv.FormatFloat(DefOptScanFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.scanFactorV2 = tidbOptFloat64(val, DefOptScanFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptDescScanFactorV2, Value: strconv.FormatFloat(DefOptDescScanFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.descScanFactorV2 = tidbOptFloat64(val, DefOptDescScanFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptTiFlashScanFactorV2, Value: strconv.FormatFloat(DefOptTiFlashScanFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.tiflashScanFactorV2 = tidbOptFloat64(val, DefOptTiFlashScanFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptSeekFactorV2, Value: strconv.FormatFloat(DefOptSeekFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.seekFactorV2 = tidbOptFloat64(val, DefOptSeekFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptMemoryFactorV2, Value: strconv.FormatFloat(DefOptMemoryFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.memoryFactorV2 = tidbOptFloat64(val, DefOptMemoryFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptDiskFactorV2, Value: strconv.FormatFloat(DefOptDiskFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.diskFactorV2 = tidbOptFloat64(val, DefOptDiskFactorV2) - return nil - }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptConcurrencyFactorV2, Value: strconv.FormatFloat(DefOptConcurrencyFactorV2, 'f', -1, 64), Hidden: true, Type: TypeFloat, MinValue: 0, MaxValue: math.MaxUint64, SetSession: func(s *SessionVars, val string) error { - s.concurrencyFactorV2 = tidbOptFloat64(val, DefOptConcurrencyFactorV2) - return nil - }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptForceInlineCTE, Value: BoolToOnOff(DefOptForceInlineCTE), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.enableForceInlineCTE = TiDBOptOn(val) return nil diff --git a/sessionctx/variable/sysvar_test.go b/sessionctx/variable/sysvar_test.go index de70fe600be78..80115daab9270 100644 --- a/sessionctx/variable/sysvar_test.go +++ b/sessionctx/variable/sysvar_test.go @@ -468,12 +468,17 @@ func TestLcTimeNamesReadOnly(t *testing.T) { require.Error(t, err) } -func TestLcMessagesReadOnly(t *testing.T) { +func TestLcMessages(t *testing.T) { sv := GetSysVar("lc_messages") vars := NewSessionVars(nil) vars.GlobalVarsAccessor = NewMockGlobalAccessor4Tests() - _, err := sv.Validate(vars, "newvalue", ScopeGlobal) - require.Error(t, err) + _, err := sv.Validate(vars, "zh_CN", ScopeGlobal) + require.NoError(t, err) + err = sv.SetSessionFromHook(vars, "zh_CN") + require.NoError(t, err) + val, err := vars.GetSessionOrGlobalSystemVar("lc_messages") + require.NoError(t, err) + require.Equal(t, val, "zh_CN") } func TestDDLWorkers(t *testing.T) { diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index c9efe6918c74c..cb1eb734e149b 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -306,30 +306,6 @@ const ( // TiDBOptForceInlineCTE is used to enable/disable inline CTE TiDBOptForceInlineCTE = "tidb_opt_force_inline_cte" - // Variables for the Cost Model Ver2 - // TiDBOptCPUFactorV2 is the CPU factor for the Cost Model Ver2 - TiDBOptCPUFactorV2 = "tidb_opt_cpu_factor_v2" - // TiDBOptCopCPUFactorV2 is the CopCPU factor for the Cost Model Ver2 - TiDBOptCopCPUFactorV2 = "tidb_opt_copcpu_factor_v2" - // TiDBOptTiFlashCPUFactorV2 is the TiFlashCPU factor for the Cost Model Ver2 - TiDBOptTiFlashCPUFactorV2 = "tidb_opt_tiflash_cpu_factor_v2" - // TiDBOptNetworkFactorV2 is the network factor for the Cost Model Ver2 - TiDBOptNetworkFactorV2 = "tidb_opt_network_factor_v2" - // TiDBOptScanFactorV2 is the scan factor for the Cost Model Ver2 - TiDBOptScanFactorV2 = "tidb_opt_scan_factor_v2" - // TiDBOptDescScanFactorV2 is the desc scan factor for the Cost Model Ver2 - TiDBOptDescScanFactorV2 = "tidb_opt_desc_factor_v2" - // TiDBOptTiFlashScanFactorV2 is the TiFlashScan factor for the Cost Model Ver2 - TiDBOptTiFlashScanFactorV2 = "tidb_opt_tiflash_scan_factor_v2" - // TiDBOptSeekFactorV2 is the seek factor for the Cost Model Ver2 - TiDBOptSeekFactorV2 = "tidb_opt_seek_factor_v2" - // TiDBOptMemoryFactorV2 is the memory factor for the Cost Model Ver2 - TiDBOptMemoryFactorV2 = "tidb_opt_memory_factor_v2" - // TiDBOptDiskFactorV2 is the disk factor for the Cost Model Ver2 - TiDBOptDiskFactorV2 = "tidb_opt_disk_factor_v2" - // TiDBOptConcurrencyFactorV2 is the concurrency factor for the Cost Model Ver2 - TiDBOptConcurrencyFactorV2 = "tidb_opt_concurrency_factor_v2" - // TiDBIndexJoinBatchSize is used to set the batch size of an index lookup join. // The index lookup join fetches batches of data from outer executor and constructs ranges for inner executor. // This value controls how much of data in a batch to do the index join. @@ -895,17 +871,6 @@ const ( DefOptMemoryFactor = 0.001 DefOptDiskFactor = 1.5 DefOptConcurrencyFactor = 3.0 - DefOptCPUFactorV2 = 30.0 - DefOptCopCPUFactorV2 = 30.0 - DefOptTiFlashCPUFactorV2 = 2.0 - DefOptNetworkFactorV2 = 4.0 - DefOptScanFactorV2 = 100.0 - DefOptDescScanFactorV2 = 150.0 - DefOptTiFlashScanFactorV2 = 15.0 - DefOptSeekFactorV2 = 9500000.0 - DefOptMemoryFactorV2 = 0.001 - DefOptDiskFactorV2 = 1.5 - DefOptConcurrencyFactorV2 = 3.0 DefOptForceInlineCTE = false DefOptInSubqToJoinAndAgg = true DefOptPreferRangeScan = false diff --git a/sessionctx/variable/variable.go b/sessionctx/variable/variable.go index b6efe4bd02f75..deca51f3e9fa3 100644 --- a/sessionctx/variable/variable.go +++ b/sessionctx/variable/variable.go @@ -42,21 +42,21 @@ const ( ScopeInstance ScopeFlag = 1 << 2 // TypeStr is the default - TypeStr TypeFlag = 0 + TypeStr TypeFlag = iota // TypeBool for boolean - TypeBool TypeFlag = 1 + TypeBool // TypeInt for integer - TypeInt TypeFlag = 2 + TypeInt // TypeEnum for Enum - TypeEnum TypeFlag = 3 + TypeEnum // TypeFloat for Double - TypeFloat TypeFlag = 4 + TypeFloat // TypeUnsigned for Unsigned integer - TypeUnsigned TypeFlag = 5 + TypeUnsigned // TypeTime for time of day (a TiDB extension) - TypeTime TypeFlag = 6 + TypeTime // TypeDuration for a golang duration (a TiDB extension) - TypeDuration TypeFlag = 7 + TypeDuration // On is the canonical string for ON On = "ON" @@ -86,6 +86,23 @@ const ( GlobalConfigEnableTopSQL = "enable_resource_metering" ) +func (s ScopeFlag) String() string { + var scopes []string + if s == ScopeNone { + return "NONE" + } + if s&ScopeSession != 0 { + scopes = append(scopes, "SESSION") + } + if s&ScopeGlobal != 0 { + scopes = append(scopes, "GLOBAL") + } + if s&ScopeInstance != 0 { + scopes = append(scopes, "INSTANCE") + } + return strings.Join(scopes, ",") +} + // SysVar is for system variable. // All the fields of SysVar should be READ ONLY after created. type SysVar struct { diff --git a/sessionctx/variable/variable_test.go b/sessionctx/variable/variable_test.go index 57052a37f6b58..ff7f5f73ecf9e 100644 --- a/sessionctx/variable/variable_test.go +++ b/sessionctx/variable/variable_test.go @@ -272,6 +272,22 @@ func TestBoolValidation(t *testing.T) { require.Equal(t, Off, val) } +func TestTimeValidation(t *testing.T) { + sv := SysVar{Scope: ScopeSession, Name: "mynewsysvar", Value: "23:59 +0000", Type: TypeTime} + vars := NewSessionVars(nil) + + val, err := sv.Validate(vars, "23:59 +0000", ScopeSession) + require.NoError(t, err) + require.Equal(t, "23:59 +0000", val) + + val, err = sv.Validate(vars, "3:00 +0000", ScopeSession) + require.NoError(t, err) + require.Equal(t, "03:00 +0000", val) + + _, err = sv.Validate(vars, "0.000", ScopeSession) + require.Error(t, err) +} + func TestGetNativeValType(t *testing.T) { sv := SysVar{Scope: ScopeGlobal | ScopeSession, Name: "mynewsysvar", Value: Off, Type: TypeBool} @@ -382,6 +398,9 @@ func TestScope(t *testing.T) { require.False(t, sv.HasGlobalScope()) require.True(t, sv.HasInstanceScope()) require.False(t, sv.HasNoneScope()) + + sv = SysVar{Scope: ScopeSession, Name: "mynewsysvar", Value: On, Type: TypeEnum, PossibleValues: []string{"OFF", "ON", "AUTO"}} + require.Error(t, sv.validateScope(ScopeGlobal)) } func TestBuiltInCase(t *testing.T) { @@ -515,11 +534,38 @@ func TestSkipInitIsUsed(t *testing.T) { } } +func TestScopeToString(t *testing.T) { + require.Equal(t, "GLOBAL", ScopeGlobal.String()) + require.Equal(t, "SESSION", ScopeSession.String()) + require.Equal(t, "INSTANCE", ScopeInstance.String()) + require.Equal(t, "NONE", ScopeNone.String()) + tmp := ScopeSession + ScopeGlobal + require.Equal(t, "SESSION,GLOBAL", tmp.String()) + // this is not currently possible, but might be in future. + // *but* global + instance is not possible. these are mutually exclusive by design. + tmp = ScopeSession + ScopeInstance + require.Equal(t, "SESSION,INSTANCE", tmp.String()) +} + func TestValidateWithRelaxedValidation(t *testing.T) { sv := GetSysVar(SecureAuth) vars := NewSessionVars(nil) val := sv.ValidateWithRelaxedValidation(vars, "1", ScopeGlobal) require.Equal(t, "ON", val) + + // Relaxed validation catches the error and squashes it. + // The incorrect value is returned as-is. + // I am not sure this is the correct behavior, we might need to + // change it to return the default instead in future. + sv = GetSysVar(DefaultAuthPlugin) + val = sv.ValidateWithRelaxedValidation(vars, "RandomText", ScopeGlobal) + require.Equal(t, "RandomText", val) + + // Validation func fails, the error is also caught and squashed. + // The incorrect value is returned as-is. + sv = GetSysVar(InitConnect) + val = sv.ValidateWithRelaxedValidation(vars, "RandomText - should be valid SQL", ScopeGlobal) + require.Equal(t, "RandomText - should be valid SQL", val) } func TestInstanceConfigHasMatchingSysvar(t *testing.T) { @@ -597,11 +643,17 @@ func TestInstanceScope(t *testing.T) { } func TestSetSysVar(t *testing.T) { - val := GetSysVar(SystemTimeZone).Value + vars := NewSessionVars(nil) + vars.GlobalVarsAccessor = NewMockGlobalAccessor4Tests() + originalVal := GetSysVar(SystemTimeZone).Value SetSysVar(SystemTimeZone, "America/New_York") require.Equal(t, "America/New_York", GetSysVar(SystemTimeZone).Value) - SetSysVar(SystemTimeZone, val) // restore - require.Equal(t, val, GetSysVar(SystemTimeZone).Value) + // Test alternative Get + val, err := GetSysVar(SystemTimeZone).GetGlobalFromHook(vars) + require.Nil(t, err) + require.Equal(t, "America/New_York", val) + SetSysVar(SystemTimeZone, originalVal) // restore + require.Equal(t, originalVal, GetSysVar(SystemTimeZone).Value) } func TestSkipSysvarCache(t *testing.T) { diff --git a/sessiontxn/isolation/BUILD.bazel b/sessiontxn/isolation/BUILD.bazel index 4627aec062c04..2f2cb5dce4e31 100644 --- a/sessiontxn/isolation/BUILD.bazel +++ b/sessiontxn/isolation/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//config", "//infoschema", "//kv", + "//metrics", "//parser/ast", "//parser/mysql", "//parser/terror", diff --git a/store/copr/BUILD.bazel b/store/copr/BUILD.bazel index d04043b6f9a76..674ca8f6c54e1 100644 --- a/store/copr/BUILD.bazel +++ b/store/copr/BUILD.bazel @@ -68,6 +68,7 @@ go_test( ], embed = [":copr"], flaky = True, + race = "on", deps = [ "//kv", "//store/driver/backoff", diff --git a/table/BUILD.bazel b/table/BUILD.bazel index 6293b37b02dd9..5f8e425822279 100644 --- a/table/BUILD.bazel +++ b/table/BUILD.bazel @@ -44,6 +44,7 @@ go_test( ], embed = [":table"], flaky = True, + race = "on", shard_count = 50, deps = [ "//errno", diff --git a/telemetry/cte_test/BUILD.bazel b/telemetry/cte_test/BUILD.bazel index 7236a4ce32559..c6d60eda945df 100644 --- a/telemetry/cte_test/BUILD.bazel +++ b/telemetry/cte_test/BUILD.bazel @@ -5,6 +5,7 @@ go_test( timeout = "short", srcs = ["cte_test.go"], flaky = True, + race = "on", deps = [ "//config", "//domain", @@ -17,6 +18,7 @@ go_test( "@com_github_jeffail_gabs_v2//:gabs", "@com_github_stretchr_testify//require", "@io_etcd_go_etcd_tests_v3//integration", + "@io_opencensus_go//stats/view", "@org_uber_go_goleak//:goleak", ], ) diff --git a/telemetry/cte_test/cte_test.go b/telemetry/cte_test/cte_test.go index 41f40e252f2a4..32eed83e8df33 100644 --- a/telemetry/cte_test/cte_test.go +++ b/telemetry/cte_test/cte_test.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/tidb/testkit/testsetup" "github.com/stretchr/testify/require" "go.etcd.io/etcd/tests/v3/integration" + "go.opencensus.io/stats/view" "go.uber.org/goleak" ) @@ -37,7 +38,6 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), - goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), } @@ -120,6 +120,7 @@ func newSuite(t *testing.T) *testSuite { suite.dom.Close() err = suite.store.Close() require.NoError(t, err) + view.Stop() } return suite diff --git a/telemetry/main_test.go b/telemetry/main_test.go index cee65c94b707e..6d4b5d2c5aaf8 100644 --- a/telemetry/main_test.go +++ b/telemetry/main_test.go @@ -38,7 +38,6 @@ func TestMain(m *testing.M) { opts := []goleak.Option{ goleak.IgnoreTopFunction("go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop"), - goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start"), goleak.IgnoreTopFunction("github.com/golang/glog.(*loggingT).flushDaemon"), }