Merge branch 'master' into resource_manager/duration_buffer

Makefile

@@ -267,7 +267,7 @@ tools/bin/gotestsum:
 	GOBIN=$(shell pwd)/tools/bin $(GO) install gotest.tools/gotestsum@v1.8.1
 
 tools/bin/mockgen:
-	GOBIN=$(shell pwd)/tools/bin $(GO) install go.uber.org/mock/mockgen@v0.2.0
+	GOBIN=$(shell pwd)/tools/bin $(GO) install go.uber.org/mock/mockgen@v0.3.0
 
 # Usage:
 #

OWNERS_ALIASES

@@ -26,6 +26,10 @@ aliases:
     - windtalker
     - XuHuaiyu
     - zanmato1984
+  sig-approvers-executor-import: # approvers for importer part of executor pkg.
+    - D3Hunter
+    - gmhdbjd
+    - lance6716    
   sig-approvers-expression: # approvers for expression pkg
     - windtalker
     - XuHuaiyu
@@ -44,11 +48,70 @@ aliases:
     - lance6716
     - tangenta
     - wjhuang2016
+  sig-approvers-ddl: # approvers for ddl pkg
+    - Benjamin2037
+    - tangenta
+    - wjhuang2016
+    - ywqzzy
+    - zimulala
   sig-approvers-disttask: # approvers for disttask pkg
     - Benjamin2037
     - D3Hunter
     - gmhdbjd
     - lance6716
     - tangenta
     - wjhuang2016
-    - ywqzzy
+    - ywqzzy
+  sig-approvers-dumpling: # approvers for dumpling module
+    - Benjamin2037
+    - gmhdbjd
+    - lichunzhu
+    - okJiang
+  sig-approvers-infoschema: # approvers for infoschema pkg
+    - zimulala
+    - wjhuang2016
+    - tangenta
+    - ywqzzy
+    - D3Hunter
+    - Benjamin2037
+    - gmhdbjd
+  sig-approvers-meta: # approvers for meta pkg
+    - Benjamin2037
+    - gmhdbjd
+    - tangenta
+    - wjhuang2016
+    - ywqzzy
+    - zimulala    
+  sig-approvers-owner: # approvers for `owner` pkg
+    - Benjamin2037
+    - lichunzhu
+    - tangenta
+    - wjhuang2016
+    - ywqzzy
+    - zimulala
+  sig-approvers-resourcemanager: # approvers for resourcemanager pkg
+    - Benjamin2037
+    - D3Hunter
+    - gmhdbjd
+    - lance6716
+    - tangenta
+    - wjhuang2016
+    - ywqzzy
+  sig-approvers-table: # approvers for table packages.
+    - Benjamin2037
+    - cfzjywxk
+    - gmhdbjd
+    - tangenta
+    - wjhuang2016
+    - ywqzzy
+    - zimulala
+  sig-approvers-lock: # approvers for lock pkg
+    - Benjamin2037
+    - lance6716
+    - tangenta
+    - wjhuang2016
+    - zimulala
+  sig-approvers-tidb-binlog: # approvers for tidb-binlog module
+    - Benjamin2037
+    - gmhdbjd
+    - lichunzhu

WORKSPACE

@@ -16,12 +16,12 @@ versions.check(minimum_bazel_version = "6.0.0")
 
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "91585017debb61982f7054c9688857a2ad1fd823fc3f9cb05048b0025c47d023",
+    sha256 = "eff2811014f8b26d91fadda335000cbfce145674368bcd732fbf782adb53a778",
+    strip_prefix = "rules_go-df20c987afcbbc721518c79f9c6489d87d73582c",
     urls = [
-        "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
-        "http://ats.apps.svc/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.42.0/rules_go-v0.42.0.zip",
+        "http://bazel-cache.pingcap.net:8080/bazelbuild/rules_go/archive/df20c987afcbbc721518c79f9c6489d87d73582c.zip",
+        "http://ats.apps.svc/bazelbuild/rules_go/archive/df20c987afcbbc721518c79f9c6489d87d73582c.zip",
+        "https://github.com/bazelbuild/rules_go/archive/df20c987afcbbc721518c79f9c6489d87d73582c.zip",
     ],
 )
 

br/OWNERS

@@ -2,7 +2,7 @@
 options:
   no_parent_owners: true
 filters:
-  "(tidb-lightning\\.tom)$":
+  "(tidb-lightning\\.toml)$":
     approvers:
       - sig-critical-approvers-tidb-lightning
   ".*":

br/pkg/lightning/backend/backend.go

@@ -114,6 +114,7 @@ type ExternalEngineConfig struct {
 	TotalFileSize int64
 	// TotalKVCount can be an estimated value.
 	TotalKVCount int64
+	CheckHotspot bool
 }
 
 // CheckCtx contains all parameters used in CheckRequirements

br/pkg/lightning/backend/external/BUILD.bazel

@@ -30,17 +30,18 @@ go_library(
         "//br/pkg/storage",
         "//pkg/kv",
         "//pkg/metrics",
-        "//pkg/sessionctx/variable",
         "//pkg/util/hack",
         "//pkg/util/logutil",
+        "//pkg/util/memory",
         "//pkg/util/size",
         "@com_github_cockroachdb_pebble//:pebble",
         "@com_github_docker_go_units//:go-units",
-        "@com_github_jfcg_sorty_v2//:sorty",
+        "@com_github_google_uuid//:uuid",
         "@com_github_pingcap_errors//:errors",
         "@org_golang_x_sync//errgroup",
         "@org_uber_go_atomic//:atomic",
         "@org_uber_go_zap//:zap",
+        "@org_uber_go_zap//zapcore",
     ],
 )
 
@@ -61,13 +62,15 @@ go_test(
     ],
     embed = [":external"],
     flaky = True,
-    shard_count = 41,
+    shard_count = 43,
     deps = [
         "//br/pkg/lightning/backend/kv",
         "//br/pkg/lightning/common",
+        "//br/pkg/membuf",
         "//br/pkg/storage",
         "//pkg/kv",
         "//pkg/util/codec",
+        "//pkg/util/logutil",
         "//pkg/util/size",
         "@com_github_aws_aws_sdk_go//aws",
         "@com_github_aws_aws_sdk_go//aws/credentials",

br/pkg/lightning/backend/external/byte_reader.go

@@ -278,6 +278,8 @@ func (r *byteReader) reload() error {
 	if err != nil {
 		switch err {
 		case io.EOF:
+			// move curBufOffset so following read will also find EOF
+			r.curBufOffset = len(r.curBuf)
 			return err
 		case io.ErrUnexpectedEOF:
 			// The last batch.