Optimizer: Add partition by support for derived TopN(filter on row_number)

DEPS.bzl

@@ -1385,8 +1385,8 @@ def go_deps():
         name = "com_github_go_toolsmith_astcopy",
         build_file_proto_mode = "disable",
         importpath = "github.com/go-toolsmith/astcopy",
-        sum = "h1:YnWf5Rnh1hUudj11kei53kI57quN/VH6Hp1n+erozn0=",
-        version = "v1.0.2",
+        sum = "h1:r0bgSRlMOAgO+BdQnVAcpMSMkrQCnV6ZJmIkrJgcJj0=",
+        version = "v1.0.3",
     )
     go_repository(
         name = "com_github_go_toolsmith_astequal",
@@ -1606,8 +1606,8 @@ def go_deps():
         name = "com_github_golangci_golangci_lint",
         build_file_proto_mode = "disable",
         importpath = "github.com/golangci/golangci-lint",
-        sum = "h1:M1bpDymgdaPKNzPwQdebCGki/nzvVkr2f/eUfk9C9oU=",
-        version = "v1.51.0",
+        sum = "h1:N5HD/x0ZrhJYsgKWyz7yJxxQ8JKR0Acc+FOP7QtGSAA=",
+        version = "v1.51.1",
     )
     go_repository(
         name = "com_github_golangci_gosec",
@@ -1812,8 +1812,8 @@ def go_deps():
         name = "com_github_gordonklaus_ineffassign",
         build_file_proto_mode = "disable",
         importpath = "github.com/gordonklaus/ineffassign",
-        sum = "h1:PVRE9d4AQKmbelZ7emNig1+NT27DUmKZn5qXxfio54U=",
-        version = "v0.0.0-20210914165742-4cc7213b9bc8",
+        sum = "h1:9alfqbrhuD+9fLZ4iaAVwhlp5PEhmnBt7yvK2Oy5C1U=",
+        version = "v0.0.0-20230107090616-13ace0543b28",
     )
     go_repository(
         name = "com_github_gorilla_context",
@@ -2432,8 +2432,8 @@ def go_deps():
         name = "com_github_junk1tm_musttag",
         build_file_proto_mode = "disable",
         importpath = "github.com/junk1tm/musttag",
-        sum = "h1:I8UHQkDj2u/MClcGU8PbMoYwhykiSQFEbXKKMjixPyk=",
-        version = "v0.4.3",
+        sum = "h1:VK4L7v7lvWAhKDDx0cUJgbb0UBNipYinv8pPeHJzH9Q=",
+        version = "v0.4.4",
     )
 
     go_repository(
@@ -3110,8 +3110,8 @@ def go_deps():
         name = "com_github_nunnatsa_ginkgolinter",
         build_file_proto_mode = "disable",
         importpath = "github.com/nunnatsa/ginkgolinter",
-        sum = "h1:j4mzqx1hkE75mXHs3AUlWghZBi59c3GDWXp6zzcI+kE=",
-        version = "v0.7.1",
+        sum = "h1:/y4o/0hV+ruUHj4xXh89xlFjoaitnI4LnkpuYs02q1c=",
+        version = "v0.8.1",
     )
 
     go_repository(
@@ -3633,8 +3633,8 @@ def go_deps():
         name = "com_github_ryanrolds_sqlclosecheck",
         build_file_proto_mode = "disable",
         importpath = "github.com/ryanrolds/sqlclosecheck",
-        sum = "h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=",
-        version = "v0.3.0",
+        sum = "h1:i8SX60Rppc1wRuyQjMciLqIzV3xnoHB7/tXbr6RGYNI=",
+        version = "v0.4.0",
     )
 
     go_repository(
@@ -3656,8 +3656,8 @@ def go_deps():
         name = "com_github_sanposhiho_wastedassign_v2",
         build_file_proto_mode = "disable",
         importpath = "github.com/sanposhiho/wastedassign/v2",
-        sum = "h1:+6/hQIHKNJAUixEj6EmOngGIisyeI+T3335lYTyxRoA=",
-        version = "v2.0.6",
+        sum = "h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc=",
+        version = "v2.0.7",
     )
     go_repository(
         name = "com_github_sasha_s_go_deadlock",
@@ -3737,8 +3737,8 @@ def go_deps():
         name = "com_github_shirou_gopsutil_v3",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/shirou/gopsutil/v3",
-        sum = "h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs=",
-        version = "v3.22.12",
+        sum = "h1:a9KKO+kGLKEvcPIs4W62v0nu3sciVDOOOPUD0Hz7z/4=",
+        version = "v3.23.1",
     )
     go_repository(
         name = "com_github_shopify_goreferrer",
@@ -3866,8 +3866,8 @@ def go_deps():
         name = "com_github_sourcegraph_go_diff",
         build_file_proto_mode = "disable",
         importpath = "github.com/sourcegraph/go-diff",
-        sum = "h1:FEIBISvqa2IsyC4KQQBQ1Ef2QvweGUgEIjCdE3gz+zs=",
-        version = "v0.6.2-0.20221031073116-7ef5f68ebea1",
+        sum = "h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0=",
+        version = "v0.7.0",
     )
 
     go_repository(
@@ -4151,8 +4151,8 @@ def go_deps():
         name = "com_github_tomarrell_wrapcheck_v2",
         build_file_proto_mode = "disable",
         importpath = "github.com/tomarrell/wrapcheck/v2",
-        sum = "h1:J/F8DbSKJC83bAvC6FoZaRjZiZ/iKoueSdrEkmGeacA=",
-        version = "v2.7.0",
+        sum = "h1:qDzbir0xmoE+aNxGCPrn+rUSxAX+nG6vREgbbXAR81I=",
+        version = "v2.8.0",
     )
     go_repository(
         name = "com_github_tommy_muehle_go_mnd_v2",

Makefile

@@ -31,7 +31,7 @@ dev: checklist check explaintest gogenerate br_unit_test test_part_parser_dev ut
 # Install the check tools.
 check-setup:tools/bin/revive
 
-check: parser_yacc check-parallel lint tidy testSuite errdoc license check-bazel-prepare
+check: check-bazel-prepare parser_yacc check-parallel lint tidy testSuite errdoc license
 
 fmt:
 	@echo "gofmt (simplify)"
@@ -409,7 +409,7 @@ bazel_test: failpoint-enable bazel_ci_prepare
 		-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test
 
 
-bazel_coverage_test: failpoint-enable bazel_ci_prepare
+bazel_coverage_test: check-bazel-prepare failpoint-enable bazel_ci_prepare
 	bazel $(BAZEL_GLOBAL_CONFIG) coverage $(BAZEL_CMD_CONFIG) --test_keep_going=false \
 		--@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest \
 		-- //... -//cmd/... -//tests/graceshutdown/... \

br/docker/Dockerfile

@@ -1,5 +1,5 @@
 # For loading data to TiDB
-FROM golang:1.18.1-alpine as go-ycsb-builder
+FROM golang:1.18.9-alpine as go-ycsb-builder
 WORKDIR /go/src/github.com/pingcap/
 
 RUN apk add --no-cache \
@@ -17,7 +17,7 @@ RUN git clone https://github.com/pingcap/go-ycsb.git --depth=1 && \
 # For operating minio S3 compatible storage
 FROM minio/mc as mc-builder
 
-FROM golang:1.18.1-alpine
+FROM golang:1.18.9-alpine
 
 WORKDIR /go/src/github.com/pingcap/br
 COPY . .

br/pkg/gluetidb/BUILD.bazel

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "gluetidb",
@@ -25,3 +25,21 @@ go_library(
         "@org_uber_go_zap//:zap",
     ],
 )
+
+go_test(
+    name = "gluetidb_test",
+    timeout = "short",
+    srcs = ["glue_test.go"],
+    embed = [":gluetidb"],
+    flaky = True,
+    deps = [
+        "//ddl",
+        "//kv",
+        "//meta",
+        "//parser/model",
+        "//sessionctx",
+        "//testkit",
+        "@com_github_pingcap_failpoint//:failpoint",
+        "@com_github_stretchr_testify//require",
+    ],
+)

br/pkg/gluetidb/glue.go

@@ -218,19 +218,21 @@ func (gs *tidbSession) CreatePlacementPolicy(ctx context.Context, policy *model.
 // SplitBatchCreateTable provide a way to split batch into small batch when batch size is large than 6 MB.
 // The raft entry has limit size of 6 MB, a batch of CreateTables may hit this limitation
 // TODO: shall query string be set for each split batch create, it looks does not matter if we set once for all.
-func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, info []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
+func (gs *tidbSession) SplitBatchCreateTable(schema model.CIStr, infos []*model.TableInfo, cs ...ddl.CreateTableWithInfoConfigurier) error {
 	var err error
 	d := domain.GetDomain(gs.se).DDL()
-	if err = d.BatchCreateTableWithInfo(gs.se, schema, info, append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
-		if len(info) == 1 {
+
+	if err = d.BatchCreateTableWithInfo(gs.se, schema, infos, append(cs, ddl.OnExistIgnore)...); kv.ErrEntryTooLarge.Equal(err) {
+		log.Info("entry too large, split batch create table", zap.Int("num table", len(infos)))
+		if len(infos) == 1 {
 			return err
 		}
-		mid := len(info) / 2
-		err = gs.SplitBatchCreateTable(schema, info[:mid])
+		mid := len(infos) / 2
+		err = gs.SplitBatchCreateTable(schema, infos[:mid], cs...)
 		if err != nil {
 			return err
 		}
-		err = gs.SplitBatchCreateTable(schema, info[mid:])
+		err = gs.SplitBatchCreateTable(schema, infos[mid:], cs...)
 		if err != nil {
 			return err
 		}
@@ -268,7 +270,7 @@ func (gs *tidbSession) CreateTables(ctx context.Context, tables map[string][]*mo
 			cloneTables = append(cloneTables, table)
 		}
 		gs.se.SetValue(sessionctx.QueryString, queryBuilder.String())
-		if err := gs.SplitBatchCreateTable(dbName, cloneTables); err != nil {
+		if err := gs.SplitBatchCreateTable(dbName, cloneTables, cs...); err != nil {
 			//It is possible to failure when TiDB does not support model.ActionCreateTables.
 			//In this circumstance, BatchCreateTableWithInfo returns errno.ErrInvalidDDLJob,
 			//we fall back to old way that creating table one by one