Merge branch 'master' into master-cost-model-2-mpp-agg

.github/licenserc.yml

@@ -23,7 +23,7 @@ header:
     - '**/*.result'
     - '**/*.example'
     - '**/*.patch'
-    - 'DEPS.bzl'
+    - '**/*.bzl'
     - '.codecov.yml'
     - 'Jenkinsfile'
     - '.editorconfig'

Makefile

@@ -25,9 +25,6 @@ buildsucc:
 
 all: dev server benchkv
 
-parser:
-	@echo "remove this command later, when our CI script doesn't call it"
-
 dev: checklist check explaintest gogenerate br_unit_test test_part_parser_dev ut
 	@>&2 echo "Great, all tests passed."
 
@@ -96,6 +93,9 @@ test_part_parser: parser_yacc test_part_parser_dev
 
 test_part_parser_dev: parser_fmt parser_unit_test
 
+parser:
+	@cd parser && make parser
+
 parser_yacc:
 	@cd parser && mv parser.go parser.go.committed && make parser && diff -u parser.go.committed parser.go && rm parser.go.committed
 

WORKSPACE

@@ -19,16 +19,22 @@ http_archive(
 )
 
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
-
 load("//:DEPS.bzl", "go_deps")
+load("//build:lint.bzl", "nogo_deps")
 
 # gazelle:repository_macro DEPS.bzl%go_deps
 go_deps()
 
+nogo_deps()
+
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
+
 go_rules_dependencies()
 
-go_register_toolchains(version = "1.18.3")
+go_register_toolchains(
+    nogo = "@//build:tidb_nogo",
+    version = "1.18.3",
+)
 
 gazelle_dependencies()
 

build/BUILD.bazel

@@ -1,7 +1,49 @@
 package(default_visibility = ["//visibility:public"])
 
-filegroup(
-    name = "all_patches",
-    srcs = glob(["*.patch"]),
-    visibility = ["//visibility:public"],
+load("@io_bazel_rules_go//go:def.bzl", "nogo")
+
+nogo(
+    name = "tidb_nogo",
+    config = ":nogo_config.json",
+    visibility = ["//visibility:public"],  # must have public visibility
+    deps = [
+        # https://github.com/golang/go/issues/48525
+        # "@com_github_timakin_bodyclose//passes/bodyclose:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/asmdecl:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/assign:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/atomic:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/atomicalign:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/bools:go_default_library",
+        # https://github.com/golang/go/issues/48525
+        # "@org_golang_x_tools//go/analysis/passes/buildssa:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/buildtag:go_default_library",
+        # https://github.com/bazelbuild/rules_go/issues/2396
+        # "@org_golang_x_tools//go/analysis/passes/cgocall:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/composite:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/copylock:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/ctrlflow:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/deepequalerrors:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/errorsas:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/findcall:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/httpresponse:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/ifaceassert:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/loopclosure:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/lostcancel:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/nilfunc:go_default_library",
+        #"@org_golang_x_tools//go/analysis/passes/nilness:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/pkgfact:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/printf:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/shift:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/sortslice:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/stdmethods:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/stringintconv:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/structtag:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/testinggoroutine:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/tests:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unmarshal:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unreachable:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
+        "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library",
+    ],
 )

build/lint.bzl

@@ -0,0 +1,21 @@
+load("@bazel_gazelle//:deps.bzl", "go_repository")
+
+def nogo_deps():
+    go_repository(
+        name = "com_github_gostaticanalysis_analysisutil",
+        importpath = "github.com/gostaticanalysis/analysisutil",
+        sum = "h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk=",
+        version = "v0.7.1",
+    )
+    go_repository(
+        name = "com_github_gostaticanalysis_comment",
+        importpath = "github.com/gostaticanalysis/comment",
+        sum = "h1:hlnx5+S2fY9Zo9ePo4AhgYsYHbM2+eAv8m/s1JiCd6Q=",
+        version = "v1.4.2",
+    )
+    go_repository(
+        name = "com_github_timakin_bodyclose",
+        importpath = "github.com/timakin/bodyclose",
+        sum = "h1:kl4KhGNsJIbDHS9/4U9yQo1UcPQM0kOMJHn29EoH/Ro=",
+        version = "v0.0.0-20210704033933-f49887972144",
+    )

build/nogo_config.json

@@ -0,0 +1,205 @@
+{
+  "asmdecl": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "assign": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "atomic": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "atomicalign": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "bodyclose": {
+    "exclude_files": {
+      "external/": "no need to vet third party code"
+    }
+  },
+  "bools": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "buildtag": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "printf": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "unreachable": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "composites": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code",
+      "br/pkg/glue/console_glue_test.go": "ignore code",
+      "br/pkg/restore/db_test.go": "ignore code",
+      ".*_/testmain\\.go$": "ignore code"
+    }
+  },
+  "copylocks": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code",
+      "/cgo/": "ignore cgo code"
+    }
+  },
+  "ctrlflow": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "deepequalerrors": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "errorsas": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "findcall": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "httpresponse": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "ifaceassert": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "inspect": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "loopclosure": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "lostcancel": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "nilfunc": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "nilness": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "pkgfact": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "shift": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "sortslice": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "stdmethods": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "stringintconv": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "structtag": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "testinggoroutine": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "tests": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "unmarshal": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    }
+  },
+  "unsafeptr": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code",
+      "parser/digester.go": "ignore code"
+    }
+  },
+  "unusedresult": {
+    "exclude_files": {
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code",
+      "parser/digester_test.go": "ignore code"
+    }
+  }
+}

cmd/explaintest/r/explain_cte.result

@@ -479,3 +479,22 @@ CTE_1	8000.00	root		Non-Recursive CTE
 CTE_0	10000.00	root		Non-Recursive CTE
 └─TableReader(Seed Part)	10000.00	root		data:TableFullScan
   └─TableFullScan	10000.00	cop[tikv]	table:t1	keep order:false, stats:pseudo
+drop table if exists tbl;
+create table tbl (id int);
+explain with t1 as (select id from tbl), t2 as (select a.id from t1 a join t1 b on a.id = b.id) select * from t2 where id in (select id from t2);
+id	estRows	task	access object	operator info
+HashJoin_33	8000.00	root		inner join, equal:[eq(test.tbl.id, test.tbl.id)]
+├─HashAgg_37(Build)	5120.00	root		group by:test.tbl.id, funcs:firstrow(test.tbl.id)->test.tbl.id
+│ └─Selection_38	8000.00	root		not(isnull(test.tbl.id))
+│   └─CTEFullScan_39	10000.00	root	CTE:t2	data:CTE_1
+└─Selection_35(Probe)	8000.00	root		not(isnull(test.tbl.id))
+  └─CTEFullScan_36	10000.00	root	CTE:t2	data:CTE_1
+CTE_1	10000.00	root		Non-Recursive CTE
+└─HashJoin_25(Seed Part)	10000.00	root		inner join, equal:[eq(test.tbl.id, test.tbl.id)]
+  ├─Selection_29(Build)	8000.00	root		not(isnull(test.tbl.id))
+  │ └─CTEFullScan_30	10000.00	root	CTE:b	data:CTE_0
+  └─Selection_27(Probe)	8000.00	root		not(isnull(test.tbl.id))
+    └─CTEFullScan_28	10000.00	root	CTE:a	data:CTE_0
+CTE_0	10000.00	root		Non-Recursive CTE
+└─TableReader_22(Seed Part)	10000.00	root		data:TableFullScan_21
+  └─TableFullScan_21	10000.00	cop[tikv]	table:tbl	keep order:false, stats:pseudo

cmd/explaintest/t/explain_cte.test

@@ -258,3 +258,8 @@ desc format='brief' with all_data as
 select v1.tps v1_tps,v2.tps v2_tps
 from version1 v1, version2 v2
 where v1.bench_type =v2.bench_type;
+
+# issue 35404
+drop table if exists tbl;
+create table tbl (id int);
+explain with t1 as (select id from tbl), t2 as (select a.id from t1 a join t1 b on a.id = b.id) select * from t2 where id in (select id from t2);

ddl/column.go

@@ -1548,7 +1548,7 @@ func (w *worker) doModifyColumn(
 	newCol, oldCol *model.ColumnInfo, pos *ast.ColumnPosition) (ver int64, _ error) {
 	if oldCol.ID != newCol.ID {
 		job.State = model.JobStateRollingback
-		return ver, dbterror.ErrKeyColumnDoesNotExits.GenWithStack("column %s id %d does not exist, this column may have been updated by other DDL ran in parallel", oldCol.Name, newCol.ID)
+		return ver, dbterror.ErrColumnInChange.GenWithStackByArgs(oldCol.Name, newCol.ID)
 	}
 	// Column from null to not null.
 	if !mysql.HasNotNullFlag(oldCol.GetFlag()) && mysql.HasNotNullFlag(newCol.GetFlag()) {

ddl/db_change_test.go

@@ -983,7 +983,7 @@ func TestParallelAlterModifyColumnWithData(t *testing.T) {
 	sql := "ALTER TABLE t MODIFY COLUMN c int;"
 	f := func(err1, err2 error) {
 		require.NoError(t, err1)
-		require.EqualError(t, err2, "[ddl:1072]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel")
+		require.EqualError(t, err2, "[ddl:8245]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel")
 		rs, err := tk.Exec("select * from t")
 		require.NoError(t, err)
 		sRows, err := session.ResultSetToStringSlice(context.Background(), tk.Session(), rs)
@@ -1057,7 +1057,7 @@ func TestParallelAlterModifyColumnToNotNullWithData(t *testing.T) {
 	sql := "ALTER TABLE t MODIFY COLUMN c int not null;"
 	f := func(err1, err2 error) {
 		require.NoError(t, err1)
-		require.EqualError(t, err2, "[ddl:1072]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel")
+		require.EqualError(t, err2, "[ddl:8245]column c id 3 does not exist, this column may have been updated by other DDL ran in parallel")
 		rs, err := tk.Exec("select * from t")
 		require.NoError(t, err)
 		sRows, err := session.ResultSetToStringSlice(context.Background(), tk.Session(), rs)