Merge remote-tracking branch 'pingcap/master' into analyze-concurrenc…

…y-hotfix

.bazelrc

@@ -1,18 +1,27 @@
 startup --host_jvm_args=-Xmx8g
 startup --unlimit_coredumps
 
+run:ci --color=yes
+
+build --announce_rc
 build --java_language_version=17
 build --java_runtime_version=17
 build --tool_java_language_version=17
 build --tool_java_runtime_version=17
-
-run --color=yes
-build:release --workspace_status_command=./build/print-workspace-status.sh --stamp
-build:release --config=ci
 build --incompatible_strict_action_env --incompatible_enable_cc_toolchain_resolution
+build:ci --color=yes
 build:ci --experimental_remote_cache_compression
-test:ci --verbose_failures
-test:ci --test_env=GO_TEST_WRAP_TESTV=1 --test_verbose_timeout_warnings
-test:ci --test_env=TZ=Asia/Shanghai --test_output=errors --experimental_ui_max_stdouterr_bytes=104857600
+build:release --workspace_status_command=./build/print-workspace-status.sh --stamp
+build:release --config=ci
+build:race --config=ci
+build:race --@io_bazel_rules_go//go/config:race --test_env=GORACE=halt_on_error=1 --test_sharding_strategy=disabled
+
+test --test_env=TZ=Asia/Shanghai
+test --test_output=errors --test_summary=testcase
+test:ci --color=yes
+test:ci --verbose_failures --test_verbose_timeout_warnings
+test:ci --test_env=GO_TEST_WRAP_TESTV=1
+test:ci --experimental_ui_max_stdouterr_bytes=104857600
+test:race --test_timeout=1200,6000,18000,72000
 
 try-import /data/bazel

.github/workflows/integration-test-dumpling-common.yml

@@ -0,0 +1,56 @@
+name: DumplingCommon
+on:
+  workflow_call:
+    inputs:
+      debug:
+        type: boolean
+        description: 'set tmate on failure'
+        required: true
+      mysql_version:
+        type: string
+        description: 'specify mysql version'
+        required: true
+
+jobs:
+  integration-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+    services:
+      mysql:
+        image: mysql:${{ inputs.mysql_version }}
+        env:
+          MYSQL_ALLOW_EMPTY_PASSWORD: yes
+        ports:
+          - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+      - name: 'checkout repository'
+        uses: actions/checkout@v3
+      - name: 'set up golang'
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18
+      - name: 'try to use build cache'
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+            ~/Library/Caches/go-build
+            ~\AppData\Local\go-build
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: 'download binary dependencies'
+        run: sh dumpling/install.sh
+      - name: 'build tidb'
+        run: make server
+      - name: 'build lightning'
+        run: make build_lightning
+      - name: 'integration test'
+        run: make dumpling_integration_test VERBOSE="true"
+      - name: 'set up tmate session if necessary'
+        if: ${{ failure() && inputs.debug }}
+        uses: mxschmitt/action-tmate@v3

.github/workflows/integration-test-dumpling.yml

@@ -1,5 +1,12 @@
 name: Dumpling
 on:
+  workflow_dispatch:
+    inputs:
+      debug:
+        type: boolean
+        description: 'Run the build with tmate debugging enabled'
+        required: false
+        default: false
   push:
     branches:
       - master
@@ -32,113 +39,20 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  integration-test-mysql-5735:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:5.7.35
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-5735:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 5.7.35
 
-  integration-test-mysql-8026:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:8.0.26
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-8022:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 8.0.22
 
-  integration-test-mysql-8022:
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    strategy:
-      fail-fast: true
-    services:
-      mysql:
-        image: mysql:8.0.22
-        env:
-          MYSQL_ALLOW_EMPTY_PASSWORD: yes
-        ports:
-          - 3306:3306
-        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
-    steps:
-      - uses: actions/checkout@v2
-      - name: Shutdown Ubuntu MySQL (SUDO)
-        run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it
-      - name: Set up Go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.18
-      - uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Get dependencies
-        run: go mod download
-      - name: Download dependencies
-        run: sh dumpling/install.sh
-      - name: Integration test
-        run: make dumpling_integration_test VERBOSE="true"
-      - name: Set up tmate session
-        if: ${{ failure() }}
-        uses: mxschmitt/action-tmate@v3
+  mysql-8026:
+    uses: ./.github/workflows/integration-test-dumpling-common.yml
+    with:
+      debug: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug }}
+      mysql_version: 8.0.26

BUILD.bazel

@@ -13,17 +13,3 @@ xcode_version(
     name = "xcode_version",
     version = "10.0",
 )
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(
-        ["**"],
-        exclude = [
-            "bazel-*/**",
-            ".git/**",
-            ".idea/**",
-        ],
-    ),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)

DEPS.bzl

@@ -3366,8 +3366,8 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sum = "h1:tugwRUqH0gYZ0noHr17AVSKxNVd+Jl2BmNqgANeDIok=",
-        version = "v2.0.1-0.20220809034808-2ed2113d1090",
+        sum = "h1:/nr7P8uzJQ7u3wPEBHCokrsVmuDvi/1x/zI/ydk5n8U=",
+        version = "v2.0.1-0.20220818084834-0d0ae0dcfb1f",
     )
     go_repository(
         name = "com_github_tikv_pd_client",

Makefile

@@ -253,6 +253,7 @@ endif
 bench-daily:
 	go test github.com/pingcap/tidb/session -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/executor -run TestBenchDaily -bench Ignore --outfile bench_daily.json
+	go test github.com/pingcap/tidb/executor/splittest -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/tablecodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/expression -run TestBenchDaily -bench Ignore --outfile bench_daily.json
 	go test github.com/pingcap/tidb/util/rowcodec -run TestBenchDaily -bench Ignore --outfile bench_daily.json
@@ -371,7 +372,8 @@ dumpling_unit_test_in_verify_ci: failpoint-enable tools/bin/gotestsum
 	$(RACE_FLAG) -coverprofile="$(TEST_COVERAGE_DIR)/dumpling_cov.unit_test.out" || ( make failpoint-disable && exit 1 )
 	@make failpoint-disable
 
-dumpling_integration_test: dumpling_bins failpoint-enable build_dumpling
+dumpling_integration_test: dumpling_bins failpoint-enable
+	@make build_dumpling
 	@make failpoint-disable
 	./dumpling/tests/run.sh $(CASE)
 
@@ -390,7 +392,7 @@ bazel_ci_prepare:
 
 bazel_prepare:
 	bazel run //:gazelle
-	bazel run //:gazelle -- update-repos -from_file=go.mod -build_file_proto_mode=disable
+	bazel run //:gazelle -- update-repos -from_file=go.mod -to_macro DEPS.bzl%go_deps  -build_file_proto_mode=disable
 
 bazel_test: failpoint-enable bazel_ci_prepare
 	bazel $(BAZEL_GLOBAL_CONFIG) test $(BAZEL_CMD_CONFIG) \

bindinfo/optimize_test.go

@@ -30,7 +30,7 @@ func TestOptimizeOnlyOnce(t *testing.T) {
 	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t(a int, b int, index idxa(a))")
 	tk.MustExec("create global binding for select * from t using select * from t use index(idxa)")
-	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/planner/checkOptimizeCountOne", "return"))
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/planner/checkOptimizeCountOne", "return(\"select * from t\")"))
 	defer func() {
 		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/planner/checkOptimizeCountOne"))
 	}()

br/pkg/lightning/backend/kv/sql2kv.go

@@ -88,10 +88,10 @@ func NewTableKVEncoder(
 	if meta.PKIsHandle && meta.ContainsAutoRandomBits() {
 		for _, col := range cols {
 			if mysql.HasPriKeyFlag(col.GetFlag()) {
-				incrementalBits := autoRandomIncrementBits(col, int(meta.AutoRandomBits))
-				autoRandomBits := rand.New(rand.NewSource(options.AutoRandomSeed)).Int63n(1<<meta.AutoRandomBits) << incrementalBits // nolint:gosec
+				shardFmt := autoid.NewShardIDFormat(&col.FieldType, meta.AutoRandomBits, meta.AutoRandomRangeBits)
+				shard := rand.New(rand.NewSource(options.AutoRandomSeed)).Int63()
 				autoIDFn = func(id int64) int64 {
-					return autoRandomBits | id
+					return shardFmt.Compose(shard, id)
 				}
 				break
 			}
@@ -122,16 +122,6 @@ func NewTableKVEncoder(
 	}, nil
 }
 
-func autoRandomIncrementBits(col *table.Column, randomBits int) int {
-	typeBitsLength := mysql.DefaultLengthOfMysqlTypes[col.GetType()] * 8
-	incrementalBits := typeBitsLength - randomBits
-	hasSignBit := !mysql.HasUnsignedFlag(col.GetFlag())
-	if hasSignBit {
-		incrementalBits--
-	}
-	return incrementalBits
-}
-
 // collectGeneratedColumns collects all expressions required to evaluate the
 // results of all generated columns. The returning slice is in evaluation order.
 func collectGeneratedColumns(se *session, meta *model.TableInfo, cols []*table.Column) ([]genCol, error) {
@@ -391,9 +381,9 @@ func (kvcodec *tableKVEncoder) Encode(
 		record = append(record, value)
 
 		if isTableAutoRandom(meta) && isPKCol(col.ToInfo()) {
-			incrementalBits := autoRandomIncrementBits(col, int(meta.AutoRandomBits))
+			shardFmt := autoid.NewShardIDFormat(&col.FieldType, meta.AutoRandomBits, meta.AutoRandomRangeBits)
 			alloc := kvcodec.tbl.Allocators(kvcodec.se).Get(autoid.AutoRandomType)
-			if err := alloc.Rebase(context.Background(), value.GetInt64()&((1<<incrementalBits)-1), false); err != nil {
+			if err := alloc.Rebase(context.Background(), value.GetInt64()&shardFmt.IncrementalMask(), false); err != nil {
 				return nil, errors.Trace(err)
 			}
 		}

br/pkg/lightning/backend/kv/sql2kv_test.go

@@ -380,14 +380,10 @@ func TestEncodeMissingAutoValue(t *testing.T) {
 		},
 		{
 			AllocType:  autoid.AutoRandomType,
-			CreateStmt: "create table t (id integer primary key auto_random(3));",
+			CreateStmt: "create table t (id bigint primary key auto_random(3));",
 		},
 	} {
 		tblInfo := mockTableInfo(t, testTblInfo.CreateStmt)
-		if testTblInfo.AllocType == autoid.AutoRandomType {
-			// seems parser can't parse auto_random properly.
-			tblInfo.AutoRandomBits = 3
-		}
 		tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo)
 		require.NoError(t, err)
 
@@ -447,8 +443,6 @@ func mockTableInfo(t *testing.T, createSQL string) *model.TableInfo {
 
 func TestDefaultAutoRandoms(t *testing.T) {
 	tblInfo := mockTableInfo(t, "create table t (id bigint unsigned NOT NULL auto_random primary key clustered, a varchar(100));")
-	// seems parser can't parse auto_random properly.
-	tblInfo.AutoRandomBits = 5
 	tbl, err := tables.TableFromMeta(lkv.NewPanickingAllocators(0), tblInfo)
 	require.NoError(t, err)
 	encoder, err := lkv.NewTableKVEncoder(tbl, &lkv.SessionOptions{

br/pkg/lightning/backend/local/BUILD.bazel

@@ -42,6 +42,7 @@ go_library(
         "//parser/mysql",
         "//table",
         "//tablecodec",
+        "//types",
         "//util/codec",
         "//util/engine",
         "//util/hack",

br/pkg/lightning/backend/local/local.go

@@ -58,6 +58,7 @@ import (
 	"github.com/pingcap/tidb/parser/mysql"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/tablecodec"
+	"github.com/pingcap/tidb/types"
 	"github.com/pingcap/tidb/util/codec"
 	"github.com/pingcap/tidb/util/engine"
 	"github.com/pingcap/tidb/util/mathutil"
@@ -1693,6 +1694,10 @@ func (local *local) ResolveDuplicateRows(ctx context.Context, tbl table.Table, t
 				if err == nil {
 					return nil
 				}
+				if types.ErrBadNumber.Equal(err) {
+					logger.Warn("delete duplicate rows encounter error", log.ShortError(err))
+					return common.ErrResolveDuplicateRows.Wrap(err).GenWithStackByArgs(tableName)
+				}
 				if log.IsContextCanceledError(err) {
 					return err
 				}