Merge branch 'master' into pr0

Makefile

@@ -14,7 +14,7 @@ export PATH := $(path_to_add):$(PATH)
 GO              := GO111MODULE=on go
 GOBUILD         := $(GO) build $(BUILD_FLAG) -tags codes -trimpath
 GOBUILDCOVERAGE := GOPATH=$(GOPATH) cd tidb-server; $(GO) test -coverpkg="../..." -c .
-GOTEST          := $(GO) test -p 4
+GOTEST          := $(GO) test -p 8
 OVERALLS        := GO111MODULE=on overalls
 
 ARCH      := "`uname -s`"

config/config.go

@@ -404,7 +404,7 @@ var defaultConf = Config{
 		Reporter: OpenTracingReporter{},
 	},
 	TiKVClient: TiKVClient{
-		GrpcConnectionCount:  16,
+		GrpcConnectionCount:  4,
 		GrpcKeepAliveTime:    10,
 		GrpcKeepAliveTimeout: 3,
 		CommitTimeout:        "41s",

config/config.toml.example

@@ -259,7 +259,7 @@ local-agent-host-port = ""
 
 [tikv-client]
 # Max gRPC connections that will be established with each tikv-server.
-grpc-connection-count = 16
+grpc-connection-count = 4
 
 # After a duration of this time in seconds if the client doesn't see any activity it pings
 # the server to see if the transport is still alive.

domain/domain.go

@@ -421,7 +421,7 @@ func (do *Domain) topNSlowQueryLoop() {
 func (do *Domain) infoSyncerKeeper() {
 	defer do.wg.Done()
 	defer recoverInDomain("infoSyncerKeeper", false)
-	ticker := time.NewTicker(time.Second * time.Duration(infosync.InfoSessionTTL) / 2)
+	ticker := time.NewTicker(infosync.ReportInterval)
 	defer ticker.Stop()
 	for {
 		select {

domain/infosync/info.go

@@ -43,14 +43,15 @@ const (
 	// ServerMinStartTSPath store the server min start timestamp.
 	ServerMinStartTSPath = "/tidb/server/minstartts"
 	// keyOpDefaultRetryCnt is the default retry count for etcd store.
-	keyOpDefaultRetryCnt = 2
+	keyOpDefaultRetryCnt = 5
 	// keyOpDefaultTimeout is the default time out for etcd store.
 	keyOpDefaultTimeout = 1 * time.Second
+	// InfoSessionTTL is the ETCD session's TTL in seconds.
+	InfoSessionTTL = 10 * 60
+	// ReportInterval is interval of infoSyncerKeeper reporting min startTS.
+	ReportInterval = 30 * time.Second
 )
 
-// InfoSessionTTL is the etcd session's TTL in seconds. It's exported for testing.
-var InfoSessionTTL = 1 * 60
-
 // InfoSyncer stores server info to etcd when the tidb-server starts and delete when tidb-server shuts down.
 type InfoSyncer struct {
 	etcdCli        *clientv3.Client

executor/adapter_test.go

@@ -20,7 +20,7 @@ import (
 	"github.com/pingcap/tidb/util/testkit"
 )
 
-func (s *testSuiteP1) TestQueryTime(c *C) {
+func (s *testSuiteP2) TestQueryTime(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 

executor/admin_test.go

@@ -44,7 +44,7 @@ func (s *testSuite1) TestAdminCheckIndexRange(c *C) {
 	result.Check(testkit.Rows("-1 hi 4", "2 cd 2"))
 }
 
-func (s *testSuite2) TestAdminRecoverIndex(c *C) {
+func (s *testSuite5) TestAdminRecoverIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -158,7 +158,7 @@ func (s *testSuite2) TestAdminRecoverIndex(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminRecoverIndex1(c *C) {
+func (s *testSuite5) TestAdminRecoverIndex1(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	s.ctx = mock.NewContext()
 	s.ctx.Store = s.store
@@ -209,7 +209,7 @@ func (s *testSuite2) TestAdminRecoverIndex1(c *C) {
 	tk.MustExec("admin check index admin_test `primary`")
 }
 
-func (s *testSuite2) TestAdminCleanupIndex(c *C) {
+func (s *testSuite5) TestAdminCleanupIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -288,7 +288,7 @@ func (s *testSuite2) TestAdminCleanupIndex(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
+func (s *testSuite5) TestAdminCleanupIndexPKNotHandle(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -336,7 +336,7 @@ func (s *testSuite2) TestAdminCleanupIndexPKNotHandle(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
+func (s *testSuite5) TestAdminCleanupIndexMore(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -399,7 +399,7 @@ func (s *testSuite2) TestAdminCleanupIndexMore(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
-func (s *testSuite2) TestAdminCheckTableFailed(c *C) {
+func (s *testSuite5) TestAdminCheckTableFailed(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_test")
@@ -631,7 +631,7 @@ func (s *testSuite1) TestAdminCheckPrimaryIndex(c *C) {
 	tk.MustExec("admin check index t idx;")
 }
 
-func (s *testSuite2) TestAdminCheckWithSnapshot(c *C) {
+func (s *testSuite5) TestAdminCheckWithSnapshot(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists admin_t_s")

executor/builder.go

@@ -865,8 +865,9 @@ func (b *executorBuilder) buildUnionScanFromReader(reader Executor, v *plannerco
 		// GetDirtyDB() is safe here. If this table has been modified in the transaction, non-nil DirtyTable
 		// can be found in DirtyDB now, so GetDirtyTable is safe; if this table has not been modified in the
 		// transaction, empty DirtyTable would be inserted into DirtyDB, it does not matter when multiple
-		// goroutines write empty DirtyTable to DirtyDB for this table concurrently. Thus we don't use lock
-		// to synchronize here.
+		// goroutines write empty DirtyTable to DirtyDB for this table concurrently. Although the DirtyDB looks
+		// safe for data race in all the cases, the map of golang will throw panic when it's accessed in parallel.
+		// So we lock it when getting dirty table.
 		physicalTableID := getPhysicalTableID(x.table)
 		us.dirty = GetDirtyDB(b.ctx).GetDirtyTable(physicalTableID)
 		us.conditions = v.Conditions

executor/ddl_test.go

@@ -42,7 +42,7 @@ import (
 	"github.com/pingcap/tidb/util/testutil"
 )
 
-func (s *testSuite3) TestTruncateTable(c *C) {
+func (s *testSuite6) TestTruncateTable(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec(`drop table if exists truncate_test;`)
@@ -59,7 +59,7 @@ func (s *testSuite3) TestTruncateTable(c *C) {
 //  1. Execute the SQL of "begin";
 //  2. A SQL that will fail to execute;
 //  3. Execute DDL.
-func (s *testSuite3) TestInTxnExecDDLFail(c *C) {
+func (s *testSuite6) TestInTxnExecDDLFail(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create table t (i int key);")
@@ -72,7 +72,7 @@ func (s *testSuite3) TestInTxnExecDDLFail(c *C) {
 	result.Check(testkit.Rows("1"))
 }
 
-func (s *testSuite3) TestCreateTable(c *C) {
+func (s *testSuite6) TestCreateTable(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	// Test create an exist database
@@ -173,7 +173,7 @@ func (s *testSuite3) TestCreateTable(c *C) {
 	r.Check(testkit.Rows("1000 aa"))
 }
 
-func (s *testSuite3) TestCreateView(c *C) {
+func (s *testSuite6) TestCreateView(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	//create an source table
@@ -226,7 +226,7 @@ func (s *testSuite3) TestCreateView(c *C) {
 	tk.MustExec("execute stmt")
 }
 
-func (s *testSuite3) TestCreateDropDatabase(c *C) {
+func (s *testSuite6) TestCreateDropDatabase(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("create database if not exists drop_test;")
 	tk.MustExec("drop database if exists drop_test;")
@@ -242,7 +242,7 @@ func (s *testSuite3) TestCreateDropDatabase(c *C) {
 	c.Assert(err, NotNil)
 }
 
-func (s *testSuite3) TestCreateDropTable(c *C) {
+func (s *testSuite6) TestCreateDropTable(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create table if not exists drop_test (a int)")
@@ -254,7 +254,7 @@ func (s *testSuite3) TestCreateDropTable(c *C) {
 	c.Assert(err, NotNil)
 }
 
-func (s *testSuite3) TestCreateDropView(c *C) {
+func (s *testSuite6) TestCreateDropView(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create or replace view drop_test as select 1,2")
@@ -272,7 +272,7 @@ func (s *testSuite3) TestCreateDropView(c *C) {
 	c.Assert(err.Error(), Equals, "[ddl:1347]'test.t_v' is not VIEW")
 }
 
-func (s *testSuite3) TestCreateDropIndex(c *C) {
+func (s *testSuite6) TestCreateDropIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create table if not exists drop_test (a int)")
@@ -281,7 +281,7 @@ func (s *testSuite3) TestCreateDropIndex(c *C) {
 	tk.MustExec("drop table drop_test")
 }
 
-func (s *testSuite3) TestAlterTableAddColumn(c *C) {
+func (s *testSuite6) TestAlterTableAddColumn(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create table if not exists alter_test (c1 int)")
@@ -306,7 +306,7 @@ func (s *testSuite3) TestAlterTableAddColumn(c *C) {
 	tk.MustExec("drop view alter_view")
 }
 
-func (s *testSuite3) TestAddNotNullColumnNoDefault(c *C) {
+func (s *testSuite6) TestAddNotNullColumnNoDefault(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("create table nn (c1 int)")
@@ -327,7 +327,7 @@ func (s *testSuite3) TestAddNotNullColumnNoDefault(c *C) {
 	tk.MustQuery("select * from nn").Check(testkit.Rows("1 0", "2 0", "3 0"))
 }
 
-func (s *testSuite3) TestAlterTableModifyColumn(c *C) {
+func (s *testSuite6) TestAlterTableModifyColumn(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists mc")
@@ -392,7 +392,7 @@ func (s *testSuite3) TestAlterTableModifyColumn(c *C) {
 
 }
 
-func (s *testSuite3) TestDefaultDBAfterDropCurDB(c *C) {
+func (s *testSuite6) TestDefaultDBAfterDropCurDB(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 
 	testSQL := `create database if not exists test_db CHARACTER SET latin1 COLLATE latin1_swedish_ci;`
@@ -411,7 +411,7 @@ func (s *testSuite3) TestDefaultDBAfterDropCurDB(c *C) {
 	tk.MustQuery(`select @@collation_database;`).Check(testkit.Rows("utf8_unicode_ci"))
 }
 
-func (s *testSuite3) TestRenameTable(c *C) {
+func (s *testSuite6) TestRenameTable(c *C) {
 	c.Assert(failpoint.Enable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange", `return(true)`), IsNil)
 	defer func() {
 		c.Assert(failpoint.Disable("github.com/pingcap/tidb/meta/autoid/mockAutoIDChange"), IsNil)
@@ -471,7 +471,7 @@ func (s *testSuite3) TestRenameTable(c *C) {
 	tk.MustExec("drop database rename2")
 }
 
-func (s *testSuite3) TestColumnCharsetAndCollate(c *C) {
+func (s *testSuite6) TestColumnCharsetAndCollate(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	dbName := "col_charset_collate"
 	tk.MustExec("create database " + dbName)
@@ -555,7 +555,7 @@ func (s *testSuite3) TestColumnCharsetAndCollate(c *C) {
 	tk.MustExec("drop database " + dbName)
 }
 
-func (s *testSuite3) TestTooLargeIdentifierLength(c *C) {
+func (s *testSuite6) TestTooLargeIdentifierLength(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 
 	// for database.
@@ -590,7 +590,7 @@ func (s *testSuite3) TestTooLargeIdentifierLength(c *C) {
 	c.Assert(err.Error(), Equals, fmt.Sprintf("[ddl:1059]Identifier name '%s' is too long", indexName2))
 }
 
-func (s *testSuite3) TestShardRowIDBits(c *C) {
+func (s *testSuite6) TestShardRowIDBits(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 
 	tk.MustExec("use test")
@@ -703,7 +703,7 @@ func (s *testSuite3) TestShardRowIDBits(c *C) {
 	c.Assert(autoid.ErrAutoincReadFailed.Equal(err), IsTrue, Commentf("err:%v", err))
 }
 
-func (s *testSuite3) TestMaxHandleAddIndex(c *C) {
+func (s *testSuite6) TestMaxHandleAddIndex(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 
 	tk.MustExec("use test")
@@ -720,7 +720,7 @@ func (s *testSuite3) TestMaxHandleAddIndex(c *C) {
 	tk.MustExec("admin check table t1")
 }
 
-func (s *testSuite3) TestSetDDLReorgWorkerCnt(c *C) {
+func (s *testSuite6) TestSetDDLReorgWorkerCnt(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	err := ddlutil.LoadDDLReorgVars(tk.Se)
@@ -754,7 +754,7 @@ func (s *testSuite3) TestSetDDLReorgWorkerCnt(c *C) {
 	res.Check(testkit.Rows("100"))
 }
 
-func (s *testSuite3) TestSetDDLReorgBatchSize(c *C) {
+func (s *testSuite6) TestSetDDLReorgBatchSize(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	err := ddlutil.LoadDDLReorgVars(tk.Se)
@@ -791,7 +791,7 @@ func (s *testSuite3) TestSetDDLReorgBatchSize(c *C) {
 	res.Check(testkit.Rows("1000"))
 }
 
-func (s *testSuite3) TestIllegalFunctionCall4GeneratedColumns(c *C) {
+func (s *testSuite6) TestIllegalFunctionCall4GeneratedColumns(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	// Test create an exist database
@@ -836,7 +836,7 @@ func (s *testSuite3) TestIllegalFunctionCall4GeneratedColumns(c *C) {
 	c.Assert(err.Error(), Equals, ddl.ErrGeneratedColumnFunctionIsNotAllowed.GenWithStackByArgs("cc").Error())
 }
 
-func (s *testSuite3) TestGeneratedColumnRelatedDDL(c *C) {
+func (s *testSuite6) TestGeneratedColumnRelatedDDL(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	// Test create an exist database
@@ -862,7 +862,7 @@ func (s *testSuite3) TestGeneratedColumnRelatedDDL(c *C) {
 	tk.MustExec("drop table t1;")
 }
 
-func (s *testSuite3) TestSetDDLErrorCountLimit(c *C) {
+func (s *testSuite6) TestSetDDLErrorCountLimit(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	err := ddlutil.LoadDDLVars(tk.Se)
@@ -891,7 +891,7 @@ func (s *testSuite3) TestSetDDLErrorCountLimit(c *C) {
 
 // Test issue #9205, fix the precision problem for time type default values
 // See https://github.com/pingcap/tidb/issues/9205 for details
-func (s *testSuite3) TestIssue9205(c *C) {
+func (s *testSuite6) TestIssue9205(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec(`drop table if exists t;`)
@@ -930,7 +930,7 @@ func (s *testSuite3) TestIssue9205(c *C) {
 	))
 }
 
-func (s *testSuite3) TestCheckDefaultFsp(c *C) {
+func (s *testSuite6) TestCheckDefaultFsp(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec(`drop table if exists t;`)
@@ -970,7 +970,7 @@ func (s *testSuite3) TestCheckDefaultFsp(c *C) {
 	c.Assert(err.Error(), Equals, "[ddl:1067]Invalid default value for 'tttt'")
 }
 
-func (s *testSuite3) TestTimestampMinDefaultValue(c *C) {
+func (s *testSuite6) TestTimestampMinDefaultValue(c *C) {
 	tk := testkit.NewTestKit(c, s.store)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists tdv;")