lightning: add retryable error on checksum (#37691)

close #37690
pingcap · Sep 8, 2022 · ec9e43e · ec9e43e
1 parent 7eb7ca9
commit ec9e43e
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 9 deletions.
diff --git a/br/pkg/lightning/common/retry.go b/br/pkg/lightning/common/retry.go
@@ -25,8 +25,8 @@ import (
 
 	"github.com/go-sql-driver/mysql"
 	"github.com/pingcap/errors"
-	berrors "github.com/pingcap/tidb/br/pkg/errors"
 	tmysql "github.com/pingcap/tidb/errno"
+	drivererr "github.com/pingcap/tidb/store/driver/error"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 )
@@ -67,6 +67,26 @@ func IsRetryableError(err error) bool {
 	return true
 }
 
+var retryableErrorIDs = map[errors.ErrorID]struct{}{
+	ErrKVEpochNotMatch.ID():  {},
+	ErrKVNotLeader.ID():      {},
+	ErrKVRegionNotFound.ID(): {},
+	// common.ErrKVServerIsBusy is a little duplication with tmysql.ErrTiKVServerBusy
+	// it's because the response of sst.ingest gives us a sst.IngestResponse which doesn't contain error code,
+	// so we have to transform it into a defined code
+	ErrKVServerIsBusy.ID():        {},
+	ErrKVReadIndexNotReady.ID():   {},
+	ErrKVIngestFailed.ID():        {},
+	ErrKVRaftProposalDropped.ID(): {},
+	// during checksum coprocessor will transform error into driver error in handleCopResponse using ToTiDBErr
+	// met ErrRegionUnavailable on free-tier import during checksum, others hasn't met yet
+	drivererr.ErrRegionUnavailable.ID(): {},
+	drivererr.ErrTiKVStaleCommand.ID():  {},
+	drivererr.ErrTiKVServerTimeout.ID(): {},
+	drivererr.ErrTiKVServerBusy.ID():    {},
+	drivererr.ErrUnknown.ID():           {},
+}
+
 func isSingleRetryableError(err error) bool {
 	err = errors.Cause(err)
 
@@ -101,14 +121,7 @@ func isSingleRetryableError(err error) bool {
 			return false
 		}
 	case *errors.Error:
-		switch {
-		case berrors.Is(nerr, ErrKVEpochNotMatch), berrors.Is(nerr, ErrKVNotLeader),
-			berrors.Is(nerr, ErrKVRegionNotFound), berrors.Is(nerr, ErrKVServerIsBusy),
-			berrors.Is(nerr, ErrKVReadIndexNotReady), berrors.Is(nerr, ErrKVIngestFailed),
-			berrors.Is(nerr, ErrKVRaftProposalDropped):
-			// common.ErrKVServerIsBusy is a little duplication with tmysql.ErrTiKVServerBusy
-			// it's because the response of sst.ingest gives us a sst.IngestResponse which doesn't contain error code,
-			// so we have to transform it into a defined code
+		if _, ok := retryableErrorIDs[nerr.ID()]; ok {
 			return true
 		}
 		return false

diff --git a/br/pkg/lightning/common/retry_test.go b/br/pkg/lightning/common/retry_test.go
@@ -24,6 +24,7 @@ import (
 	"github.com/go-sql-driver/mysql"
 	"github.com/pingcap/errors"
 	tmysql "github.com/pingcap/tidb/errno"
+	drivererr "github.com/pingcap/tidb/store/driver/error"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/multierr"
 	"google.golang.org/grpc/codes"
@@ -54,6 +55,13 @@ func TestIsRetryableError(t *testing.T) {
 	require.True(t, IsRetryableError(ErrKVIngestFailed.GenWithStack("test")))
 	require.True(t, IsRetryableError(ErrKVRaftProposalDropped.GenWithStack("test")))
 
+	// tidb error
+	require.True(t, IsRetryableError(drivererr.ErrRegionUnavailable))
+	require.True(t, IsRetryableError(drivererr.ErrTiKVStaleCommand))
+	require.True(t, IsRetryableError(drivererr.ErrTiKVServerTimeout))
+	require.True(t, IsRetryableError(drivererr.ErrTiKVServerBusy))
+	require.True(t, IsRetryableError(drivererr.ErrUnknown))
+
 	// net: connection refused
 	_, err := net.Dial("tcp", "localhost:65533")
 	require.Error(t, err)