ddl: fix flaky test TestGC (pingcap#46932)

close pingcap#46928
mjonss · Sep 13, 2023 · 762b2df · 762b2df
1 parent 0937f1d
commit 762b2df
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 14 deletions.
diff --git a/disttask/framework/dispatcher/dispatcher_manager.go b/disttask/framework/dispatcher/dispatcher_manager.go
@@ -36,6 +36,9 @@ var (
 	checkTaskRunningInterval   = 3 * time.Second
 )
 
+// WaitTaskFinished is used to sync the test.
+var WaitTaskFinished = make(chan struct{})
+
 func (dm *Manager) getRunningTaskCnt() int {
 	dm.runningTasks.RLock()
 	defer dm.runningTasks.RUnlock()
@@ -199,13 +202,16 @@ func (dm *Manager) dispatchTaskLoop() {
 }
 
 func (dm *Manager) gcSubtaskHistoryTable() {
-	logutil.Logger(dm.ctx).Info("task table gc loop start")
 	historySubtaskTableGcInterval := defaultHistorySubtaskTableGcInterval
 	failpoint.Inject("historySubtaskTableGcInterval", func(val failpoint.Value) {
 		if seconds, ok := val.(int); ok {
 			historySubtaskTableGcInterval = time.Second * time.Duration(seconds)
 		}
+
+		<-WaitTaskFinished
 	})
+
+	logutil.Logger(dm.ctx).Info("task table gc loop start")
 	ticker := time.NewTicker(historySubtaskTableGcInterval)
 	defer ticker.Stop()
 	for {

diff --git a/disttask/framework/framework_test.go b/disttask/framework/framework_test.go
@@ -575,9 +575,8 @@ func TestGC(t *testing.T) {
 	defer ctrl.Finish()
 	RegisterTaskMeta(t, ctrl, &m, &testDispatcherExt{})
 
-	failpoint.Enable("github.com/pingcap/tidb/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)")
-	// 10s to wait all subtask completed
-	failpoint.Enable("github.com/pingcap/tidb/disttask/framework/dispatcher/historySubtaskTableGcInterval", "return(10)")
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/disttask/framework/storage/subtaskHistoryKeepSeconds", "return(1)"))
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/disttask/framework/dispatcher/historySubtaskTableGcInterval", "return(1)"))
 	defer func() {
 		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/disttask/framework/storage/subtaskHistoryKeepSeconds"))
 		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/disttask/framework/dispatcher/historySubtaskTableGcInterval"))
@@ -589,18 +588,24 @@ func TestGC(t *testing.T) {
 	mgr, err := storage.GetTaskManager()
 	require.NoError(t, err)
 
-	// check the subtask history table
-	historySubTasksCnt, err := storage.GetSubtasksFromHistoryForTest(mgr)
-	require.NoError(t, err)
-	require.Equal(t, 4, historySubTasksCnt)
+	var historySubTasksCnt int
+	require.Eventually(t, func() bool {
+		historySubTasksCnt, err = storage.GetSubtasksFromHistoryForTest(mgr)
+		if err != nil {
+			return false
+		}
+		return historySubTasksCnt == 4
+	}, 10*time.Second, 500*time.Millisecond)
 
-	// wait for gc
-	time.Sleep(10 * time.Second)
+	dispatcher.WaitTaskFinished <- struct{}{}
 
-	// check the subtask in history table removed.
-	historySubTasksCnt, err = storage.GetSubtasksFromHistoryForTest(mgr)
-	require.NoError(t, err)
-	require.Equal(t, 0, historySubTasksCnt)
+	require.Eventually(t, func() bool {
+		historySubTasksCnt, err := storage.GetSubtasksFromHistoryForTest(mgr)
+		if err != nil {
+			return false
+		}
+		return historySubTasksCnt == 0
+	}, 10*time.Second, 500*time.Millisecond)
 
 	distContext.Close()
 }