enhance: Fast close msgdispatcher target (#34803)

/kind improvement

issue: #34075

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

pkg/mq/msgdispatcher/dispatcher_test.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/milvus-io/milvus/pkg/mq/common"
 	"github.com/milvus-io/milvus/pkg/mq/msgstream"
+	"github.com/milvus-io/milvus/pkg/util/lifetime"
 )
 
 func TestDispatcher(t *testing.T) {
@@ -77,11 +78,13 @@ func TestDispatcher(t *testing.T) {
 			vchannel: "mock_vchannel_0",
 			pos:      nil,
 			ch:       output,
+			cancelCh: lifetime.NewSafeChan(),
 		})
 		d.AddTarget(&target{
 			vchannel: "mock_vchannel_1",
 			pos:      nil,
 			ch:       nil,
+			cancelCh: lifetime.NewSafeChan(),
 		})
 		num := d.TargetNum()
 		assert.Equal(t, 2, num)
@@ -110,6 +113,7 @@ func TestDispatcher(t *testing.T) {
 				vchannel: "mock_vchannel_0",
 				pos:      nil,
 				ch:       output,
+				cancelCh: lifetime.NewSafeChan(),
 			}
 			assert.Equal(t, cap(output), cap(target.ch))
 			wg := &sync.WaitGroup{}

pkg/mq/msgdispatcher/target.go

@@ -21,6 +21,10 @@ import (
 	"sync"
 	"time"
 
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/util/lifetime"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
 )
 
@@ -32,19 +36,23 @@ type target struct {
 	closeMu   sync.Mutex
 	closeOnce sync.Once
 	closed    bool
+
+	cancelCh lifetime.SafeChan
 }
 
 func newTarget(vchannel string, pos *Pos) *target {
 	t := &target{
 		vchannel: vchannel,
 		ch:       make(chan *MsgPack, paramtable.Get().MQCfg.TargetBufSize.GetAsInt()),
 		pos:      pos,
+		cancelCh: lifetime.NewSafeChan(),
 	}
 	t.closed = false
 	return t
 }
 
 func (t *target) close() {
+	t.cancelCh.Close()
 	t.closeMu.Lock()
 	defer t.closeMu.Unlock()
 	t.closeOnce.Do(func() {
@@ -61,6 +69,9 @@ func (t *target) send(pack *MsgPack) error {
 	}
 	maxTolerantLag := paramtable.Get().MQCfg.MaxTolerantLag.GetAsDuration(time.Second)
 	select {
+	case <-t.cancelCh.CloseCh():
+		log.Info("target closed", zap.String("vchannel", t.vchannel))
+		return nil
 	case <-time.After(maxTolerantLag):
 		return fmt.Errorf("send target timeout, vchannel=%s, timeout=%s", t.vchannel, maxTolerantLag)
 	case t.ch <- pack: