diff --git a/executor/index_lookup_merge_join.go b/executor/index_lookup_merge_join.go
index c0283deafada5..c3b10df60b85b 100644
--- a/executor/index_lookup_merge_join.go
+++ b/executor/index_lookup_merge_join.go
@@ -563,6 +563,9 @@ func (imw *innerMergeWorker) doMergeJoin(ctx context.Context, task *lookUpMergeJ
 	for _, outerIdx := range task.outerOrderIdx {
 		outerRow := task.outerResult.GetRow(outerIdx)
 		hasMatch, hasNull, cmpResult := false, false, initCmpResult
+		if task.outerMatch != nil && !task.outerMatch[outerIdx.ChkIdx][outerIdx.RowIdx] {
+			goto missMatch
+		}
 		// If it has iterated out all inner rows and the inner rows with same key is empty,
 		// that means the outer row needn't match any inner rows.
 		if noneInnerRowsRemain && len(task.sameKeyInnerRows) == 0 {