supporing hybrid search group_by(#35982)

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
milvus-io · Sep 5, 2024 · 3249e4d · 3249e4d
1 parent 6eb8b3f
commit 3249e4d
Show file tree

Hide file tree

Showing 20 changed files with 929 additions and 312 deletions.
diff --git a/internal/datacoord/index_meta_test.go b/internal/datacoord/index_meta_test.go
@@ -170,7 +170,8 @@ func TestMeta_ScalarAutoIndex(t *testing.T) {
 			{
 				Key:   common.IndexTypeKey,
 				Value: "HYBRID",
-			}},
+			},
+		},
 		Timestamp:       0,
 		IsAutoIndex:     true,
 		UserIndexParams: userIndexParams,
@@ -205,7 +206,6 @@ func TestMeta_ScalarAutoIndex(t *testing.T) {
 		assert.Equal(t, newIndexParams[0].Key, common.IndexTypeKey)
 		assert.Equal(t, newIndexParams[0].Value, "INVERTED")
 	})
-
 }
 
 func TestMeta_CanCreateIndex(t *testing.T) {

diff --git a/internal/datacoord/task_scheduler_test.go b/internal/datacoord/task_scheduler_test.go
@@ -769,7 +769,7 @@ func (s *taskSchedulerSuite) scheduler(handler Handler) {
 		return nil
 	})
 	catalog.EXPECT().AlterSegmentIndexes(mock.Anything, mock.Anything).Return(nil)
-	//catalog.EXPECT().SaveStatsTask(mock.Anything, mock.Anything).Return(nil)
+	// catalog.EXPECT().SaveStatsTask(mock.Anything, mock.Anything).Return(nil)
 
 	in := mocks.NewMockIndexNodeClient(s.T())
 	in.EXPECT().CreateJobV2(mock.Anything, mock.Anything).Return(merr.Success(), nil)

diff --git a/internal/datanode/compaction/merge_sort.go b/internal/datanode/compaction/merge_sort.go
@@ -63,7 +63,7 @@ func mergeSortMultipleSegments(ctx context.Context,
 		return nil, err
 	}
 
-	//SegmentDeserializeReaderTest(binlogPaths, t.binlogIO, writer.GetPkID())
+	// SegmentDeserializeReaderTest(binlogPaths, t.binlogIO, writer.GetPkID())
 	segmentReaders := make([]*SegmentDeserializeReader, len(binlogs))
 	for i, s := range binlogs {
 		var binlogBatchCount int

diff --git a/internal/datanode/compaction/priority_queue_test.go b/internal/datanode/compaction/priority_queue_test.go
@@ -4,8 +4,9 @@ import (
 	"container/heap"
 	"testing"
 
-	"github.com/milvus-io/milvus/internal/storage"
 	"github.com/stretchr/testify/suite"
+
+	"github.com/milvus-io/milvus/internal/storage"
 )
 
 type PriorityQueueSuite struct {
@@ -118,7 +119,6 @@ func (s *PriorityQueueSuite) PriorityQueueMergeSort() {
 			heap.Push(&pq, next)
 		}
 	}
-
 }
 
 func TestNewPriorityQueueSuite(t *testing.T) {

diff --git a/internal/datanode/compaction/segment_writer.go b/internal/datanode/compaction/segment_writer.go
@@ -8,6 +8,7 @@ import (
 	"context"
 	"math"
 
+	"github.com/samber/lo"
 	"go.uber.org/atomic"
 	"go.uber.org/zap"
 
@@ -21,7 +22,6 @@ import (
 	"github.com/milvus-io/milvus/pkg/log"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
-	"github.com/samber/lo"
 )
 
 // Not concurrent safe.

diff --git a/internal/proto/internal.proto b/internal/proto/internal.proto
@@ -94,11 +94,8 @@ message SubSearchRequest {
   int64  topk = 7;
   int64 offset = 8;
   string metricType = 9;
-}
-
-message ExtraSearchParam {
-  int64 group_by_field_id = 1;
-  int64 group_size = 2;
+  int64 group_by_field_id = 10;
+  int64 group_size = 11;
 }
 
 message SearchRequest {
@@ -125,7 +122,8 @@ message SearchRequest {
   bool   is_advanced = 20;
   int64 offset = 21;
   common.ConsistencyLevel consistency_level = 22;
-  ExtraSearchParam extra_search_param = 23;
+  int64 group_by_field_id = 23;
+  int64 group_size = 24;
 }
 
 message SubSearchResults {