feat: support milvus orderby (#41675) (#47222)

related: #41675
design doc:
https://github.com/milvus-io/milvus-design-docs/blob/main/design_docs/20260129-orderby.md

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
This commit is contained in:
Chun Han
2026-02-03 15:30:25 +08:00
committed by GitHub
co-authored by MrPresent-Han
parent fd2b397292
commit 0958b0f907
7 changed files with 4050 additions and 19 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+258 -7
View File
@@ -78,11 +78,249 @@ func (r *rankParams) String() string {
return fmt.Sprintf("limit: %d, offset: %d, roundDecimal: %d", r.GetLimit(), r.GetOffset(), r.GetRoundDecimal())
}
// OrderByField represents a field to order by with its direction
// Supports JSON subfield paths like metadata["price"] and dynamic fields
//
// Dynamic field handling:
// When ordering by a dynamic field (e.g., "age" stored in $meta), the field lookup uses FieldName ("$meta"),
// NOT OutputFieldName ("age"). This is because:
// 1. C++ segcore creates DataArray with field_id only (no field_name)
// 2. Delegator reduction copies the empty field_name from source
// 3. Proxy's default_limit_reducer sets FieldName from schema (field.GetName() = "$meta")
//
// So after requery, the returned FieldData has FieldName="$meta", and JSONPath="/age" is used
// to extract the actual value from the JSON data for comparison.
type OrderByField struct {
FieldName string // Top-level field name for result lookup (e.g., "metadata" or "$meta" for dynamic fields)
FieldID int64 // Field ID for validation
JSONPath string // JSON Pointer format: "/price" or "/user/age" (empty for non-JSON fields)
Ascending bool // true for ASC, false for DESC
OutputFieldName string // Field name to request in requery (e.g., "age" for dynamic fields, "metadata" for JSON fields)
IsDynamicField bool // true if this is a dynamic field (uses $meta extraction at QueryNode)
}
type SearchInfo struct {
planInfo *planpb.QueryInfo
offset int64
isIterator bool
collectionID int64
planInfo *planpb.QueryInfo
offset int64
isIterator bool
collectionID int64
orderByFields []OrderByField
}
// parseOrderByFields parses the order_by_fields parameter from search params.
// Format: "field1:asc,field2:desc" or "field1,field2" (default is asc)
// Supports JSON subfield paths: metadata["price"]:asc, metadata["user"]["score"]:desc
// Supports dynamic fields: age:desc (maps to $meta["age"])
// Validates that fields exist in schema and are sortable types.
func parseOrderByFields(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb.CollectionSchema) ([]OrderByField, error) {
orderByStr, err := funcutil.GetAttrByKeyFromRepeatedKV(OrderByFieldsKey, searchParamsPair)
if err != nil || orderByStr == "" {
return nil, nil
}
// Build field name to schema map and find dynamic field
fieldSchemaMap := make(map[string]*schemapb.FieldSchema)
var dynamicField *schemapb.FieldSchema
for _, field := range schema.GetFields() {
fieldSchemaMap[field.GetName()] = field
if field.GetIsDynamic() {
dynamicField = field
}
}
var orderByFields []OrderByField
pairs := strings.Split(orderByStr, ",")
for _, pair := range pairs {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
// Split field spec and direction, handling brackets in field spec
// e.g., "metadata[\"price\"]:asc" -> fieldSpec="metadata[\"price\"]", direction="asc"
fieldSpec, direction := splitOrderByFieldAndDirection(pair)
if fieldSpec == "" {
return nil, fmt.Errorf("empty field name in order_by_fields")
}
// Parse direction
ascending := true // default is ascending
if direction != "" {
switch strings.ToLower(direction) {
case "asc", "ascending":
ascending = true
case "desc", "descending":
ascending = false
default:
return nil, fmt.Errorf("invalid order direction '%s' for field '%s', expected 'asc' or 'desc'", direction, fieldSpec)
}
}
// Parse field spec to extract field name, field ID, JSON path, and requery info
fieldName, fieldID, jsonPath, outputFieldName, isDynamic, err := parseOrderByFieldSpec(fieldSpec, fieldSchemaMap, dynamicField, schema)
if err != nil {
return nil, err
}
orderByFields = append(orderByFields, OrderByField{
FieldName: fieldName,
FieldID: fieldID,
JSONPath: jsonPath,
Ascending: ascending,
OutputFieldName: outputFieldName,
IsDynamicField: isDynamic,
})
}
return orderByFields, nil
}
// splitOrderByFieldAndDirection splits "fieldSpec:direction" handling brackets in fieldSpec
// e.g., "metadata[\"price\"]:asc" -> ("metadata[\"price\"]", "asc")
// e.g., "name:desc" -> ("name", "desc")
// e.g., "name" -> ("name", "")
//
// Limitation: This simple bracket-depth tracking does not handle:
// - Brackets inside quoted strings: metadata["key]value"] would incorrectly parse
// - Escaped quotes inside strings: metadata["key\"with\"quotes"] may misbehave
//
// These edge cases are rare in practice. Field names containing unbalanced brackets
// or complex escape sequences are not supported.
func splitOrderByFieldAndDirection(pair string) (fieldSpec, direction string) {
// Find the last colon that's not inside brackets
bracketDepth := 0
lastColonIdx := -1
for i, ch := range pair {
switch ch {
case '[':
bracketDepth++
case ']':
bracketDepth--
case ':':
if bracketDepth == 0 {
lastColonIdx = i
}
}
}
if lastColonIdx == -1 {
return strings.TrimSpace(pair), ""
}
return strings.TrimSpace(pair[:lastColonIdx]), strings.TrimSpace(pair[lastColonIdx+1:])
}
// parseOrderByFieldSpec parses a field specification and returns field name, ID, JSON path, and requery info
// Handles: regular fields, JSON fields with paths, and dynamic fields
// Returns:
// - fieldName: top-level field name ($meta for dynamic, actual name for others)
// - fieldID: field ID
// - jsonPath: JSON Pointer format path (empty for non-JSON fields)
// - outputFieldName: field name to use in requery (original key for dynamic fields)
// - isDynamicField: true if this uses dynamic field extraction at QueryNode
func parseOrderByFieldSpec(fieldSpec string, fieldSchemaMap map[string]*schemapb.FieldSchema, dynamicField *schemapb.FieldSchema, schema *schemapb.CollectionSchema) (fieldName string, fieldID int64, jsonPath string, outputFieldName string, isDynamicField bool, err error) {
// Check for JSON path syntax (brackets)
hasBrackets := strings.Contains(fieldSpec, "[") && strings.Contains(fieldSpec, "]")
if hasBrackets {
// Extract base field name (part before the first '[')
baseName := strings.Split(fieldSpec, "[")[0]
field, exists := fieldSchemaMap[baseName]
if exists {
// Field exists in schema
if field.GetIsDynamic() {
// This is $meta["key"] - explicit dynamic field access
// Use QueryNode-level extraction for dynamic fields
fieldName = common.MetaFieldName
fieldID = field.GetFieldID()
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
if err != nil {
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
}
outputFieldName = fieldSpec // Pass full spec for dynamic field extraction
isDynamicField = true
} else if typeutil.IsJSONType(field.GetDataType()) {
// Regular JSON field with path: metadata["price"]
// Regular JSON fields don't support QueryNode-level extraction
fieldName = baseName
fieldID = field.GetFieldID()
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
if err != nil {
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
}
outputFieldName = baseName // Request the whole JSON field
isDynamicField = false
} else {
// Non-JSON field with brackets - not supported
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' has brackets but is not a JSON type", fieldSpec)
}
} else if dynamicField != nil {
// Unknown field name with brackets, treat as dynamic field path
// e.g., unknown["key"] -> $meta with path /unknown/key
fieldName = common.MetaFieldName
fieldID = dynamicField.GetFieldID()
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
if err != nil {
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
}
// For dynamic fields, pass the original spec so translateOutputFields can extract subfields
outputFieldName = fieldSpec
isDynamicField = true
} else {
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' not found in schema and no dynamic field available", baseName)
}
} else {
// No brackets - regular field name or dynamic field key
field, exists := fieldSchemaMap[fieldSpec]
if exists {
// Regular field
fieldName = fieldSpec
fieldID = field.GetFieldID()
outputFieldName = fieldSpec
isDynamicField = false
// Validate sortable type
if !isSortableFieldType(field.GetDataType()) {
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' has unsortable type %s; supported types: bool, int8/16/32/64, float, double, string, varchar; for JSON fields use path syntax like field[\"key\"]",
fieldSpec, field.GetDataType().String())
}
} else if dynamicField != nil {
// Treat as dynamic field key: age -> $meta["age"]
fieldName = common.MetaFieldName
fieldID = dynamicField.GetFieldID()
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
if err != nil {
return "", 0, "", "", false, fmt.Errorf("invalid dynamic field key '%s': %w", fieldSpec, err)
}
// For dynamic fields, pass the original key so translateOutputFields can extract it
outputFieldName = fieldSpec
isDynamicField = true
} else {
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' does not exist in collection schema", fieldSpec)
}
}
return fieldName, fieldID, jsonPath, outputFieldName, isDynamicField, nil
}
// isSortableFieldType returns true if the data type can be used for order_by
// Note: JSON type is not directly sortable. Use JSON path syntax (e.g., metadata["price"])
// to sort by specific JSON subfields, or use dynamic fields for schema-less data.
func isSortableFieldType(dataType schemapb.DataType) bool {
switch dataType {
case schemapb.DataType_Bool,
schemapb.DataType_Int8,
schemapb.DataType_Int16,
schemapb.DataType_Int32,
schemapb.DataType_Int64,
schemapb.DataType_Float,
schemapb.DataType_Double,
schemapb.DataType_String,
schemapb.DataType_VarChar:
return true
default:
// Vectors, Arrays, JSON (without path), etc. are not sortable
return false
}
}
func parseSearchIteratorV2Info(searchParamsPair []*commonpb.KeyValuePair, groupByFieldId int64, isIterator bool, offset int64, queryTopK *int64) (*planpb.SearchIteratorV2Info, error) {
@@ -314,6 +552,18 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
}
}
// 8. parse order_by_fields
orderByFields, err := parseOrderByFields(searchParamsPair, schema)
if err != nil {
return nil, err
}
// 9. validate iterator + order_by combination is not allowed
if isIterator && len(orderByFields) > 0 {
return nil, merr.WrapErrParameterInvalid("", "",
"order_by is not supported when using search iterator")
}
return &SearchInfo{
planInfo: &planpb.QueryInfo{
Topk: queryTopK,
@@ -329,9 +579,10 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
JsonType: jsonType,
StrictCast: strictCast,
},
offset: offset,
isIterator: isIterator,
collectionID: collectionId,
offset: offset,
isIterator: isIterator,
collectionID: collectionId,
orderByFields: orderByFields,
}, nil
}
+1
View File
@@ -81,6 +81,7 @@ const (
SearchIterLastBoundKey = "search_iter_last_bound"
SearchIterIdKey = "search_iter_id"
QueryGroupByFieldsKey = "group_by_fields"
OrderByFieldsKey = "order_by_fields"
InsertTaskName = "InsertTask"
CreateCollectionTaskName = "CreateCollectionTask"
+31 -10
View File
@@ -99,6 +99,9 @@ type searchTask struct {
functionScore *rerank.FunctionScore
rankParams *rankParams
// Order by fields for sorting results
orderByFields []OrderByField
resolvedTimezoneStr string
isIterator bool
@@ -427,6 +430,17 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error {
return err
}
// Parse order_by_fields from main search params
if t.orderByFields, err = parseOrderByFields(t.request.GetSearchParams(), t.schema.CollectionSchema); err != nil {
log.Error("parseOrderByFields failed", zap.Error(err))
return err
}
// order_by is not supported for hybrid search
if len(t.orderByFields) > 0 {
return merr.WrapErrParameterInvalidMsg("order_by is not supported for hybrid search")
}
switch strings.ToLower(paramtable.Get().CommonCfg.HybridSearchRequeryPolicy.GetValue()) {
case "always":
t.needRequery = true
@@ -448,7 +462,8 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error {
t.queryInfos = make([]*planpb.QueryInfo, len(t.request.GetSubReqs()))
queryFieldIDs := []int64{}
for index, subReq := range t.request.GetSubReqs() {
plan, queryInfo, offset, _, err := t.tryGeneratePlan(subReq.GetSearchParams(), subReq.GetDsl(), subReq.GetExprTemplateValues())
// For hybrid search, order_by_fields comes from main search params, not sub-search params
plan, queryInfo, offset, _, _, err := t.tryGeneratePlan(subReq.GetSearchParams(), subReq.GetDsl(), subReq.GetExprTemplateValues())
if err != nil {
return err
}
@@ -647,10 +662,11 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error {
log := log.Ctx(ctx).With(zap.Int64("collID", t.GetCollectionID()), zap.String("collName", t.collectionName))
plan, queryInfo, offset, isIterator, err := t.tryGeneratePlan(t.request.GetSearchParams(), t.request.GetDsl(), t.request.GetExprTemplateValues())
plan, queryInfo, offset, isIterator, orderByFields, err := t.tryGeneratePlan(t.request.GetSearchParams(), t.request.GetDsl(), t.request.GetExprTemplateValues())
if err != nil {
return err
}
t.orderByFields = orderByFields
if t.request.FunctionScore != nil {
if t.functionScore, err = rerank.NewFunctionScore(t.schema.CollectionSchema, t.request.FunctionScore, &models.ModelExtraInfo{ClusterID: paramtable.Get().CommonCfg.ClusterPrefix.GetValue(), DBName: t.request.GetDbName()}); err != nil {
@@ -663,6 +679,11 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error {
}
}
// order_by and function_score cannot be used together
if len(t.orderByFields) > 0 && t.functionScore != nil {
return merr.WrapErrParameterInvalidMsg("order_by and function_score cannot be used together: they specify conflicting sort criteria")
}
analyzer, err := funcutil.GetAttrByKeyFromRepeatedKV(AnalyzerKey, t.request.GetSearchParams())
if err == nil {
t.SearchRequest.AnalyzerName = analyzer
@@ -765,31 +786,31 @@ func (t *searchTask) convertPlaceholderIfNeeded(phgBytes []byte, fieldID int64)
return ConvertPlaceholderGroup(phgBytes, field)
}
func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string, exprTemplateValues map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, error) {
func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string, exprTemplateValues map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, []OrderByField, error) {
annsFieldName, err := funcutil.GetAttrByKeyFromRepeatedKV(AnnsFieldKey, params)
if err != nil || len(annsFieldName) == 0 {
vecFields := typeutil.GetVectorFieldSchemas(t.schema.CollectionSchema)
if len(vecFields) == 0 {
return nil, nil, 0, false, errors.New(AnnsFieldKey + " not found in schema")
return nil, nil, 0, false, nil, errors.New(AnnsFieldKey + " not found in schema")
}
if enableMultipleVectorFields && len(vecFields) > 1 {
return nil, nil, 0, false, errors.New("multiple anns_fields exist, please specify a anns_field in search_params")
return nil, nil, 0, false, nil, errors.New("multiple anns_fields exist, please specify a anns_field in search_params")
}
annsFieldName = vecFields[0].Name
}
searchInfo, err := parseSearchInfo(params, t.schema.CollectionSchema, t.rankParams)
if err != nil {
return nil, nil, 0, false, err
return nil, nil, 0, false, nil, err
}
if searchInfo.collectionID > 0 && searchInfo.collectionID != t.GetCollectionID() {
return nil, nil, 0, false, merr.WrapErrParameterInvalidMsg("collection id:%d in the request is not consistent to that in the search context,"+
return nil, nil, 0, false, nil, merr.WrapErrParameterInvalidMsg("collection id:%d in the request is not consistent to that in the search context,"+
"alias or database may have been changed: %d", searchInfo.collectionID, t.GetCollectionID())
}
annField := typeutil.GetFieldByName(t.schema.CollectionSchema, annsFieldName)
if searchInfo.planInfo.GetGroupByFieldId() != -1 && annField.GetDataType() == schemapb.DataType_BinaryVector {
return nil, nil, 0, false, errors.New("not support search_group_by operation based on binary vector column")
return nil, nil, 0, false, nil, errors.New("not support search_group_by operation based on binary vector column")
}
searchInfo.planInfo.QueryFieldId = annField.GetFieldID()
@@ -800,13 +821,13 @@ func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string
zap.String("dsl", dsl), // may be very large if large term passed.
zap.String("anns field", annsFieldName), zap.Any("query info", searchInfo.planInfo))
metrics.ProxyParseExpressionLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "search", metrics.FailLabel).Observe(float64(time.Since(start).Milliseconds()))
return nil, nil, 0, false, merr.WrapErrParameterInvalidMsg("failed to create query plan: %v", planErr)
return nil, nil, 0, false, nil, merr.WrapErrParameterInvalidMsg("failed to create query plan: %v", planErr)
}
metrics.ProxyParseExpressionLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "search", metrics.SuccessLabel).Observe(float64(time.Since(start).Milliseconds()))
log.Ctx(t.ctx).Debug("create query plan",
zap.String("dsl", t.request.Dsl), // may be very large if large term passed.
zap.String("anns field", annsFieldName), zap.Any("query info", searchInfo.planInfo))
return plan, searchInfo.planInfo, searchInfo.offset, searchInfo.isIterator, nil
return plan, searchInfo.planInfo, searchInfo.offset, searchInfo.isIterator, searchInfo.orderByFields, nil
}
func (t *searchTask) tryParsePartitionIDsFromPlan(plan *planpb.PlanNode) ([]int64, error) {
+2 -2
View File
@@ -46,10 +46,10 @@ func TestSearchTask_PlanNamespace_AfterPreExecute(t *testing.T) {
// Capture plan to verify namespace by mocking tryGeneratePlan
var capturedPlan *planpb.PlanNode
mockey.Mock((*searchTask).tryGeneratePlan).To(func(_ *searchTask, _ []*commonpb.KeyValuePair, _ string, _ map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, error) {
mockey.Mock((*searchTask).tryGeneratePlan).To(func(_ *searchTask, _ []*commonpb.KeyValuePair, _ string, _ map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, []OrderByField, error) {
capturedPlan = &planpb.PlanNode{}
qi := &planpb.QueryInfo{Topk: 10, MetricType: "L2", QueryFieldId: 101, GroupByFieldId: -1}
return capturedPlan, qi, 0, false, nil
return capturedPlan, qi, 0, false, nil, nil
}).Build()
// Build task
+268
View File
@@ -5092,3 +5092,271 @@ func TestSearchTask_AddHighlightTask(t *testing.T) {
require.NotNil(t, task.highlighter)
})
}
func TestSearchTask_OrderByValidation(t *testing.T) {
t.Run("hybrid search with order_by should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: LimitKey, Value: "10"},
{Key: OrderByFieldsKey, Value: "price:asc"},
},
SubReqs: []*milvuspb.SubSearchRequest{
{
Dsl: "",
PlaceholderGroup: nil,
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
},
},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initAdvancedSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "order_by is not supported for hybrid search")
})
t.Run("regular search with order_by and function_score should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: OrderByFieldsKey, Value: "price:asc"},
},
FunctionScore: &schemapb.FunctionScore{
Functions: []*schemapb.FunctionSchema{
{
Name: "decay",
Type: schemapb.FunctionType_Rerank,
InputFieldNames: []string{"price"},
OutputFieldNames: []string{},
Params: []*commonpb.KeyValuePair{
{Key: "reranker", Value: "decay"},
{Key: "origin", Value: "100"},
{Key: "scale", Value: "10"},
{Key: "offset", Value: "0"},
{Key: "decay", Value: "0.5"},
{Key: "function", Value: "gauss"},
},
},
},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "order_by and function_score cannot be used together")
})
t.Run("regular search with invalid order_by direction should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: OrderByFieldsKey, Value: "price:invalid"},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid order direction")
})
t.Run("regular search with non-existent order_by field should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: OrderByFieldsKey, Value: "nonexistent_field:asc"},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "does not exist in collection schema")
})
t.Run("regular search with unsortable order_by field should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: OrderByFieldsKey, Value: "vec:asc"},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "unsortable type")
})
t.Run("search with iterator and order_by should fail", func(t *testing.T) {
ctx := context.Background()
qt := &searchTask{
ctx: ctx,
SearchRequest: &internalpb.SearchRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Search,
Timestamp: uint64(time.Now().UnixNano()),
},
},
request: &milvuspb.SearchRequest{
CollectionName: "test_collection",
SearchParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: metric.L2},
{Key: ParamsKey, Value: `{"nprobe": 10}`},
{Key: AnnsFieldKey, Value: "vec"},
{Key: TopKKey, Value: "10"},
{Key: RoundDecimalKey, Value: "-1"},
{Key: IteratorField, Value: "True"},
{Key: OrderByFieldsKey, Value: "price:asc"},
},
},
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
},
},
},
}
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
err := qt.initSearchRequest(ctx)
assert.Error(t, err)
assert.Contains(t, err.Error(), "order_by is not supported when using search iterator")
})
}