mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
related: #41675 design doc: https://github.com/milvus-io/milvus-design-docs/blob/main/design_docs/20260129-orderby.md Signed-off-by: MrPresent-Han <chun.han@gmail.com> Co-authored-by: MrPresent-Han <chun.han@gmail.com>
This commit is contained in:
co-authored by
MrPresent-Han
parent
fd2b397292
commit
0958b0f907
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -78,11 +78,249 @@ func (r *rankParams) String() string {
|
||||
return fmt.Sprintf("limit: %d, offset: %d, roundDecimal: %d", r.GetLimit(), r.GetOffset(), r.GetRoundDecimal())
|
||||
}
|
||||
|
||||
// OrderByField represents a field to order by with its direction
|
||||
// Supports JSON subfield paths like metadata["price"] and dynamic fields
|
||||
//
|
||||
// Dynamic field handling:
|
||||
// When ordering by a dynamic field (e.g., "age" stored in $meta), the field lookup uses FieldName ("$meta"),
|
||||
// NOT OutputFieldName ("age"). This is because:
|
||||
// 1. C++ segcore creates DataArray with field_id only (no field_name)
|
||||
// 2. Delegator reduction copies the empty field_name from source
|
||||
// 3. Proxy's default_limit_reducer sets FieldName from schema (field.GetName() = "$meta")
|
||||
//
|
||||
// So after requery, the returned FieldData has FieldName="$meta", and JSONPath="/age" is used
|
||||
// to extract the actual value from the JSON data for comparison.
|
||||
type OrderByField struct {
|
||||
FieldName string // Top-level field name for result lookup (e.g., "metadata" or "$meta" for dynamic fields)
|
||||
FieldID int64 // Field ID for validation
|
||||
JSONPath string // JSON Pointer format: "/price" or "/user/age" (empty for non-JSON fields)
|
||||
Ascending bool // true for ASC, false for DESC
|
||||
OutputFieldName string // Field name to request in requery (e.g., "age" for dynamic fields, "metadata" for JSON fields)
|
||||
IsDynamicField bool // true if this is a dynamic field (uses $meta extraction at QueryNode)
|
||||
}
|
||||
|
||||
type SearchInfo struct {
|
||||
planInfo *planpb.QueryInfo
|
||||
offset int64
|
||||
isIterator bool
|
||||
collectionID int64
|
||||
planInfo *planpb.QueryInfo
|
||||
offset int64
|
||||
isIterator bool
|
||||
collectionID int64
|
||||
orderByFields []OrderByField
|
||||
}
|
||||
|
||||
// parseOrderByFields parses the order_by_fields parameter from search params.
|
||||
// Format: "field1:asc,field2:desc" or "field1,field2" (default is asc)
|
||||
// Supports JSON subfield paths: metadata["price"]:asc, metadata["user"]["score"]:desc
|
||||
// Supports dynamic fields: age:desc (maps to $meta["age"])
|
||||
// Validates that fields exist in schema and are sortable types.
|
||||
func parseOrderByFields(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb.CollectionSchema) ([]OrderByField, error) {
|
||||
orderByStr, err := funcutil.GetAttrByKeyFromRepeatedKV(OrderByFieldsKey, searchParamsPair)
|
||||
if err != nil || orderByStr == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Build field name to schema map and find dynamic field
|
||||
fieldSchemaMap := make(map[string]*schemapb.FieldSchema)
|
||||
var dynamicField *schemapb.FieldSchema
|
||||
for _, field := range schema.GetFields() {
|
||||
fieldSchemaMap[field.GetName()] = field
|
||||
if field.GetIsDynamic() {
|
||||
dynamicField = field
|
||||
}
|
||||
}
|
||||
|
||||
var orderByFields []OrderByField
|
||||
pairs := strings.Split(orderByStr, ",")
|
||||
for _, pair := range pairs {
|
||||
pair = strings.TrimSpace(pair)
|
||||
if pair == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Split field spec and direction, handling brackets in field spec
|
||||
// e.g., "metadata[\"price\"]:asc" -> fieldSpec="metadata[\"price\"]", direction="asc"
|
||||
fieldSpec, direction := splitOrderByFieldAndDirection(pair)
|
||||
if fieldSpec == "" {
|
||||
return nil, fmt.Errorf("empty field name in order_by_fields")
|
||||
}
|
||||
|
||||
// Parse direction
|
||||
ascending := true // default is ascending
|
||||
if direction != "" {
|
||||
switch strings.ToLower(direction) {
|
||||
case "asc", "ascending":
|
||||
ascending = true
|
||||
case "desc", "descending":
|
||||
ascending = false
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid order direction '%s' for field '%s', expected 'asc' or 'desc'", direction, fieldSpec)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse field spec to extract field name, field ID, JSON path, and requery info
|
||||
fieldName, fieldID, jsonPath, outputFieldName, isDynamic, err := parseOrderByFieldSpec(fieldSpec, fieldSchemaMap, dynamicField, schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
orderByFields = append(orderByFields, OrderByField{
|
||||
FieldName: fieldName,
|
||||
FieldID: fieldID,
|
||||
JSONPath: jsonPath,
|
||||
Ascending: ascending,
|
||||
OutputFieldName: outputFieldName,
|
||||
IsDynamicField: isDynamic,
|
||||
})
|
||||
}
|
||||
|
||||
return orderByFields, nil
|
||||
}
|
||||
|
||||
// splitOrderByFieldAndDirection splits "fieldSpec:direction" handling brackets in fieldSpec
|
||||
// e.g., "metadata[\"price\"]:asc" -> ("metadata[\"price\"]", "asc")
|
||||
// e.g., "name:desc" -> ("name", "desc")
|
||||
// e.g., "name" -> ("name", "")
|
||||
//
|
||||
// Limitation: This simple bracket-depth tracking does not handle:
|
||||
// - Brackets inside quoted strings: metadata["key]value"] would incorrectly parse
|
||||
// - Escaped quotes inside strings: metadata["key\"with\"quotes"] may misbehave
|
||||
//
|
||||
// These edge cases are rare in practice. Field names containing unbalanced brackets
|
||||
// or complex escape sequences are not supported.
|
||||
func splitOrderByFieldAndDirection(pair string) (fieldSpec, direction string) {
|
||||
// Find the last colon that's not inside brackets
|
||||
bracketDepth := 0
|
||||
lastColonIdx := -1
|
||||
for i, ch := range pair {
|
||||
switch ch {
|
||||
case '[':
|
||||
bracketDepth++
|
||||
case ']':
|
||||
bracketDepth--
|
||||
case ':':
|
||||
if bracketDepth == 0 {
|
||||
lastColonIdx = i
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lastColonIdx == -1 {
|
||||
return strings.TrimSpace(pair), ""
|
||||
}
|
||||
return strings.TrimSpace(pair[:lastColonIdx]), strings.TrimSpace(pair[lastColonIdx+1:])
|
||||
}
|
||||
|
||||
// parseOrderByFieldSpec parses a field specification and returns field name, ID, JSON path, and requery info
|
||||
// Handles: regular fields, JSON fields with paths, and dynamic fields
|
||||
// Returns:
|
||||
// - fieldName: top-level field name ($meta for dynamic, actual name for others)
|
||||
// - fieldID: field ID
|
||||
// - jsonPath: JSON Pointer format path (empty for non-JSON fields)
|
||||
// - outputFieldName: field name to use in requery (original key for dynamic fields)
|
||||
// - isDynamicField: true if this uses dynamic field extraction at QueryNode
|
||||
func parseOrderByFieldSpec(fieldSpec string, fieldSchemaMap map[string]*schemapb.FieldSchema, dynamicField *schemapb.FieldSchema, schema *schemapb.CollectionSchema) (fieldName string, fieldID int64, jsonPath string, outputFieldName string, isDynamicField bool, err error) {
|
||||
// Check for JSON path syntax (brackets)
|
||||
hasBrackets := strings.Contains(fieldSpec, "[") && strings.Contains(fieldSpec, "]")
|
||||
|
||||
if hasBrackets {
|
||||
// Extract base field name (part before the first '[')
|
||||
baseName := strings.Split(fieldSpec, "[")[0]
|
||||
field, exists := fieldSchemaMap[baseName]
|
||||
|
||||
if exists {
|
||||
// Field exists in schema
|
||||
if field.GetIsDynamic() {
|
||||
// This is $meta["key"] - explicit dynamic field access
|
||||
// Use QueryNode-level extraction for dynamic fields
|
||||
fieldName = common.MetaFieldName
|
||||
fieldID = field.GetFieldID()
|
||||
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
|
||||
if err != nil {
|
||||
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
|
||||
}
|
||||
outputFieldName = fieldSpec // Pass full spec for dynamic field extraction
|
||||
isDynamicField = true
|
||||
} else if typeutil.IsJSONType(field.GetDataType()) {
|
||||
// Regular JSON field with path: metadata["price"]
|
||||
// Regular JSON fields don't support QueryNode-level extraction
|
||||
fieldName = baseName
|
||||
fieldID = field.GetFieldID()
|
||||
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
|
||||
if err != nil {
|
||||
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
|
||||
}
|
||||
outputFieldName = baseName // Request the whole JSON field
|
||||
isDynamicField = false
|
||||
} else {
|
||||
// Non-JSON field with brackets - not supported
|
||||
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' has brackets but is not a JSON type", fieldSpec)
|
||||
}
|
||||
} else if dynamicField != nil {
|
||||
// Unknown field name with brackets, treat as dynamic field path
|
||||
// e.g., unknown["key"] -> $meta with path /unknown/key
|
||||
fieldName = common.MetaFieldName
|
||||
fieldID = dynamicField.GetFieldID()
|
||||
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
|
||||
if err != nil {
|
||||
return "", 0, "", "", false, fmt.Errorf("invalid JSON path in order_by field '%s': %w", fieldSpec, err)
|
||||
}
|
||||
// For dynamic fields, pass the original spec so translateOutputFields can extract subfields
|
||||
outputFieldName = fieldSpec
|
||||
isDynamicField = true
|
||||
} else {
|
||||
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' not found in schema and no dynamic field available", baseName)
|
||||
}
|
||||
} else {
|
||||
// No brackets - regular field name or dynamic field key
|
||||
field, exists := fieldSchemaMap[fieldSpec]
|
||||
if exists {
|
||||
// Regular field
|
||||
fieldName = fieldSpec
|
||||
fieldID = field.GetFieldID()
|
||||
outputFieldName = fieldSpec
|
||||
isDynamicField = false
|
||||
// Validate sortable type
|
||||
if !isSortableFieldType(field.GetDataType()) {
|
||||
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' has unsortable type %s; supported types: bool, int8/16/32/64, float, double, string, varchar; for JSON fields use path syntax like field[\"key\"]",
|
||||
fieldSpec, field.GetDataType().String())
|
||||
}
|
||||
} else if dynamicField != nil {
|
||||
// Treat as dynamic field key: age -> $meta["age"]
|
||||
fieldName = common.MetaFieldName
|
||||
fieldID = dynamicField.GetFieldID()
|
||||
jsonPath, err = typeutil2.ParseAndVerifyNestedPath(fieldSpec, schema, fieldID)
|
||||
if err != nil {
|
||||
return "", 0, "", "", false, fmt.Errorf("invalid dynamic field key '%s': %w", fieldSpec, err)
|
||||
}
|
||||
// For dynamic fields, pass the original key so translateOutputFields can extract it
|
||||
outputFieldName = fieldSpec
|
||||
isDynamicField = true
|
||||
} else {
|
||||
return "", 0, "", "", false, fmt.Errorf("order_by field '%s' does not exist in collection schema", fieldSpec)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldName, fieldID, jsonPath, outputFieldName, isDynamicField, nil
|
||||
}
|
||||
|
||||
// isSortableFieldType returns true if the data type can be used for order_by
|
||||
// Note: JSON type is not directly sortable. Use JSON path syntax (e.g., metadata["price"])
|
||||
// to sort by specific JSON subfields, or use dynamic fields for schema-less data.
|
||||
func isSortableFieldType(dataType schemapb.DataType) bool {
|
||||
switch dataType {
|
||||
case schemapb.DataType_Bool,
|
||||
schemapb.DataType_Int8,
|
||||
schemapb.DataType_Int16,
|
||||
schemapb.DataType_Int32,
|
||||
schemapb.DataType_Int64,
|
||||
schemapb.DataType_Float,
|
||||
schemapb.DataType_Double,
|
||||
schemapb.DataType_String,
|
||||
schemapb.DataType_VarChar:
|
||||
return true
|
||||
default:
|
||||
// Vectors, Arrays, JSON (without path), etc. are not sortable
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func parseSearchIteratorV2Info(searchParamsPair []*commonpb.KeyValuePair, groupByFieldId int64, isIterator bool, offset int64, queryTopK *int64) (*planpb.SearchIteratorV2Info, error) {
|
||||
@@ -314,6 +552,18 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
|
||||
}
|
||||
}
|
||||
|
||||
// 8. parse order_by_fields
|
||||
orderByFields, err := parseOrderByFields(searchParamsPair, schema)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 9. validate iterator + order_by combination is not allowed
|
||||
if isIterator && len(orderByFields) > 0 {
|
||||
return nil, merr.WrapErrParameterInvalid("", "",
|
||||
"order_by is not supported when using search iterator")
|
||||
}
|
||||
|
||||
return &SearchInfo{
|
||||
planInfo: &planpb.QueryInfo{
|
||||
Topk: queryTopK,
|
||||
@@ -329,9 +579,10 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
|
||||
JsonType: jsonType,
|
||||
StrictCast: strictCast,
|
||||
},
|
||||
offset: offset,
|
||||
isIterator: isIterator,
|
||||
collectionID: collectionId,
|
||||
offset: offset,
|
||||
isIterator: isIterator,
|
||||
collectionID: collectionId,
|
||||
orderByFields: orderByFields,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -81,6 +81,7 @@ const (
|
||||
SearchIterLastBoundKey = "search_iter_last_bound"
|
||||
SearchIterIdKey = "search_iter_id"
|
||||
QueryGroupByFieldsKey = "group_by_fields"
|
||||
OrderByFieldsKey = "order_by_fields"
|
||||
|
||||
InsertTaskName = "InsertTask"
|
||||
CreateCollectionTaskName = "CreateCollectionTask"
|
||||
|
||||
@@ -99,6 +99,9 @@ type searchTask struct {
|
||||
functionScore *rerank.FunctionScore
|
||||
rankParams *rankParams
|
||||
|
||||
// Order by fields for sorting results
|
||||
orderByFields []OrderByField
|
||||
|
||||
resolvedTimezoneStr string
|
||||
|
||||
isIterator bool
|
||||
@@ -427,6 +430,17 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Parse order_by_fields from main search params
|
||||
if t.orderByFields, err = parseOrderByFields(t.request.GetSearchParams(), t.schema.CollectionSchema); err != nil {
|
||||
log.Error("parseOrderByFields failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
// order_by is not supported for hybrid search
|
||||
if len(t.orderByFields) > 0 {
|
||||
return merr.WrapErrParameterInvalidMsg("order_by is not supported for hybrid search")
|
||||
}
|
||||
|
||||
switch strings.ToLower(paramtable.Get().CommonCfg.HybridSearchRequeryPolicy.GetValue()) {
|
||||
case "always":
|
||||
t.needRequery = true
|
||||
@@ -448,7 +462,8 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error {
|
||||
t.queryInfos = make([]*planpb.QueryInfo, len(t.request.GetSubReqs()))
|
||||
queryFieldIDs := []int64{}
|
||||
for index, subReq := range t.request.GetSubReqs() {
|
||||
plan, queryInfo, offset, _, err := t.tryGeneratePlan(subReq.GetSearchParams(), subReq.GetDsl(), subReq.GetExprTemplateValues())
|
||||
// For hybrid search, order_by_fields comes from main search params, not sub-search params
|
||||
plan, queryInfo, offset, _, _, err := t.tryGeneratePlan(subReq.GetSearchParams(), subReq.GetDsl(), subReq.GetExprTemplateValues())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -647,10 +662,11 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error {
|
||||
|
||||
log := log.Ctx(ctx).With(zap.Int64("collID", t.GetCollectionID()), zap.String("collName", t.collectionName))
|
||||
|
||||
plan, queryInfo, offset, isIterator, err := t.tryGeneratePlan(t.request.GetSearchParams(), t.request.GetDsl(), t.request.GetExprTemplateValues())
|
||||
plan, queryInfo, offset, isIterator, orderByFields, err := t.tryGeneratePlan(t.request.GetSearchParams(), t.request.GetDsl(), t.request.GetExprTemplateValues())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.orderByFields = orderByFields
|
||||
|
||||
if t.request.FunctionScore != nil {
|
||||
if t.functionScore, err = rerank.NewFunctionScore(t.schema.CollectionSchema, t.request.FunctionScore, &models.ModelExtraInfo{ClusterID: paramtable.Get().CommonCfg.ClusterPrefix.GetValue(), DBName: t.request.GetDbName()}); err != nil {
|
||||
@@ -663,6 +679,11 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// order_by and function_score cannot be used together
|
||||
if len(t.orderByFields) > 0 && t.functionScore != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("order_by and function_score cannot be used together: they specify conflicting sort criteria")
|
||||
}
|
||||
|
||||
analyzer, err := funcutil.GetAttrByKeyFromRepeatedKV(AnalyzerKey, t.request.GetSearchParams())
|
||||
if err == nil {
|
||||
t.SearchRequest.AnalyzerName = analyzer
|
||||
@@ -765,31 +786,31 @@ func (t *searchTask) convertPlaceholderIfNeeded(phgBytes []byte, fieldID int64)
|
||||
return ConvertPlaceholderGroup(phgBytes, field)
|
||||
}
|
||||
|
||||
func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string, exprTemplateValues map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, error) {
|
||||
func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string, exprTemplateValues map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, []OrderByField, error) {
|
||||
annsFieldName, err := funcutil.GetAttrByKeyFromRepeatedKV(AnnsFieldKey, params)
|
||||
if err != nil || len(annsFieldName) == 0 {
|
||||
vecFields := typeutil.GetVectorFieldSchemas(t.schema.CollectionSchema)
|
||||
if len(vecFields) == 0 {
|
||||
return nil, nil, 0, false, errors.New(AnnsFieldKey + " not found in schema")
|
||||
return nil, nil, 0, false, nil, errors.New(AnnsFieldKey + " not found in schema")
|
||||
}
|
||||
|
||||
if enableMultipleVectorFields && len(vecFields) > 1 {
|
||||
return nil, nil, 0, false, errors.New("multiple anns_fields exist, please specify a anns_field in search_params")
|
||||
return nil, nil, 0, false, nil, errors.New("multiple anns_fields exist, please specify a anns_field in search_params")
|
||||
}
|
||||
annsFieldName = vecFields[0].Name
|
||||
}
|
||||
searchInfo, err := parseSearchInfo(params, t.schema.CollectionSchema, t.rankParams)
|
||||
if err != nil {
|
||||
return nil, nil, 0, false, err
|
||||
return nil, nil, 0, false, nil, err
|
||||
}
|
||||
if searchInfo.collectionID > 0 && searchInfo.collectionID != t.GetCollectionID() {
|
||||
return nil, nil, 0, false, merr.WrapErrParameterInvalidMsg("collection id:%d in the request is not consistent to that in the search context,"+
|
||||
return nil, nil, 0, false, nil, merr.WrapErrParameterInvalidMsg("collection id:%d in the request is not consistent to that in the search context,"+
|
||||
"alias or database may have been changed: %d", searchInfo.collectionID, t.GetCollectionID())
|
||||
}
|
||||
|
||||
annField := typeutil.GetFieldByName(t.schema.CollectionSchema, annsFieldName)
|
||||
if searchInfo.planInfo.GetGroupByFieldId() != -1 && annField.GetDataType() == schemapb.DataType_BinaryVector {
|
||||
return nil, nil, 0, false, errors.New("not support search_group_by operation based on binary vector column")
|
||||
return nil, nil, 0, false, nil, errors.New("not support search_group_by operation based on binary vector column")
|
||||
}
|
||||
|
||||
searchInfo.planInfo.QueryFieldId = annField.GetFieldID()
|
||||
@@ -800,13 +821,13 @@ func (t *searchTask) tryGeneratePlan(params []*commonpb.KeyValuePair, dsl string
|
||||
zap.String("dsl", dsl), // may be very large if large term passed.
|
||||
zap.String("anns field", annsFieldName), zap.Any("query info", searchInfo.planInfo))
|
||||
metrics.ProxyParseExpressionLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "search", metrics.FailLabel).Observe(float64(time.Since(start).Milliseconds()))
|
||||
return nil, nil, 0, false, merr.WrapErrParameterInvalidMsg("failed to create query plan: %v", planErr)
|
||||
return nil, nil, 0, false, nil, merr.WrapErrParameterInvalidMsg("failed to create query plan: %v", planErr)
|
||||
}
|
||||
metrics.ProxyParseExpressionLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "search", metrics.SuccessLabel).Observe(float64(time.Since(start).Milliseconds()))
|
||||
log.Ctx(t.ctx).Debug("create query plan",
|
||||
zap.String("dsl", t.request.Dsl), // may be very large if large term passed.
|
||||
zap.String("anns field", annsFieldName), zap.Any("query info", searchInfo.planInfo))
|
||||
return plan, searchInfo.planInfo, searchInfo.offset, searchInfo.isIterator, nil
|
||||
return plan, searchInfo.planInfo, searchInfo.offset, searchInfo.isIterator, searchInfo.orderByFields, nil
|
||||
}
|
||||
|
||||
func (t *searchTask) tryParsePartitionIDsFromPlan(plan *planpb.PlanNode) ([]int64, error) {
|
||||
|
||||
@@ -46,10 +46,10 @@ func TestSearchTask_PlanNamespace_AfterPreExecute(t *testing.T) {
|
||||
|
||||
// Capture plan to verify namespace by mocking tryGeneratePlan
|
||||
var capturedPlan *planpb.PlanNode
|
||||
mockey.Mock((*searchTask).tryGeneratePlan).To(func(_ *searchTask, _ []*commonpb.KeyValuePair, _ string, _ map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, error) {
|
||||
mockey.Mock((*searchTask).tryGeneratePlan).To(func(_ *searchTask, _ []*commonpb.KeyValuePair, _ string, _ map[string]*schemapb.TemplateValue) (*planpb.PlanNode, *planpb.QueryInfo, int64, bool, []OrderByField, error) {
|
||||
capturedPlan = &planpb.PlanNode{}
|
||||
qi := &planpb.QueryInfo{Topk: 10, MetricType: "L2", QueryFieldId: 101, GroupByFieldId: -1}
|
||||
return capturedPlan, qi, 0, false, nil
|
||||
return capturedPlan, qi, 0, false, nil, nil
|
||||
}).Build()
|
||||
|
||||
// Build task
|
||||
|
||||
@@ -5092,3 +5092,271 @@ func TestSearchTask_AddHighlightTask(t *testing.T) {
|
||||
require.NotNil(t, task.highlighter)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSearchTask_OrderByValidation(t *testing.T) {
|
||||
t.Run("hybrid search with order_by should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: LimitKey, Value: "10"},
|
||||
{Key: OrderByFieldsKey, Value: "price:asc"},
|
||||
},
|
||||
SubReqs: []*milvuspb.SubSearchRequest{
|
||||
{
|
||||
Dsl: "",
|
||||
PlaceholderGroup: nil,
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initAdvancedSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "order_by is not supported for hybrid search")
|
||||
})
|
||||
|
||||
t.Run("regular search with order_by and function_score should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: OrderByFieldsKey, Value: "price:asc"},
|
||||
},
|
||||
FunctionScore: &schemapb.FunctionScore{
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "decay",
|
||||
Type: schemapb.FunctionType_Rerank,
|
||||
InputFieldNames: []string{"price"},
|
||||
OutputFieldNames: []string{},
|
||||
Params: []*commonpb.KeyValuePair{
|
||||
{Key: "reranker", Value: "decay"},
|
||||
{Key: "origin", Value: "100"},
|
||||
{Key: "scale", Value: "10"},
|
||||
{Key: "offset", Value: "0"},
|
||||
{Key: "decay", Value: "0.5"},
|
||||
{Key: "function", Value: "gauss"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "order_by and function_score cannot be used together")
|
||||
})
|
||||
|
||||
t.Run("regular search with invalid order_by direction should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: OrderByFieldsKey, Value: "price:invalid"},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid order direction")
|
||||
})
|
||||
|
||||
t.Run("regular search with non-existent order_by field should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: OrderByFieldsKey, Value: "nonexistent_field:asc"},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "does not exist in collection schema")
|
||||
})
|
||||
|
||||
t.Run("regular search with unsortable order_by field should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: OrderByFieldsKey, Value: "vec:asc"},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unsortable type")
|
||||
})
|
||||
|
||||
t.Run("search with iterator and order_by should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: IteratorField, Value: "True"},
|
||||
{Key: OrderByFieldsKey, Value: "price:asc"},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "order_by is not supported when using search iterator")
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user