mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
fix: apply search order by offset after scalar sort (#50143)
issue: https://github.com/milvus-io/milvus/issues/49879 ## Summary - Keep `limit + offset` candidates during reduce for common search with `order_by_fields`. - Apply final offset/limit after scalar order_by sorting in the search pipeline. - Add targeted proxy unit tests for row-aligned reorder, group pagination, string IDs, and vector slicing. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: MrPresent-Han <chun.han@gmail.com> Co-authored-by: MrPresent-Han <chun.han@gmail.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
co-authored by
MrPresent-Han
Claude Opus 4.7
parent
8dad593ba5
commit
a7fae63d78
+204
-104
@@ -179,17 +179,23 @@ type searchReduceOperator struct {
|
||||
isSearchAggregation bool
|
||||
}
|
||||
|
||||
func newSearchReduceOperator(t *searchTask, _ map[string]any) (operator, error) {
|
||||
const reduceOffsetParamKey = "reduce_offset"
|
||||
|
||||
func newSearchReduceOperator(t *searchTask, params map[string]any) (operator, error) {
|
||||
pkField, err := t.schema.GetPkField()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
offset := t.GetOffset()
|
||||
if v, ok := params[reduceOffsetParamKey].(int64); ok {
|
||||
offset = v
|
||||
}
|
||||
return &searchReduceOperator{
|
||||
traceCtx: t.TraceCtx(),
|
||||
primaryFieldSchema: pkField,
|
||||
nq: t.GetNq(),
|
||||
topK: t.GetTopk(),
|
||||
offset: t.GetOffset(),
|
||||
offset: offset,
|
||||
collectionID: t.GetCollectionID(),
|
||||
partitionIDs: t.GetPartitionIDs(),
|
||||
queryInfos: t.queryInfos,
|
||||
@@ -1784,6 +1790,8 @@ type orderByOperator struct {
|
||||
orderByFields []OrderByField
|
||||
groupByFieldId int64
|
||||
groupSize int64
|
||||
limit int64
|
||||
offset int64
|
||||
}
|
||||
|
||||
func newOrderByOperator(t *searchTask, _ map[string]any) (operator, error) {
|
||||
@@ -1801,6 +1809,8 @@ func newOrderByOperator(t *searchTask, _ map[string]any) (operator, error) {
|
||||
orderByFields: t.orderByFields,
|
||||
groupByFieldId: groupByFieldId,
|
||||
groupSize: groupSize,
|
||||
limit: t.GetTopk() - t.GetOffset(),
|
||||
offset: t.GetOffset(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -1809,12 +1819,13 @@ func (op *orderByOperator) run(ctx context.Context, span trace.Span, inputs ...a
|
||||
defer sp.End()
|
||||
|
||||
result := inputs[0].(*milvuspb.SearchResults)
|
||||
resultData := result.GetResults()
|
||||
|
||||
if len(op.orderByFields) == 0 {
|
||||
return []any{result}, nil
|
||||
}
|
||||
|
||||
numResults := len(result.GetResults().GetScores())
|
||||
numResults := len(resultData.GetScores())
|
||||
if numResults == 0 {
|
||||
return []any{result}, nil
|
||||
}
|
||||
@@ -1826,7 +1837,7 @@ func (op *orderByOperator) run(ctx context.Context, span trace.Span, inputs ...a
|
||||
|
||||
// Get per-query result counts from Topks
|
||||
// Topks[i] contains the number of results for the i-th query
|
||||
topks := result.GetResults().GetTopks()
|
||||
topks := resultData.GetTopks()
|
||||
|
||||
// Validate that sum(Topks) matches numResults to prevent slice bounds panic
|
||||
var sumTopks int64
|
||||
@@ -1843,50 +1854,76 @@ func (op *orderByOperator) run(ctx context.Context, span trace.Span, inputs ...a
|
||||
indices[i] = i
|
||||
}
|
||||
|
||||
// Sort results per query - each query's results should be sorted independently
|
||||
// Results are stored sequentially: [q1_results..., q2_results..., ...]
|
||||
if len(topks) <= 1 {
|
||||
// Single query (nq=1) or no topks info - sort all results together
|
||||
if err := op.sortQueryResults(result, indices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
// Multiple queries (nq>1) - sort each query's results independently
|
||||
offset := 0
|
||||
for _, topk := range topks {
|
||||
if topk > 0 {
|
||||
// Extract the slice of indices for this query
|
||||
queryIndices := indices[offset : offset+int(topk)]
|
||||
if err := op.sortQueryResults(result, queryIndices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
selectedIndices := make([]int, 0, numResults)
|
||||
newTopks := make([]int64, 0, len(topks))
|
||||
queryOffset := 0
|
||||
for _, topk := range topks {
|
||||
queryIndices := indices[queryOffset : queryOffset+int(topk)]
|
||||
if topk > 0 {
|
||||
var err error
|
||||
queryIndices, err = op.sortQueryResults(result, queryIndices)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
offset += int(topk)
|
||||
}
|
||||
selectedIndices = append(selectedIndices, queryIndices...)
|
||||
newTopks = append(newTopks, int64(len(queryIndices)))
|
||||
queryOffset += int(topk)
|
||||
}
|
||||
|
||||
// Reorder results based on sorted indices
|
||||
if err := op.reorderResults(result, indices); err != nil {
|
||||
if err := op.reorderResults(result, selectedIndices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var maxTopK int64
|
||||
for _, topk := range newTopks {
|
||||
if topk > maxTopK {
|
||||
maxTopK = topk
|
||||
}
|
||||
}
|
||||
resultData.Topks = newTopks
|
||||
resultData.TopK = maxTopK
|
||||
|
||||
return []any{result}, nil
|
||||
}
|
||||
|
||||
func getOrderByGroupByFieldValue(resultData *schemapb.SearchResultData) *schemapb.FieldData {
|
||||
if resultData == nil {
|
||||
return nil
|
||||
}
|
||||
if gbvs := resultData.GetGroupByFieldValues(); len(gbvs) > 0 {
|
||||
return gbvs[0]
|
||||
}
|
||||
return resultData.GetGroupByFieldValue()
|
||||
}
|
||||
|
||||
func paginateSortedRows(indices []int, offset, limit int64) []int {
|
||||
start := int(offset)
|
||||
if start > len(indices) {
|
||||
start = len(indices)
|
||||
}
|
||||
end := len(indices)
|
||||
if limit > 0 && start+int(limit) < end {
|
||||
end = start + int(limit)
|
||||
}
|
||||
return indices[start:end]
|
||||
}
|
||||
|
||||
// sortQueryResults sorts the given indices slice based on order_by fields.
|
||||
// This handles both regular and group-by cases for a single query's results.
|
||||
// Returns an error if comparison fails (e.g., index out of bounds).
|
||||
func (op *orderByOperator) sortQueryResults(result *milvuspb.SearchResults, indices []int) error {
|
||||
// Returns the sorted and paginated indices, or an error if comparison fails.
|
||||
func (op *orderByOperator) sortQueryResults(result *milvuspb.SearchResults, indices []int) ([]int, error) {
|
||||
if len(indices) == 0 {
|
||||
return nil
|
||||
return indices, nil
|
||||
}
|
||||
|
||||
if op.groupByFieldId >= 0 && op.groupSize > 0 {
|
||||
// Group-by case: sort groups by the first row's value in each group
|
||||
return op.sortGroupsByOrderByFields(result, indices)
|
||||
}
|
||||
// Regular case: sort all results
|
||||
return op.sortResultsByOrderByFields(result, indices)
|
||||
if err := op.sortResultsByOrderByFields(result, indices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return paginateSortedRows(indices, op.offset, op.limit), nil
|
||||
}
|
||||
|
||||
// validateOrderByFields checks that all order_by fields exist in the result
|
||||
@@ -2187,23 +2224,19 @@ func (op *orderByOperator) sortResultsByOrderByFields(result *milvuspb.SearchRes
|
||||
// This invariant is guaranteed by the upstream search/reduce pipeline which groups results before
|
||||
// returning them. If this invariant is violated (e.g., [A, B, A] instead of [A, A, B]), the function
|
||||
// will treat non-contiguous occurrences as separate groups and produce incorrect ordering.
|
||||
func (op *orderByOperator) sortGroupsByOrderByFields(result *milvuspb.SearchResults, indices []int) error {
|
||||
func (op *orderByOperator) sortGroupsByOrderByFields(result *milvuspb.SearchResults, indices []int) ([]int, error) {
|
||||
numResults := len(indices)
|
||||
if numResults == 0 {
|
||||
return nil
|
||||
return indices, nil
|
||||
}
|
||||
|
||||
// All internal pipeline stages emit to the plural channel. The task
|
||||
// output boundary downgrades to singular for legacy-wire SDK clients,
|
||||
// which runs after orderBy, so this reader sees plural only. orderBy
|
||||
// inspects column 0 because orderBy + multi-field composite key is not
|
||||
// a pipeline combination constructed today.
|
||||
gbvs := result.GetResults().GetGroupByFieldValues()
|
||||
if len(gbvs) == 0 {
|
||||
// No group by field value, fall back to regular sort
|
||||
return op.sortResultsByOrderByFields(result, indices)
|
||||
groupByValue := getOrderByGroupByFieldValue(result.GetResults())
|
||||
if groupByValue == nil {
|
||||
if err := op.sortResultsByOrderByFields(result, indices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return paginateSortedRows(indices, op.offset, op.limit), nil
|
||||
}
|
||||
groupByValue := gbvs[0]
|
||||
|
||||
// Find group boundaries by detecting when GroupByFieldValue changes
|
||||
// Each group is represented as [startLocalIdx, endLocalIdx) - indices into the 'indices' slice
|
||||
@@ -2259,19 +2292,20 @@ func (op *orderByOperator) sortGroupsByOrderByFields(result *milvuspb.SearchResu
|
||||
return false
|
||||
})
|
||||
if sortErr != nil {
|
||||
return sortErr
|
||||
return nil, sortErr
|
||||
}
|
||||
|
||||
// Rebuild indices array based on sorted groups
|
||||
// Collect actual data indices in the new sorted order
|
||||
newIndices := make([]int, 0, numResults)
|
||||
for _, g := range groups {
|
||||
for localIdx := g.start; localIdx < g.end; localIdx++ {
|
||||
newIndices = append(newIndices, indices[localIdx])
|
||||
selected := make([]int, 0, numResults)
|
||||
for groupIdx, g := range groups {
|
||||
if int64(groupIdx) < op.offset {
|
||||
continue
|
||||
}
|
||||
if op.limit > 0 && int64(groupIdx) >= op.offset+op.limit {
|
||||
break
|
||||
}
|
||||
selected = append(selected, indices[g.start:g.end]...)
|
||||
}
|
||||
copy(indices, newIndices)
|
||||
return nil
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
// isSameGroupByValue checks if two indices have the same group by value.
|
||||
@@ -2406,40 +2440,96 @@ func (op *orderByOperator) reorderResults(result *milvuspb.SearchResults, indice
|
||||
results := result.GetResults()
|
||||
n := len(indices)
|
||||
|
||||
// Reorder IDs
|
||||
var intIDs *schemapb.LongArray
|
||||
var strIDs *schemapb.StringArray
|
||||
var newIntIDs []int64
|
||||
var newStrIDs []string
|
||||
if ids := results.GetIds(); ids != nil {
|
||||
if intIds := ids.GetIntId(); intIds != nil {
|
||||
newData := make([]int64, n)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= len(intIds.Data) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for int IDs (len=%d)", oldIdx, len(intIds.Data))
|
||||
}
|
||||
newData[newIdx] = intIds.Data[oldIdx]
|
||||
}
|
||||
intIds.Data = newData
|
||||
} else if strIds := ids.GetStrId(); strIds != nil {
|
||||
newData := make([]string, n)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= len(strIds.Data) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for string IDs (len=%d)", oldIdx, len(strIds.Data))
|
||||
}
|
||||
newData[newIdx] = strIds.Data[oldIdx]
|
||||
}
|
||||
strIds.Data = newData
|
||||
if intIDData := ids.GetIntId(); intIDData != nil {
|
||||
intIDs = intIDData
|
||||
newIntIDs = make([]int64, n)
|
||||
} else if strIDData := ids.GetStrId(); strIDData != nil {
|
||||
strIDs = strIDData
|
||||
newStrIDs = make([]string, n)
|
||||
}
|
||||
}
|
||||
|
||||
// Reorder scores
|
||||
var newScores []float32
|
||||
if len(results.Scores) > 0 {
|
||||
newScores := make([]float32, n)
|
||||
for newIdx, oldIdx := range indices {
|
||||
newScores = make([]float32, n)
|
||||
}
|
||||
var newDistances []float32
|
||||
if len(results.Distances) > 0 {
|
||||
newDistances = make([]float32, n)
|
||||
}
|
||||
var newRecalls []float32
|
||||
if len(results.Recalls) > 0 {
|
||||
newRecalls = make([]float32, n)
|
||||
}
|
||||
var elemIndices *schemapb.LongArray
|
||||
var newElementIndices []int64
|
||||
if data := results.GetElementIndices(); data != nil && len(data.GetData()) > 0 {
|
||||
elemIndices = data
|
||||
newElementIndices = make([]int64, n)
|
||||
}
|
||||
|
||||
for newIdx, oldIdx := range indices {
|
||||
if intIDs != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(intIDs.Data) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for int IDs (len=%d)", oldIdx, len(intIDs.Data))
|
||||
}
|
||||
newIntIDs[newIdx] = intIDs.Data[oldIdx]
|
||||
}
|
||||
if strIDs != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(strIDs.Data) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for string IDs (len=%d)", oldIdx, len(strIDs.Data))
|
||||
}
|
||||
newStrIDs[newIdx] = strIDs.Data[oldIdx]
|
||||
}
|
||||
if newScores != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(results.Scores) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for scores (len=%d)", oldIdx, len(results.Scores))
|
||||
}
|
||||
newScores[newIdx] = results.Scores[oldIdx]
|
||||
}
|
||||
if newDistances != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(results.Distances) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for distances (len=%d)", oldIdx, len(results.Distances))
|
||||
}
|
||||
newDistances[newIdx] = results.Distances[oldIdx]
|
||||
}
|
||||
if newRecalls != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(results.Recalls) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for recalls (len=%d)", oldIdx, len(results.Recalls))
|
||||
}
|
||||
newRecalls[newIdx] = results.Recalls[oldIdx]
|
||||
}
|
||||
if newElementIndices != nil {
|
||||
if oldIdx < 0 || oldIdx >= len(elemIndices.GetData()) {
|
||||
return merr.WrapErrServiceInternalMsg("reorderResults: index %d out of bounds for element indices (len=%d)", oldIdx, len(elemIndices.GetData()))
|
||||
}
|
||||
newElementIndices[newIdx] = elemIndices.GetData()[oldIdx]
|
||||
}
|
||||
}
|
||||
|
||||
if intIDs != nil {
|
||||
intIDs.Data = newIntIDs
|
||||
}
|
||||
if strIDs != nil {
|
||||
strIDs.Data = newStrIDs
|
||||
}
|
||||
if newScores != nil {
|
||||
results.Scores = newScores
|
||||
}
|
||||
if newDistances != nil {
|
||||
results.Distances = newDistances
|
||||
}
|
||||
if newRecalls != nil {
|
||||
results.Recalls = newRecalls
|
||||
}
|
||||
if newElementIndices != nil {
|
||||
elemIndices.Data = newElementIndices
|
||||
}
|
||||
|
||||
// Reorder field data
|
||||
for _, field := range results.FieldsData {
|
||||
@@ -2448,9 +2538,7 @@ func (op *orderByOperator) reorderResults(result *milvuspb.SearchResults, indice
|
||||
}
|
||||
}
|
||||
|
||||
// Reorder every group-by column — all internal stages emit plural; the
|
||||
// task output boundary handles legacy-wire singular downgrade after.
|
||||
for _, gbv := range results.GetGroupByFieldValues() {
|
||||
if gbv := getOrderByGroupByFieldValue(results); gbv != nil {
|
||||
if err := reorderFieldData(gbv, indices); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2486,6 +2574,13 @@ func countValidRows(validData []bool) int {
|
||||
return validCount
|
||||
}
|
||||
|
||||
func validateReorderIndex(oldIdx, originalRows int, field *schemapb.FieldData) error {
|
||||
if oldIdx < 0 || oldIdx >= originalRows {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for %s field %s (rows=%d)", oldIdx, field.GetType().String(), field.GetFieldName(), originalRows)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reorderNullableFloatVectorData(field *schemapb.FieldData, data []float32, width int, indices []int, newValidData []bool, logicalToPhysical []int, validCount int) ([]float32, error) {
|
||||
expected := validCount * width
|
||||
if len(data) != expected {
|
||||
@@ -2649,13 +2744,14 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: newData}}
|
||||
break
|
||||
}
|
||||
if len(data) != n*dim {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: FloatVector field %s has %d elements, expected %d (n=%d, dim=%d)", field.GetFieldName(), len(data), n*dim, n, dim)
|
||||
if len(data)%dim != 0 {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: FloatVector field %s has %d elements not divisible by dim %d", field.GetFieldName(), len(data), dim)
|
||||
}
|
||||
originalRows := len(data) / dim
|
||||
newData := make([]float32, n*dim)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for FloatVector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, originalRows, field); err != nil {
|
||||
return err
|
||||
}
|
||||
srcStart := oldIdx * dim
|
||||
dstStart := newIdx * dim
|
||||
@@ -2680,13 +2776,14 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_BinaryVector{BinaryVector: newData}
|
||||
break
|
||||
}
|
||||
if len(data) != n*bytesPerVector {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: BinaryVector field %s has %d bytes, expected %d (n=%d, bytesPerVector=%d)", field.GetFieldName(), len(data), n*bytesPerVector, n, bytesPerVector)
|
||||
if len(data)%bytesPerVector != 0 {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: BinaryVector field %s has %d bytes not divisible by width %d", field.GetFieldName(), len(data), bytesPerVector)
|
||||
}
|
||||
originalRows := len(data) / bytesPerVector
|
||||
newData := make([]byte, n*bytesPerVector)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for BinaryVector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, originalRows, field); err != nil {
|
||||
return err
|
||||
}
|
||||
srcStart := oldIdx * bytesPerVector
|
||||
dstStart := newIdx * bytesPerVector
|
||||
@@ -2711,13 +2808,14 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_Float16Vector{Float16Vector: newData}
|
||||
break
|
||||
}
|
||||
if len(data) != n*bytesPerVector {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: Float16Vector field %s has %d bytes, expected %d (n=%d, bytesPerVector=%d)", field.GetFieldName(), len(data), n*bytesPerVector, n, bytesPerVector)
|
||||
if len(data)%bytesPerVector != 0 {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: Float16Vector field %s has %d bytes not divisible by width %d", field.GetFieldName(), len(data), bytesPerVector)
|
||||
}
|
||||
originalRows := len(data) / bytesPerVector
|
||||
newData := make([]byte, n*bytesPerVector)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for Float16Vector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, originalRows, field); err != nil {
|
||||
return err
|
||||
}
|
||||
srcStart := oldIdx * bytesPerVector
|
||||
dstStart := newIdx * bytesPerVector
|
||||
@@ -2742,13 +2840,14 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: newData}
|
||||
break
|
||||
}
|
||||
if len(data) != n*bytesPerVector {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: BFloat16Vector field %s has %d bytes, expected %d (n=%d, bytesPerVector=%d)", field.GetFieldName(), len(data), n*bytesPerVector, n, bytesPerVector)
|
||||
if len(data)%bytesPerVector != 0 {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: BFloat16Vector field %s has %d bytes not divisible by width %d", field.GetFieldName(), len(data), bytesPerVector)
|
||||
}
|
||||
originalRows := len(data) / bytesPerVector
|
||||
newData := make([]byte, n*bytesPerVector)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for BFloat16Vector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, originalRows, field); err != nil {
|
||||
return err
|
||||
}
|
||||
srcStart := oldIdx * bytesPerVector
|
||||
dstStart := newIdx * bytesPerVector
|
||||
@@ -2773,13 +2872,10 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_SparseFloatVector{SparseFloatVector: sparseData}
|
||||
break
|
||||
}
|
||||
if len(contents) != n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: SparseFloatVector field %s has %d elements, expected %d", field.GetFieldName(), len(contents), n)
|
||||
}
|
||||
newContents := make([][]byte, n)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for SparseFloatVector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, len(contents), field); err != nil {
|
||||
return err
|
||||
}
|
||||
newContents[newIdx] = contents[oldIdx]
|
||||
}
|
||||
@@ -2801,13 +2897,14 @@ func reorderFieldData(field *schemapb.FieldData, indices []int) error {
|
||||
vectors.Data = &schemapb.VectorField_Int8Vector{Int8Vector: newData}
|
||||
break
|
||||
}
|
||||
if len(data) != n*dim {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: Int8Vector field %s has %d bytes, expected %d (n=%d, dim=%d)", field.GetFieldName(), len(data), n*dim, n, dim)
|
||||
if len(data)%dim != 0 {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: Int8Vector field %s has %d bytes not divisible by dim %d", field.GetFieldName(), len(data), dim)
|
||||
}
|
||||
originalRows := len(data) / dim
|
||||
newData := make([]byte, n*dim)
|
||||
for newIdx, oldIdx := range indices {
|
||||
if oldIdx < 0 || oldIdx >= n {
|
||||
return merr.WrapErrServiceInternalMsg("reorderFieldData: index %d out of bounds for Int8Vector field %s (n=%d)", oldIdx, field.GetFieldName(), n)
|
||||
if err := validateReorderIndex(oldIdx, originalRows, field); err != nil {
|
||||
return err
|
||||
}
|
||||
srcStart := oldIdx * dim
|
||||
dstStart := newIdx * dim
|
||||
@@ -3388,7 +3485,7 @@ var hybridSearchWithRequeryPipe = &pipelineDef{
|
||||
},
|
||||
}
|
||||
|
||||
// searchWithOrderByPipe: reduce → requery → organize → order_by
|
||||
// searchWithOrderByPipe: reduce without offset → requery → organize → order_by
|
||||
// For common search with order_by_fields
|
||||
var searchWithOrderByPipe = &pipelineDef{
|
||||
name: "searchWithOrderBy",
|
||||
@@ -3397,7 +3494,10 @@ var searchWithOrderByPipe = &pipelineDef{
|
||||
name: "reduce",
|
||||
inputs: []string{pipelineInput, pipelineStorageCost},
|
||||
outputs: []string{"reduced", "metrics"},
|
||||
opName: searchReduceOp,
|
||||
params: map[string]any{
|
||||
reduceOffsetParamKey: int64(0),
|
||||
},
|
||||
opName: searchReduceOp,
|
||||
},
|
||||
{
|
||||
name: "merge",
|
||||
@@ -3461,7 +3561,7 @@ func newBuiltInPipeline(t *searchTask) (*pipeline, error) {
|
||||
|
||||
hasOrderBy := len(t.orderByFields) > 0
|
||||
|
||||
// Common search with order_by: reduce → requery → order_by
|
||||
// Common search with order_by: reduce without offset, then order and slice.
|
||||
if !t.GetIsAdvanced() && hasOrderBy {
|
||||
return newPipeline(searchWithOrderByPipe, t)
|
||||
}
|
||||
|
||||
@@ -3228,6 +3228,30 @@ func (s *SearchPipelineSuite) TestIsSameGroupByValue() {
|
||||
s.False(isSameGroupByValue(boolField, 1, 2)) // true != false
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestNewSearchReduceOperatorUsesPipelineOffsetParam() {
|
||||
task := &searchTask{
|
||||
ctx: context.Background(),
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Nq: 1,
|
||||
Topk: 3,
|
||||
Offset: 2,
|
||||
CollectionID: 100,
|
||||
PartitionIDs: []int64{10},
|
||||
},
|
||||
schema: newSchemaInfo(&schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 101, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
},
|
||||
}),
|
||||
queryInfos: []*planpb.QueryInfo{{}},
|
||||
}
|
||||
|
||||
op, err := newSearchReduceOperator(task, map[string]any{reduceOffsetParamKey: int64(0)})
|
||||
|
||||
s.NoError(err)
|
||||
s.Equal(int64(0), op.(*searchReduceOperator).offset)
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestNewOrderByOperatorUsesPluralGroupByFieldIDs() {
|
||||
task := &searchTask{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
@@ -3296,6 +3320,172 @@ func (s *SearchPipelineSuite) TestOrderByOperator() {
|
||||
s.Equal(expectedPrices, actualPrices)
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorAppliesOffsetAfterOrderBy() {
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_IntId{
|
||||
IntId: &schemapb.LongArray{Data: []int64{1, 2, 3, 4}},
|
||||
},
|
||||
},
|
||||
Scores: []float32{0.9, 0.8, 0.7, 0.6},
|
||||
Distances: []float32{9, 8, 7, 6},
|
||||
Recalls: []float32{90, 80, 70, 60},
|
||||
ElementIndices: &schemapb.LongArray{Data: []int64{10, 20, 30, 40}},
|
||||
TopK: 4,
|
||||
Topks: []int64{4},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
{
|
||||
Type: schemapb.DataType_Int64,
|
||||
FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{
|
||||
Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{
|
||||
LongData: &schemapb.LongArray{Data: []int64{30, 10, 20, 40}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
limit: 2,
|
||||
offset: 1,
|
||||
}
|
||||
outputs, err := op.run(context.Background(), s.span, result)
|
||||
|
||||
s.NoError(err)
|
||||
sliced := outputs[0].(*milvuspb.SearchResults).GetResults()
|
||||
s.Equal([]int64{3, 1}, sliced.GetIds().GetIntId().GetData())
|
||||
s.Equal([]float32{0.7, 0.9}, sliced.GetScores())
|
||||
s.Equal([]float32{7, 9}, sliced.GetDistances())
|
||||
s.Equal([]float32{70, 90}, sliced.GetRecalls())
|
||||
s.Equal([]int64{30, 10}, sliced.GetElementIndices().GetData())
|
||||
s.Equal([]int64{20, 30}, sliced.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
s.Equal([]int64{2}, sliced.GetTopks())
|
||||
s.Equal(int64(2), sliced.GetTopK())
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorSlicesStringIDsAndNonNullableVectorAfterOrderBy() {
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_StrId{
|
||||
StrId: &schemapb.StringArray{Data: []string{"a", "b", "c", "d"}},
|
||||
},
|
||||
},
|
||||
Scores: []float32{0.9, 0.8, 0.7, 0.6},
|
||||
TopK: 4,
|
||||
Topks: []int64{4},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
{
|
||||
Type: schemapb.DataType_Int64,
|
||||
FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{
|
||||
Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{
|
||||
LongData: &schemapb.LongArray{Data: []int64{30, 10, 20, 40}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: schemapb.DataType_FloatVector,
|
||||
FieldName: "vec",
|
||||
Field: &schemapb.FieldData_Vectors{
|
||||
Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_FloatVector{
|
||||
FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4, 5, 6, 7, 8}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
limit: 2,
|
||||
offset: 1,
|
||||
}
|
||||
outputs, err := op.run(context.Background(), s.span, result)
|
||||
|
||||
s.NoError(err)
|
||||
sliced := outputs[0].(*milvuspb.SearchResults).GetResults()
|
||||
s.Equal([]string{"c", "a"}, sliced.GetIds().GetStrId().GetData())
|
||||
s.Equal([]float32{0.7, 0.9}, sliced.GetScores())
|
||||
s.Equal([]int64{20, 30}, sliced.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
s.Equal([]float32{5, 6, 1, 2}, sliced.GetFieldsData()[1].GetVectors().GetFloatVector().GetData())
|
||||
s.Equal([]int64{2}, sliced.GetTopks())
|
||||
s.Equal(int64(2), sliced.GetTopK())
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorAppliesGroupOffsetAfterOrderBy() {
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{
|
||||
IdField: &schemapb.IDs_IntId{
|
||||
IntId: &schemapb.LongArray{Data: []int64{1, 2, 3, 4, 5, 6}},
|
||||
},
|
||||
},
|
||||
Scores: []float32{0.9, 0.85, 0.8, 0.75, 0.7, 0.65},
|
||||
TopK: 6,
|
||||
Topks: []int64{6},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
{
|
||||
Type: schemapb.DataType_Int64,
|
||||
FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{
|
||||
Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{
|
||||
LongData: &schemapb.LongArray{Data: []int64{30, 25, 10, 20, 15, 18}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
GroupByFieldValues: []*schemapb.FieldData{
|
||||
{
|
||||
Type: schemapb.DataType_VarChar,
|
||||
Field: &schemapb.FieldData_Scalars{
|
||||
Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_StringData{
|
||||
StringData: &schemapb.StringArray{Data: []string{"A", "A", "B", "C", "C", "C"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: 100,
|
||||
groupSize: 3,
|
||||
limit: 1,
|
||||
offset: 1,
|
||||
}
|
||||
outputs, err := op.run(context.Background(), s.span, result)
|
||||
|
||||
s.NoError(err)
|
||||
sliced := outputs[0].(*milvuspb.SearchResults).GetResults()
|
||||
s.Equal([]int64{4, 5, 6}, sliced.GetIds().GetIntId().GetData())
|
||||
s.Equal([]float32{0.75, 0.7, 0.65}, sliced.GetScores())
|
||||
s.Equal([]int64{20, 15, 18}, sliced.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
s.Equal([]string{"C", "C", "C"}, sliced.GetGroupByFieldValues()[0].GetScalars().GetStringData().GetData())
|
||||
s.Nil(sliced.GetGroupByFieldValue())
|
||||
s.Equal([]int64{3}, sliced.GetTopks())
|
||||
s.Equal(int64(3), sliced.GetTopK())
|
||||
}
|
||||
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorReordersNullableVectorCompactOutput() {
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
@@ -3496,6 +3686,374 @@ func (s *SearchPipelineSuite) TestOrderByOperatorMultipleQueriesWithGroupBy() {
|
||||
s.Equal(expectedPrices, actualPrices)
|
||||
}
|
||||
|
||||
// Per liliu-z review (search_pipeline.go:1855): the per-query offset/limit loop was only
|
||||
// exercised with nq=1. Assert offset/limit apply PER QUERY, not globally over the flattened
|
||||
// result, so a regression that slices the offset window globally returns misaligned rows for nq>1.
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorAppliesOffsetPerQueryForMultipleQueries() {
|
||||
// nq=2, topk=4 each.
|
||||
// q1 ids[1,2,3,4] price[40,10,30,20] -> sort asc [2,4,3,1] -> offset1/limit2 -> [4,3]
|
||||
// q2 ids[5,6,7,8] price[80,50,70,60] -> sort asc [6,8,7,5] -> offset1/limit2 -> [8,7]
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2, 3, 4, 5, 6, 7, 8}}}},
|
||||
Scores: []float32{0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2},
|
||||
TopK: 4,
|
||||
Topks: []int64{4, 4},
|
||||
FieldsData: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64,
|
||||
FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{40, 10, 30, 20, 80, 50, 70, 60}}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
limit: 2,
|
||||
offset: 1,
|
||||
}
|
||||
outputs, err := op.run(context.Background(), s.span, result)
|
||||
s.NoError(err)
|
||||
sliced := outputs[0].(*milvuspb.SearchResults).GetResults()
|
||||
|
||||
// per-query pages: q1 -> [4,3], q2 -> [8,7]; a global slice would drop q2's page entirely.
|
||||
s.Equal([]int64{4, 3, 8, 7}, sliced.GetIds().GetIntId().GetData())
|
||||
s.Equal([]int64{20, 30, 60, 70}, sliced.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
s.Equal([]float32{0.6, 0.7, 0.2, 0.3}, sliced.GetScores())
|
||||
s.Equal([]int64{2, 2}, sliced.GetTopks()) // each query keeps its own page size
|
||||
}
|
||||
|
||||
// Same per-query invariant as above, combined with group-by: the group offset/limit must
|
||||
// paginate groups PER QUERY, not across the flattened multi-query result.
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorAppliesGroupOffsetPerQueryForMultipleQueries() {
|
||||
// nq=2, topk=4 each, 2 groups (size 2) per query.
|
||||
// q1 A(ids1,2 p20,25) B(ids3,4 p10,15) -> groups asc by first price [B,A] -> off1/lim1 group -> A=[1,2]
|
||||
// q2 C(ids5,6 p50,55) D(ids7,8 p30,35) -> groups asc by first price [D,C] -> off1/lim1 group -> C=[5,6]
|
||||
result := &milvuspb.SearchResults{
|
||||
Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2, 3, 4, 5, 6, 7, 8}}}},
|
||||
Scores: []float32{0.9, 0.85, 0.8, 0.75, 0.7, 0.65, 0.6, 0.55},
|
||||
TopK: 4,
|
||||
Topks: []int64{4, 4},
|
||||
GroupByFieldValues: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64,
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{100, 100, 200, 200, 300, 300, 400, 400}}},
|
||||
}},
|
||||
}},
|
||||
FieldsData: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64,
|
||||
FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{20, 25, 10, 15, 50, 55, 30, 35}}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: 1,
|
||||
groupSize: 2,
|
||||
limit: 1,
|
||||
offset: 1,
|
||||
}
|
||||
outputs, err := op.run(context.Background(), s.span, result)
|
||||
s.NoError(err)
|
||||
sliced := outputs[0].(*milvuspb.SearchResults).GetResults()
|
||||
|
||||
s.Equal([]int64{1, 2, 5, 6}, sliced.GetIds().GetIntId().GetData())
|
||||
s.Equal([]int64{20, 25, 50, 55}, sliced.GetFieldsData()[0].GetScalars().GetLongData().GetData())
|
||||
s.Equal([]int64{2, 2}, sliced.GetTopks())
|
||||
}
|
||||
|
||||
// paginateSortedRows: offset at/beyond the row count clamps to an empty page.
|
||||
func (s *SearchPipelineSuite) TestPaginateSortedRowsOffsetBeyondLength() {
|
||||
indices := []int{0, 1, 2}
|
||||
s.Empty(paginateSortedRows(indices, 5, 2)) // offset > len -> empty
|
||||
s.Empty(paginateSortedRows(indices, 3, 0)) // offset == len -> empty
|
||||
s.Equal([]int{1, 2}, paginateSortedRows(indices, 1, 10)) // limit beyond end -> offset..end
|
||||
s.Equal([]int{0, 1, 2}, paginateSortedRows(indices, 0, 0)) // no limit -> all
|
||||
}
|
||||
|
||||
// getOrderByGroupByFieldValue: nil input, plural-preferred, legacy fallback.
|
||||
func (s *SearchPipelineSuite) TestGetOrderByGroupByFieldValueNilAndSelection() {
|
||||
s.Nil(getOrderByGroupByFieldValue(nil))
|
||||
|
||||
plural := &schemapb.SearchResultData{
|
||||
GroupByFieldValues: []*schemapb.FieldData{{FieldName: "g"}},
|
||||
}
|
||||
s.Equal("g", getOrderByGroupByFieldValue(plural).GetFieldName())
|
||||
|
||||
legacy := &schemapb.SearchResultData{
|
||||
GroupByFieldValue: &schemapb.FieldData{FieldName: "l"},
|
||||
}
|
||||
s.Equal("l", getOrderByGroupByFieldValue(legacy).GetFieldName())
|
||||
}
|
||||
|
||||
// sortQueryResults returns early on empty indices (op.run guards with topk>0, so cover directly).
|
||||
func (s *SearchPipelineSuite) TestSortQueryResultsEmptyIndices() {
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
}
|
||||
out, err := op.sortQueryResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{}}, []int{})
|
||||
s.NoError(err)
|
||||
s.Empty(out)
|
||||
}
|
||||
|
||||
// sortGroupsByOrderByFields returns early on empty indices.
|
||||
func (s *SearchPipelineSuite) TestSortGroupsByOrderByFieldsEmptyIndices() {
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: 0,
|
||||
groupSize: 1,
|
||||
}
|
||||
out, err := op.sortGroupsByOrderByFields(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{}}, []int{})
|
||||
s.NoError(err)
|
||||
s.Empty(out)
|
||||
}
|
||||
|
||||
// reorderFieldData: vector payload length not divisible by per-vector width -> error (defensive guard).
|
||||
func (s *SearchPipelineSuite) TestReorderFieldDataDimNotDivisibleErrors() {
|
||||
indices := []int{0}
|
||||
cases := []struct {
|
||||
name string
|
||||
field *schemapb.FieldData
|
||||
}{
|
||||
{"float_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_FloatVector, FieldName: "fv",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3}}},
|
||||
}},
|
||||
}},
|
||||
{"binary_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_BinaryVector, FieldName: "bv",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 16,
|
||||
Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02, 0x03}},
|
||||
}},
|
||||
}},
|
||||
{"float16_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_Float16Vector, FieldName: "f16",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}},
|
||||
}},
|
||||
}},
|
||||
{"bfloat16_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_BFloat16Vector, FieldName: "bf16",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}},
|
||||
}},
|
||||
}},
|
||||
{"int8_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_Int8Vector, FieldName: "i8",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 4,
|
||||
Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s.Error(reorderFieldData(c.field, indices), c.name)
|
||||
}
|
||||
}
|
||||
|
||||
// reorderFieldData: index out of range for the vector payload -> validateReorderIndex error (defensive guard).
|
||||
func (s *SearchPipelineSuite) TestReorderFieldDataIndexOutOfBoundsErrors() {
|
||||
oob := []int{5} // every field below holds 2 vectors; index 5 is out of range
|
||||
cases := []struct {
|
||||
name string
|
||||
field *schemapb.FieldData
|
||||
}{
|
||||
{"float_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_FloatVector, FieldName: "fv",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3, 4}}},
|
||||
}},
|
||||
}},
|
||||
{"binary_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_BinaryVector, FieldName: "bv",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 16,
|
||||
Data: &schemapb.VectorField_BinaryVector{BinaryVector: []byte{0x01, 0x02, 0x03, 0x04}},
|
||||
}},
|
||||
}},
|
||||
{"float16_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_Float16Vector, FieldName: "f16",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_Float16Vector{Float16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}},
|
||||
}},
|
||||
}},
|
||||
{"bfloat16_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_BFloat16Vector, FieldName: "bf16",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_Bfloat16Vector{Bfloat16Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}},
|
||||
}},
|
||||
}},
|
||||
{"sparse_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_SparseFloatVector, FieldName: "sv",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 100,
|
||||
Data: &schemapb.VectorField_SparseFloatVector{SparseFloatVector: &schemapb.SparseFloatArray{
|
||||
Contents: [][]byte{{0x01, 0x02}, {0x03, 0x04}},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
{"int8_vector", &schemapb.FieldData{
|
||||
Type: schemapb.DataType_Int8Vector, FieldName: "i8",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 4,
|
||||
Data: &schemapb.VectorField_Int8Vector{Int8Vector: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}},
|
||||
}},
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
s.Error(reorderFieldData(c.field, oob), c.name)
|
||||
}
|
||||
}
|
||||
|
||||
// reorderResults: out-of-range index for each result column -> internal error (defensive guard).
|
||||
func (s *SearchPipelineSuite) TestReorderResultsIndexOutOfBounds() {
|
||||
op := &orderByOperator{}
|
||||
oob := []int{5}
|
||||
longIDs := func() *schemapb.IDs {
|
||||
return &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}}}}
|
||||
}
|
||||
|
||||
// int IDs
|
||||
s.Error(op.reorderResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1}}}},
|
||||
}}, oob), "int_ids")
|
||||
|
||||
// string IDs
|
||||
s.Error(op.reorderResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_StrId{StrId: &schemapb.StringArray{Data: []string{"a"}}}},
|
||||
}}, oob), "str_ids")
|
||||
|
||||
// distances (IDs long enough so the distances check is the one that trips)
|
||||
s.Error(op.reorderResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: longIDs(), Distances: []float32{0.1},
|
||||
}}, oob), "distances")
|
||||
|
||||
// recalls
|
||||
s.Error(op.reorderResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: longIDs(), Recalls: []float32{0.1},
|
||||
}}, oob), "recalls")
|
||||
|
||||
// element indices
|
||||
s.Error(op.reorderResults(&milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: longIDs(), ElementIndices: &schemapb.LongArray{Data: []int64{10}},
|
||||
}}, oob), "element_indices")
|
||||
}
|
||||
|
||||
// op.run propagates a sort error: the order_by field has fewer values than rows, so
|
||||
// compareFieldDataAt hits an out-of-bounds index during the per-query sort.
|
||||
func (s *SearchPipelineSuite) TestOrderByRunReturnsSortError() {
|
||||
result := &milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2, 3}}}},
|
||||
Scores: []float32{0.9, 0.8, 0.7},
|
||||
Topks: []int64{3},
|
||||
FieldsData: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64, FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10}}}, // 1 value for 3 rows
|
||||
}},
|
||||
}},
|
||||
}}
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
}
|
||||
_, err := op.run(context.Background(), s.span, result)
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// op.run propagates a reorder error: the sort succeeds but a malformed vector field fails reorderResults.
|
||||
func (s *SearchPipelineSuite) TestOrderByRunReturnsReorderError() {
|
||||
result := &milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2}}}},
|
||||
Scores: []float32{0.9, 0.8},
|
||||
Topks: []int64{2},
|
||||
FieldsData: []*schemapb.FieldData{
|
||||
{
|
||||
Type: schemapb.DataType_Int64, FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{20, 10}}},
|
||||
}},
|
||||
},
|
||||
{
|
||||
Type: schemapb.DataType_FloatVector, FieldName: "vec",
|
||||
Field: &schemapb.FieldData_Vectors{Vectors: &schemapb.VectorField{
|
||||
Dim: 2,
|
||||
Data: &schemapb.VectorField_FloatVector{FloatVector: &schemapb.FloatArray{Data: []float32{1, 2, 3}}}, // 3 not divisible by dim 2
|
||||
}},
|
||||
},
|
||||
},
|
||||
}}
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: -1,
|
||||
}
|
||||
_, err := op.run(context.Background(), s.span, result)
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// sortGroupsByOrderByFields propagates a sort error during group ordering (group value present).
|
||||
func (s *SearchPipelineSuite) TestSortGroupsByOrderByFieldsReturnsSortError() {
|
||||
result := &milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2, 3, 4}}}},
|
||||
GroupByFieldValues: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64,
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{100, 100, 200, 200}}},
|
||||
}},
|
||||
}},
|
||||
FieldsData: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64, FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10}}}, // 1 value for 4 rows
|
||||
}},
|
||||
}},
|
||||
}}
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: 1,
|
||||
groupSize: 2,
|
||||
}
|
||||
_, err := op.sortGroupsByOrderByFields(result, []int{0, 1, 2, 3})
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// sortGroupsByOrderByFields with no group value falls back to row sort and propagates its error.
|
||||
func (s *SearchPipelineSuite) TestSortGroupsByOrderByFieldsNilGroupValueReturnsSortError() {
|
||||
result := &milvuspb.SearchResults{Results: &schemapb.SearchResultData{
|
||||
Ids: &schemapb.IDs{IdField: &schemapb.IDs_IntId{IntId: &schemapb.LongArray{Data: []int64{1, 2, 3}}}},
|
||||
FieldsData: []*schemapb.FieldData{{
|
||||
Type: schemapb.DataType_Int64, FieldName: "price",
|
||||
Field: &schemapb.FieldData_Scalars{Scalars: &schemapb.ScalarField{
|
||||
Data: &schemapb.ScalarField_LongData{LongData: &schemapb.LongArray{Data: []int64{10}}},
|
||||
}},
|
||||
}},
|
||||
}}
|
||||
op := &orderByOperator{
|
||||
orderByFields: []OrderByField{{FieldName: "price", Ascending: true}},
|
||||
groupByFieldId: 1,
|
||||
groupSize: 2,
|
||||
}
|
||||
_, err := op.sortGroupsByOrderByFields(result, []int{0, 1, 2})
|
||||
s.Error(err)
|
||||
}
|
||||
|
||||
// Test orderByOperator with descending sort
|
||||
func (s *SearchPipelineSuite) TestOrderByOperatorDescending() {
|
||||
result := &milvuspb.SearchResults{
|
||||
|
||||
@@ -606,6 +606,10 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(groupByFieldIds) > 1 && len(orderByFields) > 0 {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(
|
||||
"order_by_fields is not supported with multi-field group_by_fields")
|
||||
}
|
||||
|
||||
// 8. validate iterator + order_by combination is not allowed
|
||||
if isIterator && len(orderByFields) > 0 {
|
||||
|
||||
@@ -5741,6 +5741,49 @@ func TestSearchTask_OrderByValidation(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "order_by and function_score cannot be used together")
|
||||
})
|
||||
|
||||
t.Run("regular search with multi-field group_by_fields and order_by should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
ctx: ctx,
|
||||
SearchRequest: &internalpb.SearchRequest{
|
||||
Base: &commonpb.MsgBase{
|
||||
MsgType: commonpb.MsgType_Search,
|
||||
Timestamp: uint64(time.Now().UnixNano()),
|
||||
},
|
||||
},
|
||||
request: &milvuspb.SearchRequest{
|
||||
CollectionName: "test_collection",
|
||||
SearchParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MetricTypeKey, Value: metric.L2},
|
||||
{Key: ParamsKey, Value: `{"nprobe": 10}`},
|
||||
{Key: AnnsFieldKey, Value: "vec"},
|
||||
{Key: TopKKey, Value: "10"},
|
||||
{Key: RoundDecimalKey, Value: "-1"},
|
||||
{Key: GroupByFieldsKey, Value: "category,brand"},
|
||||
{Key: OrderByFieldsKey, Value: "price:asc"},
|
||||
},
|
||||
},
|
||||
schema: &schemaInfo{
|
||||
CollectionSchema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
|
||||
{FieldID: 101, Name: "price", DataType: schemapb.DataType_Float},
|
||||
{FieldID: 102, Name: "category", DataType: schemapb.DataType_VarChar},
|
||||
{FieldID: 103, Name: "brand", DataType: schemapb.DataType_VarChar},
|
||||
{FieldID: 104, Name: "vec", DataType: schemapb.DataType_FloatVector, TypeParams: []*commonpb.KeyValuePair{{Key: "dim", Value: "128"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
qt.schema.schemaHelper, _ = typeutil.CreateSchemaHelper(qt.schema.CollectionSchema)
|
||||
qt.schema.pkField = &schemapb.FieldSchema{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true}
|
||||
|
||||
err := qt.initSearchRequest(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
assert.Contains(t, err.Error(), "order_by_fields is not supported with multi-field group_by_fields")
|
||||
})
|
||||
|
||||
t.Run("regular search with invalid order_by direction should fail", func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
qt := &searchTask{
|
||||
|
||||
Reference in New Issue
Block a user