enhance: bind index meta to add function field DDL (#51105)

issue: #51104
companion SDK PR (merge first): milvus-io/pymilvus#3670

## Summary

`add_function_field` previously accepted a vector output field without
any index meta; bump-schema-version compaction then stalled forever (no
segment index task can be created for a field with no index defined, and
`GetQueryVChanPositions` never hands off unindexed compacted-to segments
— the fail-closed chain is silent). This PR makes the index meta a
mandatory, atomically-bound part of the DDL:

- **Mandatory at every layer**: SDK requires `index_params`; proxy and
rootcoord reject vector function-output fields without an explicit
`index_type` strictly before the broadcast (no silent AUTOINDEX).
- **create_index-aligned materialization**: proxy normalizes the params
with the same logic as `create_index` (name rules via
`validateIndexName`, checker existence, `CheckTrain` incl. dimension
filling, function-type defaults such as `bm25_k1/bm25_b/bm25_avgdl`) and
writes them back into the request; rootcoord prepare allocates index
id/name, rejects name conflicts and unknown index types, and serializes
a complete `indexpb.FieldIndex` into the new
`AlterCollectionMessageUpdates.bound_field_indexes` WAL field.
- **Atomic apply**: the alter-collection ack callback commits the schema
and then applies the bound index by dispatching a synthetic
`CreateIndexMessage` to datacoord's existing `createIndexV2AckCallback`
(same pattern as `cascadeDropFieldIndexesInline`). The callback is
replayed until success across crashes and is fully idempotent (all ids
come from the message body), so `DDL success ⇒ index meta exists` holds
with no partial terminal state.
- **Reuse promotion**: `ValidateIndexParams`/`CheckDuplidateKey` moved
from datacoord to `internal/util/indexparamcheck`; shared
`ExpandIndexParams` / `FillFunctionOutputIndexParams` /
`PrepareFunctionOutputIndexParams` helpers now back both the
create_index path and this DDL.
- **Dead fan-out removed**: the ordinary `CreateIndex` broadcast is
reverted to control-channel-only; the vchannel fan-out had no consumer
and wrote one dead WAL entry per vchannel per index creation.
- **No milvus-proto change**: the request already carried
`FieldInfo.index_name`/`extra_params` (previously unused); this feature
activates them.
- e2e suites updated to the new semantics:
`test_add_function_field_feature.py` rewritten (explicit `index_params`
everywhere, post-DDL `create_index` blocks removed since the bound index
conflicts with a second distinct index per existing semantics, new
negative cases for the mandatory checks); the external-table negative
case updated as well.

## Verification

- New/updated unit tests: rootcoord DDL callbacks
(mandatory/AUTOINDEX/unknown-index-type rejections, bound-index
materialization + ack-callback application via a recording fake), proxy
task validation (invalid index name, normalization write-back), shared
helper tests.
- End-to-end on a live cluster (StorageV3 + bump compaction enabled):
request without params rejected; index meta visible with
create_index-aligned params immediately after the DDL returns; bump
compaction rewrote 2000-row sealed segments and the bound index built on
the results (the exact chain that previously stalled); BM25 search over
pre-DDL rows succeeds once the compacted segments are loaded.

## Limitations (follow-ups)

- Live propagation of the new index meta to already-loaded QueryNode
delegators is intentionally out of scope; a loaded collection observes
the bound index through the segment load path (e.g. reload /
compacted-segment load). Tracked as a separate follow-up PR.
- The python e2e suites require the companion pymilvus change:
milvus-io/pymilvus#3670 is merged and
`tests/python_client/requirements.txt` is bumped to
`pymilvus==3.1.0rc61` in this PR (all 22 rewritten cases + the
external-table case verified locally against the released rc61).
- During a rolling-upgrade window an old coordinator replaying the new
message ignores `bound_field_indexes` (proto3 unknown field), i.e.
pre-existing behavior; avoid the new argument in mixed-version clusters.
- `AddCollectionFunction` on an existing field and plain
add-vector-field keep their current behavior (the WAL carrier is generic
for future extension).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Chun Han
2026-07-09 17:38:41 +08:00
committed by GitHub
co-authored by MrPresent-Han Claude Fable 5
parent 75526b2463
commit aea5d701bd
23 changed files with 1989 additions and 752 deletions
+21 -12
View File
@@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/pkg/v3/mlog"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
type indexInspector struct {
@@ -193,27 +194,35 @@ func getSegmentBinlogFields(segment *SegmentInfo) map[int64]struct{} {
}
func (i *indexInspector) canCreateIndexForSegment(ctx context.Context, segment *SegmentInfo, index *model.Index, segmentBinlogFields map[int64]struct{}) bool {
_, hasField := segmentBinlogFields[index.FieldID]
if !i.isFunctionOutputField(segment.CollectionID, index.FieldID) || hasField {
if _, hasField := segmentBinlogFields[index.FieldID]; hasField {
return true
}
mlog.Debug(ctx, "function output field has no binlog, skip create index", mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
return false
}
func (i *indexInspector) isFunctionOutputField(collectionID, fieldID int64) bool {
collection := i.meta.GetCollection(collectionID)
if collection == nil || collection.Schema == nil {
// The segment has no binlog for the indexed field. A function-output field
// receives its data only through backfill, so building now is doomed; any
// other field may proceed. The schema is resolved through the handler
// (lazy-loading on cache miss, e.g. right after a datacoord restart) and the
// check fails closed: on an unresolvable or inconsistent view, defer to the
// next inspection round instead of trusting a stale schema.
collection, err := i.handler.GetCollection(ctx, segment.CollectionID)
if err != nil || collection == nil || collection.Schema == nil {
mlog.Warn(ctx, "cannot resolve collection schema, defer index build",
mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID), mlog.Err(err))
return false
}
for _, functionSchema := range collection.Schema.GetFunctions() {
for _, outputFieldID := range functionSchema.GetOutputFieldIds() {
if outputFieldID == fieldID {
return true
if outputFieldID == index.FieldID {
mlog.Debug(ctx, "function output field has no binlog, skip create index", mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
return false
}
}
}
return false
if typeutil.GetFieldByID(collection.Schema, index.FieldID) == nil {
mlog.Warn(ctx, "indexed field not found in cached collection schema, defer index build",
mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
return false
}
return true
}
func (i *indexInspector) createIndexForSegment(ctx context.Context, segment *SegmentInfo, indexID UniqueID) error {
+52 -2
View File
@@ -22,6 +22,7 @@ import (
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -487,7 +488,7 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
inspector := newIndexInspector(ctx, notifyChan, m, scheduler, alloc, handler, storageCli, versionManager)
collID := int64(10)
m.collections.Insert(collID, &collectionInfo{
collInfo := &collectionInfo{
ID: collID,
Schema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
@@ -500,7 +501,9 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
{Name: "bm25_fn", OutputFieldIds: []int64{102}},
},
},
})
}
m.collections.Insert(collID, collInfo)
handler.EXPECT().GetCollection(mock.Anything, collID).Return(collInfo, nil).Maybe()
t.Run("create function output index when binlog exists", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
@@ -604,4 +607,51 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
assert.Contains(t, segIndexes, UniqueID(8))
assert.NotContains(t, segIndexes, UniqueID(9))
})
t.Run("defer index build when schema is unresolvable", func(t *testing.T) {
unresolvableCollID := int64(11)
handler.EXPECT().GetCollection(mock.Anything, unresolvableCollID).
Return(nil, errors.New("mock rootcoord unreachable")).Once()
m.indexMeta.indexes[unresolvableCollID] = map[UniqueID]*model.Index{
10: {CollectionID: unresolvableCollID, FieldID: 102, IndexID: 10, IndexName: "unresolvable_schema_idx"},
}
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 5,
CollectionID: unresolvableCollID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
assert.NotContains(t, m.indexMeta.GetSegmentIndexes(unresolvableCollID, segment.GetID()), UniqueID(10))
})
t.Run("defer index build when indexed field is unknown to schema", func(t *testing.T) {
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
11: {CollectionID: collID, FieldID: 104, IndexID: 11, IndexName: "stale_schema_idx"},
}
segment := &SegmentInfo{
SegmentInfo: &datapb.SegmentInfo{
ID: 6,
CollectionID: collID,
State: commonpb.SegmentState_Flushed,
IsSorted: true,
Binlogs: []*datapb.FieldBinlog{
{FieldID: 0, ChildFields: []int64{100, 101}},
},
},
}
m.segments.SetSegment(segment.GetID(), segment)
err := inspector.createIndexesForSegment(ctx, segment)
assert.NoError(t, err)
assert.NotContains(t, m.indexMeta.GetSegmentIndexes(collID, segment.GetID()), UniqueID(11))
})
}
+4 -47
View File
@@ -298,13 +298,10 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
UserIndexParams: req.GetUserIndexParams(),
}
// Validate the index params.
if err := ValidateIndexParams(index); err != nil {
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
return nil, err
}
channels := make([]string, 0, len(coll.GetVirtualChannelNames())+1)
channels = append(channels, streaming.WAL().ControlChannel())
channels = append(channels, coll.GetVirtualChannelNames()...)
if _, err = broadcaster.Broadcast(ctx, message.NewCreateIndexMessageBuilderV2().
WithHeader(&message.CreateIndexMessageHeader{
DbId: coll.GetDbId(),
@@ -316,7 +313,7 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
WithBody(&message.CreateIndexMessageBody{
FieldIndex: model.MarshalIndexModel(index),
}).
WithBroadcast(channels).
WithBroadcast([]string{streaming.WAL().ControlChannel()}).
MustBuildBroadcast(),
); err != nil {
mlog.Error(context.TODO(), "CreateIndex fail", mlog.Err(err))
@@ -325,51 +322,11 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
}
mlog.Info(context.TODO(), "CreateIndex successfully",
mlog.String("IndexName", index.IndexName), mlog.Int64("fieldID", index.FieldID),
mlog.Int64("IndexID", index.IndexID),
mlog.Strings("channels", channels))
mlog.Int64("IndexID", index.IndexID))
metrics.IndexRequestCounter.WithLabelValues(metrics.SuccessLabel).Inc()
return merr.Success(), nil
}
func ValidateIndexParams(index *model.Index) error {
if err := CheckDuplidateKey(index.IndexParams, "indexParams"); err != nil {
return err
}
if err := CheckDuplidateKey(index.UserIndexParams, "userIndexParams"); err != nil {
return err
}
if err := CheckDuplidateKey(index.TypeParams, "typeParams"); err != nil {
return err
}
indexType := GetIndexType(index.IndexParams)
indexParams := funcutil.KeyValuePair2Map(index.IndexParams)
userIndexParams := funcutil.KeyValuePair2Map(index.UserIndexParams)
if err := indexparamcheck.ValidateMmapIndexParams(indexType, indexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid mmap index params: %s", err.Error())
}
if err := indexparamcheck.ValidateMmapIndexParams(indexType, userIndexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid mmap user index params: %s", err.Error())
}
if err := indexparamcheck.ValidateOffsetCacheIndexParams(indexType, indexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
}
if err := indexparamcheck.ValidateOffsetCacheIndexParams(indexType, userIndexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
}
return nil
}
func CheckDuplidateKey(kvs []*commonpb.KeyValuePair, tag string) error {
keySet := typeutil.NewSet[string]()
for _, kv := range kvs {
if keySet.Contain(kv.GetKey()) {
return merr.WrapErrParameterInvalidMsg("duplicate %s key in %s params", kv.GetKey(), tag)
}
keySet.Insert(kv.GetKey())
}
return nil
}
func UpdateParams(index *model.Index, from []*commonpb.KeyValuePair, updates []*commonpb.KeyValuePair) []*commonpb.KeyValuePair {
params := make(map[string]string)
for _, param := range from {
@@ -498,7 +455,7 @@ func (s *Server) AlterIndex(ctx context.Context, req *indexpb.AlterIndexRequest)
index.IndexParams = newIndexParams
}
if err := ValidateIndexParams(index); err != nil {
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
return merr.Status(err), nil
}
}
-107
View File
@@ -51,7 +51,6 @@ import (
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/channel"
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/broadcast"
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/registry"
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
@@ -2868,112 +2867,6 @@ func TestMeta_GetHasUnindexTaskSegments(t *testing.T) {
})
}
func TestValidateIndexParams(t *testing.T) {
t.Run("valid", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
{
Key: common.MmapEnabledKey,
Value: "true",
},
},
}
err := ValidateIndexParams(index)
assert.NoError(t, err)
})
t.Run("invalid index param", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
{
Key: common.MmapEnabledKey,
Value: "h",
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("invalid index user param", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
},
UserIndexParams: []*commonpb.KeyValuePair{
{
Key: common.MmapEnabledKey,
Value: "h",
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_index_params", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_user_index_params", func(t *testing.T) {
index := &model.Index{
UserIndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_user_index_params", func(t *testing.T) {
index := &model.Index{
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: indexparamcheck.AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
}
func TestJsonIndex(t *testing.T) {
initStreamingSystem(t)
+1 -1
View File
@@ -884,7 +884,7 @@ func (sm *snapshotManager) RestoreIndexes(
}
// Validate the index params (basic validation without JSON path parsing)
if err := ValidateIndexParams(index); err != nil {
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
return merr.Wrapf(err, "failed to validate index %s", indexInfo.GetIndexName())
}
+24
View File
@@ -32,6 +32,7 @@ import (
"github.com/milvus-io/milvus/internal/proxy/shardclient"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/function/validator"
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
"github.com/milvus-io/milvus/internal/util/schemautil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mlog"
@@ -1183,6 +1184,29 @@ func (t *alterCollectionSchemaTask) preExecuteAdd(ctx context.Context) error {
}
}
// Validate the bound index params against the new field, mirroring the
// create_index path (index name rules, checker existence, data-type
// compatibility, train params, dimension filling). Validation-only: the
// request keeps the user's original params so the persisted UserIndexParams
// match the create_index convention; rootcoord independently derives the
// normalized IndexParams at prepare.
if plan.Kind == schemautil.AlterSchemaAddFunctionField {
if err := validateIndexName(plan.IndexName); err != nil {
return err
}
if err := indexparamcheck.ValidateIndexParamsSize(plan.IndexExtraParams...); err != nil {
return err
}
indexParamsMap, err := indexparamcheck.PrepareFunctionOutputIndexParams(
plan.Function.GetType(), plan.Field.GetName(), plan.IndexExtraParams)
if err != nil {
return err
}
if err := checkTrain(ctx, plan.Field, indexParamsMap); err != nil {
return err
}
}
// Validate function-field type compatibility (e.g., BM25 requires varchar input,
// SparseFloatVector output). Construct a merged schema with old fields + optional new field
// + new function, then validate only the new function to avoid re-checking existing
+11 -124
View File
@@ -22,7 +22,6 @@ import (
"strings"
"github.com/cockroachdb/errors"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
@@ -203,92 +202,23 @@ func wrapUserIndexParams(metricType string) []*commonpb.KeyValuePair {
}
}
func checkIndexParamsSize(size int) error {
maxIndexParamsSize := paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()
if size > maxIndexParamsSize {
return merr.WrapErrParameterInvalidMsg("index params size exceeds limit: %d > %d", size, maxIndexParamsSize)
}
return nil
}
func validateIndexParamsSize(params ...*commonpb.KeyValuePair) error {
size := 0
for _, param := range params {
size += len(param.GetKey()) + len(param.GetValue())
}
return checkIndexParamsSize(size)
}
func validateIndexParamsMapSize(params map[string]string) error {
size := 0
for k, v := range params {
size += len(k) + len(v)
}
return checkIndexParamsSize(size)
}
func (cit *createIndexTask) parseFunctionParamsToIndex(indexParamsMap map[string]string) error {
if !cit.fieldSchema.GetIsFunctionOutput() {
return nil
}
switch cit.functionSchema.GetType() {
case schemapb.FunctionType_Unknown:
return merr.WrapErrParameterInvalidMsg("unknown function type encountered")
case schemapb.FunctionType_BM25:
// set default BM25 params if not provided in index params
if _, ok := indexParamsMap["bm25_k1"]; !ok {
indexParamsMap["bm25_k1"] = "1.2"
}
if _, ok := indexParamsMap["bm25_b"]; !ok {
indexParamsMap["bm25_b"] = "0.75"
}
if _, ok := indexParamsMap["bm25_avgdl"]; !ok {
indexParamsMap["bm25_avgdl"] = "100"
}
if metricType, ok := indexParamsMap["metric_type"]; !ok {
indexParamsMap["metric_type"] = metric.BM25
} else if metricType != metric.BM25 {
return merr.WrapErrParameterInvalidMsg("index metric type of BM25 function output field must be BM25, got %s", metricType)
}
default:
return nil
}
return nil
return indexparamcheck.FillFunctionOutputIndexParams(cit.functionSchema.GetType(), indexParamsMap)
}
func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
cit.newExtraParams = cit.req.GetExtraParams()
if err := validateIndexParamsSize(cit.newExtraParams...); err != nil {
if err := indexparamcheck.ValidateIndexParamsSize(cit.newExtraParams...); err != nil {
return err
}
isVecIndex := typeutil.IsVectorType(cit.fieldSchema.DataType)
indexParamsMap := make(map[string]string)
keys := typeutil.NewSet[string]()
for _, kv := range cit.req.GetExtraParams() {
if keys.Contain(kv.GetKey()) {
return merr.WrapErrParameterInvalidMsg("duplicated index param (key=%s) (value=%s) found", kv.GetKey(), kv.GetValue())
}
keys.Insert(kv.GetKey())
if kv.Key == common.ParamsKey {
params, err := funcutil.JSONToMap(kv.Value)
if err != nil {
return err
}
for k, v := range params {
indexParamsMap[k] = v
}
} else {
indexParamsMap[kv.Key] = kv.Value
}
indexParamsMap, err := indexparamcheck.ExpandIndexParams(cit.req.GetExtraParams())
if err != nil {
return err
}
if jsonCastType, exist := indexParamsMap[common.JSONCastTypeKey]; exist {
@@ -298,7 +228,7 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
indexParamsMap[common.JSONCastFunctionKey] = strings.ToUpper(strings.TrimSpace(jsonCastFunction))
}
if err := validateIndexParamsMapSize(indexParamsMap); err != nil {
if err := indexparamcheck.ValidateIndexParamsMapSize(indexParamsMap); err != nil {
return err
}
@@ -586,11 +516,11 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
}
}
if err := validateIndexParamsMapSize(indexParamsMap); err != nil {
if err := indexparamcheck.ValidateIndexParamsMapSize(indexParamsMap); err != nil {
return err
}
err := checkTrain(ctx, cit.fieldSchema, indexParamsMap)
err = checkTrain(ctx, cit.fieldSchema, indexParamsMap)
if err != nil {
// checkTrain may propagate errors from indexparamcheck (not yet
// merr-standardized). Already-merr errors (leaves / fillDimension /
@@ -652,28 +582,6 @@ func (cit *createIndexTask) getIndexedFieldAndFunction(ctx context.Context) erro
return nil
}
func fillDimension(field *schemapb.FieldSchema, indexParams map[string]string) error {
if !typeutil.IsVectorType(field.GetDataType()) {
return nil
}
params := make([]*commonpb.KeyValuePair, 0, len(field.GetTypeParams())+len(field.GetIndexParams()))
params = append(params, field.GetTypeParams()...)
params = append(params, field.GetIndexParams()...)
dimensionInSchema, err := funcutil.GetAttrByKeyFromRepeatedKV(DimKey, params)
if err != nil {
return merr.WrapErrParameterInvalidMsg("dimension not found in schema")
}
dimension, exist := indexParams[DimKey]
if exist {
if dimensionInSchema != dimension {
return merr.WrapErrParameterInvalidMsg("dimension mismatch, dimension in schema: %s, dimension: %s", dimensionInSchema, dimension)
}
} else {
indexParams[DimKey] = dimensionInSchema
}
return nil
}
func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams map[string]string) error {
indexType := indexParams[common.IndexTypeKey]
@@ -687,8 +595,7 @@ func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams ma
indexParams[common.HybridHighCardinalityIndexTypeKey] = paramtable.Get().DataCoordCfg.HybridIndexHighCardinalityIndexType.GetValue()
}
checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)
if err != nil {
if _, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType); err != nil {
mlog.Warn(ctx, "Failed to get index checker", mlog.String(common.IndexTypeKey, indexType))
return merr.WrapErrParameterInvalidMsg("invalid index type: %s", indexType)
}
@@ -712,28 +619,8 @@ func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams ma
}
}
isSparse := typeutil.IsSparseFloatVectorType(field.DataType)
if !isSparse {
if err := fillDimension(field, indexParams); err != nil {
return err
}
}
effectiveField := field
if effectiveDataType != field.DataType {
effectiveField = proto.Clone(field).(*schemapb.FieldSchema)
effectiveField.DataType = effectiveDataType
effectiveField.ElementType = effectiveElementType
}
if err := checker.CheckValidDataType(indexType, effectiveField); err != nil {
mlog.Info(ctx, "create index with invalid data type", mlog.Err(err), mlog.String("data_type", field.GetDataType().String()))
return err
}
if err := checker.CheckTrain(effectiveDataType, effectiveElementType, indexParams); err != nil {
mlog.Info(ctx, "create index with invalid parameters", mlog.Err(err))
if err := indexparamcheck.ValidateFieldIndexParams(field, indexParams); err != nil {
mlog.Info(ctx, "create index with invalid parameters", mlog.Err(err), mlog.String("data_type", field.GetDataType().String()))
return err
}
+40 -5
View File
@@ -45,6 +45,7 @@ import (
"github.com/milvus-io/milvus/internal/proxy/shardclient"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
"github.com/milvus-io/milvus/internal/util/function/embedding"
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mq/msgstream"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
@@ -3538,18 +3539,20 @@ func Test_createIndexTask_getIndexedFieldAndFunction(t *testing.T) {
}
func Test_fillDimension(t *testing.T) {
// fillDimension moved to indexparamcheck.FillDimension (shared with the
// add-function-field bound-index path); coverage kept here for the proxy flow.
t.Run("scalar", func(t *testing.T) {
f := &schemapb.FieldSchema{
DataType: schemapb.DataType_Int64,
}
assert.NoError(t, fillDimension(f, nil))
assert.NoError(t, indexparamcheck.FillDimension(f, nil))
})
t.Run("no dim in schema", func(t *testing.T) {
f := &schemapb.FieldSchema{
DataType: schemapb.DataType_FloatVector,
}
assert.Error(t, fillDimension(f, nil))
assert.Error(t, indexparamcheck.FillDimension(f, nil))
})
t.Run("dimension mismatch", func(t *testing.T) {
@@ -3562,7 +3565,7 @@ func Test_fillDimension(t *testing.T) {
},
},
}
assert.Error(t, fillDimension(f, map[string]string{common.DimKey: "8"}))
assert.Error(t, indexparamcheck.FillDimension(f, map[string]string{common.DimKey: "8"}))
})
t.Run("normal", func(t *testing.T) {
@@ -3576,7 +3579,7 @@ func Test_fillDimension(t *testing.T) {
},
}
m := map[string]string{}
assert.NoError(t, fillDimension(f, m))
assert.NoError(t, indexparamcheck.FillDimension(f, m))
assert.Equal(t, "128", m[common.DimKey])
})
}
@@ -7687,7 +7690,13 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: proto.Clone(sparseOutputField).(*schemapb.FieldSchema)},
{
FieldSchema: proto.Clone(sparseOutputField).(*schemapb.FieldSchema),
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
},
},
},
FuncSchema: []*schemapb.FunctionSchema{proto.Clone(functionSchema).(*schemapb.FunctionSchema)},
},
@@ -7995,6 +8004,10 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
{Key: common.DimKey, Value: "4096"},
},
}
addRequest.FieldInfos[0].ExtraParams = []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
}
functionSchema := proto.Clone(addRequest.GetFuncSchema()[0]).(*schemapb.FunctionSchema)
functionSchema.Name = "minhash_func"
functionSchema.Type = schemapb.FunctionType_MinHash
@@ -8069,6 +8082,28 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
assert.NoError(t, err)
})
t.Run("PreExecute rejects invalid bound index name", func(t *testing.T) {
req := buildValidRequest()
req.GetAction().GetAddRequest().GetFieldInfos()[0].IndexName = "1bad-index-name"
task := buildTask(req, oldSchema)
err := task.PreExecute(ctx)
assert.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.ErrorContains(t, err, "Invalid index name")
})
t.Run("PreExecute keeps the user's original index params in the request", func(t *testing.T) {
req := buildValidRequest()
original := proto.Clone(req.GetAction().GetAddRequest().GetFieldInfos()[0]).(*milvuspb.AlterCollectionSchemaRequest_FieldInfo)
task := buildTask(req, oldSchema)
err := task.PreExecute(ctx)
assert.NoError(t, err)
// Validation-only: no normalization write-back, so the persisted
// UserIndexParams stay aligned with the create_index convention and a
// later create_index with identical params remains idempotent.
assert.True(t, proto.Equal(original, req.GetAction().GetAddRequest().GetFieldInfos()[0]))
})
t.Run("Execute leaves function output marker to RootCoord", func(t *testing.T) {
req := buildValidRequest()
task := buildTask(req, oldSchema)
+2 -26
View File
@@ -1789,32 +1789,8 @@ func validCharInIndexName(c byte) bool {
}
func validateIndexName(indexName string) error {
indexName = strings.TrimSpace(indexName)
if indexName == "" {
return nil
}
invalidMsg := "Invalid index name: " + indexName + ". "
if len(indexName) > Params.ProxyCfg.MaxNameLength.GetAsInt() {
msg := invalidMsg + "The length of a index name must be less than " + Params.ProxyCfg.MaxNameLength.GetValue() + " characters."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
firstChar := indexName[0]
if firstChar != '_' && !isAlpha(firstChar) {
msg := invalidMsg + "The first character of a index name must be an underscore or letter."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
indexNameSize := len(indexName)
for i := 1; i < indexNameSize; i++ {
c := indexName[i]
if !validCharInIndexName(c) {
msg := invalidMsg + "Index name can only contain numbers, letters, and underscores."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
}
return nil
// Shared with rootcoord's bound-index prepare (indexparamcheck).
return indexparamcheck.ValidateIndexName(indexName)
}
func isCollectionLoaded(ctx context.Context, mc types.MixCoordClient, collID int64) (bool, error) {
@@ -441,6 +441,18 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
}
return merr.Wrap(err, "failed to alter collection")
}
// Refresh datacoord's cached collection schema BEFORE the bound index meta
// becomes visible: creating the index signals the index inspector, whose
// function-output-field guard reads that cached schema — on a stale view it
// would schedule doomed builds on segments that have no binlog for the new
// field yet. The schema push depends only on rootcoord meta (updated above),
// never on index meta, so this order is always safe.
if err := c.broker.BroadcastAlteredCollection(ctx, header.CollectionId); err != nil {
return merr.Wrap(err, "failed to broadcast altered collection")
}
if err := c.applyBoundFieldIndexesInline(ctx, result); err != nil {
return err
}
if body.Updates.AlterLoadConfig != nil {
resp, err := c.mixCoord.UpdateLoadConfig(ctx, &querypb.UpdateLoadConfigRequest{
CollectionIDs: []int64{header.CollectionId},
@@ -461,9 +473,6 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
if err := c.cascadeDropFieldIndexesInline(ctx, result); err != nil {
return err
}
if err := c.broker.BroadcastAlteredCollection(ctx, header.CollectionId); err != nil {
return merr.Wrap(err, "failed to broadcast altered collection")
}
// If the collection was renamed or moved to a different DB, grants were migrated
// in MetaTable.AlterCollection. Refresh the RBAC policy cache on all proxies so
@@ -482,6 +491,54 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
return c.ExpireCaches(ctx, header)
}
// applyBoundFieldIndexesInline creates the index meta bound to a newly added
// function-output field by inlining the CreateIndex ack callback, same pattern as
// cascadeDropFieldIndexesInline. The FieldIndex was fully materialized (id/name
// allocated, params validated) at DDL prepare time, so this is a pure idempotent
// apply: a replayed callback rebuilds the identical synthetic message. Cannot use
// the CreateIndex RPC here because it would deadlock on the resource key lock.
// The synthetic message is never appended to the WAL; it only routes the apply
// through the registry to datacoord's createIndexV2AckCallback.
func (c *DDLCallback) applyBoundFieldIndexesInline(ctx context.Context, result message.BroadcastResultAlterCollectionMessageV2) error {
header := result.Message.Header()
boundFieldIndexes := result.Message.MustBody().GetUpdates().GetBoundFieldIndexes()
if len(boundFieldIndexes) == 0 {
return nil
}
controlChannelResult := result.GetControlChannelResult()
for _, fieldIndex := range boundFieldIndexes {
indexInfo := fieldIndex.GetIndexInfo()
mlog.Info(ctx, "applying bound field index of alter collection schema",
mlog.FieldMessage(result.Message),
mlog.FieldFieldID(indexInfo.GetFieldID()),
mlog.String("indexName", indexInfo.GetIndexName()),
mlog.FieldIndexID(indexInfo.GetIndexID()),
)
createIndexMsg := message.NewCreateIndexMessageBuilderV2().
WithHeader(&message.CreateIndexMessageHeader{
DbId: header.DbId,
CollectionId: header.CollectionId,
FieldId: indexInfo.GetFieldID(),
IndexId: indexInfo.GetIndexID(),
IndexName: indexInfo.GetIndexName(),
}).
WithBody(&message.CreateIndexMessageBody{
FieldIndex: fieldIndex,
}).
WithBroadcast([]string{streaming.WAL().ControlChannel()}).
MustBuildBroadcast().
WithBroadcastID(result.Message.BroadcastHeader().BroadcastID)
if err := registry.CallMessageAckCallback(ctx, createIndexMsg, map[string]*message.AppendResult{
streaming.WAL().ControlChannel(): controlChannelResult,
}); err != nil {
return merr.Wrap(err, "failed to apply bound field index")
}
}
return nil
}
// cascadeDropFieldIndexesInline drops indexes on dropped fields by inlining the
// DropIndex ack callback, same pattern as dropCollectionV1AckCallback.
// Cannot use DropIndex RPC here because it would deadlock on the resource key lock.
@@ -268,7 +268,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_UpdateLoadConfigRPCError(t *te
withMeta(meta),
withMixCoord(mixc),
withValidProxyManager(),
withBroker(&mockBroker{}),
withBroker(&mockBroker{
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
return nil
},
}),
)
cb := &DDLCallback{Core: c}
@@ -308,7 +312,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_UpdateLoadConfigNonRGNotFoundE
withMeta(meta),
withMixCoord(mixc),
withValidProxyManager(),
withBroker(&mockBroker{}),
withBroker(&mockBroker{
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
return nil
},
}),
)
cb := &DDLCallback{Core: c}
@@ -352,7 +360,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_StopRetryOnResourceGroupNotFou
withMeta(meta),
withMixCoord(mixc),
withValidProxyManager(),
withBroker(&mockBroker{}),
withBroker(&mockBroker{
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
return nil
},
}),
)
cb := &DDLCallback{Core: c}
@@ -435,8 +447,9 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_BroadcastAlteredCollectionErro
meta := mockrootcoord.NewIMetaTable(t)
meta.EXPECT().AlterCollection(mock.Anything, mock.Anything).Return(nil)
// UpdateLoadConfig is never reached: the schema broadcast now runs first
// (before the bound-index apply and load-config update) and fails here.
mixc := imocks.NewMixCoord(t)
mixc.On("UpdateLoadConfig", mock.Anything, mock.Anything).Return(merr.Success(), nil)
c := newTestCore(
withMeta(meta),
@@ -19,6 +19,7 @@ package rootcoord
import (
"context"
"github.com/samber/lo"
"google.golang.org/protobuf/types/known/fieldmaskpb"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
@@ -28,8 +29,10 @@ import (
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster"
"github.com/milvus-io/milvus/internal/util/function/validator"
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
"github.com/milvus-io/milvus/internal/util/schemautil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
@@ -128,6 +131,18 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
return merr.WrapErrParameterInvalidMsg("%s", err.Error())
}
// Materialize the bound index meta for the new function output field BEFORE the
// broadcast, so the WAL message carries a complete, replay-deterministic index
// definition and the ack callback stays a pure idempotent apply.
var boundFieldIndexes []*indexpb.FieldIndex
if plan.Kind == schemautil.AlterSchemaAddFunctionField {
fieldIndex, err := c.prepareBoundFieldIndex(ctx, coll, plan)
if err != nil {
return err
}
boundFieldIndexes = append(boundFieldIndexes, fieldIndex)
}
// Broadcast.
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
if err != nil {
@@ -148,8 +163,9 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
}).
WithBody(&messagespb.AlterCollectionMessageBody{
Updates: &messagespb.AlterCollectionMessageUpdates{
Schema: schema,
Properties: properties,
Schema: schema,
Properties: properties,
BoundFieldIndexes: boundFieldIndexes,
},
}).
WithBroadcast(channels).
@@ -160,6 +176,91 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
return nil
}
// prepareBoundFieldIndex materializes the index meta bound to the newly added
// function-output field, strictly BEFORE the DDL broadcast: index id/name are
// allocated here and serialized into the WAL message so that the ack-callback
// apply is a pure idempotent write (a replayed callback rebuilds the identical
// index), and every input-dependent rejection happens before anything commits.
func (c *Core) prepareBoundFieldIndex(ctx context.Context, coll *model.Collection, plan *schemautil.AlterSchemaAddPlan) (*indexpb.FieldIndex, error) {
indexParamsMap, err := indexparamcheck.PrepareFunctionOutputIndexParams(
plan.Function.GetType(), plan.Field.GetName(), plan.IndexExtraParams)
if err != nil {
return nil, err
}
// The index type must have a registered checker — an unknown type would pass
// structural validation, get persisted via the ack callback, and then never
// build. Proxy already rejects this, but the check must live pre-broadcast
// for callers that reach rootcoord directly.
indexType := indexParamsMap[common.IndexTypeKey]
if checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType); err != nil || indexparamcheck.IsHYBRIDChecker(checker) {
return nil, merr.WrapErrParameterInvalidMsg(
"invalid index type %s for the bound index of function output field %q",
indexType, plan.Field.GetName())
}
// Full field-aware validation (params size, dimension fill+match, data-type
// compatibility, train params), identical to the create_index path — an index
// that cannot build must never be persisted through the ack callback.
if err := indexparamcheck.ValidateFieldIndexParams(plan.Field, indexParamsMap); err != nil {
return nil, err
}
indexName := plan.IndexName
if indexName == "" {
indexName = plan.Field.GetName()
}
// Name-format rule, same as the proxy path — enforced here too for callers
// that reach rootcoord directly.
if err := indexparamcheck.ValidateIndexName(indexName); err != nil {
return nil, err
}
// Reject index-name conflicts with existing indexes (the field itself is new,
// so only cross-field name collisions are possible).
resp, err := c.mixCoord.DescribeIndex(ctx, &indexpb.DescribeIndexRequest{
CollectionID: coll.CollectionID,
})
if err := merr.CheckRPCCall(resp.GetStatus(), err); err != nil {
if !merr.ErrIndexNotFound.Is(err) {
return nil, merr.Wrap(err, "failed to list existing indexes for bound index preparation")
}
} else {
for _, info := range resp.GetIndexInfos() {
if info.GetIndexName() == indexName {
return nil, merr.WrapErrParameterInvalidMsg("index name %s already exists in collection", indexName)
}
}
}
indexID, err := c.idAllocator.AllocOne()
if err != nil {
return nil, merr.Wrap(err, "failed to allocate index id for bound index")
}
createTime, err := c.tsoAllocator.GenerateTSO(1)
if err != nil {
return nil, merr.Wrap(err, "failed to allocate timestamp for bound index")
}
indexParams := funcutil.Map2KeyValuePair(indexParamsMap)
// Field type params minus per-field mmap/warmup keys, mirroring datacoord CreateIndex.
typeParams := lo.Filter(plan.Field.GetTypeParams(), func(kv *commonpb.KeyValuePair, _ int) bool {
return kv.GetKey() != common.MmapEnabledKey && kv.GetKey() != common.WarmupKey
})
index := &model.Index{
CollectionID: coll.CollectionID,
FieldID: plan.Field.GetFieldID(),
IndexID: indexID,
IndexName: indexName,
TypeParams: typeParams,
IndexParams: indexParams,
CreateTime: createTime,
IsAutoIndex: false,
UserIndexParams: plan.IndexExtraParams,
}
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
return nil, err
}
return model.MarshalIndexModel(index), nil
}
func prepareAlterSchemaAddField(coll *model.Collection, plan *schemautil.AlterSchemaAddPlan) error {
if !plan.HasField() {
return nil
@@ -19,6 +19,7 @@ package rootcoord
import (
"context"
"strconv"
"strings"
"testing"
"github.com/cockroachdb/errors"
@@ -30,16 +31,23 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/distributed/streaming"
"github.com/milvus-io/milvus/internal/metastore/model"
imocks "github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/mocks/distributed/mock_streaming"
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_balancer"
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/balance"
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/registry"
"github.com/milvus-io/milvus/internal/tso"
mocktso "github.com/milvus-io/milvus/internal/tso/mocks"
"github.com/milvus-io/milvus/internal/util/schemautil"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
"github.com/milvus-io/milvus/pkg/v3/util/timestamptz"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
@@ -64,7 +72,13 @@ func buildAlterSchemaReq(dbName, collName, inputField, outputField, funcName str
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: outputFieldSchema},
{
FieldSchema: outputFieldSchema,
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
},
},
},
FuncSchema: []*schemapb.FunctionSchema{functionSchema},
},
@@ -416,7 +430,13 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
{Name: "fn_dup_field", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}, OutputFieldNames: []string{"field1"}},
},
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{FieldSchema: &schemapb.FieldSchema{Name: "field1", DataType: schemapb.DataType_SparseFloatVector}},
{
FieldSchema: &schemapb.FieldSchema{Name: "field1", DataType: schemapb.DataType_SparseFloatVector},
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
},
},
},
},
},
@@ -424,6 +444,14 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
// case 7.1: vector function output field without bound index params is rejected.
noIndexReq := buildAlterSchemaReq(dbName, collectionName, "field1", "sparse_no_index", "fn_no_index")
noIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = nil
resp, err = core.AlterCollectionSchema(ctx, noIndexReq)
noIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, noIndexErr, merr.ErrParameterInvalid)
require.ErrorContains(t, noIndexErr, "index params are required")
// case 8: output field points to an existing field while FieldInfos adds a different field
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
@@ -462,6 +490,29 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
})
require.NoError(t, merr.CheckRPCCall(addFieldResp, err))
// case 7.2: AUTOINDEX is not accepted as the bound index type in V1
// (rejected at rootcoord prepare, after function validation passes).
autoIndexReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_auto_index", "fn_auto_index")
autoIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: common.AutoIndexName},
}
resp, err = core.AlterCollectionSchema(ctx, autoIndexReq)
autoIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, autoIndexErr, merr.ErrParameterInvalid)
require.ErrorContains(t, autoIndexErr, "explicit index_type is required")
// case 7.3: an index type without a registered checker is rejected at prepare —
// it would otherwise be persisted via the ack callback and never build.
unknownIndexReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_unknown_index", "fn_unknown_index")
unknownIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NOT_A_REAL_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
}
resp, err = core.AlterCollectionSchema(ctx, unknownIndexReq)
unknownIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, unknownIndexErr, merr.ErrParameterInvalid)
require.ErrorContains(t, unknownIndexErr, "invalid index type")
// happy path: add binary vector output field + MinHash function.
minHashReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "binary_minhash_output", "minhash_fn")
minHashFieldSchema := minHashReq.GetAction().GetAddRequest().GetFieldInfos()[0].GetFieldSchema()
@@ -469,6 +520,10 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
minHashFieldSchema.TypeParams = []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "4096"},
}
minHashReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
}
minHashFunction := minHashReq.GetAction().GetAddRequest().GetFuncSchema()[0]
minHashFunction.Type = schemapb.FunctionType_MinHash
minHashFunction.Params = []*commonpb.KeyValuePair{
@@ -481,6 +536,27 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
assertSchemaVersion(t, ctx, core, dbName, collectionName, 5)
// The bound index meta must have been applied through the CreateIndex ack
// callback within the same DDL, fully materialized at prepare time.
coll, err = core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
require.NoError(t, err)
var minHashFieldID int64
for _, field := range coll.Fields {
if field.Name == "binary_minhash_output" {
minHashFieldID = field.FieldID
}
}
require.Positive(t, minHashFieldID)
boundIndexes := recordedBoundIndexes()
require.Len(t, boundIndexes, 1)
boundIndexInfo := boundIndexes[0].GetIndexInfo()
require.Equal(t, "binary_minhash_output", boundIndexInfo.GetIndexName())
require.Equal(t, minHashFieldID, boundIndexInfo.GetFieldID())
require.Positive(t, boundIndexInfo.GetIndexID())
boundIndexParams := funcutil.KeyValuePair2Map(boundIndexInfo.GetIndexParams())
require.Equal(t, "MINHASH_LSH", boundIndexParams[common.IndexTypeKey])
require.Equal(t, "MHJACCARD", boundIndexParams[common.MetricTypeKey])
minHashBadArityReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "minhash_bad_arity", "minhash_bad_arity_fn")
minHashBadArityReq.GetAction().GetAddRequest().GetFuncSchema()[0].Type = schemapb.FunctionType_MinHash
minHashBadArityReq.GetAction().GetAddRequest().GetFuncSchema()[0].InputFieldNames = []string{"text_input", "field1"}
@@ -1344,3 +1420,238 @@ func TestAlterCollectionV2AckCallbackUsesHeaderDroppedFieldIDs(t *testing.T) {
})
require.NoError(t, err)
}
func newBoundIndexTestPlan() (*model.Collection, *schemautil.AlterSchemaAddPlan) {
coll := &model.Collection{CollectionID: 1, DBID: 1, Name: "coll"}
plan := &schemautil.AlterSchemaAddPlan{
Kind: schemautil.AlterSchemaAddFunctionField,
Field: &schemapb.FieldSchema{
FieldID: 101,
Name: "sparse",
DataType: schemapb.DataType_SparseFloatVector,
},
Function: &schemapb.FunctionSchema{Name: "bm25_fn", Type: schemapb.FunctionType_BM25},
IndexExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
},
}
return coll, plan
}
func withNoIndexMixCoord(t *testing.T) *imocks.MixCoord {
mixc := imocks.NewMixCoord(t)
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(
&indexpb.DescribeIndexResponse{Status: merr.Status(merr.WrapErrIndexNotFound("sparse"))}, nil,
).Maybe()
return mixc
}
func withOkTso(t *testing.T) tso.Allocator {
alloc := mocktso.NewAllocator(t)
alloc.EXPECT().GenerateTSO(mock.Anything).Return(uint64(100), nil).Maybe()
return alloc
}
func TestPrepareBoundFieldIndex(t *testing.T) {
t.Run("happy path materializes replay-deterministic index", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
fieldIndex, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.NoError(t, err)
info := fieldIndex.GetIndexInfo()
require.Equal(t, int64(101), info.GetFieldID())
require.Equal(t, "sparse", info.GetIndexName())
require.Positive(t, info.GetIndexID())
params := funcutil.KeyValuePair2Map(info.GetIndexParams())
require.Equal(t, "SPARSE_INVERTED_INDEX", params[common.IndexTypeKey])
require.Equal(t, "1.2", params["bm25_k1"])
// UserIndexParams keep the user's ORIGINAL params (create_index
// convention), so a later create_index with identical params is
// treated as idempotent instead of a distinct index.
require.Equal(t, plan.IndexExtraParams, info.GetUserIndexParams())
})
t.Run("index name conflict rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
mixc := imocks.NewMixCoord(t)
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(
&indexpb.DescribeIndexResponse{
Status: merr.Success(),
IndexInfos: []*indexpb.IndexInfo{{FieldID: 100, IndexID: 1, IndexName: "sparse"}},
}, nil,
)
c := newTestCore(withMixCoord(mixc), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.ErrorIs(t, err, merr.ErrParameterInvalid)
require.ErrorContains(t, err, "already exists")
})
t.Run("describe index transient error rejects the DDL", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
mixc := imocks.NewMixCoord(t)
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(nil, errors.New("rpc unavailable"))
c := newTestCore(withMixCoord(mixc), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "failed to list existing indexes")
})
t.Run("index id allocation failure", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withInvalidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "failed to allocate index id")
})
t.Run("incompatible index type for field data type rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
plan.Field = &schemapb.FieldSchema{
FieldID: 102,
Name: "binary_mh",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "512"},
},
}
plan.Function = &schemapb.FunctionSchema{Name: "mh_fn", Type: schemapb.FunctionType_MinHash}
// SPARSE index on a binary vector field: must be rejected pre-broadcast.
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
})
t.Run("dimension mismatch rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
plan.Field = &schemapb.FieldSchema{
FieldID: 102,
Name: "binary_mh",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "512"},
},
}
plan.Function = &schemapb.FunctionSchema{Name: "mh_fn", Type: schemapb.FunctionType_MinHash}
plan.IndexExtraParams = []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
{Key: common.DimKey, Value: "1024"},
}
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "dimension mismatch")
})
t.Run("bogus warmup policy rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
plan.IndexExtraParams = append(plan.IndexExtraParams, &commonpb.KeyValuePair{
Key: common.WarmupKey, Value: "bogus",
})
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "warmup")
})
t.Run("malformed index name rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
plan.IndexName = "1bad-name"
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "Invalid index name")
})
t.Run("oversized index params rejected", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
plan.IndexExtraParams = append(plan.IndexExtraParams, &commonpb.KeyValuePair{
Key: "huge",
Value: strings.Repeat("x", paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()+1),
})
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "exceeds limit")
})
t.Run("tso allocation failure", func(t *testing.T) {
coll, plan := newBoundIndexTestPlan()
alloc := mocktso.NewAllocator(t)
alloc.EXPECT().GenerateTSO(mock.Anything).Return(uint64(0), errors.New("tso unavailable"))
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(alloc))
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
require.Error(t, err)
require.ErrorContains(t, err, "failed to allocate timestamp")
})
}
func TestApplyBoundFieldIndexesInline(t *testing.T) {
wal := mock_streaming.NewMockWALAccesser(t)
wal.EXPECT().ControlChannel().Return(funcutil.GetControlChannel("by-dev-rootcoord-dml_0")).Maybe()
streaming.SetWALForTest(wal)
boundFieldIndex := &indexpb.FieldIndex{
IndexInfo: &indexpb.IndexInfo{
CollectionID: 1,
FieldID: 101,
IndexID: 201,
IndexName: "sparse",
IndexParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
},
},
}
buildResult := func(bound []*indexpb.FieldIndex) message.BroadcastResultAlterCollectionMessageV2 {
raw := message.NewAlterCollectionMessageBuilderV2().
WithHeader(&messagespb.AlterCollectionMessageHeader{
DbId: 1,
CollectionId: 1,
}).
WithBody(&messagespb.AlterCollectionMessageBody{
Updates: &messagespb.AlterCollectionMessageUpdates{
BoundFieldIndexes: bound,
},
}).
WithBroadcast([]string{funcutil.GetControlChannel("by-dev-rootcoord-dml_0")}).
MustBuildBroadcast()
msg := message.MustAsBroadcastAlterCollectionMessageV2(raw.WithBroadcastID(1))
return message.BroadcastResultAlterCollectionMessageV2{
Message: msg,
Results: map[string]*message.AppendResult{},
}
}
t.Run("no bound indexes short circuits", func(t *testing.T) {
registry.ResetRegistration()
cb := &DDLCallback{Core: newTestCore()}
require.NoError(t, cb.applyBoundFieldIndexesInline(context.Background(), buildResult(nil)))
})
t.Run("dispatches synthetic create index message", func(t *testing.T) {
registry.ResetRegistration()
var applied []*indexpb.FieldIndex
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
applied = append(applied, result.Message.MustBody().GetFieldIndex())
return nil
})
cb := &DDLCallback{Core: newTestCore()}
require.NoError(t, cb.applyBoundFieldIndexesInline(context.Background(), buildResult([]*indexpb.FieldIndex{boundFieldIndex})))
require.Len(t, applied, 1)
require.Equal(t, int64(201), applied[0].GetIndexInfo().GetIndexID())
require.Equal(t, "sparse", applied[0].GetIndexInfo().GetIndexName())
})
t.Run("dispatch failure propagates so the ack callback retries", func(t *testing.T) {
registry.ResetRegistration()
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
return errors.New("catalog write failed")
})
cb := &DDLCallback{Core: newTestCore()}
err := cb.applyBoundFieldIndexesInline(context.Background(), buildResult([]*indexpb.FieldIndex{boundFieldIndex}))
require.Error(t, err)
require.ErrorContains(t, err, "failed to apply bound field index")
})
}
+31
View File
@@ -56,6 +56,7 @@ import (
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mlog"
"github.com/milvus-io/milvus/pkg/v3/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
"github.com/milvus-io/milvus/pkg/v3/proto/proxypb"
"github.com/milvus-io/milvus/pkg/v3/proto/rootcoordpb"
@@ -79,6 +80,31 @@ func TestMain(m *testing.M) {
os.Exit(code)
}
// testBoundIndexRecorder captures FieldIndexes applied through the fake
// CreateIndexV2 ack callback so DDL tests can assert on bound-index creation.
var testBoundIndexRecorder = struct {
mu sync.Mutex
indexes []*indexpb.FieldIndex
}{}
func resetRecordedBoundIndexes() {
testBoundIndexRecorder.mu.Lock()
defer testBoundIndexRecorder.mu.Unlock()
testBoundIndexRecorder.indexes = nil
}
func recordBoundIndex(fieldIndex *indexpb.FieldIndex) {
testBoundIndexRecorder.mu.Lock()
defer testBoundIndexRecorder.mu.Unlock()
testBoundIndexRecorder.indexes = append(testBoundIndexRecorder.indexes, fieldIndex)
}
func recordedBoundIndexes() []*indexpb.FieldIndex {
testBoundIndexRecorder.mu.Lock()
defer testBoundIndexRecorder.mu.Unlock()
return append([]*indexpb.FieldIndex(nil), testBoundIndexRecorder.indexes...)
}
func initStreamingSystemAndCore(t *testing.T) *Core {
kv, _ := kvfactory.GetEtcdAndPath()
path := funcutil.RandomString(10) + "/meta"
@@ -110,6 +136,11 @@ func initStreamingSystemAndCore(t *testing.T) *Core {
registry.RegisterDropIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultDropIndexMessageV2) error {
return nil
})
resetRecordedBoundIndexes()
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
recordBoundIndex(result.Message.MustBody().GetFieldIndex())
return nil
})
registry.RegisterDropLoadConfigV2AckCallback(func(ctx context.Context, result message.BroadcastResultDropLoadConfigMessageV2) error {
return nil
})
@@ -0,0 +1,302 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package indexparamcheck
import (
"strings"
"google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/metric"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// ValidateIndexParams performs structural validation on an index definition.
// Moved from internal/datacoord so that rootcoord's add-function-field bound-index
// prepare can share it with datacoord's CreateIndex/AlterIndex/snapshot restore paths.
func ValidateIndexParams(index *model.Index) error {
if err := checkDuplicateKey(index.IndexParams, "indexParams"); err != nil {
return err
}
if err := checkDuplicateKey(index.UserIndexParams, "userIndexParams"); err != nil {
return err
}
if err := checkDuplicateKey(index.TypeParams, "typeParams"); err != nil {
return err
}
indexType := common.GetIndexType(index.IndexParams)
indexParams := funcutil.KeyValuePair2Map(index.IndexParams)
userIndexParams := funcutil.KeyValuePair2Map(index.UserIndexParams)
if err := ValidateMmapIndexParams(indexType, indexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid mmap index params: %s", err.Error())
}
if err := ValidateMmapIndexParams(indexType, userIndexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid mmap user index params: %s", err.Error())
}
if err := ValidateOffsetCacheIndexParams(indexType, indexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
}
if err := ValidateOffsetCacheIndexParams(indexType, userIndexParams); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
}
return nil
}
// checkDuplicateKey rejects duplicated keys in a key-value pair list.
func checkDuplicateKey(kvs []*commonpb.KeyValuePair, tag string) error {
keySet := typeutil.NewSet[string]()
for _, kv := range kvs {
if keySet.Contain(kv.GetKey()) {
return merr.WrapErrParameterInvalidMsg("duplicate %s key in %s params", kv.GetKey(), tag)
}
keySet.Insert(kv.GetKey())
}
return nil
}
// FillFunctionOutputIndexParams applies the function-type-specific defaults and
// constraints to the index params of a function output field. Shared by the
// create_index path (proxy createIndexTask.parseFunctionParamsToIndex) and the
// add-function-field bound-index prepare (rootcoord), so both produce identical
// build params (e.g. knowhere requires bm25_k1/bm25_b/bm25_avgdl for BM25).
func FillFunctionOutputIndexParams(functionType schemapb.FunctionType, indexParamsMap map[string]string) error {
switch functionType {
case schemapb.FunctionType_Unknown:
return merr.WrapErrParameterInvalidMsg("unknown function type encountered")
case schemapb.FunctionType_BM25:
// set default BM25 params if not provided in index params
if _, ok := indexParamsMap["bm25_k1"]; !ok {
indexParamsMap["bm25_k1"] = "1.2"
}
if _, ok := indexParamsMap["bm25_b"]; !ok {
indexParamsMap["bm25_b"] = "0.75"
}
if _, ok := indexParamsMap["bm25_avgdl"]; !ok {
indexParamsMap["bm25_avgdl"] = "100"
}
if metricType, ok := indexParamsMap[common.MetricTypeKey]; !ok {
indexParamsMap[common.MetricTypeKey] = metric.BM25
} else if metricType != metric.BM25 {
return merr.WrapErrParameterInvalidMsg("index metric type of BM25 function output field must be BM25, got %s", metricType)
}
default:
return nil
}
return nil
}
// PrepareFunctionOutputIndexParams expands the user-provided extra params of a
// function output field's bound index, requires an explicit index type (no
// AUTOINDEX resolution for bound indexes), and applies function-type-specific
// defaults. Shared by proxy validation and rootcoord materialization so both
// operate on identical params.
func PrepareFunctionOutputIndexParams(functionType schemapb.FunctionType, fieldName string, extraParams []*commonpb.KeyValuePair) (map[string]string, error) {
indexParamsMap, err := ExpandIndexParams(extraParams)
if err != nil {
return nil, err
}
indexType := indexParamsMap[common.IndexTypeKey]
if indexType == "" || indexType == common.AutoIndexName {
return nil, merr.WrapErrParameterInvalidMsg(
"an explicit index_type is required for the bound index of function output field %q", fieldName)
}
if err := FillFunctionOutputIndexParams(functionType, indexParamsMap); err != nil {
return nil, err
}
return indexParamsMap, nil
}
// ExpandIndexParams flattens user-provided index extra params into a plain map,
// expanding the legacy JSON-encoded common.ParamsKey entry, and rejecting
// duplicated keys. Same convention as proxy's createIndexTask.parseIndexParams.
func ExpandIndexParams(extraParams []*commonpb.KeyValuePair) (map[string]string, error) {
indexParamsMap := make(map[string]string, len(extraParams))
keys := typeutil.NewSet[string]()
for _, kv := range extraParams {
if keys.Contain(kv.GetKey()) {
return nil, merr.WrapErrParameterInvalidMsg("duplicated index param (key=%s) (value=%s) found", kv.GetKey(), kv.GetValue())
}
keys.Insert(kv.GetKey())
if kv.GetKey() == common.ParamsKey {
params, err := funcutil.JSONToMap(kv.GetValue())
if err != nil {
return nil, err
}
for k, v := range params {
indexParamsMap[k] = v
}
} else {
indexParamsMap[kv.GetKey()] = kv.GetValue()
}
}
return indexParamsMap, nil
}
// CheckIndexParamsSize rejects index params whose total size exceeds
// proxy.maxIndexParamsSize. Moved from proxy so every DDL boundary that
// accepts index params (create_index and the add-function-field bound index)
// enforces the same limit.
func CheckIndexParamsSize(size int) error {
maxIndexParamsSize := paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()
if size > maxIndexParamsSize {
return merr.WrapErrParameterInvalidMsg("index params size exceeds limit: %d > %d", size, maxIndexParamsSize)
}
return nil
}
// ValidateIndexParamsSize checks the total size of a key-value pair list.
func ValidateIndexParamsSize(params ...*commonpb.KeyValuePair) error {
size := 0
for _, param := range params {
size += len(param.GetKey()) + len(param.GetValue())
}
return CheckIndexParamsSize(size)
}
// ValidateIndexParamsMapSize checks the total size of an expanded params map.
func ValidateIndexParamsMapSize(params map[string]string) error {
size := 0
for k, v := range params {
size += len(k) + len(v)
}
return CheckIndexParamsSize(size)
}
// FillDimension copies the dimension from the field schema into the index
// params (rejecting a mismatch), so index build always sees the field's real
// dimension. Moved from proxy for DDL-boundary reuse.
func FillDimension(field *schemapb.FieldSchema, indexParams map[string]string) error {
if !typeutil.IsVectorType(field.GetDataType()) {
return nil
}
params := make([]*commonpb.KeyValuePair, 0, len(field.GetTypeParams())+len(field.GetIndexParams()))
params = append(params, field.GetTypeParams()...)
params = append(params, field.GetIndexParams()...)
dimensionInSchema, err := funcutil.GetAttrByKeyFromRepeatedKV(common.DimKey, params)
if err != nil {
return merr.WrapErrParameterInvalidMsg("dimension not found in schema")
}
dimension, exist := indexParams[common.DimKey]
if exist {
if dimensionInSchema != dimension {
return merr.WrapErrParameterInvalidMsg("dimension mismatch, dimension in schema: %s, dimension: %s", dimensionInSchema, dimension)
}
} else {
indexParams[common.DimKey] = dimensionInSchema
}
return nil
}
// ValidateFieldIndexParams runs the field-aware index validation that the
// create_index path applies (params size, checker existence, dimension
// fill+match, data-type compatibility, train-params validation), so the
// add-function-field bound-index path enforces identical rules at every DDL
// boundary, including callers that reach rootcoord directly.
func ValidateFieldIndexParams(field *schemapb.FieldSchema, indexParamsMap map[string]string) error {
if err := ValidateIndexParamsMapSize(indexParamsMap); err != nil {
return err
}
if err := ValidateWarmupIndexParams(indexParamsMap); err != nil {
return merr.WrapErrParameterInvalidMsg("invalid warmup params: %s", err.Error())
}
indexType := indexParamsMap[common.IndexTypeKey]
checker, err := GetIndexCheckerMgrInstance().GetChecker(indexType)
if err != nil {
return merr.WrapErrParameterInvalidMsg("invalid index type: %s", indexType)
}
// For ArrayOfVector with non-EmbList metrics, each embedding in the array is
// indexed independently as a regular vector; resolve the effective data type
// used for compatibility checks (same rule as the create_index path).
effectiveDataType := field.GetDataType()
effectiveElementType := field.GetElementType()
if typeutil.IsArrayOfVectorType(field.GetDataType()) &&
!funcutil.SliceContain(EmbListMetrics, indexParamsMap[common.MetricTypeKey]) {
effectiveDataType = field.GetElementType()
effectiveElementType = schemapb.DataType_None
}
if typeutil.IsVectorType(field.GetDataType()) && !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
if err := FillDimension(field, indexParamsMap); err != nil {
return err
}
}
effectiveField := field
if effectiveDataType != field.GetDataType() {
effectiveField = proto.Clone(field).(*schemapb.FieldSchema)
effectiveField.DataType = effectiveDataType
effectiveField.ElementType = effectiveElementType
}
if err := checker.CheckValidDataType(indexType, effectiveField); err != nil {
return err
}
return checker.CheckTrain(effectiveDataType, effectiveElementType, indexParamsMap)
}
// ValidateIndexName enforces the index-name format rules. Moved from proxy so
// rootcoord's bound-index prepare applies the same rule to callers that reach
// it directly.
func ValidateIndexName(indexName string) error {
indexName = strings.TrimSpace(indexName)
if indexName == "" {
return nil
}
invalidMsg := "Invalid index name: " + indexName + ". "
if len(indexName) > paramtable.Get().ProxyCfg.MaxNameLength.GetAsInt() {
msg := invalidMsg + "The length of a index name must be less than " + paramtable.Get().ProxyCfg.MaxNameLength.GetValue() + " characters."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
firstChar := indexName[0]
if firstChar != '_' && !isIndexNameAlpha(firstChar) {
msg := invalidMsg + "The first character of a index name must be an underscore or letter."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
for i := 1; i < len(indexName); i++ {
c := indexName[i]
if !validCharInIndexName(c) {
msg := invalidMsg + "Index name can only contain numbers, letters, and underscores."
return merr.WrapErrParameterInvalidMsg("%s", msg)
}
}
return nil
}
func isIndexNameAlpha(c uint8) bool {
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
}
func validCharInIndexName(c byte) bool {
return c == '_' || c == '[' || c == ']' || isIndexNameAlpha(c) || (c >= '0' && c <= '9')
}
@@ -0,0 +1,262 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package indexparamcheck
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
)
func TestValidateIndexParams(t *testing.T) {
t.Run("valid", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
{
Key: common.MmapEnabledKey,
Value: "true",
},
},
}
err := ValidateIndexParams(index)
assert.NoError(t, err)
})
t.Run("invalid index param", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
{
Key: common.MmapEnabledKey,
Value: "h",
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("invalid index user param", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
},
UserIndexParams: []*commonpb.KeyValuePair{
{
Key: common.MmapEnabledKey,
Value: "h",
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_index_params", func(t *testing.T) {
index := &model.Index{
IndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_user_index_params", func(t *testing.T) {
index := &model.Index{
UserIndexParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
t.Run("duplicated_type_params", func(t *testing.T) {
index := &model.Index{
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
{
Key: common.IndexTypeKey,
Value: AutoIndex,
},
},
}
err := ValidateIndexParams(index)
assert.Error(t, err)
})
}
func TestExpandIndexParams(t *testing.T) {
t.Run("flat_and_json_params", func(t *testing.T) {
params, err := ExpandIndexParams([]*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.MetricTypeKey, Value: "BM25"},
{Key: common.ParamsKey, Value: `{"bm25_k1": "1.2", "bm25_b": "0.75"}`},
})
assert.NoError(t, err)
assert.Equal(t, "SPARSE_INVERTED_INDEX", params[common.IndexTypeKey])
assert.Equal(t, "BM25", params[common.MetricTypeKey])
assert.Equal(t, "1.2", params["bm25_k1"])
assert.Equal(t, "0.75", params["bm25_b"])
})
t.Run("duplicated_key_rejected", func(t *testing.T) {
_, err := ExpandIndexParams([]*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
{Key: common.IndexTypeKey, Value: "SPARSE_WAND"},
})
assert.Error(t, err)
})
t.Run("invalid_json_rejected", func(t *testing.T) {
_, err := ExpandIndexParams([]*commonpb.KeyValuePair{
{Key: common.ParamsKey, Value: `{not-json`},
})
assert.Error(t, err)
})
}
func TestValidateFieldIndexParams(t *testing.T) {
paramtable.Init()
sparseField := &schemapb.FieldSchema{
FieldID: 101,
Name: "sparse",
DataType: schemapb.DataType_SparseFloatVector,
}
binaryField := &schemapb.FieldSchema{
FieldID: 102,
Name: "binary_mh",
DataType: schemapb.DataType_BinaryVector,
TypeParams: []*commonpb.KeyValuePair{
{Key: common.DimKey, Value: "512"},
},
}
t.Run("happy path sparse bm25", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
common.MetricTypeKey: "BM25",
"bm25_k1": "1.2",
"bm25_b": "0.75",
"bm25_avgdl": "100",
}
assert.NoError(t, ValidateFieldIndexParams(sparseField, params))
})
t.Run("dimension filled from schema", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "MINHASH_LSH",
common.MetricTypeKey: "MHJACCARD",
}
assert.NoError(t, ValidateFieldIndexParams(binaryField, params))
assert.Equal(t, "512", params[common.DimKey])
})
t.Run("dimension mismatch rejected", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "MINHASH_LSH",
common.MetricTypeKey: "MHJACCARD",
common.DimKey: "1024",
}
err := ValidateFieldIndexParams(binaryField, params)
assert.Error(t, err)
assert.Contains(t, err.Error(), "dimension mismatch")
})
t.Run("incompatible index type for field data type rejected", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
common.MetricTypeKey: "BM25",
}
err := ValidateFieldIndexParams(binaryField, params)
assert.Error(t, err)
})
t.Run("unknown index type rejected", func(t *testing.T) {
params := map[string]string{common.IndexTypeKey: "NOT_A_REAL_INDEX"}
err := ValidateFieldIndexParams(sparseField, params)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid index type")
})
t.Run("bogus warmup policy rejected", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
common.MetricTypeKey: "BM25",
common.WarmupKey: "bogus",
}
err := ValidateFieldIndexParams(sparseField, params)
assert.Error(t, err)
assert.Contains(t, err.Error(), "warmup")
})
t.Run("oversized params rejected", func(t *testing.T) {
params := map[string]string{
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
"huge": strings.Repeat("x", paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()+1),
}
err := ValidateFieldIndexParams(sparseField, params)
assert.Error(t, err)
assert.Contains(t, err.Error(), "exceeds limit")
})
}
func TestValidateIndexName(t *testing.T) {
paramtable.Init()
assert.NoError(t, ValidateIndexName(""))
assert.NoError(t, ValidateIndexName("sparse_idx_1"))
assert.NoError(t, ValidateIndexName("_idx"))
assert.Error(t, ValidateIndexName("1bad"))
assert.Error(t, ValidateIndexName("bad-name"))
assert.Error(t, ValidateIndexName(strings.Repeat("x", paramtable.Get().ProxyCfg.MaxNameLength.GetAsInt()+1)))
}
+30 -1
View File
@@ -17,9 +17,11 @@
package schemautil
import (
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
type AlterSchemaAddKind int
@@ -34,6 +36,12 @@ type AlterSchemaAddPlan struct {
Kind AlterSchemaAddKind
Field *schemapb.FieldSchema
Function *schemapb.FunctionSchema
// Index meta bound to the newly added field, parsed from
// FieldInfo.index_name/extra_params. Mandatory for vector-type function
// output fields so that bump-schema-version compaction results are never
// blocked on a missing vector index.
IndexName string
IndexExtraParams []*commonpb.KeyValuePair
}
func (p *AlterSchemaAddPlan) HasField() bool {
@@ -59,12 +67,16 @@ func ParseAlterSchemaAddRequest(addRequest *milvuspb.AlterCollectionSchemaReques
}
var field *schemapb.FieldSchema
var indexName string
var indexExtraParams []*commonpb.KeyValuePair
if len(fieldInfos) == 1 {
fieldInfo := fieldInfos[0]
if fieldInfo == nil || fieldInfo.GetFieldSchema() == nil {
return nil, merr.WrapErrParameterInvalidMsg("fieldSchema is nil in fieldInfos")
}
field = fieldInfo.GetFieldSchema()
indexName = fieldInfo.GetIndexName()
indexExtraParams = fieldInfo.GetExtraParams()
}
var function *schemapb.FunctionSchema
@@ -77,7 +89,13 @@ func ParseAlterSchemaAddRequest(addRequest *milvuspb.AlterCollectionSchemaReques
switch {
case field != nil && function != nil:
return &AlterSchemaAddPlan{Kind: AlterSchemaAddFunctionField, Field: field, Function: function}, nil
return &AlterSchemaAddPlan{
Kind: AlterSchemaAddFunctionField,
Field: field,
Function: function,
IndexName: indexName,
IndexExtraParams: indexExtraParams,
}, nil
case field != nil:
return &AlterSchemaAddPlan{Kind: AlterSchemaAddField, Field: field}, nil
case function != nil:
@@ -113,6 +131,17 @@ func ValidateAlterSchemaAddFunctionPlan(plan *AlterSchemaAddPlan) error {
plan.Field.GetName(),
)
}
// A vector output field MUST come with bound index params: without an index
// meta, bump-schema-version compaction results can never pass the
// indexed-visibility check and the whole backfill pipeline silently stalls.
if typeutil.IsVectorType(plan.Field.GetDataType()) && len(plan.IndexExtraParams) == 0 {
return merr.WrapErrParameterInvalidMsg(
"index params are required when adding function output field %q of vector type %s: "+
"provide index_type/metric_type (and params) in field_info.extra_params",
plan.Field.GetName(),
plan.Field.GetDataType().String(),
)
}
return nil
default:
return merr.WrapErrParameterInvalidMsg("unknown alter schema add request kind")
+4
View File
@@ -309,6 +309,10 @@ message AlterCollectionMessageUpdates {
common.ConsistencyLevel consistency_level = 6; // consistency level should be updated.
repeated common.KeyValuePair properties = 7; // collection properties should be updated.
AlterLoadConfigOfAlterCollection alter_load_config = 8; // alter load config of alter collection.
// Index meta bound to newly added function-output/vector fields.
// Fully materialized (index id/name allocated, params validated) at DDL prepare
// stage BEFORE broadcast; applied atomically with the schema in the ack callback.
repeated index.FieldIndex bound_field_indexes = 9;
}
// AlterLoadConfigOfAlterCollection is the body of alter load config of alter collection message.
+71 -54
View File
@@ -2048,6 +2048,10 @@ type AlterCollectionMessageUpdates struct {
ConsistencyLevel commonpb.ConsistencyLevel `protobuf:"varint,6,opt,name=consistency_level,json=consistencyLevel,proto3,enum=milvus.proto.common.ConsistencyLevel" json:"consistency_level,omitempty"` // consistency level should be updated.
Properties []*commonpb.KeyValuePair `protobuf:"bytes,7,rep,name=properties,proto3" json:"properties,omitempty"` // collection properties should be updated.
AlterLoadConfig *AlterLoadConfigOfAlterCollection `protobuf:"bytes,8,opt,name=alter_load_config,json=alterLoadConfig,proto3" json:"alter_load_config,omitempty"` // alter load config of alter collection.
// Index meta bound to newly added function-output/vector fields.
// Fully materialized (index id/name allocated, params validated) at DDL prepare
// stage BEFORE broadcast; applied atomically with the schema in the ack callback.
BoundFieldIndexes []*indexpb.FieldIndex `protobuf:"bytes,9,rep,name=bound_field_indexes,json=boundFieldIndexes,proto3" json:"bound_field_indexes,omitempty"`
}
func (x *AlterCollectionMessageUpdates) Reset() {
@@ -2138,6 +2142,13 @@ func (x *AlterCollectionMessageUpdates) GetAlterLoadConfig() *AlterLoadConfigOfA
return nil
}
func (x *AlterCollectionMessageUpdates) GetBoundFieldIndexes() []*indexpb.FieldIndex {
if x != nil {
return x.BoundFieldIndexes
}
return nil
}
// AlterLoadConfigOfAlterCollection is the body of alter load config of alter collection message.
type AlterLoadConfigOfAlterCollection struct {
state protoimpl.MessageState
@@ -6870,8 +6881,8 @@ var file_messages_proto_rawDesc = []byte{
0x34, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d,
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c,
0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x55, 0x70,
0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xd3,
0x03, 0x0a, 0x1d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xa3,
0x04, 0x0a, 0x1d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73,
0x12, 0x13, 0x0a, 0x05, 0x64, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
0x04, 0x64, 0x62, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x64, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65,
@@ -6900,7 +6911,12 @@ var file_messages_proto_rawDesc = []byte{
0x65, 0x73, 0x2e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66,
0x69, 0x67, 0x4f, 0x66, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x61, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x22, 0x72, 0x0a, 0x20, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61,
0x6e, 0x66, 0x69, 0x67, 0x12, 0x4e, 0x0a, 0x13, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x66, 0x69,
0x65, 0x6c, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65,
0x78, 0x52, 0x11, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x64,
0x65, 0x78, 0x65, 0x73, 0x22, 0x72, 0x0a, 0x20, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61,
0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x66, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f,
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c,
0x69, 0x63, 0x61, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
@@ -7678,14 +7694,14 @@ var file_messages_proto_goTypes = []interface{}{
(*fieldmaskpb.FieldMask)(nil), // 130: google.protobuf.FieldMask
(commonpb.ConsistencyLevel)(0), // 131: milvus.proto.common.ConsistencyLevel
(*commonpb.KeyValuePair)(nil), // 132: milvus.proto.common.KeyValuePair
(commonpb.LoadPriority)(0), // 133: milvus.proto.common.LoadPriority
(*milvuspb.UserEntity)(nil), // 134: milvus.proto.milvus.UserEntity
(*internalpb.CredentialInfo)(nil), // 135: milvus.proto.internal.CredentialInfo
(*milvuspb.RoleEntity)(nil), // 136: milvus.proto.milvus.RoleEntity
(*milvuspb.RBACMeta)(nil), // 137: milvus.proto.milvus.RBACMeta
(*milvuspb.GrantEntity)(nil), // 138: milvus.proto.milvus.GrantEntity
(*milvuspb.PrivilegeGroupInfo)(nil), // 139: milvus.proto.milvus.PrivilegeGroupInfo
(*indexpb.FieldIndex)(nil), // 140: milvus.proto.index.FieldIndex
(*indexpb.FieldIndex)(nil), // 133: milvus.proto.index.FieldIndex
(commonpb.LoadPriority)(0), // 134: milvus.proto.common.LoadPriority
(*milvuspb.UserEntity)(nil), // 135: milvus.proto.milvus.UserEntity
(*internalpb.CredentialInfo)(nil), // 136: milvus.proto.internal.CredentialInfo
(*milvuspb.RoleEntity)(nil), // 137: milvus.proto.milvus.RoleEntity
(*milvuspb.RBACMeta)(nil), // 138: milvus.proto.milvus.RBACMeta
(*milvuspb.GrantEntity)(nil), // 139: milvus.proto.milvus.GrantEntity
(*milvuspb.PrivilegeGroupInfo)(nil), // 140: milvus.proto.milvus.PrivilegeGroupInfo
(commonpb.WALName)(0), // 141: milvus.proto.common.WALName
(commonpb.MsgType)(0), // 142: milvus.proto.common.MsgType
(*commonpb.MessageID)(nil), // 143: milvus.proto.common.MessageID
@@ -7707,49 +7723,50 @@ var file_messages_proto_depIdxs = []int32{
131, // 11: milvus.proto.messages.AlterCollectionMessageUpdates.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel
132, // 12: milvus.proto.messages.AlterCollectionMessageUpdates.properties:type_name -> milvus.proto.common.KeyValuePair
35, // 13: milvus.proto.messages.AlterCollectionMessageUpdates.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterCollection
38, // 14: milvus.proto.messages.AlterLoadConfigMessageHeader.load_fields:type_name -> milvus.proto.messages.LoadFieldConfig
39, // 15: milvus.proto.messages.AlterLoadConfigMessageHeader.replicas:type_name -> milvus.proto.messages.LoadReplicaConfig
133, // 16: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
132, // 17: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
132, // 18: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
46, // 19: milvus.proto.messages.AlterDatabaseMessageBody.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterDatabase
134, // 20: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
135, // 21: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
134, // 22: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
135, // 23: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
136, // 24: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
134, // 25: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
136, // 26: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
63, // 27: milvus.proto.messages.AlterUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
63, // 28: milvus.proto.messages.DropUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
137, // 29: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
138, // 30: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
138, // 31: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
139, // 32: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
139, // 33: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
123, // 34: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
140, // 35: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
140, // 36: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
141, // 37: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
124, // 38: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
105, // 39: milvus.proto.messages.CacheExpirations.cache_expirations:type_name -> milvus.proto.messages.CacheExpiration
106, // 40: milvus.proto.messages.CacheExpiration.legacy_proxy_collection_meta_cache:type_name -> milvus.proto.messages.LegacyProxyCollectionMetaCache
142, // 41: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
125, // 42: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
114, // 43: milvus.proto.messages.BroadcastHeader.Resource_keys:type_name -> milvus.proto.messages.ResourceKey
143, // 44: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
143, // 45: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
2, // 46: milvus.proto.messages.ResourceKey.domain:type_name -> milvus.proto.messages.ResourceDomain
120, // 47: milvus.proto.messages.BatchUpdateManifestMessageBody.items:type_name -> milvus.proto.messages.BatchUpdateManifestItem
121, // 48: milvus.proto.messages.BatchUpdateManifestItem.v2_column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
126, // 49: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
144, // 50: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
145, // 51: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry.value:type_name -> milvus.proto.data.FieldBinlog
52, // [52:52] is the sub-list for method output_type
52, // [52:52] is the sub-list for method input_type
52, // [52:52] is the sub-list for extension type_name
52, // [52:52] is the sub-list for extension extendee
0, // [0:52] is the sub-list for field type_name
133, // 14: milvus.proto.messages.AlterCollectionMessageUpdates.bound_field_indexes:type_name -> milvus.proto.index.FieldIndex
38, // 15: milvus.proto.messages.AlterLoadConfigMessageHeader.load_fields:type_name -> milvus.proto.messages.LoadFieldConfig
39, // 16: milvus.proto.messages.AlterLoadConfigMessageHeader.replicas:type_name -> milvus.proto.messages.LoadReplicaConfig
134, // 17: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
132, // 18: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
132, // 19: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
46, // 20: milvus.proto.messages.AlterDatabaseMessageBody.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterDatabase
135, // 21: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
136, // 22: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
135, // 23: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
136, // 24: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
137, // 25: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
135, // 26: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
137, // 27: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
63, // 28: milvus.proto.messages.AlterUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
63, // 29: milvus.proto.messages.DropUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
138, // 30: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
139, // 31: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
139, // 32: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
140, // 33: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
140, // 34: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
123, // 35: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
133, // 36: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
133, // 37: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
141, // 38: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
124, // 39: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
105, // 40: milvus.proto.messages.CacheExpirations.cache_expirations:type_name -> milvus.proto.messages.CacheExpiration
106, // 41: milvus.proto.messages.CacheExpiration.legacy_proxy_collection_meta_cache:type_name -> milvus.proto.messages.LegacyProxyCollectionMetaCache
142, // 42: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
125, // 43: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
114, // 44: milvus.proto.messages.BroadcastHeader.Resource_keys:type_name -> milvus.proto.messages.ResourceKey
143, // 45: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
143, // 46: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
2, // 47: milvus.proto.messages.ResourceKey.domain:type_name -> milvus.proto.messages.ResourceDomain
120, // 48: milvus.proto.messages.BatchUpdateManifestMessageBody.items:type_name -> milvus.proto.messages.BatchUpdateManifestItem
121, // 49: milvus.proto.messages.BatchUpdateManifestItem.v2_column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
126, // 50: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
144, // 51: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
145, // 52: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry.value:type_name -> milvus.proto.data.FieldBinlog
53, // [53:53] is the sub-list for method output_type
53, // [53:53] is the sub-list for method input_type
53, // [53:53] is the sub-list for extension type_name
53, // [53:53] is the sub-list for extension extendee
0, // [0:53] is the sub-list for field type_name
}
func init() { file_messages_proto_init() }
@@ -6,7 +6,16 @@ from base.client_v2_base import TestMilvusClientV2Base
from common import common_func as cf
from common import common_type as ct
from common.common_type import CaseLabel, CheckTasks
from pymilvus import AnnSearchRequest, DataType, FieldSchema, Function, FunctionType, RRFRanker, WeightedRanker
from pymilvus import (
AnnSearchRequest,
DataType,
FieldSchema,
Function,
FunctionType,
MilvusException,
RRFRanker,
WeightedRanker,
)
from utils.util_log import test_log as log
@@ -101,7 +110,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
client.load_collection(collection_name)
# Step 1: Execute add_function_field.
# Step 1: Execute add_function_field with the bound sparse index params.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -109,17 +118,16 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
# Step 2: Verify the new output field and function are visible in collection schema.
desc = client.describe_collection(collection_name)
assert "sparse" in [field["name"] for field in desc["fields"]]
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
# Step 3: Explicitly create sparse index for the added BM25 output field.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params)
# Step 3: The bound sparse index is created atomically by add_function_field; wait for build.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
# Step 4: Load collection after sparse index is ready.
@@ -192,13 +200,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
self.drop_collection(client, collection_name)
@pytest.mark.tags(CaseLabel.L0)
def test_add_bm25_function_field_no_auto_index(self):
def test_add_bm25_function_field_creates_bound_index(self):
"""
TC-02: add_function_field does not auto-create index.
TC-02: add_function_field atomically creates the bound index meta.
target: verify schema evolution and index management boundary
method: add BM25 function field, search before index, then create index and search again
expected: add_function_field does not create index implicitly; BM25 search works after explicit index/load
target: verify the bound index meta is created together with the schema change
method: add BM25 function field with bound index params, then describe the index right away
expected: describe_index shows the bound SPARSE_INVERTED_INDEX immediately; BM25 search works after load
"""
client = self._client()
collection_name = cf.gen_collection_name_by_testcase_name()
@@ -228,7 +236,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
client.load_collection(collection_name)
# Step 1: Execute add_function_field.
# Step 1: Execute add_function_field with the bound sparse index params.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -236,41 +244,26 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
assert "sparse" in [field["name"] for field in desc["fields"]]
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
# Expected: add_function_field should not create sparse index implicitly.
assert client.describe_index(collection_name, index_name="sparse") is None
# Step 2: The bound index meta is created atomically with the schema change,
# so describe_index shows it right after add_function_field returns.
bound_index_info = client.describe_index(collection_name, index_name="sparse")
assert bound_index_info is not None
assert bound_index_info["index_type"] == "SPARSE_INVERTED_INDEX"
# Step 2: Search on anns_field="sparse" before creating sparse index.
# Expected: no crash; field-not-loaded, no-index, empty result, or equivalent behavior is acceptable.
search_res, is_succ = self.search(
client,
collection_name,
data=["alpha"],
anns_field="sparse",
limit=5,
output_fields=["id"],
check_task=CheckTasks.check_nothing,
)
if is_succ:
assert len(search_res[0]) == 0
else:
err = str(search_res).lower()
assert any(token in err for token in ["sparse", "index", "load", "loaded", "not exist", "not found"]), err
# Step 3: Create sparse index and load collection.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params)
# Step 3: Wait for the bound index build and load collection.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
client.load_collection(collection_name)
# Step 4: Search again and wait until BM25 search becomes ready.
# Expected: BM25 search succeeds after explicit index creation and load.
# Step 4: Search and wait until BM25 search becomes ready.
# Expected: BM25 search succeeds through the bound index after load.
search_timeout = 30
poll_interval = 1
poll_start = time.time()
@@ -352,7 +345,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
]
client.insert(collection_name=collection_name, data=growing_rows)
# Step 4: Execute add_function_field while pre-add growing rows exist.
# Step 4: Execute add_function_field with bound sparse index params while pre-add growing rows exist.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -360,16 +353,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
assert "sparse" in [field["name"] for field in desc["fields"]]
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
# Step 5: Create sparse index and load collection.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params)
# Step 5: Wait for the bound sparse index build and load collection.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=30)
client.load_collection(collection_name)
@@ -454,7 +446,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert "bm25_fn" not in [func["name"] for func in before_desc.get("functions", [])]
assert int(before_desc["properties"]["max_field_id"]) == before_max_field_id
# Step 2: Execute add_function_field.
# Step 2: Execute add_function_field with the bound sparse index params.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -462,7 +454,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
# Step 3: Describe collection again and verify metadata has been updated.
after_desc = client.describe_collection(collection_name)
@@ -714,6 +708,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
),
]
# Valid bound index params so the field/function validation error stays the trigger.
invalid_case_index_params = client.prepare_index_params()
invalid_case_index_params.add_index(field_name="", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
for case_name, field_schema, function, error in invalid_cases:
# Step 2: Each invalid request must fail clearly.
self.add_function_field(
@@ -721,6 +719,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
collection_name,
field_schema,
function,
index_params=invalid_case_index_params,
check_task=CheckTasks.err_res,
check_items=error,
)
@@ -732,6 +731,61 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert [func["name"] for func in after_desc.get("functions", [])] == before_function_names, case_name
assert after_desc["properties"]["max_field_id"] == before_max_field_id, case_name
# Step 3.5: Bound index params negative matrix.
bound_params_field = FieldSchema(name="sparse_bound_neg", dtype=DataType.SPARSE_FLOAT_VECTOR)
bound_params_function = Function(
name="bm25_bound_neg",
function_type=FunctionType.BM25,
input_field_names=["text"],
output_field_names=["sparse_bound_neg"],
)
# Case a: empty IndexParams is rejected client-side with ParamError.
self.add_function_field(
client,
collection_name,
bound_params_field,
bound_params_function,
index_params=client.prepare_index_params(),
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1,
ct.err_msg: "index_params must contain exactly one index for the function output field",
},
)
# Case b: AUTOINDEX index_type is rejected; the bound index requires an explicit index_type.
autoindex_params = client.prepare_index_params()
autoindex_params.add_index(field_name="sparse_bound_neg", index_type="AUTOINDEX", metric_type="BM25")
self.add_function_field(
client,
collection_name,
bound_params_field,
bound_params_function,
index_params=autoindex_params,
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1100,
ct.err_msg: "an explicit index_type is required for the bound index of function output field",
},
)
# Case c: the server mandates bound index params for vector function output fields;
# a raw alter_collection_schema without index params must be rejected.
with pytest.raises(MilvusException, match="index params are required"):
client._get_connection().alter_collection_schema(
collection_name=collection_name,
field_schema=bound_params_field,
func=bound_params_function,
)
# Bound index params failures must not partially mutate schema either.
after_desc = client.describe_collection(collection_name)
assert after_desc["schema_version"] == before_schema_version
assert [field["name"] for field in after_desc["fields"]] == before_field_names
assert [func["name"] for func in after_desc.get("functions", [])] == before_function_names
assert after_desc["properties"]["max_field_id"] == before_max_field_id
# Step 4: Non-existent collection should fail and not affect the original collection.
sparse_field = FieldSchema(name="sparse_missing_collection", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
@@ -745,6 +799,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
f"{collection_name}_missing",
sparse_field,
bm25_function,
index_params=invalid_case_index_params,
check_task=CheckTasks.err_res,
check_items={ct.err_code: 100, ct.err_msg: "can't find collection"},
)
@@ -791,6 +846,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse_duplicate_function"],
),
index_params=invalid_case_index_params,
check_task=CheckTasks.err_res,
check_items={ct.err_code: 1100, ct.err_msg: "duplicate function name: bm25_fn"},
)
@@ -812,7 +868,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
assert "sparse" in [field["name"] for field in desc["fields"]]
@@ -856,11 +914,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
self.release_collection(client, collection_name)
client.load_collection(collection_name)
@@ -1006,7 +1063,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
client.load_collection(collection_name)
# Step 3: Execute add_function_field.
# Step 3: Execute add_function_field with the bound sparse index params.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -1014,12 +1071,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
# Step 4: Create sparse BM25 index and reload collection.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=30)
# Step 4: Wait for the bound sparse index build and reload collection.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=30)
self.release_collection(client, collection_name)
client.load_collection(collection_name)
@@ -1166,11 +1222,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
client.load_collection(collection_name)
@@ -1307,7 +1362,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
client.load_collection(collection_name)
# Step 1: add_function_field adds bm25_fn and sparse.
# Step 1: add_function_field adds bm25_fn and sparse with the bound sparse index.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -1315,7 +1370,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
fields = {field["name"]: field for field in desc["fields"]}
@@ -1324,9 +1381,6 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert "bm25_fn" in functions
old_sparse_field_id = fields["sparse"]["field_id"]
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
self.release_collection(client, collection_name)
client.load_collection(collection_name)
@@ -1412,7 +1466,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
fields = {field["name"]: field for field in desc["fields"]}
@@ -1421,9 +1477,6 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert "bm25_fn" in functions
assert fields["sparse"]["field_id"] > old_sparse_field_id
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
self.release_collection(client, collection_name)
client.load_collection(collection_name)
@@ -1487,7 +1540,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert "sparse" not in [field["name"] for field in alias_desc["fields"]]
assert "bm25_fn" not in [func["name"] for func in alias_desc.get("functions", [])]
# Step 2: Execute add_function_field through collection name.
# Step 2: Execute add_function_field through collection name with the bound sparse index.
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
bm25_function = Function(
name="bm25_fn",
@@ -1495,16 +1548,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
# Step 3: Verify alias describe/create_index/load/search observes the new schema.
# Step 3: Verify alias describe/load/search observes the new schema and bound index.
alias_desc = client.describe_collection(alias_name)
assert "sparse" in [field["name"] for field in alias_desc["fields"]]
assert "bm25_fn" in [func["name"] for func in alias_desc.get("functions", [])]
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(alias_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, alias_name, index_name="sparse", timeout=60)
client.load_collection(alias_name)
@@ -1562,15 +1614,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(reverse_alias_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(reverse_alias_name, sparse_field, bm25_function, index_params=bound_index_params)
reverse_desc = client.describe_collection(reverse_collection_name)
assert "sparse" in [field["name"] for field in reverse_desc["fields"]]
assert "bm25_fn" in [func["name"] for func in reverse_desc.get("functions", [])]
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(reverse_collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, reverse_collection_name, index_name="sparse", timeout=60)
client.load_collection(reverse_collection_name)
@@ -1655,12 +1706,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
# Step 5: Create sparse BM25 index and load the added function output field.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
# Step 5: Wait for the bound sparse index build and load the added function output field.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
self.release_collection(client, collection_name)
client.load_collection(collection_name)
@@ -1764,16 +1814,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
assert "sparse" in [field["name"] for field in desc["fields"]]
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
# Step 4: Create sparse BM25 index for the added output field.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
# Step 4: Wait for the bound sparse index build of the added output field.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
# Step 5: Do not call load_collection/release_collection here.
@@ -1872,12 +1921,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse_added"],
)
client.add_function_field(bm25_collection_name, sparse_added, bm25_added_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse_added", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(
bm25_collection_name, sparse_added, bm25_added_function, index_params=bound_index_params
)
# Step 6 and Step 7: Create index for added output and wait for backfill/search convergence.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse_added", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(bm25_collection_name, index_params, timeout=60)
# Step 6 and Step 7: Wait for the bound index build and backfill/search convergence.
assert self.wait_for_index_ready(client, bm25_collection_name, index_name="sparse_added", timeout=60)
# Step 8 and Step 9: Search added output and compare topK id order and distances when present.
@@ -2013,17 +2063,18 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
output_field_names=["mh_added"],
params=minhash_params,
)
client.add_function_field(minhash_collection_name, mh_added, minhash_added_function)
# Step 6 and Step 7: Create index for added MinHash output and wait for backfill/search convergence.
index_params = client.prepare_index_params()
index_params.add_index(
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(
field_name="mh_added",
index_type="MINHASH_LSH",
metric_type="MHJACCARD",
params={"mh_lsh_band": 8},
)
client.create_index(minhash_collection_name, index_params, timeout=60)
client.add_function_field(
minhash_collection_name, mh_added, minhash_added_function, index_params=bound_index_params
)
# Step 6 and Step 7: Wait for the bound MinHash index build and backfill/search convergence.
assert self.wait_for_index_ready(client, minhash_collection_name, index_name="mh_added", timeout=60)
# Step 8 and Step 9: Search added output and compare topK id order and distances when present.
@@ -2143,7 +2194,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["title"],
output_field_names=["sparse_title"],
)
client.add_function_field(collection_name, sparse_title, title_function)
title_bound_index_params = client.prepare_index_params()
title_bound_index_params.add_index(
field_name="sparse_title", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
)
client.add_function_field(collection_name, sparse_title, title_function, index_params=title_bound_index_params)
# Step 3: Add body -> sparse_body.
sparse_body = FieldSchema(name="sparse_body", dtype=DataType.SPARSE_FLOAT_VECTOR)
@@ -2153,7 +2208,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["body"],
output_field_names=["sparse_body"],
)
client.add_function_field(collection_name, sparse_body, body_function)
body_bound_index_params = client.prepare_index_params()
body_bound_index_params.add_index(
field_name="sparse_body", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
)
client.add_function_field(collection_name, sparse_body, body_function, index_params=body_bound_index_params)
desc = client.describe_collection(collection_name)
field_names = [field["name"] for field in desc["fields"]]
@@ -2163,11 +2222,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert "bm25_title_fn" in function_names
assert "bm25_body_fn" in function_names
# Step 4: Create indexes for both added BM25 output fields.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse_title", index_type="AUTOINDEX", metric_type="BM25")
index_params.add_index(field_name="sparse_body", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
# Step 4: Wait for the bound index builds of both added BM25 output fields.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_title", timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_body", timeout=60)
@@ -2309,7 +2364,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["standard_text"],
output_field_names=["sparse_standard"],
)
client.add_function_field(collection_name, sparse_standard, standard_function)
standard_bound_index_params = client.prepare_index_params()
standard_bound_index_params.add_index(
field_name="sparse_standard", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
)
client.add_function_field(
collection_name, sparse_standard, standard_function, index_params=standard_bound_index_params
)
sparse_filtered = FieldSchema(name="sparse_filtered", dtype=DataType.SPARSE_FLOAT_VECTOR)
filtered_function = Function(
@@ -2318,7 +2379,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["filtered_text"],
output_field_names=["sparse_filtered"],
)
client.add_function_field(collection_name, sparse_filtered, filtered_function)
filtered_bound_index_params = client.prepare_index_params()
filtered_bound_index_params.add_index(
field_name="sparse_filtered", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
)
client.add_function_field(
collection_name, sparse_filtered, filtered_function, index_params=filtered_bound_index_params
)
after_add_desc = client.describe_collection(collection_name)
field_names = [field["name"] for field in after_add_desc["fields"]]
@@ -2330,11 +2397,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
assert analyzer_params_from_desc(after_add_desc, "standard_text") == standard_analyzer_params
assert analyzer_params_from_desc(after_add_desc, "filtered_text") == filtered_analyzer_params
# Step 3: Create indexes and verify analyzer-specific BM25 behavior.
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse_standard", index_type="AUTOINDEX", metric_type="BM25")
index_params.add_index(field_name="sparse_filtered", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
# Step 3: Wait for the bound index builds and verify analyzer-specific BM25 behavior.
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_standard", timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_filtered", timeout=60)
@@ -2485,11 +2548,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
self.release_collection(client, collection_name)
@@ -2593,11 +2655,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
self.release_collection(client, collection_name)
@@ -2744,15 +2805,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
after_add_desc = client.describe_collection(collection_name)
text_field_after = next(field for field in after_add_desc["fields"] if field["name"] == "text")
assert json.loads(text_field_after["params"]["multi_analyzer_params"]) == multi_analyzer_params
index_params = client.prepare_index_params()
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, index_params, timeout=60)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
self.release_collection(client, collection_name)
@@ -2839,7 +2899,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(bm25_collection_name, bm25_field, bm25_function)
bm25_bound_index_params = client.prepare_index_params()
bm25_bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(bm25_collection_name, bm25_field, bm25_function, index_params=bm25_bound_index_params)
bm25_desc = client.describe_collection(bm25_collection_name)
assert "sparse" in [field["name"] for field in bm25_desc["fields"]]
@@ -2872,7 +2934,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
output_field_names=["mh"],
params={"num_hashes": minhash_num_hashes, "shingle_size": 3},
)
client.add_function_field(minhash_collection_name, minhash_field, minhash_function)
minhash_bound_index_params = client.prepare_index_params()
minhash_bound_index_params.add_index(field_name="mh", index_type="MINHASH_LSH", metric_type="MHJACCARD")
client.add_function_field(
minhash_collection_name, minhash_field, minhash_function, index_params=minhash_bound_index_params
)
minhash_desc = client.describe_collection(minhash_collection_name)
assert "mh" in [field["name"] for field in minhash_desc["fields"]]
@@ -2938,12 +3004,17 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
),
]
# Valid bound index params so the unsupported-function-type error stays the trigger.
unsupported_case_index_params = client.prepare_index_params()
unsupported_case_index_params.add_index(field_name="", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
for case_name, field_schema, function, function_type_value in unsupported_cases:
self.add_function_field(
client,
unsupported_collection_name,
field_schema,
function,
index_params=unsupported_case_index_params,
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1,
@@ -3046,20 +3117,19 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
output_field_names=["mh"],
params={"num_hashes": num_hashes, "shingle_size": 5},
)
client.add_function_field(collection_name, mh_field, mh_function)
desc = client.describe_collection(collection_name)
assert "mh" in [field["name"] for field in desc["fields"]]
assert "minhash_params_backfill_fn" in [func["name"] for func in desc.get("functions", [])]
index_params = client.prepare_index_params()
index_params.add_index(
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(
field_name="mh",
index_type="MINHASH_LSH",
metric_type="MHJACCARD",
params={"mh_lsh_band": 8},
)
client.create_index(collection_name, index_params, timeout=60)
client.add_function_field(collection_name, mh_field, mh_function, index_params=bound_index_params)
desc = client.describe_collection(collection_name)
assert "mh" in [field["name"] for field in desc["fields"]]
assert "minhash_params_backfill_fn" in [func["name"] for func in desc.get("functions", [])]
assert self.wait_for_index_ready(client, collection_name, index_name="mh", timeout=120)
self.release_collection(client, collection_name)
@@ -3133,11 +3203,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
output_field_names=["mh"],
params={"num_hashes": 16, "shingle_size": 3},
)
mismatch_bound_index_params = client.prepare_index_params()
mismatch_bound_index_params.add_index(field_name="mh", index_type="MINHASH_LSH", metric_type="MHJACCARD")
self.add_function_field(
client,
mismatch_collection_name,
mismatch_field,
mismatch_function,
index_params=mismatch_bound_index_params,
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1100,
@@ -3160,11 +3233,16 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
output_field_names=["mh_bad_shingle"],
params={"num_hashes": 16, "shingle_size": 0},
)
bad_shingle_bound_index_params = client.prepare_index_params()
bad_shingle_bound_index_params.add_index(
field_name="mh_bad_shingle", index_type="MINHASH_LSH", metric_type="MHJACCARD"
)
self.add_function_field(
client,
mismatch_collection_name,
bad_shingle_field,
bad_shingle_function,
index_params=bad_shingle_bound_index_params,
check_task=CheckTasks.err_res,
check_items={
ct.err_code: 1100,
@@ -3233,11 +3311,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
sparse_index = client.prepare_index_params()
sparse_index.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, sparse_index, timeout=180)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=240)
sparse_index_info = client.describe_index(collection_name, index_name="sparse")
@@ -3346,11 +3422,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
input_field_names=["text"],
output_field_names=["sparse"],
)
client.add_function_field(collection_name, sparse_field, bm25_function)
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
sparse_index = client.prepare_index_params()
sparse_index.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
client.create_index(collection_name, sparse_index, timeout=180)
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=240)
post_add_rows = [
@@ -2936,11 +2936,18 @@ class TestMilvusClientExternalTableAddField(ExternalTableTestBase):
ct.err_code: 1100,
ct.err_msg: "alter collection schema operation is not supported for external collection",
}
bound_index_params = client.prepare_index_params()
bound_index_params.add_index(
field_name="bm25_sparse",
index_type="SPARSE_INVERTED_INDEX",
metric_type="BM25",
)
self.add_function_field(
client,
collection_name=coll,
field_schema=bm25_field,
func=bm25_function,
index_params=bound_index_params,
check_task=CheckTasks.err_res,
check_items=error,
)
@@ -1,17 +1,15 @@
import pytest
from base.client_v2_base import TestMilvusClientV2Base
from common import common_func as cf
from common import common_type as ct
from common.common_type import CaseLabel, CheckTasks
from utils.util_pymilvus import *
from utils.util_pymilvus import DataType, gen_vectors
default_dim = ct.default_dim
class TestPartitionKeyParams(TestMilvusClientV2Base):
""" Test case of partition key params """
"""Test case of partition key params"""
@pytest.mark.tags(CaseLabel.L0)
@pytest.mark.parametrize("par_key_field", [ct.default_int64_field_name, ct.default_string_field_name])
@@ -25,10 +23,19 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
client = self._client()
schema = self.create_schema(client, auto_id=True)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64,
is_partition_key=(par_key_field == ct.default_int64_field_name))
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
is_partition_key=(par_key_field == ct.default_string_field_name))
self.add_field(
schema,
ct.default_int64_field_name,
DataType.INT64,
is_partition_key=(par_key_field == ct.default_int64_field_name),
)
self.add_field(
schema,
ct.default_string_field_name,
DataType.VARCHAR,
max_length=ct.default_length,
is_partition_key=(par_key_field == ct.default_string_field_name),
)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema)
@@ -41,19 +48,26 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
entities_per_parkey = 10
for _ in range(entities_per_parkey):
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
self.insert(client, c_name, data)
# flush
self.flush(client, c_name)
# build index
index_params = self.prepare_index_params(client)[0]
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="IVF_SQ8",
metric_type="COSINE", params={"nlist": 128})
index_params.add_index(
field_name=ct.default_float_vec_field_name,
index_type="IVF_SQ8",
metric_type="COSINE",
params={"nlist": 128},
)
self.create_index(client, c_name, index_params)
# load
self.load_collection(client, c_name)
@@ -61,46 +75,81 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
nq = 10
search_vectors = gen_vectors(nq, ct.default_dim)
# search with mixed filtered
res1 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey})[0]
res1 = self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey},
)[0]
# search with partition key filter only or with non partition key
res2 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey})[0]
res2 = self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f"{ct.default_int64_field_name} in [1,3,5]",
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey},
)[0]
# search with partition key filter only or with non partition key
res3 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey})[0]
res3 = self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f'{ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey},
)[0]
# assert the results persist
for i in range(nq):
assert res1[i].ids == res2[i].ids == res3[i].ids
# search with 'or' to verify no partition key optimization local with or binary expr
query_res1 = self.query(client, c_name,
filter=f'{ct.default_string_field_name} == "{string_prefix}5" || {ct.default_int64_field_name} in [2,4,6]',
output_fields=['count(*)'])[0]
query_res2 = self.query(client, c_name,
filter=f'{ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"] || {ct.default_int64_field_name}==5',
output_fields=['count(*)'])[0]
query_res3 = self.query(client, c_name,
filter=f'{ct.default_int64_field_name}==5 or {ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"]',
output_fields=['count(*)'])[0]
query_res4 = self.query(client, c_name,
filter=f'{ct.default_int64_field_name} in [2,4,6] || {ct.default_string_field_name} == "{string_prefix}5"',
output_fields=['count(*)'])[0]
query_res1 = self.query(
client,
c_name,
filter=f'{ct.default_string_field_name} == "{string_prefix}5" || {ct.default_int64_field_name} in [2,4,6]',
output_fields=["count(*)"],
)[0]
query_res2 = self.query(
client,
c_name,
filter=f'{ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"] || {ct.default_int64_field_name}==5',
output_fields=["count(*)"],
)[0]
query_res3 = self.query(
client,
c_name,
filter=f'{ct.default_int64_field_name}==5 or {ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"]',
output_fields=["count(*)"],
)[0]
query_res4 = self.query(
client,
c_name,
filter=f'{ct.default_int64_field_name} in [2,4,6] || {ct.default_string_field_name} == "{string_prefix}5"',
output_fields=["count(*)"],
)[0]
# assert the results persist
assert query_res1[0].get('count(*)') == query_res2[0].get('count(*)') \
== query_res3[0].get('count(*)') == query_res4[0].get('count(*)') == 40
assert (
query_res1[0].get("count(*)")
== query_res2[0].get("count(*)")
== query_res3[0].get("count(*)")
== query_res4[0].get("count(*)")
== 40
)
@pytest.mark.tags(CaseLabel.L0)
@pytest.mark.parametrize("par_key_field", [ct.default_int64_field_name, ct.default_string_field_name])
@@ -130,20 +179,24 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
entities_per_parkey = 20
for n in range(entities_per_parkey):
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
"pk": str(n * nb + i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
"pk": str(n * nb + i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
self.insert(client, c_name, data)
# flush
self.flush(client, c_name)
# build index
index_params = self.prepare_index_params(client)[0]
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="FLAT",
metric_type="COSINE", params={})
index_params.add_index(
field_name=ct.default_float_vec_field_name, index_type="FLAT", metric_type="COSINE", params={}
)
if index_on_par_key_field:
index_params.add_index(field_name=par_key_field)
self.create_index(client, c_name, index_params)
@@ -153,12 +206,18 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
nq = 10
search_vectors = gen_vectors(nq, ct.default_dim)
# search with mixed filtered
self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey})
self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": entities_per_parkey},
)
@pytest.mark.tags(CaseLabel.L2)
def test_partition_key_off_in_field_but_enable_in_schema(self):
@@ -168,8 +227,7 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
2. verify the collection created successfully and partition key is enabled
"""
client = self._client()
schema = self.create_schema(client, auto_id=True,
partition_key_field=ct.default_int64_field_name)[0]
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_int64_field_name)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=False)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
@@ -187,7 +245,7 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
""" Test case of partition key invalid params """
"""Test case of partition key invalid params"""
@pytest.mark.tags(CaseLabel.L2)
def test_max_partitions(self):
@@ -204,8 +262,9 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
schema = self.create_schema(client, auto_id=True)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
is_partition_key=True)
self.add_field(
schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True
)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema, num_partitions=max_partition)
@@ -217,11 +276,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
string_prefix = cf.gen_str_by_length(length=6)
for _ in range(5):
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
self.insert(client, c_name, data)
# drop collection
@@ -231,9 +293,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
num_partitions = max_partition + 1
err_msg = f"partition number ({num_partitions}) exceeds max configuration ({max_partition})"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema, num_partitions=num_partitions,
check_task=CheckTasks.err_res,
check_items={"err_code": 1100, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
num_partitions=num_partitions,
check_task=CheckTasks.err_res,
check_items={"err_code": 1100, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L1)
def test_min_partitions(self):
@@ -262,12 +329,15 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
string_prefix = cf.gen_str_by_length(length=6)
for n in range(5):
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
"pk": str(n * nb + i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
"pk": str(n * nb + i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
self.insert(client, c_name, data)
self.flush(client, c_name)
@@ -277,12 +347,22 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
# create a collection with min partitions - 1
err_msg = "The specified num_partitions should be greater than or equal to 1"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema, num_partitions=min_partition - 1,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(client, c_name, schema=schema, num_partitions=min_partition - 3,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
num_partitions=min_partition - 1,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
self.create_collection(
client,
c_name,
schema=schema,
num_partitions=min_partition - 3,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L0)
@pytest.mark.parametrize("is_par_key", [None, "", "invalid", 0.1, [], {}, ()])
@@ -295,10 +375,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
client = self._client()
schema = self.create_schema(client, auto_id=True)[0]
err_msg = "Param is_partition_key must be bool type"
self.add_field(schema, ct.default_int64_field_name, DataType.INT64,
is_partition_key=is_par_key,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.add_field(
schema,
ct.default_int64_field_name,
DataType.INT64,
is_partition_key=is_par_key,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L2)
@pytest.mark.parametrize("num_partitions", [True, False, "", "invalid", 0.1, [], {}, ()])
@@ -316,9 +400,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "invalid num_partitions type"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema, num_partitions=num_partitions,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
num_partitions=num_partitions,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L0)
def test_partition_key_on_multi_fields(self):
@@ -330,37 +419,45 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
client = self._client()
# sub-case 1: both defined in field schema via add_field
# pymilvus(>=3.1.0rc61) validates partition key uniqueness client-side at add_field
schema = self.create_schema(client, auto_id=True)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
is_partition_key=True)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "Expected only one partition key field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.add_field(
schema,
ct.default_string_field_name,
DataType.VARCHAR,
max_length=ct.default_length,
is_partition_key=True,
check_task=CheckTasks.err_res,
check_items={"err_code": 1, "err_msg": err_msg},
)
# sub-case 2: partition_key_field passed as list in create_schema
err_msg = "Param partition_key_field must be str type"
self.create_schema(client, auto_id=True,
partition_key_field=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_schema(
client,
auto_id=True,
partition_key_field=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# sub-case 3: one defined in field schema, one defined in create_schema
schema = self.create_schema(client, auto_id=True,
partition_key_field=ct.default_string_field_name)[0]
# pymilvus raises client-side when adding the field named by partition_key_field
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_string_field_name)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "Expected only one partition key field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.add_field(
schema,
ct.default_string_field_name,
DataType.VARCHAR,
max_length=ct.default_length,
check_task=CheckTasks.err_res,
check_items={"err_code": 1, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L0)
@pytest.mark.parametrize("is_int64_primary", [True, False])
@@ -377,16 +474,21 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
if is_int64_primary:
self.add_field(schema, "pk", DataType.INT64, is_primary=True, is_partition_key=True)
else:
self.add_field(schema, "pk", DataType.VARCHAR, max_length=ct.default_length,
is_primary=True, is_partition_key=True)
self.add_field(
schema, "pk", DataType.VARCHAR, max_length=ct.default_length, is_primary=True, is_partition_key=True
)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "the partition key field must not be primary field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# sub-case 2: partition key set on primary field via create_schema partition_key_field
schema = self.create_schema(client, auto_id=False, partition_key_field="pk")[0]
@@ -399,9 +501,13 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "the partition key field must not be primary field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L1)
def test_partition_key_on_and_off(self):
@@ -413,35 +519,51 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
client = self._client()
# sub-case 1: int64 field is_partition_key=True, schema partition_key_field=vector field
schema = self.create_schema(client, auto_id=True,
partition_key_field=ct.default_float_vec_field_name)[0]
# pymilvus raises client-side when adding the field named by partition_key_field
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_float_vec_field_name)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "Expected only one partition key field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.add_field(
schema,
ct.default_float_vec_field_name,
DataType.FLOAT_VECTOR,
dim=default_dim,
check_task=CheckTasks.err_res,
check_items={"err_code": 1, "err_msg": err_msg},
)
# sub-case 2: string1 is_partition_key=True, schema partition_key_field=string2
schema = self.create_schema(client, auto_id=True,
partition_key_field="string2")[0]
# pymilvus raises client-side when adding the field named by partition_key_field
schema = self.create_schema(client, auto_id=True, partition_key_field="string2")[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, "string1", DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True)
self.add_field(schema, "string2", DataType.VARCHAR, max_length=ct.default_length)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "Expected only one partition key field"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.add_field(
schema,
"string2",
DataType.VARCHAR,
max_length=ct.default_length,
check_task=CheckTasks.err_res,
check_items={"err_code": 1, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L2)
@pytest.mark.parametrize("field_type", [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR, DataType.FLOAT,
DataType.DOUBLE, DataType.BOOL, DataType.INT8,
DataType.INT16, DataType.INT32, DataType.JSON])
@pytest.mark.parametrize(
"field_type",
[
DataType.FLOAT_VECTOR,
DataType.BINARY_VECTOR,
DataType.FLOAT,
DataType.DOUBLE,
DataType.BOOL,
DataType.INT8,
DataType.INT16,
DataType.INT32,
DataType.JSON,
],
)
def test_partition_key_on_invalid_type_fields(self, field_type):
"""
Method
@@ -461,16 +583,26 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
if field_type == DataType.BINARY_VECTOR:
self.add_field(schema, ct.default_binary_vec_field_name, DataType.BINARY_VECTOR,
dim=default_dim, is_partition_key=True)
self.add_field(
schema, ct.default_binary_vec_field_name, DataType.BINARY_VECTOR, dim=default_dim, is_partition_key=True
)
else:
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim,
is_partition_key=(field_type == DataType.FLOAT_VECTOR))
self.add_field(
schema,
ct.default_float_vec_field_name,
DataType.FLOAT_VECTOR,
dim=default_dim,
is_partition_key=(field_type == DataType.FLOAT_VECTOR),
)
err_msg = "Partition key field type must be DataType.INT64 or DataType.VARCHAR"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L2)
def test_partition_key_on_not_existed_fields(self):
@@ -480,17 +612,20 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
2. verify the error raised
"""
client = self._client()
schema = self.create_schema(client, auto_id=True,
partition_key_field="non_existing_field")[0]
schema = self.create_schema(client, auto_id=True, partition_key_field="non_existing_field")[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "the specified partition key field {non_existing_field} not exist"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
@pytest.mark.tags(CaseLabel.L1)
def test_partition_key_on_empty_and_num_partitions_set(self):
@@ -509,9 +644,13 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "the specified partition key field {} not exist"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# sub-case 2: no partition key but num_partitions set → server error
schema = self.create_schema(client, auto_id=True)[0]
@@ -521,13 +660,18 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
err_msg = "num_partitions should only be specified with partition key field enabled"
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema, num_partitions=200,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_collection(
client,
c_name,
schema=schema,
num_partitions=200,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
""" Test case of partition key insert invalid data """
"""Test case of partition key insert invalid data"""
@pytest.mark.tags(CaseLabel.L2)
@pytest.mark.parametrize("invalid_data", [99, True, None, [], {}, ()])
@@ -539,8 +683,7 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
3. verify the error raised
"""
client = self._client()
schema = self.create_schema(client, auto_id=False,
partition_key_field=ct.default_string_field_name)[0]
schema = self.create_schema(client, auto_id=False, partition_key_field=ct.default_string_field_name)[0]
self.add_field(schema, "pk", DataType.VARCHAR, max_length=ct.default_length, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
@@ -552,12 +695,15 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
nb = 10
string_prefix = cf.gen_str_by_length(length=6)
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
"pk": str(i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
"pk": str(i),
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
data[1][ct.default_string_field_name] = invalid_data # inject invalid data
if invalid_data is None:
@@ -568,13 +714,13 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
# non-string types trigger DataNotMatchException in row-based insert
err_msg = "The Input data type is inconsistent with defined schema"
err_code = 1
self.insert(client, c_name, data,
check_task=CheckTasks.err_res,
check_items={"err_code": err_code, "err_msg": err_msg})
self.insert(
client, c_name, data, check_task=CheckTasks.err_res, check_items={"err_code": err_code, "err_msg": err_msg}
)
class TestPartitionApiForbidden(TestMilvusClientV2Base):
""" Test case of partition api forbidden when partition key is on """
"""Test case of partition api forbidden when partition key is on"""
@pytest.mark.tags(CaseLabel.L1)
def test_create_partition(self):
@@ -593,8 +739,9 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
schema = self.create_schema(client, auto_id=True)[0]
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
is_partition_key=True)
self.add_field(
schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True
)
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
c_name = cf.gen_collection_name_by_testcase_name()
self.create_collection(client, c_name, schema=schema)
@@ -602,9 +749,13 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
# create partition → error
err_msg = "disable create partition if partition key mode is used"
partition_name = cf.gen_unique_str("partition")
self.create_partition(client, c_name, partition_name,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.create_partition(
client,
c_name,
partition_name,
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# list/has partition → allowed
partitions = self.list_partitions(client, c_name)[0]
@@ -617,53 +768,81 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
entities_per_parkey = 10
for _ in range(entities_per_parkey):
float_vec_values = gen_vectors(nb, ct.default_dim)
data = [{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i]
} for i in range(nb)]
data = [
{
ct.default_int64_field_name: i,
ct.default_string_field_name: string_prefix + str(i),
ct.default_float_vec_field_name: float_vec_values[i],
}
for i in range(nb)
]
self.insert(client, c_name, data)
# insert with partition_name → error
err_msg = "not support manually specifying the partition names if partition key mode is used"
self.insert(client, c_name, data, partition_name=partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.insert(
client,
c_name,
data,
partition_name=partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# load partitions → error
err_msg = "disable load partitions if partition key mode is used"
self.load_partitions(client, c_name, [partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.load_partitions(
client,
c_name,
[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# flush + index + load collection → allowed
self.flush(client, c_name)
index_params = self.prepare_index_params(client)[0]
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="IVF_SQ8",
metric_type="COSINE", params={"nlist": 128})
index_params.add_index(
field_name=ct.default_float_vec_field_name,
index_type="IVF_SQ8",
metric_type="COSINE",
params={"nlist": 128},
)
self.create_index(client, c_name, index_params)
self.load_collection(client, c_name)
# search without partition_names → allowed
nq = 10
search_vectors = gen_vectors(nq, ct.default_dim)
res1 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": ct.default_limit})[0]
res1 = self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
check_task=CheckTasks.check_search_results,
check_items={"nq": nq, "limit": ct.default_limit},
)[0]
pks = res1[0].ids[:3]
# search with partition_names → error
err_msg = "not support manually specifying the partition names if partition key mode is used"
self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params, limit=entities_per_parkey,
filter=f'{ct.default_int64_field_name} in [1,3,5]',
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
partition_names=[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.search(
client,
c_name,
data=search_vectors,
anns_field=ct.default_float_vec_field_name,
search_params=ct.default_search_params,
limit=entities_per_parkey,
filter=f"{ct.default_int64_field_name} in [1,3,5]",
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
partition_names=[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# get_load_state with partition → allowed (v1: loading_progress/wait_for_loading_complete)
load_state = self.get_load_state(client, c_name, partition_name=partitions[0])[0]
@@ -678,23 +857,41 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
# delete with partition_name → error
err_msg = "not support manually specifying the partition names if partition key mode is used"
self.delete(client, c_name, filter=f'pk in {pks}', partition_name=partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.delete(
client,
c_name,
filter=f"pk in {pks}",
partition_name=partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# query with partition_names → error
self.query(client, c_name, filter=f'pk in {pks}', partition_names=[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.query(
client,
c_name,
filter=f"pk in {pks}",
partition_names=[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# release partitions → error
err_msg = "disable release partitions if partition key mode is used"
self.release_partitions(client, c_name, [partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.release_partitions(
client,
c_name,
[partitions[0]],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
# drop partition → error
err_msg = "disable drop partition if partition key mode is used"
self.drop_partition(client, c_name, partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg})
self.drop_partition(
client,
c_name,
partitions[0],
check_task=CheckTasks.err_res,
check_items={"err_code": 2, "err_msg": err_msg},
)
+2 -2
View File
@@ -23,8 +23,8 @@ pytest-sugar==0.9.5
pytest-random-order
# pymilvus
pymilvus==3.1.0rc54
pymilvus[bulk_writer]==3.1.0rc54
pymilvus==3.1.0rc61
pymilvus[bulk_writer]==3.1.0rc61
# for protobuf
protobuf>=5.29.5