mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 10:15:43 +00:00
enhance: bind index meta to add function field DDL (#51105)
issue: #51104 companion SDK PR (merge first): milvus-io/pymilvus#3670 ## Summary `add_function_field` previously accepted a vector output field without any index meta; bump-schema-version compaction then stalled forever (no segment index task can be created for a field with no index defined, and `GetQueryVChanPositions` never hands off unindexed compacted-to segments — the fail-closed chain is silent). This PR makes the index meta a mandatory, atomically-bound part of the DDL: - **Mandatory at every layer**: SDK requires `index_params`; proxy and rootcoord reject vector function-output fields without an explicit `index_type` strictly before the broadcast (no silent AUTOINDEX). - **create_index-aligned materialization**: proxy normalizes the params with the same logic as `create_index` (name rules via `validateIndexName`, checker existence, `CheckTrain` incl. dimension filling, function-type defaults such as `bm25_k1/bm25_b/bm25_avgdl`) and writes them back into the request; rootcoord prepare allocates index id/name, rejects name conflicts and unknown index types, and serializes a complete `indexpb.FieldIndex` into the new `AlterCollectionMessageUpdates.bound_field_indexes` WAL field. - **Atomic apply**: the alter-collection ack callback commits the schema and then applies the bound index by dispatching a synthetic `CreateIndexMessage` to datacoord's existing `createIndexV2AckCallback` (same pattern as `cascadeDropFieldIndexesInline`). The callback is replayed until success across crashes and is fully idempotent (all ids come from the message body), so `DDL success ⇒ index meta exists` holds with no partial terminal state. - **Reuse promotion**: `ValidateIndexParams`/`CheckDuplidateKey` moved from datacoord to `internal/util/indexparamcheck`; shared `ExpandIndexParams` / `FillFunctionOutputIndexParams` / `PrepareFunctionOutputIndexParams` helpers now back both the create_index path and this DDL. - **Dead fan-out removed**: the ordinary `CreateIndex` broadcast is reverted to control-channel-only; the vchannel fan-out had no consumer and wrote one dead WAL entry per vchannel per index creation. - **No milvus-proto change**: the request already carried `FieldInfo.index_name`/`extra_params` (previously unused); this feature activates them. - e2e suites updated to the new semantics: `test_add_function_field_feature.py` rewritten (explicit `index_params` everywhere, post-DDL `create_index` blocks removed since the bound index conflicts with a second distinct index per existing semantics, new negative cases for the mandatory checks); the external-table negative case updated as well. ## Verification - New/updated unit tests: rootcoord DDL callbacks (mandatory/AUTOINDEX/unknown-index-type rejections, bound-index materialization + ack-callback application via a recording fake), proxy task validation (invalid index name, normalization write-back), shared helper tests. - End-to-end on a live cluster (StorageV3 + bump compaction enabled): request without params rejected; index meta visible with create_index-aligned params immediately after the DDL returns; bump compaction rewrote 2000-row sealed segments and the bound index built on the results (the exact chain that previously stalled); BM25 search over pre-DDL rows succeeds once the compacted segments are loaded. ## Limitations (follow-ups) - Live propagation of the new index meta to already-loaded QueryNode delegators is intentionally out of scope; a loaded collection observes the bound index through the segment load path (e.g. reload / compacted-segment load). Tracked as a separate follow-up PR. - The python e2e suites require the companion pymilvus change: milvus-io/pymilvus#3670 is merged and `tests/python_client/requirements.txt` is bumped to `pymilvus==3.1.0rc61` in this PR (all 22 rewritten cases + the external-table case verified locally against the released rc61). - During a rolling-upgrade window an old coordinator replaying the new message ignores `bound_field_indexes` (proto3 unknown field), i.e. pre-existing behavior; avoid the new argument in mixed-version clusters. - `AddCollectionFunction` on an existing field and plain add-vector-field keep their current behavior (the WAL carrier is generic for future extension). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Signed-off-by: MrPresent-Han <chun.han@gmail.com> Co-authored-by: MrPresent-Han <chun.han@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
MrPresent-Han
Claude Fable 5
parent
75526b2463
commit
aea5d701bd
@@ -30,6 +30,7 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
type indexInspector struct {
|
||||
@@ -193,27 +194,35 @@ func getSegmentBinlogFields(segment *SegmentInfo) map[int64]struct{} {
|
||||
}
|
||||
|
||||
func (i *indexInspector) canCreateIndexForSegment(ctx context.Context, segment *SegmentInfo, index *model.Index, segmentBinlogFields map[int64]struct{}) bool {
|
||||
_, hasField := segmentBinlogFields[index.FieldID]
|
||||
if !i.isFunctionOutputField(segment.CollectionID, index.FieldID) || hasField {
|
||||
if _, hasField := segmentBinlogFields[index.FieldID]; hasField {
|
||||
return true
|
||||
}
|
||||
mlog.Debug(ctx, "function output field has no binlog, skip create index", mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *indexInspector) isFunctionOutputField(collectionID, fieldID int64) bool {
|
||||
collection := i.meta.GetCollection(collectionID)
|
||||
if collection == nil || collection.Schema == nil {
|
||||
// The segment has no binlog for the indexed field. A function-output field
|
||||
// receives its data only through backfill, so building now is doomed; any
|
||||
// other field may proceed. The schema is resolved through the handler
|
||||
// (lazy-loading on cache miss, e.g. right after a datacoord restart) and the
|
||||
// check fails closed: on an unresolvable or inconsistent view, defer to the
|
||||
// next inspection round instead of trusting a stale schema.
|
||||
collection, err := i.handler.GetCollection(ctx, segment.CollectionID)
|
||||
if err != nil || collection == nil || collection.Schema == nil {
|
||||
mlog.Warn(ctx, "cannot resolve collection schema, defer index build",
|
||||
mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID), mlog.Err(err))
|
||||
return false
|
||||
}
|
||||
for _, functionSchema := range collection.Schema.GetFunctions() {
|
||||
for _, outputFieldID := range functionSchema.GetOutputFieldIds() {
|
||||
if outputFieldID == fieldID {
|
||||
return true
|
||||
if outputFieldID == index.FieldID {
|
||||
mlog.Debug(ctx, "function output field has no binlog, skip create index", mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
if typeutil.GetFieldByID(collection.Schema, index.FieldID) == nil {
|
||||
mlog.Warn(ctx, "indexed field not found in cached collection schema, defer index build",
|
||||
mlog.FieldSegmentID(segment.ID), mlog.FieldFieldID(index.FieldID), mlog.FieldIndexID(index.IndexID))
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (i *indexInspector) createIndexForSegment(ctx context.Context, segment *SegmentInfo, indexID UniqueID) error {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
|
||||
@@ -487,7 +488,7 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
|
||||
|
||||
inspector := newIndexInspector(ctx, notifyChan, m, scheduler, alloc, handler, storageCli, versionManager)
|
||||
collID := int64(10)
|
||||
m.collections.Insert(collID, &collectionInfo{
|
||||
collInfo := &collectionInfo{
|
||||
ID: collID,
|
||||
Schema: &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
@@ -500,7 +501,9 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
|
||||
{Name: "bm25_fn", OutputFieldIds: []int64{102}},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
m.collections.Insert(collID, collInfo)
|
||||
handler.EXPECT().GetCollection(mock.Anything, collID).Return(collInfo, nil).Maybe()
|
||||
|
||||
t.Run("create function output index when binlog exists", func(t *testing.T) {
|
||||
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
|
||||
@@ -604,4 +607,51 @@ func TestIndexInspector_FunctionOutputBinlogGate(t *testing.T) {
|
||||
assert.Contains(t, segIndexes, UniqueID(8))
|
||||
assert.NotContains(t, segIndexes, UniqueID(9))
|
||||
})
|
||||
|
||||
t.Run("defer index build when schema is unresolvable", func(t *testing.T) {
|
||||
unresolvableCollID := int64(11)
|
||||
handler.EXPECT().GetCollection(mock.Anything, unresolvableCollID).
|
||||
Return(nil, errors.New("mock rootcoord unreachable")).Once()
|
||||
m.indexMeta.indexes[unresolvableCollID] = map[UniqueID]*model.Index{
|
||||
10: {CollectionID: unresolvableCollID, FieldID: 102, IndexID: 10, IndexName: "unresolvable_schema_idx"},
|
||||
}
|
||||
segment := &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 5,
|
||||
CollectionID: unresolvableCollID,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
IsSorted: true,
|
||||
Binlogs: []*datapb.FieldBinlog{
|
||||
{FieldID: 0, ChildFields: []int64{100, 101}},
|
||||
},
|
||||
},
|
||||
}
|
||||
m.segments.SetSegment(segment.GetID(), segment)
|
||||
|
||||
err := inspector.createIndexesForSegment(ctx, segment)
|
||||
assert.NoError(t, err)
|
||||
assert.NotContains(t, m.indexMeta.GetSegmentIndexes(unresolvableCollID, segment.GetID()), UniqueID(10))
|
||||
})
|
||||
|
||||
t.Run("defer index build when indexed field is unknown to schema", func(t *testing.T) {
|
||||
m.indexMeta.indexes[collID] = map[UniqueID]*model.Index{
|
||||
11: {CollectionID: collID, FieldID: 104, IndexID: 11, IndexName: "stale_schema_idx"},
|
||||
}
|
||||
segment := &SegmentInfo{
|
||||
SegmentInfo: &datapb.SegmentInfo{
|
||||
ID: 6,
|
||||
CollectionID: collID,
|
||||
State: commonpb.SegmentState_Flushed,
|
||||
IsSorted: true,
|
||||
Binlogs: []*datapb.FieldBinlog{
|
||||
{FieldID: 0, ChildFields: []int64{100, 101}},
|
||||
},
|
||||
},
|
||||
}
|
||||
m.segments.SetSegment(segment.GetID(), segment)
|
||||
|
||||
err := inspector.createIndexesForSegment(ctx, segment)
|
||||
assert.NoError(t, err)
|
||||
assert.NotContains(t, m.indexMeta.GetSegmentIndexes(collID, segment.GetID()), UniqueID(11))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -298,13 +298,10 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
|
||||
UserIndexParams: req.GetUserIndexParams(),
|
||||
}
|
||||
// Validate the index params.
|
||||
if err := ValidateIndexParams(index); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
channels := make([]string, 0, len(coll.GetVirtualChannelNames())+1)
|
||||
channels = append(channels, streaming.WAL().ControlChannel())
|
||||
channels = append(channels, coll.GetVirtualChannelNames()...)
|
||||
if _, err = broadcaster.Broadcast(ctx, message.NewCreateIndexMessageBuilderV2().
|
||||
WithHeader(&message.CreateIndexMessageHeader{
|
||||
DbId: coll.GetDbId(),
|
||||
@@ -316,7 +313,7 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
|
||||
WithBody(&message.CreateIndexMessageBody{
|
||||
FieldIndex: model.MarshalIndexModel(index),
|
||||
}).
|
||||
WithBroadcast(channels).
|
||||
WithBroadcast([]string{streaming.WAL().ControlChannel()}).
|
||||
MustBuildBroadcast(),
|
||||
); err != nil {
|
||||
mlog.Error(context.TODO(), "CreateIndex fail", mlog.Err(err))
|
||||
@@ -325,51 +322,11 @@ func (s *Server) CreateIndex(ctx context.Context, req *indexpb.CreateIndexReques
|
||||
}
|
||||
mlog.Info(context.TODO(), "CreateIndex successfully",
|
||||
mlog.String("IndexName", index.IndexName), mlog.Int64("fieldID", index.FieldID),
|
||||
mlog.Int64("IndexID", index.IndexID),
|
||||
mlog.Strings("channels", channels))
|
||||
mlog.Int64("IndexID", index.IndexID))
|
||||
metrics.IndexRequestCounter.WithLabelValues(metrics.SuccessLabel).Inc()
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
func ValidateIndexParams(index *model.Index) error {
|
||||
if err := CheckDuplidateKey(index.IndexParams, "indexParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := CheckDuplidateKey(index.UserIndexParams, "userIndexParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := CheckDuplidateKey(index.TypeParams, "typeParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
indexType := GetIndexType(index.IndexParams)
|
||||
indexParams := funcutil.KeyValuePair2Map(index.IndexParams)
|
||||
userIndexParams := funcutil.KeyValuePair2Map(index.UserIndexParams)
|
||||
if err := indexparamcheck.ValidateMmapIndexParams(indexType, indexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid mmap index params: %s", err.Error())
|
||||
}
|
||||
if err := indexparamcheck.ValidateMmapIndexParams(indexType, userIndexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid mmap user index params: %s", err.Error())
|
||||
}
|
||||
if err := indexparamcheck.ValidateOffsetCacheIndexParams(indexType, indexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
|
||||
}
|
||||
if err := indexparamcheck.ValidateOffsetCacheIndexParams(indexType, userIndexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CheckDuplidateKey(kvs []*commonpb.KeyValuePair, tag string) error {
|
||||
keySet := typeutil.NewSet[string]()
|
||||
for _, kv := range kvs {
|
||||
if keySet.Contain(kv.GetKey()) {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicate %s key in %s params", kv.GetKey(), tag)
|
||||
}
|
||||
keySet.Insert(kv.GetKey())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateParams(index *model.Index, from []*commonpb.KeyValuePair, updates []*commonpb.KeyValuePair) []*commonpb.KeyValuePair {
|
||||
params := make(map[string]string)
|
||||
for _, param := range from {
|
||||
@@ -498,7 +455,7 @@ func (s *Server) AlterIndex(ctx context.Context, req *indexpb.AlterIndexRequest)
|
||||
index.IndexParams = newIndexParams
|
||||
}
|
||||
|
||||
if err := ValidateIndexParams(index); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,6 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/channel"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/broadcast"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/registry"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/internal/util/sessionutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/datapb"
|
||||
@@ -2868,112 +2867,6 @@ func TestMeta_GetHasUnindexTaskSegments(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateIndexParams(t *testing.T) {
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "true",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid index param", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "h",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid index user param", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
},
|
||||
UserIndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "h",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_index_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_user_index_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
UserIndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_user_index_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: indexparamcheck.AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestJsonIndex(t *testing.T) {
|
||||
initStreamingSystem(t)
|
||||
|
||||
|
||||
@@ -884,7 +884,7 @@ func (sm *snapshotManager) RestoreIndexes(
|
||||
}
|
||||
|
||||
// Validate the index params (basic validation without JSON path parsing)
|
||||
if err := ValidateIndexParams(index); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
|
||||
return merr.Wrapf(err, "failed to validate index %s", indexInfo.GetIndexName())
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/proxy/shardclient"
|
||||
"github.com/milvus-io/milvus/internal/types"
|
||||
"github.com/milvus-io/milvus/internal/util/function/validator"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/internal/util/schemautil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
@@ -1183,6 +1184,29 @@ func (t *alterCollectionSchemaTask) preExecuteAdd(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the bound index params against the new field, mirroring the
|
||||
// create_index path (index name rules, checker existence, data-type
|
||||
// compatibility, train params, dimension filling). Validation-only: the
|
||||
// request keeps the user's original params so the persisted UserIndexParams
|
||||
// match the create_index convention; rootcoord independently derives the
|
||||
// normalized IndexParams at prepare.
|
||||
if plan.Kind == schemautil.AlterSchemaAddFunctionField {
|
||||
if err := validateIndexName(plan.IndexName); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := indexparamcheck.ValidateIndexParamsSize(plan.IndexExtraParams...); err != nil {
|
||||
return err
|
||||
}
|
||||
indexParamsMap, err := indexparamcheck.PrepareFunctionOutputIndexParams(
|
||||
plan.Function.GetType(), plan.Field.GetName(), plan.IndexExtraParams)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkTrain(ctx, plan.Field, indexParamsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Validate function-field type compatibility (e.g., BM25 requires varchar input,
|
||||
// SparseFloatVector output). Construct a merged schema with old fields + optional new field
|
||||
// + new function, then validate only the new function to avoid re-checking existing
|
||||
|
||||
+11
-124
@@ -22,7 +22,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
@@ -203,92 +202,23 @@ func wrapUserIndexParams(metricType string) []*commonpb.KeyValuePair {
|
||||
}
|
||||
}
|
||||
|
||||
func checkIndexParamsSize(size int) error {
|
||||
maxIndexParamsSize := paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()
|
||||
if size > maxIndexParamsSize {
|
||||
return merr.WrapErrParameterInvalidMsg("index params size exceeds limit: %d > %d", size, maxIndexParamsSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateIndexParamsSize(params ...*commonpb.KeyValuePair) error {
|
||||
size := 0
|
||||
for _, param := range params {
|
||||
size += len(param.GetKey()) + len(param.GetValue())
|
||||
}
|
||||
return checkIndexParamsSize(size)
|
||||
}
|
||||
|
||||
func validateIndexParamsMapSize(params map[string]string) error {
|
||||
size := 0
|
||||
for k, v := range params {
|
||||
size += len(k) + len(v)
|
||||
}
|
||||
return checkIndexParamsSize(size)
|
||||
}
|
||||
|
||||
func (cit *createIndexTask) parseFunctionParamsToIndex(indexParamsMap map[string]string) error {
|
||||
if !cit.fieldSchema.GetIsFunctionOutput() {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch cit.functionSchema.GetType() {
|
||||
case schemapb.FunctionType_Unknown:
|
||||
return merr.WrapErrParameterInvalidMsg("unknown function type encountered")
|
||||
|
||||
case schemapb.FunctionType_BM25:
|
||||
// set default BM25 params if not provided in index params
|
||||
if _, ok := indexParamsMap["bm25_k1"]; !ok {
|
||||
indexParamsMap["bm25_k1"] = "1.2"
|
||||
}
|
||||
|
||||
if _, ok := indexParamsMap["bm25_b"]; !ok {
|
||||
indexParamsMap["bm25_b"] = "0.75"
|
||||
}
|
||||
|
||||
if _, ok := indexParamsMap["bm25_avgdl"]; !ok {
|
||||
indexParamsMap["bm25_avgdl"] = "100"
|
||||
}
|
||||
|
||||
if metricType, ok := indexParamsMap["metric_type"]; !ok {
|
||||
indexParamsMap["metric_type"] = metric.BM25
|
||||
} else if metricType != metric.BM25 {
|
||||
return merr.WrapErrParameterInvalidMsg("index metric type of BM25 function output field must be BM25, got %s", metricType)
|
||||
}
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
return indexparamcheck.FillFunctionOutputIndexParams(cit.functionSchema.GetType(), indexParamsMap)
|
||||
}
|
||||
|
||||
func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
|
||||
cit.newExtraParams = cit.req.GetExtraParams()
|
||||
if err := validateIndexParamsSize(cit.newExtraParams...); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParamsSize(cit.newExtraParams...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isVecIndex := typeutil.IsVectorType(cit.fieldSchema.DataType)
|
||||
indexParamsMap := make(map[string]string)
|
||||
|
||||
keys := typeutil.NewSet[string]()
|
||||
for _, kv := range cit.req.GetExtraParams() {
|
||||
if keys.Contain(kv.GetKey()) {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicated index param (key=%s) (value=%s) found", kv.GetKey(), kv.GetValue())
|
||||
}
|
||||
keys.Insert(kv.GetKey())
|
||||
if kv.Key == common.ParamsKey {
|
||||
params, err := funcutil.JSONToMap(kv.Value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for k, v := range params {
|
||||
indexParamsMap[k] = v
|
||||
}
|
||||
} else {
|
||||
indexParamsMap[kv.Key] = kv.Value
|
||||
}
|
||||
indexParamsMap, err := indexparamcheck.ExpandIndexParams(cit.req.GetExtraParams())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if jsonCastType, exist := indexParamsMap[common.JSONCastTypeKey]; exist {
|
||||
@@ -298,7 +228,7 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
|
||||
indexParamsMap[common.JSONCastFunctionKey] = strings.ToUpper(strings.TrimSpace(jsonCastFunction))
|
||||
}
|
||||
|
||||
if err := validateIndexParamsMapSize(indexParamsMap); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParamsMapSize(indexParamsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -586,11 +516,11 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := validateIndexParamsMapSize(indexParamsMap); err != nil {
|
||||
if err := indexparamcheck.ValidateIndexParamsMapSize(indexParamsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err := checkTrain(ctx, cit.fieldSchema, indexParamsMap)
|
||||
err = checkTrain(ctx, cit.fieldSchema, indexParamsMap)
|
||||
if err != nil {
|
||||
// checkTrain may propagate errors from indexparamcheck (not yet
|
||||
// merr-standardized). Already-merr errors (leaves / fillDimension /
|
||||
@@ -652,28 +582,6 @@ func (cit *createIndexTask) getIndexedFieldAndFunction(ctx context.Context) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func fillDimension(field *schemapb.FieldSchema, indexParams map[string]string) error {
|
||||
if !typeutil.IsVectorType(field.GetDataType()) {
|
||||
return nil
|
||||
}
|
||||
params := make([]*commonpb.KeyValuePair, 0, len(field.GetTypeParams())+len(field.GetIndexParams()))
|
||||
params = append(params, field.GetTypeParams()...)
|
||||
params = append(params, field.GetIndexParams()...)
|
||||
dimensionInSchema, err := funcutil.GetAttrByKeyFromRepeatedKV(DimKey, params)
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("dimension not found in schema")
|
||||
}
|
||||
dimension, exist := indexParams[DimKey]
|
||||
if exist {
|
||||
if dimensionInSchema != dimension {
|
||||
return merr.WrapErrParameterInvalidMsg("dimension mismatch, dimension in schema: %s, dimension: %s", dimensionInSchema, dimension)
|
||||
}
|
||||
} else {
|
||||
indexParams[DimKey] = dimensionInSchema
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams map[string]string) error {
|
||||
indexType := indexParams[common.IndexTypeKey]
|
||||
|
||||
@@ -687,8 +595,7 @@ func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams ma
|
||||
indexParams[common.HybridHighCardinalityIndexTypeKey] = paramtable.Get().DataCoordCfg.HybridIndexHighCardinalityIndexType.GetValue()
|
||||
}
|
||||
|
||||
checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)
|
||||
if err != nil {
|
||||
if _, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType); err != nil {
|
||||
mlog.Warn(ctx, "Failed to get index checker", mlog.String(common.IndexTypeKey, indexType))
|
||||
return merr.WrapErrParameterInvalidMsg("invalid index type: %s", indexType)
|
||||
}
|
||||
@@ -712,28 +619,8 @@ func checkTrain(ctx context.Context, field *schemapb.FieldSchema, indexParams ma
|
||||
}
|
||||
}
|
||||
|
||||
isSparse := typeutil.IsSparseFloatVectorType(field.DataType)
|
||||
|
||||
if !isSparse {
|
||||
if err := fillDimension(field, indexParams); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
effectiveField := field
|
||||
if effectiveDataType != field.DataType {
|
||||
effectiveField = proto.Clone(field).(*schemapb.FieldSchema)
|
||||
effectiveField.DataType = effectiveDataType
|
||||
effectiveField.ElementType = effectiveElementType
|
||||
}
|
||||
|
||||
if err := checker.CheckValidDataType(indexType, effectiveField); err != nil {
|
||||
mlog.Info(ctx, "create index with invalid data type", mlog.Err(err), mlog.String("data_type", field.GetDataType().String()))
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checker.CheckTrain(effectiveDataType, effectiveElementType, indexParams); err != nil {
|
||||
mlog.Info(ctx, "create index with invalid parameters", mlog.Err(err))
|
||||
if err := indexparamcheck.ValidateFieldIndexParams(field, indexParams); err != nil {
|
||||
mlog.Info(ctx, "create index with invalid parameters", mlog.Err(err), mlog.String("data_type", field.GetDataType().String()))
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/proxy/shardclient"
|
||||
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
|
||||
"github.com/milvus-io/milvus/internal/util/function/embedding"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mq/msgstream"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
|
||||
@@ -3538,18 +3539,20 @@ func Test_createIndexTask_getIndexedFieldAndFunction(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_fillDimension(t *testing.T) {
|
||||
// fillDimension moved to indexparamcheck.FillDimension (shared with the
|
||||
// add-function-field bound-index path); coverage kept here for the proxy flow.
|
||||
t.Run("scalar", func(t *testing.T) {
|
||||
f := &schemapb.FieldSchema{
|
||||
DataType: schemapb.DataType_Int64,
|
||||
}
|
||||
assert.NoError(t, fillDimension(f, nil))
|
||||
assert.NoError(t, indexparamcheck.FillDimension(f, nil))
|
||||
})
|
||||
|
||||
t.Run("no dim in schema", func(t *testing.T) {
|
||||
f := &schemapb.FieldSchema{
|
||||
DataType: schemapb.DataType_FloatVector,
|
||||
}
|
||||
assert.Error(t, fillDimension(f, nil))
|
||||
assert.Error(t, indexparamcheck.FillDimension(f, nil))
|
||||
})
|
||||
|
||||
t.Run("dimension mismatch", func(t *testing.T) {
|
||||
@@ -3562,7 +3565,7 @@ func Test_fillDimension(t *testing.T) {
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.Error(t, fillDimension(f, map[string]string{common.DimKey: "8"}))
|
||||
assert.Error(t, indexparamcheck.FillDimension(f, map[string]string{common.DimKey: "8"}))
|
||||
})
|
||||
|
||||
t.Run("normal", func(t *testing.T) {
|
||||
@@ -3576,7 +3579,7 @@ func Test_fillDimension(t *testing.T) {
|
||||
},
|
||||
}
|
||||
m := map[string]string{}
|
||||
assert.NoError(t, fillDimension(f, m))
|
||||
assert.NoError(t, indexparamcheck.FillDimension(f, m))
|
||||
assert.Equal(t, "128", m[common.DimKey])
|
||||
})
|
||||
}
|
||||
@@ -7687,7 +7690,13 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
|
||||
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
|
||||
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
|
||||
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
|
||||
{FieldSchema: proto.Clone(sparseOutputField).(*schemapb.FieldSchema)},
|
||||
{
|
||||
FieldSchema: proto.Clone(sparseOutputField).(*schemapb.FieldSchema),
|
||||
ExtraParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
},
|
||||
},
|
||||
},
|
||||
FuncSchema: []*schemapb.FunctionSchema{proto.Clone(functionSchema).(*schemapb.FunctionSchema)},
|
||||
},
|
||||
@@ -7995,6 +8004,10 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
|
||||
{Key: common.DimKey, Value: "4096"},
|
||||
},
|
||||
}
|
||||
addRequest.FieldInfos[0].ExtraParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
|
||||
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
|
||||
}
|
||||
functionSchema := proto.Clone(addRequest.GetFuncSchema()[0]).(*schemapb.FunctionSchema)
|
||||
functionSchema.Name = "minhash_func"
|
||||
functionSchema.Type = schemapb.FunctionType_MinHash
|
||||
@@ -8069,6 +8082,28 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("PreExecute rejects invalid bound index name", func(t *testing.T) {
|
||||
req := buildValidRequest()
|
||||
req.GetAction().GetAddRequest().GetFieldInfos()[0].IndexName = "1bad-index-name"
|
||||
task := buildTask(req, oldSchema)
|
||||
err := task.PreExecute(ctx)
|
||||
assert.Error(t, err)
|
||||
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
assert.ErrorContains(t, err, "Invalid index name")
|
||||
})
|
||||
|
||||
t.Run("PreExecute keeps the user's original index params in the request", func(t *testing.T) {
|
||||
req := buildValidRequest()
|
||||
original := proto.Clone(req.GetAction().GetAddRequest().GetFieldInfos()[0]).(*milvuspb.AlterCollectionSchemaRequest_FieldInfo)
|
||||
task := buildTask(req, oldSchema)
|
||||
err := task.PreExecute(ctx)
|
||||
assert.NoError(t, err)
|
||||
// Validation-only: no normalization write-back, so the persisted
|
||||
// UserIndexParams stay aligned with the create_index convention and a
|
||||
// later create_index with identical params remains idempotent.
|
||||
assert.True(t, proto.Equal(original, req.GetAction().GetAddRequest().GetFieldInfos()[0]))
|
||||
})
|
||||
|
||||
t.Run("Execute leaves function output marker to RootCoord", func(t *testing.T) {
|
||||
req := buildValidRequest()
|
||||
task := buildTask(req, oldSchema)
|
||||
|
||||
+2
-26
@@ -1789,32 +1789,8 @@ func validCharInIndexName(c byte) bool {
|
||||
}
|
||||
|
||||
func validateIndexName(indexName string) error {
|
||||
indexName = strings.TrimSpace(indexName)
|
||||
|
||||
if indexName == "" {
|
||||
return nil
|
||||
}
|
||||
invalidMsg := "Invalid index name: " + indexName + ". "
|
||||
if len(indexName) > Params.ProxyCfg.MaxNameLength.GetAsInt() {
|
||||
msg := invalidMsg + "The length of a index name must be less than " + Params.ProxyCfg.MaxNameLength.GetValue() + " characters."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
|
||||
firstChar := indexName[0]
|
||||
if firstChar != '_' && !isAlpha(firstChar) {
|
||||
msg := invalidMsg + "The first character of a index name must be an underscore or letter."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
|
||||
indexNameSize := len(indexName)
|
||||
for i := 1; i < indexNameSize; i++ {
|
||||
c := indexName[i]
|
||||
if !validCharInIndexName(c) {
|
||||
msg := invalidMsg + "Index name can only contain numbers, letters, and underscores."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
// Shared with rootcoord's bound-index prepare (indexparamcheck).
|
||||
return indexparamcheck.ValidateIndexName(indexName)
|
||||
}
|
||||
|
||||
func isCollectionLoaded(ctx context.Context, mc types.MixCoordClient, collID int64) (bool, error) {
|
||||
|
||||
@@ -441,6 +441,18 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
|
||||
}
|
||||
return merr.Wrap(err, "failed to alter collection")
|
||||
}
|
||||
// Refresh datacoord's cached collection schema BEFORE the bound index meta
|
||||
// becomes visible: creating the index signals the index inspector, whose
|
||||
// function-output-field guard reads that cached schema — on a stale view it
|
||||
// would schedule doomed builds on segments that have no binlog for the new
|
||||
// field yet. The schema push depends only on rootcoord meta (updated above),
|
||||
// never on index meta, so this order is always safe.
|
||||
if err := c.broker.BroadcastAlteredCollection(ctx, header.CollectionId); err != nil {
|
||||
return merr.Wrap(err, "failed to broadcast altered collection")
|
||||
}
|
||||
if err := c.applyBoundFieldIndexesInline(ctx, result); err != nil {
|
||||
return err
|
||||
}
|
||||
if body.Updates.AlterLoadConfig != nil {
|
||||
resp, err := c.mixCoord.UpdateLoadConfig(ctx, &querypb.UpdateLoadConfigRequest{
|
||||
CollectionIDs: []int64{header.CollectionId},
|
||||
@@ -461,9 +473,6 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
|
||||
if err := c.cascadeDropFieldIndexesInline(ctx, result); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.broker.BroadcastAlteredCollection(ctx, header.CollectionId); err != nil {
|
||||
return merr.Wrap(err, "failed to broadcast altered collection")
|
||||
}
|
||||
|
||||
// If the collection was renamed or moved to a different DB, grants were migrated
|
||||
// in MetaTable.AlterCollection. Refresh the RBAC policy cache on all proxies so
|
||||
@@ -482,6 +491,54 @@ func (c *DDLCallback) alterCollectionV2AckCallback(ctx context.Context, result m
|
||||
return c.ExpireCaches(ctx, header)
|
||||
}
|
||||
|
||||
// applyBoundFieldIndexesInline creates the index meta bound to a newly added
|
||||
// function-output field by inlining the CreateIndex ack callback, same pattern as
|
||||
// cascadeDropFieldIndexesInline. The FieldIndex was fully materialized (id/name
|
||||
// allocated, params validated) at DDL prepare time, so this is a pure idempotent
|
||||
// apply: a replayed callback rebuilds the identical synthetic message. Cannot use
|
||||
// the CreateIndex RPC here because it would deadlock on the resource key lock.
|
||||
// The synthetic message is never appended to the WAL; it only routes the apply
|
||||
// through the registry to datacoord's createIndexV2AckCallback.
|
||||
func (c *DDLCallback) applyBoundFieldIndexesInline(ctx context.Context, result message.BroadcastResultAlterCollectionMessageV2) error {
|
||||
header := result.Message.Header()
|
||||
boundFieldIndexes := result.Message.MustBody().GetUpdates().GetBoundFieldIndexes()
|
||||
if len(boundFieldIndexes) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
controlChannelResult := result.GetControlChannelResult()
|
||||
for _, fieldIndex := range boundFieldIndexes {
|
||||
indexInfo := fieldIndex.GetIndexInfo()
|
||||
mlog.Info(ctx, "applying bound field index of alter collection schema",
|
||||
mlog.FieldMessage(result.Message),
|
||||
mlog.FieldFieldID(indexInfo.GetFieldID()),
|
||||
mlog.String("indexName", indexInfo.GetIndexName()),
|
||||
mlog.FieldIndexID(indexInfo.GetIndexID()),
|
||||
)
|
||||
createIndexMsg := message.NewCreateIndexMessageBuilderV2().
|
||||
WithHeader(&message.CreateIndexMessageHeader{
|
||||
DbId: header.DbId,
|
||||
CollectionId: header.CollectionId,
|
||||
FieldId: indexInfo.GetFieldID(),
|
||||
IndexId: indexInfo.GetIndexID(),
|
||||
IndexName: indexInfo.GetIndexName(),
|
||||
}).
|
||||
WithBody(&message.CreateIndexMessageBody{
|
||||
FieldIndex: fieldIndex,
|
||||
}).
|
||||
WithBroadcast([]string{streaming.WAL().ControlChannel()}).
|
||||
MustBuildBroadcast().
|
||||
WithBroadcastID(result.Message.BroadcastHeader().BroadcastID)
|
||||
|
||||
if err := registry.CallMessageAckCallback(ctx, createIndexMsg, map[string]*message.AppendResult{
|
||||
streaming.WAL().ControlChannel(): controlChannelResult,
|
||||
}); err != nil {
|
||||
return merr.Wrap(err, "failed to apply bound field index")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cascadeDropFieldIndexesInline drops indexes on dropped fields by inlining the
|
||||
// DropIndex ack callback, same pattern as dropCollectionV1AckCallback.
|
||||
// Cannot use DropIndex RPC here because it would deadlock on the resource key lock.
|
||||
|
||||
@@ -268,7 +268,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_UpdateLoadConfigRPCError(t *te
|
||||
withMeta(meta),
|
||||
withMixCoord(mixc),
|
||||
withValidProxyManager(),
|
||||
withBroker(&mockBroker{}),
|
||||
withBroker(&mockBroker{
|
||||
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
|
||||
return nil
|
||||
},
|
||||
}),
|
||||
)
|
||||
cb := &DDLCallback{Core: c}
|
||||
|
||||
@@ -308,7 +312,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_UpdateLoadConfigNonRGNotFoundE
|
||||
withMeta(meta),
|
||||
withMixCoord(mixc),
|
||||
withValidProxyManager(),
|
||||
withBroker(&mockBroker{}),
|
||||
withBroker(&mockBroker{
|
||||
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
|
||||
return nil
|
||||
},
|
||||
}),
|
||||
)
|
||||
cb := &DDLCallback{Core: c}
|
||||
|
||||
@@ -352,7 +360,11 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_StopRetryOnResourceGroupNotFou
|
||||
withMeta(meta),
|
||||
withMixCoord(mixc),
|
||||
withValidProxyManager(),
|
||||
withBroker(&mockBroker{}),
|
||||
withBroker(&mockBroker{
|
||||
BroadcastAlteredCollectionFunc: func(ctx context.Context, collectionID int64) error {
|
||||
return nil
|
||||
},
|
||||
}),
|
||||
)
|
||||
cb := &DDLCallback{Core: c}
|
||||
|
||||
@@ -435,8 +447,9 @@ func TestDDLCallbacksAlterCollectionV2AckCallback_BroadcastAlteredCollectionErro
|
||||
meta := mockrootcoord.NewIMetaTable(t)
|
||||
meta.EXPECT().AlterCollection(mock.Anything, mock.Anything).Return(nil)
|
||||
|
||||
// UpdateLoadConfig is never reached: the schema broadcast now runs first
|
||||
// (before the bound-index apply and load-config update) and fails here.
|
||||
mixc := imocks.NewMixCoord(t)
|
||||
mixc.On("UpdateLoadConfig", mock.Anything, mock.Anything).Return(merr.Success(), nil)
|
||||
|
||||
c := newTestCore(
|
||||
withMeta(meta),
|
||||
|
||||
@@ -19,6 +19,7 @@ package rootcoord
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/samber/lo"
|
||||
"google.golang.org/protobuf/types/known/fieldmaskpb"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
@@ -28,8 +29,10 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster"
|
||||
"github.com/milvus-io/milvus/internal/util/function/validator"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
"github.com/milvus-io/milvus/internal/util/schemautil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
@@ -128,6 +131,18 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
|
||||
return merr.WrapErrParameterInvalidMsg("%s", err.Error())
|
||||
}
|
||||
|
||||
// Materialize the bound index meta for the new function output field BEFORE the
|
||||
// broadcast, so the WAL message carries a complete, replay-deterministic index
|
||||
// definition and the ack callback stays a pure idempotent apply.
|
||||
var boundFieldIndexes []*indexpb.FieldIndex
|
||||
if plan.Kind == schemautil.AlterSchemaAddFunctionField {
|
||||
fieldIndex, err := c.prepareBoundFieldIndex(ctx, coll, plan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
boundFieldIndexes = append(boundFieldIndexes, fieldIndex)
|
||||
}
|
||||
|
||||
// Broadcast.
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
@@ -148,8 +163,9 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
|
||||
}).
|
||||
WithBody(&messagespb.AlterCollectionMessageBody{
|
||||
Updates: &messagespb.AlterCollectionMessageUpdates{
|
||||
Schema: schema,
|
||||
Properties: properties,
|
||||
Schema: schema,
|
||||
Properties: properties,
|
||||
BoundFieldIndexes: boundFieldIndexes,
|
||||
},
|
||||
}).
|
||||
WithBroadcast(channels).
|
||||
@@ -160,6 +176,91 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
|
||||
return nil
|
||||
}
|
||||
|
||||
// prepareBoundFieldIndex materializes the index meta bound to the newly added
|
||||
// function-output field, strictly BEFORE the DDL broadcast: index id/name are
|
||||
// allocated here and serialized into the WAL message so that the ack-callback
|
||||
// apply is a pure idempotent write (a replayed callback rebuilds the identical
|
||||
// index), and every input-dependent rejection happens before anything commits.
|
||||
func (c *Core) prepareBoundFieldIndex(ctx context.Context, coll *model.Collection, plan *schemautil.AlterSchemaAddPlan) (*indexpb.FieldIndex, error) {
|
||||
indexParamsMap, err := indexparamcheck.PrepareFunctionOutputIndexParams(
|
||||
plan.Function.GetType(), plan.Field.GetName(), plan.IndexExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The index type must have a registered checker — an unknown type would pass
|
||||
// structural validation, get persisted via the ack callback, and then never
|
||||
// build. Proxy already rejects this, but the check must live pre-broadcast
|
||||
// for callers that reach rootcoord directly.
|
||||
indexType := indexParamsMap[common.IndexTypeKey]
|
||||
if checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType); err != nil || indexparamcheck.IsHYBRIDChecker(checker) {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(
|
||||
"invalid index type %s for the bound index of function output field %q",
|
||||
indexType, plan.Field.GetName())
|
||||
}
|
||||
// Full field-aware validation (params size, dimension fill+match, data-type
|
||||
// compatibility, train params), identical to the create_index path — an index
|
||||
// that cannot build must never be persisted through the ack callback.
|
||||
if err := indexparamcheck.ValidateFieldIndexParams(plan.Field, indexParamsMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indexName := plan.IndexName
|
||||
if indexName == "" {
|
||||
indexName = plan.Field.GetName()
|
||||
}
|
||||
// Name-format rule, same as the proxy path — enforced here too for callers
|
||||
// that reach rootcoord directly.
|
||||
if err := indexparamcheck.ValidateIndexName(indexName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reject index-name conflicts with existing indexes (the field itself is new,
|
||||
// so only cross-field name collisions are possible).
|
||||
resp, err := c.mixCoord.DescribeIndex(ctx, &indexpb.DescribeIndexRequest{
|
||||
CollectionID: coll.CollectionID,
|
||||
})
|
||||
if err := merr.CheckRPCCall(resp.GetStatus(), err); err != nil {
|
||||
if !merr.ErrIndexNotFound.Is(err) {
|
||||
return nil, merr.Wrap(err, "failed to list existing indexes for bound index preparation")
|
||||
}
|
||||
} else {
|
||||
for _, info := range resp.GetIndexInfos() {
|
||||
if info.GetIndexName() == indexName {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("index name %s already exists in collection", indexName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
indexID, err := c.idAllocator.AllocOne()
|
||||
if err != nil {
|
||||
return nil, merr.Wrap(err, "failed to allocate index id for bound index")
|
||||
}
|
||||
createTime, err := c.tsoAllocator.GenerateTSO(1)
|
||||
if err != nil {
|
||||
return nil, merr.Wrap(err, "failed to allocate timestamp for bound index")
|
||||
}
|
||||
|
||||
indexParams := funcutil.Map2KeyValuePair(indexParamsMap)
|
||||
// Field type params minus per-field mmap/warmup keys, mirroring datacoord CreateIndex.
|
||||
typeParams := lo.Filter(plan.Field.GetTypeParams(), func(kv *commonpb.KeyValuePair, _ int) bool {
|
||||
return kv.GetKey() != common.MmapEnabledKey && kv.GetKey() != common.WarmupKey
|
||||
})
|
||||
index := &model.Index{
|
||||
CollectionID: coll.CollectionID,
|
||||
FieldID: plan.Field.GetFieldID(),
|
||||
IndexID: indexID,
|
||||
IndexName: indexName,
|
||||
TypeParams: typeParams,
|
||||
IndexParams: indexParams,
|
||||
CreateTime: createTime,
|
||||
IsAutoIndex: false,
|
||||
UserIndexParams: plan.IndexExtraParams,
|
||||
}
|
||||
if err := indexparamcheck.ValidateIndexParams(index); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return model.MarshalIndexModel(index), nil
|
||||
}
|
||||
|
||||
func prepareAlterSchemaAddField(coll *model.Collection, plan *schemautil.AlterSchemaAddPlan) error {
|
||||
if !plan.HasField() {
|
||||
return nil
|
||||
|
||||
@@ -19,6 +19,7 @@ package rootcoord
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/cockroachdb/errors"
|
||||
@@ -30,16 +31,23 @@ import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/distributed/streaming"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
imocks "github.com/milvus-io/milvus/internal/mocks"
|
||||
"github.com/milvus-io/milvus/internal/mocks/distributed/mock_streaming"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_balancer"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/balance"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster/registry"
|
||||
"github.com/milvus-io/milvus/internal/tso"
|
||||
mocktso "github.com/milvus-io/milvus/internal/tso/mocks"
|
||||
"github.com/milvus-io/milvus/internal/util/schemautil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/timestamptz"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
@@ -64,7 +72,13 @@ func buildAlterSchemaReq(dbName, collName, inputField, outputField, funcName str
|
||||
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
|
||||
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
|
||||
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
|
||||
{FieldSchema: outputFieldSchema},
|
||||
{
|
||||
FieldSchema: outputFieldSchema,
|
||||
ExtraParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
},
|
||||
},
|
||||
},
|
||||
FuncSchema: []*schemapb.FunctionSchema{functionSchema},
|
||||
},
|
||||
@@ -416,7 +430,13 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
{Name: "fn_dup_field", Type: schemapb.FunctionType_BM25, InputFieldNames: []string{"field1"}, OutputFieldNames: []string{"field1"}},
|
||||
},
|
||||
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
|
||||
{FieldSchema: &schemapb.FieldSchema{Name: "field1", DataType: schemapb.DataType_SparseFloatVector}},
|
||||
{
|
||||
FieldSchema: &schemapb.FieldSchema{Name: "field1", DataType: schemapb.DataType_SparseFloatVector},
|
||||
ExtraParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -424,6 +444,14 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
|
||||
|
||||
// case 7.1: vector function output field without bound index params is rejected.
|
||||
noIndexReq := buildAlterSchemaReq(dbName, collectionName, "field1", "sparse_no_index", "fn_no_index")
|
||||
noIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = nil
|
||||
resp, err = core.AlterCollectionSchema(ctx, noIndexReq)
|
||||
noIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
|
||||
require.ErrorIs(t, noIndexErr, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, noIndexErr, "index params are required")
|
||||
|
||||
// case 8: output field points to an existing field while FieldInfos adds a different field
|
||||
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
|
||||
DbName: dbName,
|
||||
@@ -462,6 +490,29 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, merr.CheckRPCCall(addFieldResp, err))
|
||||
|
||||
// case 7.2: AUTOINDEX is not accepted as the bound index type in V1
|
||||
// (rejected at rootcoord prepare, after function validation passes).
|
||||
autoIndexReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_auto_index", "fn_auto_index")
|
||||
autoIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: common.AutoIndexName},
|
||||
}
|
||||
resp, err = core.AlterCollectionSchema(ctx, autoIndexReq)
|
||||
autoIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
|
||||
require.ErrorIs(t, autoIndexErr, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, autoIndexErr, "explicit index_type is required")
|
||||
|
||||
// case 7.3: an index type without a registered checker is rejected at prepare —
|
||||
// it would otherwise be persisted via the ack callback and never build.
|
||||
unknownIndexReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_unknown_index", "fn_unknown_index")
|
||||
unknownIndexReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "NOT_A_REAL_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
}
|
||||
resp, err = core.AlterCollectionSchema(ctx, unknownIndexReq)
|
||||
unknownIndexErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
|
||||
require.ErrorIs(t, unknownIndexErr, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, unknownIndexErr, "invalid index type")
|
||||
|
||||
// happy path: add binary vector output field + MinHash function.
|
||||
minHashReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "binary_minhash_output", "minhash_fn")
|
||||
minHashFieldSchema := minHashReq.GetAction().GetAddRequest().GetFieldInfos()[0].GetFieldSchema()
|
||||
@@ -469,6 +520,10 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
minHashFieldSchema.TypeParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "4096"},
|
||||
}
|
||||
minHashReq.GetAction().GetAddRequest().GetFieldInfos()[0].ExtraParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
|
||||
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
|
||||
}
|
||||
minHashFunction := minHashReq.GetAction().GetAddRequest().GetFuncSchema()[0]
|
||||
minHashFunction.Type = schemapb.FunctionType_MinHash
|
||||
minHashFunction.Params = []*commonpb.KeyValuePair{
|
||||
@@ -481,6 +536,27 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 5)
|
||||
|
||||
// The bound index meta must have been applied through the CreateIndex ack
|
||||
// callback within the same DDL, fully materialized at prepare time.
|
||||
coll, err = core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
var minHashFieldID int64
|
||||
for _, field := range coll.Fields {
|
||||
if field.Name == "binary_minhash_output" {
|
||||
minHashFieldID = field.FieldID
|
||||
}
|
||||
}
|
||||
require.Positive(t, minHashFieldID)
|
||||
boundIndexes := recordedBoundIndexes()
|
||||
require.Len(t, boundIndexes, 1)
|
||||
boundIndexInfo := boundIndexes[0].GetIndexInfo()
|
||||
require.Equal(t, "binary_minhash_output", boundIndexInfo.GetIndexName())
|
||||
require.Equal(t, minHashFieldID, boundIndexInfo.GetFieldID())
|
||||
require.Positive(t, boundIndexInfo.GetIndexID())
|
||||
boundIndexParams := funcutil.KeyValuePair2Map(boundIndexInfo.GetIndexParams())
|
||||
require.Equal(t, "MINHASH_LSH", boundIndexParams[common.IndexTypeKey])
|
||||
require.Equal(t, "MHJACCARD", boundIndexParams[common.MetricTypeKey])
|
||||
|
||||
minHashBadArityReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "minhash_bad_arity", "minhash_bad_arity_fn")
|
||||
minHashBadArityReq.GetAction().GetAddRequest().GetFuncSchema()[0].Type = schemapb.FunctionType_MinHash
|
||||
minHashBadArityReq.GetAction().GetAddRequest().GetFuncSchema()[0].InputFieldNames = []string{"text_input", "field1"}
|
||||
@@ -1344,3 +1420,238 @@ func TestAlterCollectionV2AckCallbackUsesHeaderDroppedFieldIDs(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func newBoundIndexTestPlan() (*model.Collection, *schemautil.AlterSchemaAddPlan) {
|
||||
coll := &model.Collection{CollectionID: 1, DBID: 1, Name: "coll"}
|
||||
plan := &schemautil.AlterSchemaAddPlan{
|
||||
Kind: schemautil.AlterSchemaAddFunctionField,
|
||||
Field: &schemapb.FieldSchema{
|
||||
FieldID: 101,
|
||||
Name: "sparse",
|
||||
DataType: schemapb.DataType_SparseFloatVector,
|
||||
},
|
||||
Function: &schemapb.FunctionSchema{Name: "bm25_fn", Type: schemapb.FunctionType_BM25},
|
||||
IndexExtraParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
},
|
||||
}
|
||||
return coll, plan
|
||||
}
|
||||
|
||||
func withNoIndexMixCoord(t *testing.T) *imocks.MixCoord {
|
||||
mixc := imocks.NewMixCoord(t)
|
||||
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(
|
||||
&indexpb.DescribeIndexResponse{Status: merr.Status(merr.WrapErrIndexNotFound("sparse"))}, nil,
|
||||
).Maybe()
|
||||
return mixc
|
||||
}
|
||||
|
||||
func withOkTso(t *testing.T) tso.Allocator {
|
||||
alloc := mocktso.NewAllocator(t)
|
||||
alloc.EXPECT().GenerateTSO(mock.Anything).Return(uint64(100), nil).Maybe()
|
||||
return alloc
|
||||
}
|
||||
|
||||
func TestPrepareBoundFieldIndex(t *testing.T) {
|
||||
t.Run("happy path materializes replay-deterministic index", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
fieldIndex, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.NoError(t, err)
|
||||
info := fieldIndex.GetIndexInfo()
|
||||
require.Equal(t, int64(101), info.GetFieldID())
|
||||
require.Equal(t, "sparse", info.GetIndexName())
|
||||
require.Positive(t, info.GetIndexID())
|
||||
params := funcutil.KeyValuePair2Map(info.GetIndexParams())
|
||||
require.Equal(t, "SPARSE_INVERTED_INDEX", params[common.IndexTypeKey])
|
||||
require.Equal(t, "1.2", params["bm25_k1"])
|
||||
// UserIndexParams keep the user's ORIGINAL params (create_index
|
||||
// convention), so a later create_index with identical params is
|
||||
// treated as idempotent instead of a distinct index.
|
||||
require.Equal(t, plan.IndexExtraParams, info.GetUserIndexParams())
|
||||
})
|
||||
|
||||
t.Run("index name conflict rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
mixc := imocks.NewMixCoord(t)
|
||||
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(
|
||||
&indexpb.DescribeIndexResponse{
|
||||
Status: merr.Success(),
|
||||
IndexInfos: []*indexpb.IndexInfo{{FieldID: 100, IndexID: 1, IndexName: "sparse"}},
|
||||
}, nil,
|
||||
)
|
||||
c := newTestCore(withMixCoord(mixc), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, err, "already exists")
|
||||
})
|
||||
|
||||
t.Run("describe index transient error rejects the DDL", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
mixc := imocks.NewMixCoord(t)
|
||||
mixc.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(nil, errors.New("rpc unavailable"))
|
||||
c := newTestCore(withMixCoord(mixc), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "failed to list existing indexes")
|
||||
})
|
||||
|
||||
t.Run("index id allocation failure", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withInvalidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "failed to allocate index id")
|
||||
})
|
||||
|
||||
t.Run("incompatible index type for field data type rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
plan.Field = &schemapb.FieldSchema{
|
||||
FieldID: 102,
|
||||
Name: "binary_mh",
|
||||
DataType: schemapb.DataType_BinaryVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "512"},
|
||||
},
|
||||
}
|
||||
plan.Function = &schemapb.FunctionSchema{Name: "mh_fn", Type: schemapb.FunctionType_MinHash}
|
||||
// SPARSE index on a binary vector field: must be rejected pre-broadcast.
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("dimension mismatch rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
plan.Field = &schemapb.FieldSchema{
|
||||
FieldID: 102,
|
||||
Name: "binary_mh",
|
||||
DataType: schemapb.DataType_BinaryVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "512"},
|
||||
},
|
||||
}
|
||||
plan.Function = &schemapb.FunctionSchema{Name: "mh_fn", Type: schemapb.FunctionType_MinHash}
|
||||
plan.IndexExtraParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "MINHASH_LSH"},
|
||||
{Key: common.MetricTypeKey, Value: "MHJACCARD"},
|
||||
{Key: common.DimKey, Value: "1024"},
|
||||
}
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "dimension mismatch")
|
||||
})
|
||||
|
||||
t.Run("bogus warmup policy rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
plan.IndexExtraParams = append(plan.IndexExtraParams, &commonpb.KeyValuePair{
|
||||
Key: common.WarmupKey, Value: "bogus",
|
||||
})
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "warmup")
|
||||
})
|
||||
|
||||
t.Run("malformed index name rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
plan.IndexName = "1bad-name"
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "Invalid index name")
|
||||
})
|
||||
|
||||
t.Run("oversized index params rejected", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
plan.IndexExtraParams = append(plan.IndexExtraParams, &commonpb.KeyValuePair{
|
||||
Key: "huge",
|
||||
Value: strings.Repeat("x", paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()+1),
|
||||
})
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(withOkTso(t)))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "exceeds limit")
|
||||
})
|
||||
|
||||
t.Run("tso allocation failure", func(t *testing.T) {
|
||||
coll, plan := newBoundIndexTestPlan()
|
||||
alloc := mocktso.NewAllocator(t)
|
||||
alloc.EXPECT().GenerateTSO(mock.Anything).Return(uint64(0), errors.New("tso unavailable"))
|
||||
c := newTestCore(withMixCoord(withNoIndexMixCoord(t)), withValidIDAllocator(), withTsoAllocator(alloc))
|
||||
_, err := c.prepareBoundFieldIndex(context.Background(), coll, plan)
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "failed to allocate timestamp")
|
||||
})
|
||||
}
|
||||
|
||||
func TestApplyBoundFieldIndexesInline(t *testing.T) {
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
wal.EXPECT().ControlChannel().Return(funcutil.GetControlChannel("by-dev-rootcoord-dml_0")).Maybe()
|
||||
streaming.SetWALForTest(wal)
|
||||
|
||||
boundFieldIndex := &indexpb.FieldIndex{
|
||||
IndexInfo: &indexpb.IndexInfo{
|
||||
CollectionID: 1,
|
||||
FieldID: 101,
|
||||
IndexID: 201,
|
||||
IndexName: "sparse",
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
buildResult := func(bound []*indexpb.FieldIndex) message.BroadcastResultAlterCollectionMessageV2 {
|
||||
raw := message.NewAlterCollectionMessageBuilderV2().
|
||||
WithHeader(&messagespb.AlterCollectionMessageHeader{
|
||||
DbId: 1,
|
||||
CollectionId: 1,
|
||||
}).
|
||||
WithBody(&messagespb.AlterCollectionMessageBody{
|
||||
Updates: &messagespb.AlterCollectionMessageUpdates{
|
||||
BoundFieldIndexes: bound,
|
||||
},
|
||||
}).
|
||||
WithBroadcast([]string{funcutil.GetControlChannel("by-dev-rootcoord-dml_0")}).
|
||||
MustBuildBroadcast()
|
||||
msg := message.MustAsBroadcastAlterCollectionMessageV2(raw.WithBroadcastID(1))
|
||||
return message.BroadcastResultAlterCollectionMessageV2{
|
||||
Message: msg,
|
||||
Results: map[string]*message.AppendResult{},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("no bound indexes short circuits", func(t *testing.T) {
|
||||
registry.ResetRegistration()
|
||||
cb := &DDLCallback{Core: newTestCore()}
|
||||
require.NoError(t, cb.applyBoundFieldIndexesInline(context.Background(), buildResult(nil)))
|
||||
})
|
||||
|
||||
t.Run("dispatches synthetic create index message", func(t *testing.T) {
|
||||
registry.ResetRegistration()
|
||||
var applied []*indexpb.FieldIndex
|
||||
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
|
||||
applied = append(applied, result.Message.MustBody().GetFieldIndex())
|
||||
return nil
|
||||
})
|
||||
cb := &DDLCallback{Core: newTestCore()}
|
||||
require.NoError(t, cb.applyBoundFieldIndexesInline(context.Background(), buildResult([]*indexpb.FieldIndex{boundFieldIndex})))
|
||||
require.Len(t, applied, 1)
|
||||
require.Equal(t, int64(201), applied[0].GetIndexInfo().GetIndexID())
|
||||
require.Equal(t, "sparse", applied[0].GetIndexInfo().GetIndexName())
|
||||
})
|
||||
|
||||
t.Run("dispatch failure propagates so the ack callback retries", func(t *testing.T) {
|
||||
registry.ResetRegistration()
|
||||
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
|
||||
return errors.New("catalog write failed")
|
||||
})
|
||||
cb := &DDLCallback{Core: newTestCore()}
|
||||
err := cb.applyBoundFieldIndexesInline(context.Background(), buildResult([]*indexpb.FieldIndex{boundFieldIndex}))
|
||||
require.Error(t, err)
|
||||
require.ErrorContains(t, err, "failed to apply bound field index")
|
||||
})
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/mlog"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/etcdpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/indexpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/internalpb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/proxypb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/rootcoordpb"
|
||||
@@ -79,6 +80,31 @@ func TestMain(m *testing.M) {
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
// testBoundIndexRecorder captures FieldIndexes applied through the fake
|
||||
// CreateIndexV2 ack callback so DDL tests can assert on bound-index creation.
|
||||
var testBoundIndexRecorder = struct {
|
||||
mu sync.Mutex
|
||||
indexes []*indexpb.FieldIndex
|
||||
}{}
|
||||
|
||||
func resetRecordedBoundIndexes() {
|
||||
testBoundIndexRecorder.mu.Lock()
|
||||
defer testBoundIndexRecorder.mu.Unlock()
|
||||
testBoundIndexRecorder.indexes = nil
|
||||
}
|
||||
|
||||
func recordBoundIndex(fieldIndex *indexpb.FieldIndex) {
|
||||
testBoundIndexRecorder.mu.Lock()
|
||||
defer testBoundIndexRecorder.mu.Unlock()
|
||||
testBoundIndexRecorder.indexes = append(testBoundIndexRecorder.indexes, fieldIndex)
|
||||
}
|
||||
|
||||
func recordedBoundIndexes() []*indexpb.FieldIndex {
|
||||
testBoundIndexRecorder.mu.Lock()
|
||||
defer testBoundIndexRecorder.mu.Unlock()
|
||||
return append([]*indexpb.FieldIndex(nil), testBoundIndexRecorder.indexes...)
|
||||
}
|
||||
|
||||
func initStreamingSystemAndCore(t *testing.T) *Core {
|
||||
kv, _ := kvfactory.GetEtcdAndPath()
|
||||
path := funcutil.RandomString(10) + "/meta"
|
||||
@@ -110,6 +136,11 @@ func initStreamingSystemAndCore(t *testing.T) *Core {
|
||||
registry.RegisterDropIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultDropIndexMessageV2) error {
|
||||
return nil
|
||||
})
|
||||
resetRecordedBoundIndexes()
|
||||
registry.RegisterCreateIndexV2AckCallback(func(ctx context.Context, result message.BroadcastResultCreateIndexMessageV2) error {
|
||||
recordBoundIndex(result.Message.MustBody().GetFieldIndex())
|
||||
return nil
|
||||
})
|
||||
registry.RegisterDropLoadConfigV2AckCallback(func(ctx context.Context, result message.BroadcastResultDropLoadConfigMessageV2) error {
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package indexparamcheck
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/metric"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
// ValidateIndexParams performs structural validation on an index definition.
|
||||
// Moved from internal/datacoord so that rootcoord's add-function-field bound-index
|
||||
// prepare can share it with datacoord's CreateIndex/AlterIndex/snapshot restore paths.
|
||||
func ValidateIndexParams(index *model.Index) error {
|
||||
if err := checkDuplicateKey(index.IndexParams, "indexParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkDuplicateKey(index.UserIndexParams, "userIndexParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkDuplicateKey(index.TypeParams, "typeParams"); err != nil {
|
||||
return err
|
||||
}
|
||||
indexType := common.GetIndexType(index.IndexParams)
|
||||
indexParams := funcutil.KeyValuePair2Map(index.IndexParams)
|
||||
userIndexParams := funcutil.KeyValuePair2Map(index.UserIndexParams)
|
||||
if err := ValidateMmapIndexParams(indexType, indexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid mmap index params: %s", err.Error())
|
||||
}
|
||||
if err := ValidateMmapIndexParams(indexType, userIndexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid mmap user index params: %s", err.Error())
|
||||
}
|
||||
if err := ValidateOffsetCacheIndexParams(indexType, indexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
|
||||
}
|
||||
if err := ValidateOffsetCacheIndexParams(indexType, userIndexParams); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid offset cache index params: %s", err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkDuplicateKey rejects duplicated keys in a key-value pair list.
|
||||
func checkDuplicateKey(kvs []*commonpb.KeyValuePair, tag string) error {
|
||||
keySet := typeutil.NewSet[string]()
|
||||
for _, kv := range kvs {
|
||||
if keySet.Contain(kv.GetKey()) {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicate %s key in %s params", kv.GetKey(), tag)
|
||||
}
|
||||
keySet.Insert(kv.GetKey())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// FillFunctionOutputIndexParams applies the function-type-specific defaults and
|
||||
// constraints to the index params of a function output field. Shared by the
|
||||
// create_index path (proxy createIndexTask.parseFunctionParamsToIndex) and the
|
||||
// add-function-field bound-index prepare (rootcoord), so both produce identical
|
||||
// build params (e.g. knowhere requires bm25_k1/bm25_b/bm25_avgdl for BM25).
|
||||
func FillFunctionOutputIndexParams(functionType schemapb.FunctionType, indexParamsMap map[string]string) error {
|
||||
switch functionType {
|
||||
case schemapb.FunctionType_Unknown:
|
||||
return merr.WrapErrParameterInvalidMsg("unknown function type encountered")
|
||||
|
||||
case schemapb.FunctionType_BM25:
|
||||
// set default BM25 params if not provided in index params
|
||||
if _, ok := indexParamsMap["bm25_k1"]; !ok {
|
||||
indexParamsMap["bm25_k1"] = "1.2"
|
||||
}
|
||||
|
||||
if _, ok := indexParamsMap["bm25_b"]; !ok {
|
||||
indexParamsMap["bm25_b"] = "0.75"
|
||||
}
|
||||
|
||||
if _, ok := indexParamsMap["bm25_avgdl"]; !ok {
|
||||
indexParamsMap["bm25_avgdl"] = "100"
|
||||
}
|
||||
|
||||
if metricType, ok := indexParamsMap[common.MetricTypeKey]; !ok {
|
||||
indexParamsMap[common.MetricTypeKey] = metric.BM25
|
||||
} else if metricType != metric.BM25 {
|
||||
return merr.WrapErrParameterInvalidMsg("index metric type of BM25 function output field must be BM25, got %s", metricType)
|
||||
}
|
||||
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrepareFunctionOutputIndexParams expands the user-provided extra params of a
|
||||
// function output field's bound index, requires an explicit index type (no
|
||||
// AUTOINDEX resolution for bound indexes), and applies function-type-specific
|
||||
// defaults. Shared by proxy validation and rootcoord materialization so both
|
||||
// operate on identical params.
|
||||
func PrepareFunctionOutputIndexParams(functionType schemapb.FunctionType, fieldName string, extraParams []*commonpb.KeyValuePair) (map[string]string, error) {
|
||||
indexParamsMap, err := ExpandIndexParams(extraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
indexType := indexParamsMap[common.IndexTypeKey]
|
||||
if indexType == "" || indexType == common.AutoIndexName {
|
||||
return nil, merr.WrapErrParameterInvalidMsg(
|
||||
"an explicit index_type is required for the bound index of function output field %q", fieldName)
|
||||
}
|
||||
if err := FillFunctionOutputIndexParams(functionType, indexParamsMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return indexParamsMap, nil
|
||||
}
|
||||
|
||||
// ExpandIndexParams flattens user-provided index extra params into a plain map,
|
||||
// expanding the legacy JSON-encoded common.ParamsKey entry, and rejecting
|
||||
// duplicated keys. Same convention as proxy's createIndexTask.parseIndexParams.
|
||||
func ExpandIndexParams(extraParams []*commonpb.KeyValuePair) (map[string]string, error) {
|
||||
indexParamsMap := make(map[string]string, len(extraParams))
|
||||
keys := typeutil.NewSet[string]()
|
||||
for _, kv := range extraParams {
|
||||
if keys.Contain(kv.GetKey()) {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("duplicated index param (key=%s) (value=%s) found", kv.GetKey(), kv.GetValue())
|
||||
}
|
||||
keys.Insert(kv.GetKey())
|
||||
if kv.GetKey() == common.ParamsKey {
|
||||
params, err := funcutil.JSONToMap(kv.GetValue())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range params {
|
||||
indexParamsMap[k] = v
|
||||
}
|
||||
} else {
|
||||
indexParamsMap[kv.GetKey()] = kv.GetValue()
|
||||
}
|
||||
}
|
||||
return indexParamsMap, nil
|
||||
}
|
||||
|
||||
// CheckIndexParamsSize rejects index params whose total size exceeds
|
||||
// proxy.maxIndexParamsSize. Moved from proxy so every DDL boundary that
|
||||
// accepts index params (create_index and the add-function-field bound index)
|
||||
// enforces the same limit.
|
||||
func CheckIndexParamsSize(size int) error {
|
||||
maxIndexParamsSize := paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()
|
||||
if size > maxIndexParamsSize {
|
||||
return merr.WrapErrParameterInvalidMsg("index params size exceeds limit: %d > %d", size, maxIndexParamsSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateIndexParamsSize checks the total size of a key-value pair list.
|
||||
func ValidateIndexParamsSize(params ...*commonpb.KeyValuePair) error {
|
||||
size := 0
|
||||
for _, param := range params {
|
||||
size += len(param.GetKey()) + len(param.GetValue())
|
||||
}
|
||||
return CheckIndexParamsSize(size)
|
||||
}
|
||||
|
||||
// ValidateIndexParamsMapSize checks the total size of an expanded params map.
|
||||
func ValidateIndexParamsMapSize(params map[string]string) error {
|
||||
size := 0
|
||||
for k, v := range params {
|
||||
size += len(k) + len(v)
|
||||
}
|
||||
return CheckIndexParamsSize(size)
|
||||
}
|
||||
|
||||
// FillDimension copies the dimension from the field schema into the index
|
||||
// params (rejecting a mismatch), so index build always sees the field's real
|
||||
// dimension. Moved from proxy for DDL-boundary reuse.
|
||||
func FillDimension(field *schemapb.FieldSchema, indexParams map[string]string) error {
|
||||
if !typeutil.IsVectorType(field.GetDataType()) {
|
||||
return nil
|
||||
}
|
||||
params := make([]*commonpb.KeyValuePair, 0, len(field.GetTypeParams())+len(field.GetIndexParams()))
|
||||
params = append(params, field.GetTypeParams()...)
|
||||
params = append(params, field.GetIndexParams()...)
|
||||
dimensionInSchema, err := funcutil.GetAttrByKeyFromRepeatedKV(common.DimKey, params)
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("dimension not found in schema")
|
||||
}
|
||||
dimension, exist := indexParams[common.DimKey]
|
||||
if exist {
|
||||
if dimensionInSchema != dimension {
|
||||
return merr.WrapErrParameterInvalidMsg("dimension mismatch, dimension in schema: %s, dimension: %s", dimensionInSchema, dimension)
|
||||
}
|
||||
} else {
|
||||
indexParams[common.DimKey] = dimensionInSchema
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateFieldIndexParams runs the field-aware index validation that the
|
||||
// create_index path applies (params size, checker existence, dimension
|
||||
// fill+match, data-type compatibility, train-params validation), so the
|
||||
// add-function-field bound-index path enforces identical rules at every DDL
|
||||
// boundary, including callers that reach rootcoord directly.
|
||||
func ValidateFieldIndexParams(field *schemapb.FieldSchema, indexParamsMap map[string]string) error {
|
||||
if err := ValidateIndexParamsMapSize(indexParamsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ValidateWarmupIndexParams(indexParamsMap); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid warmup params: %s", err.Error())
|
||||
}
|
||||
indexType := indexParamsMap[common.IndexTypeKey]
|
||||
checker, err := GetIndexCheckerMgrInstance().GetChecker(indexType)
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("invalid index type: %s", indexType)
|
||||
}
|
||||
|
||||
// For ArrayOfVector with non-EmbList metrics, each embedding in the array is
|
||||
// indexed independently as a regular vector; resolve the effective data type
|
||||
// used for compatibility checks (same rule as the create_index path).
|
||||
effectiveDataType := field.GetDataType()
|
||||
effectiveElementType := field.GetElementType()
|
||||
if typeutil.IsArrayOfVectorType(field.GetDataType()) &&
|
||||
!funcutil.SliceContain(EmbListMetrics, indexParamsMap[common.MetricTypeKey]) {
|
||||
effectiveDataType = field.GetElementType()
|
||||
effectiveElementType = schemapb.DataType_None
|
||||
}
|
||||
|
||||
if typeutil.IsVectorType(field.GetDataType()) && !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
|
||||
if err := FillDimension(field, indexParamsMap); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
effectiveField := field
|
||||
if effectiveDataType != field.GetDataType() {
|
||||
effectiveField = proto.Clone(field).(*schemapb.FieldSchema)
|
||||
effectiveField.DataType = effectiveDataType
|
||||
effectiveField.ElementType = effectiveElementType
|
||||
}
|
||||
|
||||
if err := checker.CheckValidDataType(indexType, effectiveField); err != nil {
|
||||
return err
|
||||
}
|
||||
return checker.CheckTrain(effectiveDataType, effectiveElementType, indexParamsMap)
|
||||
}
|
||||
|
||||
// ValidateIndexName enforces the index-name format rules. Moved from proxy so
|
||||
// rootcoord's bound-index prepare applies the same rule to callers that reach
|
||||
// it directly.
|
||||
func ValidateIndexName(indexName string) error {
|
||||
indexName = strings.TrimSpace(indexName)
|
||||
|
||||
if indexName == "" {
|
||||
return nil
|
||||
}
|
||||
invalidMsg := "Invalid index name: " + indexName + ". "
|
||||
if len(indexName) > paramtable.Get().ProxyCfg.MaxNameLength.GetAsInt() {
|
||||
msg := invalidMsg + "The length of a index name must be less than " + paramtable.Get().ProxyCfg.MaxNameLength.GetValue() + " characters."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
|
||||
firstChar := indexName[0]
|
||||
if firstChar != '_' && !isIndexNameAlpha(firstChar) {
|
||||
msg := invalidMsg + "The first character of a index name must be an underscore or letter."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
|
||||
for i := 1; i < len(indexName); i++ {
|
||||
c := indexName[i]
|
||||
if !validCharInIndexName(c) {
|
||||
msg := invalidMsg + "Index name can only contain numbers, letters, and underscores."
|
||||
return merr.WrapErrParameterInvalidMsg("%s", msg)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isIndexNameAlpha(c uint8) bool {
|
||||
return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')
|
||||
}
|
||||
|
||||
func validCharInIndexName(c byte) bool {
|
||||
return c == '_' || c == '[' || c == ']' || isIndexNameAlpha(c) || (c >= '0' && c <= '9')
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package indexparamcheck
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
|
||||
)
|
||||
|
||||
func TestValidateIndexParams(t *testing.T) {
|
||||
t.Run("valid", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "true",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid index param", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "h",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid index user param", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
},
|
||||
UserIndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.MmapEnabledKey,
|
||||
Value: "h",
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_index_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
IndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_user_index_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
UserIndexParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("duplicated_type_params", func(t *testing.T) {
|
||||
index := &model.Index{
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
{
|
||||
Key: common.IndexTypeKey,
|
||||
Value: AutoIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
err := ValidateIndexParams(index)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestExpandIndexParams(t *testing.T) {
|
||||
t.Run("flat_and_json_params", func(t *testing.T) {
|
||||
params, err := ExpandIndexParams([]*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.MetricTypeKey, Value: "BM25"},
|
||||
{Key: common.ParamsKey, Value: `{"bm25_k1": "1.2", "bm25_b": "0.75"}`},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "SPARSE_INVERTED_INDEX", params[common.IndexTypeKey])
|
||||
assert.Equal(t, "BM25", params[common.MetricTypeKey])
|
||||
assert.Equal(t, "1.2", params["bm25_k1"])
|
||||
assert.Equal(t, "0.75", params["bm25_b"])
|
||||
})
|
||||
|
||||
t.Run("duplicated_key_rejected", func(t *testing.T) {
|
||||
_, err := ExpandIndexParams([]*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_INVERTED_INDEX"},
|
||||
{Key: common.IndexTypeKey, Value: "SPARSE_WAND"},
|
||||
})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid_json_rejected", func(t *testing.T) {
|
||||
_, err := ExpandIndexParams([]*commonpb.KeyValuePair{
|
||||
{Key: common.ParamsKey, Value: `{not-json`},
|
||||
})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateFieldIndexParams(t *testing.T) {
|
||||
paramtable.Init()
|
||||
|
||||
sparseField := &schemapb.FieldSchema{
|
||||
FieldID: 101,
|
||||
Name: "sparse",
|
||||
DataType: schemapb.DataType_SparseFloatVector,
|
||||
}
|
||||
binaryField := &schemapb.FieldSchema{
|
||||
FieldID: 102,
|
||||
Name: "binary_mh",
|
||||
DataType: schemapb.DataType_BinaryVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "512"},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("happy path sparse bm25", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
|
||||
common.MetricTypeKey: "BM25",
|
||||
"bm25_k1": "1.2",
|
||||
"bm25_b": "0.75",
|
||||
"bm25_avgdl": "100",
|
||||
}
|
||||
assert.NoError(t, ValidateFieldIndexParams(sparseField, params))
|
||||
})
|
||||
|
||||
t.Run("dimension filled from schema", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "MINHASH_LSH",
|
||||
common.MetricTypeKey: "MHJACCARD",
|
||||
}
|
||||
assert.NoError(t, ValidateFieldIndexParams(binaryField, params))
|
||||
assert.Equal(t, "512", params[common.DimKey])
|
||||
})
|
||||
|
||||
t.Run("dimension mismatch rejected", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "MINHASH_LSH",
|
||||
common.MetricTypeKey: "MHJACCARD",
|
||||
common.DimKey: "1024",
|
||||
}
|
||||
err := ValidateFieldIndexParams(binaryField, params)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "dimension mismatch")
|
||||
})
|
||||
|
||||
t.Run("incompatible index type for field data type rejected", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
|
||||
common.MetricTypeKey: "BM25",
|
||||
}
|
||||
err := ValidateFieldIndexParams(binaryField, params)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("unknown index type rejected", func(t *testing.T) {
|
||||
params := map[string]string{common.IndexTypeKey: "NOT_A_REAL_INDEX"}
|
||||
err := ValidateFieldIndexParams(sparseField, params)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "invalid index type")
|
||||
})
|
||||
|
||||
t.Run("bogus warmup policy rejected", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
|
||||
common.MetricTypeKey: "BM25",
|
||||
common.WarmupKey: "bogus",
|
||||
}
|
||||
err := ValidateFieldIndexParams(sparseField, params)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "warmup")
|
||||
})
|
||||
|
||||
t.Run("oversized params rejected", func(t *testing.T) {
|
||||
params := map[string]string{
|
||||
common.IndexTypeKey: "SPARSE_INVERTED_INDEX",
|
||||
"huge": strings.Repeat("x", paramtable.Get().ProxyCfg.MaxIndexParamsSize.GetAsInt()+1),
|
||||
}
|
||||
err := ValidateFieldIndexParams(sparseField, params)
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "exceeds limit")
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateIndexName(t *testing.T) {
|
||||
paramtable.Init()
|
||||
assert.NoError(t, ValidateIndexName(""))
|
||||
assert.NoError(t, ValidateIndexName("sparse_idx_1"))
|
||||
assert.NoError(t, ValidateIndexName("_idx"))
|
||||
assert.Error(t, ValidateIndexName("1bad"))
|
||||
assert.Error(t, ValidateIndexName("bad-name"))
|
||||
assert.Error(t, ValidateIndexName(strings.Repeat("x", paramtable.Get().ProxyCfg.MaxNameLength.GetAsInt()+1)))
|
||||
}
|
||||
@@ -17,9 +17,11 @@
|
||||
package schemautil
|
||||
|
||||
import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
type AlterSchemaAddKind int
|
||||
@@ -34,6 +36,12 @@ type AlterSchemaAddPlan struct {
|
||||
Kind AlterSchemaAddKind
|
||||
Field *schemapb.FieldSchema
|
||||
Function *schemapb.FunctionSchema
|
||||
// Index meta bound to the newly added field, parsed from
|
||||
// FieldInfo.index_name/extra_params. Mandatory for vector-type function
|
||||
// output fields so that bump-schema-version compaction results are never
|
||||
// blocked on a missing vector index.
|
||||
IndexName string
|
||||
IndexExtraParams []*commonpb.KeyValuePair
|
||||
}
|
||||
|
||||
func (p *AlterSchemaAddPlan) HasField() bool {
|
||||
@@ -59,12 +67,16 @@ func ParseAlterSchemaAddRequest(addRequest *milvuspb.AlterCollectionSchemaReques
|
||||
}
|
||||
|
||||
var field *schemapb.FieldSchema
|
||||
var indexName string
|
||||
var indexExtraParams []*commonpb.KeyValuePair
|
||||
if len(fieldInfos) == 1 {
|
||||
fieldInfo := fieldInfos[0]
|
||||
if fieldInfo == nil || fieldInfo.GetFieldSchema() == nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("fieldSchema is nil in fieldInfos")
|
||||
}
|
||||
field = fieldInfo.GetFieldSchema()
|
||||
indexName = fieldInfo.GetIndexName()
|
||||
indexExtraParams = fieldInfo.GetExtraParams()
|
||||
}
|
||||
|
||||
var function *schemapb.FunctionSchema
|
||||
@@ -77,7 +89,13 @@ func ParseAlterSchemaAddRequest(addRequest *milvuspb.AlterCollectionSchemaReques
|
||||
|
||||
switch {
|
||||
case field != nil && function != nil:
|
||||
return &AlterSchemaAddPlan{Kind: AlterSchemaAddFunctionField, Field: field, Function: function}, nil
|
||||
return &AlterSchemaAddPlan{
|
||||
Kind: AlterSchemaAddFunctionField,
|
||||
Field: field,
|
||||
Function: function,
|
||||
IndexName: indexName,
|
||||
IndexExtraParams: indexExtraParams,
|
||||
}, nil
|
||||
case field != nil:
|
||||
return &AlterSchemaAddPlan{Kind: AlterSchemaAddField, Field: field}, nil
|
||||
case function != nil:
|
||||
@@ -113,6 +131,17 @@ func ValidateAlterSchemaAddFunctionPlan(plan *AlterSchemaAddPlan) error {
|
||||
plan.Field.GetName(),
|
||||
)
|
||||
}
|
||||
// A vector output field MUST come with bound index params: without an index
|
||||
// meta, bump-schema-version compaction results can never pass the
|
||||
// indexed-visibility check and the whole backfill pipeline silently stalls.
|
||||
if typeutil.IsVectorType(plan.Field.GetDataType()) && len(plan.IndexExtraParams) == 0 {
|
||||
return merr.WrapErrParameterInvalidMsg(
|
||||
"index params are required when adding function output field %q of vector type %s: "+
|
||||
"provide index_type/metric_type (and params) in field_info.extra_params",
|
||||
plan.Field.GetName(),
|
||||
plan.Field.GetDataType().String(),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return merr.WrapErrParameterInvalidMsg("unknown alter schema add request kind")
|
||||
|
||||
@@ -309,6 +309,10 @@ message AlterCollectionMessageUpdates {
|
||||
common.ConsistencyLevel consistency_level = 6; // consistency level should be updated.
|
||||
repeated common.KeyValuePair properties = 7; // collection properties should be updated.
|
||||
AlterLoadConfigOfAlterCollection alter_load_config = 8; // alter load config of alter collection.
|
||||
// Index meta bound to newly added function-output/vector fields.
|
||||
// Fully materialized (index id/name allocated, params validated) at DDL prepare
|
||||
// stage BEFORE broadcast; applied atomically with the schema in the ack callback.
|
||||
repeated index.FieldIndex bound_field_indexes = 9;
|
||||
}
|
||||
|
||||
// AlterLoadConfigOfAlterCollection is the body of alter load config of alter collection message.
|
||||
|
||||
@@ -2048,6 +2048,10 @@ type AlterCollectionMessageUpdates struct {
|
||||
ConsistencyLevel commonpb.ConsistencyLevel `protobuf:"varint,6,opt,name=consistency_level,json=consistencyLevel,proto3,enum=milvus.proto.common.ConsistencyLevel" json:"consistency_level,omitempty"` // consistency level should be updated.
|
||||
Properties []*commonpb.KeyValuePair `protobuf:"bytes,7,rep,name=properties,proto3" json:"properties,omitempty"` // collection properties should be updated.
|
||||
AlterLoadConfig *AlterLoadConfigOfAlterCollection `protobuf:"bytes,8,opt,name=alter_load_config,json=alterLoadConfig,proto3" json:"alter_load_config,omitempty"` // alter load config of alter collection.
|
||||
// Index meta bound to newly added function-output/vector fields.
|
||||
// Fully materialized (index id/name allocated, params validated) at DDL prepare
|
||||
// stage BEFORE broadcast; applied atomically with the schema in the ack callback.
|
||||
BoundFieldIndexes []*indexpb.FieldIndex `protobuf:"bytes,9,rep,name=bound_field_indexes,json=boundFieldIndexes,proto3" json:"bound_field_indexes,omitempty"`
|
||||
}
|
||||
|
||||
func (x *AlterCollectionMessageUpdates) Reset() {
|
||||
@@ -2138,6 +2142,13 @@ func (x *AlterCollectionMessageUpdates) GetAlterLoadConfig() *AlterLoadConfigOfA
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AlterCollectionMessageUpdates) GetBoundFieldIndexes() []*indexpb.FieldIndex {
|
||||
if x != nil {
|
||||
return x.BoundFieldIndexes
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AlterLoadConfigOfAlterCollection is the body of alter load config of alter collection message.
|
||||
type AlterLoadConfigOfAlterCollection struct {
|
||||
state protoimpl.MessageState
|
||||
@@ -6870,8 +6881,8 @@ var file_messages_proto_rawDesc = []byte{
|
||||
0x34, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d,
|
||||
0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c,
|
||||
0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x55, 0x70,
|
||||
0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xd3,
|
||||
0x03, 0x0a, 0x1d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
|
||||
0x64, 0x61, 0x74, 0x65, 0x73, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xa3,
|
||||
0x04, 0x0a, 0x1d, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x73,
|
||||
0x12, 0x13, 0x0a, 0x05, 0x64, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
|
||||
0x04, 0x64, 0x62, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x64, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65,
|
||||
@@ -6900,7 +6911,12 @@ var file_messages_proto_rawDesc = []byte{
|
||||
0x65, 0x73, 0x2e, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66,
|
||||
0x69, 0x67, 0x4f, 0x66, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x61, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61, 0x64, 0x43, 0x6f,
|
||||
0x6e, 0x66, 0x69, 0x67, 0x22, 0x72, 0x0a, 0x20, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61,
|
||||
0x6e, 0x66, 0x69, 0x67, 0x12, 0x4e, 0x0a, 0x13, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x66, 0x69,
|
||||
0x65, 0x6c, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28,
|
||||
0x0b, 0x32, 0x1e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65,
|
||||
0x78, 0x52, 0x11, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x6e, 0x64,
|
||||
0x65, 0x78, 0x65, 0x73, 0x22, 0x72, 0x0a, 0x20, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x4c, 0x6f, 0x61,
|
||||
0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x66, 0x41, 0x6c, 0x74, 0x65, 0x72, 0x43, 0x6f,
|
||||
0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c,
|
||||
0x69, 0x63, 0x61, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05,
|
||||
@@ -7678,14 +7694,14 @@ var file_messages_proto_goTypes = []interface{}{
|
||||
(*fieldmaskpb.FieldMask)(nil), // 130: google.protobuf.FieldMask
|
||||
(commonpb.ConsistencyLevel)(0), // 131: milvus.proto.common.ConsistencyLevel
|
||||
(*commonpb.KeyValuePair)(nil), // 132: milvus.proto.common.KeyValuePair
|
||||
(commonpb.LoadPriority)(0), // 133: milvus.proto.common.LoadPriority
|
||||
(*milvuspb.UserEntity)(nil), // 134: milvus.proto.milvus.UserEntity
|
||||
(*internalpb.CredentialInfo)(nil), // 135: milvus.proto.internal.CredentialInfo
|
||||
(*milvuspb.RoleEntity)(nil), // 136: milvus.proto.milvus.RoleEntity
|
||||
(*milvuspb.RBACMeta)(nil), // 137: milvus.proto.milvus.RBACMeta
|
||||
(*milvuspb.GrantEntity)(nil), // 138: milvus.proto.milvus.GrantEntity
|
||||
(*milvuspb.PrivilegeGroupInfo)(nil), // 139: milvus.proto.milvus.PrivilegeGroupInfo
|
||||
(*indexpb.FieldIndex)(nil), // 140: milvus.proto.index.FieldIndex
|
||||
(*indexpb.FieldIndex)(nil), // 133: milvus.proto.index.FieldIndex
|
||||
(commonpb.LoadPriority)(0), // 134: milvus.proto.common.LoadPriority
|
||||
(*milvuspb.UserEntity)(nil), // 135: milvus.proto.milvus.UserEntity
|
||||
(*internalpb.CredentialInfo)(nil), // 136: milvus.proto.internal.CredentialInfo
|
||||
(*milvuspb.RoleEntity)(nil), // 137: milvus.proto.milvus.RoleEntity
|
||||
(*milvuspb.RBACMeta)(nil), // 138: milvus.proto.milvus.RBACMeta
|
||||
(*milvuspb.GrantEntity)(nil), // 139: milvus.proto.milvus.GrantEntity
|
||||
(*milvuspb.PrivilegeGroupInfo)(nil), // 140: milvus.proto.milvus.PrivilegeGroupInfo
|
||||
(commonpb.WALName)(0), // 141: milvus.proto.common.WALName
|
||||
(commonpb.MsgType)(0), // 142: milvus.proto.common.MsgType
|
||||
(*commonpb.MessageID)(nil), // 143: milvus.proto.common.MessageID
|
||||
@@ -7707,49 +7723,50 @@ var file_messages_proto_depIdxs = []int32{
|
||||
131, // 11: milvus.proto.messages.AlterCollectionMessageUpdates.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel
|
||||
132, // 12: milvus.proto.messages.AlterCollectionMessageUpdates.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
35, // 13: milvus.proto.messages.AlterCollectionMessageUpdates.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterCollection
|
||||
38, // 14: milvus.proto.messages.AlterLoadConfigMessageHeader.load_fields:type_name -> milvus.proto.messages.LoadFieldConfig
|
||||
39, // 15: milvus.proto.messages.AlterLoadConfigMessageHeader.replicas:type_name -> milvus.proto.messages.LoadReplicaConfig
|
||||
133, // 16: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
132, // 17: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
132, // 18: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
46, // 19: milvus.proto.messages.AlterDatabaseMessageBody.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterDatabase
|
||||
134, // 20: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
135, // 21: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
134, // 22: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
135, // 23: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
136, // 24: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
134, // 25: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
136, // 26: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
63, // 27: milvus.proto.messages.AlterUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
63, // 28: milvus.proto.messages.DropUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
137, // 29: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
|
||||
138, // 30: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
138, // 31: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
139, // 32: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
139, // 33: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
123, // 34: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
140, // 35: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
|
||||
140, // 36: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
|
||||
141, // 37: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
|
||||
124, // 38: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
105, // 39: milvus.proto.messages.CacheExpirations.cache_expirations:type_name -> milvus.proto.messages.CacheExpiration
|
||||
106, // 40: milvus.proto.messages.CacheExpiration.legacy_proxy_collection_meta_cache:type_name -> milvus.proto.messages.LegacyProxyCollectionMetaCache
|
||||
142, // 41: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
|
||||
125, // 42: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
114, // 43: milvus.proto.messages.BroadcastHeader.Resource_keys:type_name -> milvus.proto.messages.ResourceKey
|
||||
143, // 44: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
|
||||
143, // 45: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
|
||||
2, // 46: milvus.proto.messages.ResourceKey.domain:type_name -> milvus.proto.messages.ResourceDomain
|
||||
120, // 47: milvus.proto.messages.BatchUpdateManifestMessageBody.items:type_name -> milvus.proto.messages.BatchUpdateManifestItem
|
||||
121, // 48: milvus.proto.messages.BatchUpdateManifestItem.v2_column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
|
||||
126, // 49: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
|
||||
144, // 50: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
|
||||
145, // 51: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry.value:type_name -> milvus.proto.data.FieldBinlog
|
||||
52, // [52:52] is the sub-list for method output_type
|
||||
52, // [52:52] is the sub-list for method input_type
|
||||
52, // [52:52] is the sub-list for extension type_name
|
||||
52, // [52:52] is the sub-list for extension extendee
|
||||
0, // [0:52] is the sub-list for field type_name
|
||||
133, // 14: milvus.proto.messages.AlterCollectionMessageUpdates.bound_field_indexes:type_name -> milvus.proto.index.FieldIndex
|
||||
38, // 15: milvus.proto.messages.AlterLoadConfigMessageHeader.load_fields:type_name -> milvus.proto.messages.LoadFieldConfig
|
||||
39, // 16: milvus.proto.messages.AlterLoadConfigMessageHeader.replicas:type_name -> milvus.proto.messages.LoadReplicaConfig
|
||||
134, // 17: milvus.proto.messages.LoadReplicaConfig.priority:type_name -> milvus.proto.common.LoadPriority
|
||||
132, // 18: milvus.proto.messages.CreateDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
132, // 19: milvus.proto.messages.AlterDatabaseMessageBody.properties:type_name -> milvus.proto.common.KeyValuePair
|
||||
46, // 20: milvus.proto.messages.AlterDatabaseMessageBody.alter_load_config:type_name -> milvus.proto.messages.AlterLoadConfigOfAlterDatabase
|
||||
135, // 21: milvus.proto.messages.CreateUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
136, // 22: milvus.proto.messages.CreateUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
135, // 23: milvus.proto.messages.AlterUserMessageHeader.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
136, // 24: milvus.proto.messages.AlterUserMessageBody.credential_info:type_name -> milvus.proto.internal.CredentialInfo
|
||||
137, // 25: milvus.proto.messages.AlterRoleMessageHeader.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
135, // 26: milvus.proto.messages.RoleBinding.user_entity:type_name -> milvus.proto.milvus.UserEntity
|
||||
137, // 27: milvus.proto.messages.RoleBinding.role_entity:type_name -> milvus.proto.milvus.RoleEntity
|
||||
63, // 28: milvus.proto.messages.AlterUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
63, // 29: milvus.proto.messages.DropUserRoleMessageHeader.role_binding:type_name -> milvus.proto.messages.RoleBinding
|
||||
138, // 30: milvus.proto.messages.RestoreRBACMessageBody.rbac_meta:type_name -> milvus.proto.milvus.RBACMeta
|
||||
139, // 31: milvus.proto.messages.AlterPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
139, // 32: milvus.proto.messages.DropPrivilegeMessageHeader.entity:type_name -> milvus.proto.milvus.GrantEntity
|
||||
140, // 33: milvus.proto.messages.AlterPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
140, // 34: milvus.proto.messages.DropPrivilegeGroupMessageHeader.privilege_group_info:type_name -> milvus.proto.milvus.PrivilegeGroupInfo
|
||||
123, // 35: milvus.proto.messages.AlterResourceGroupMessageHeader.resource_group_configs:type_name -> milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry
|
||||
133, // 36: milvus.proto.messages.CreateIndexMessageBody.field_index:type_name -> milvus.proto.index.FieldIndex
|
||||
133, // 37: milvus.proto.messages.AlterIndexMessageBody.field_indexes:type_name -> milvus.proto.index.FieldIndex
|
||||
141, // 38: milvus.proto.messages.AlterWALMessageHeader.target_wal_name:type_name -> milvus.proto.common.WALName
|
||||
124, // 39: milvus.proto.messages.AlterWALMessageHeader.config:type_name -> milvus.proto.messages.AlterWALMessageHeader.ConfigEntry
|
||||
105, // 40: milvus.proto.messages.CacheExpirations.cache_expirations:type_name -> milvus.proto.messages.CacheExpiration
|
||||
106, // 41: milvus.proto.messages.CacheExpiration.legacy_proxy_collection_meta_cache:type_name -> milvus.proto.messages.LegacyProxyCollectionMetaCache
|
||||
142, // 42: milvus.proto.messages.LegacyProxyCollectionMetaCache.msg_type:type_name -> milvus.proto.common.MsgType
|
||||
125, // 43: milvus.proto.messages.RMQMessageLayout.properties:type_name -> milvus.proto.messages.RMQMessageLayout.PropertiesEntry
|
||||
114, // 44: milvus.proto.messages.BroadcastHeader.Resource_keys:type_name -> milvus.proto.messages.ResourceKey
|
||||
143, // 45: milvus.proto.messages.ReplicateHeader.message_id:type_name -> milvus.proto.common.MessageID
|
||||
143, // 46: milvus.proto.messages.ReplicateHeader.last_confirmed_message_id:type_name -> milvus.proto.common.MessageID
|
||||
2, // 47: milvus.proto.messages.ResourceKey.domain:type_name -> milvus.proto.messages.ResourceDomain
|
||||
120, // 48: milvus.proto.messages.BatchUpdateManifestMessageBody.items:type_name -> milvus.proto.messages.BatchUpdateManifestItem
|
||||
121, // 49: milvus.proto.messages.BatchUpdateManifestItem.v2_column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups
|
||||
126, // 50: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.column_groups:type_name -> milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry
|
||||
144, // 51: milvus.proto.messages.AlterResourceGroupMessageHeader.ResourceGroupConfigsEntry.value:type_name -> milvus.proto.rg.ResourceGroupConfig
|
||||
145, // 52: milvus.proto.messages.BatchUpdateManifestV2ColumnGroups.ColumnGroupsEntry.value:type_name -> milvus.proto.data.FieldBinlog
|
||||
53, // [53:53] is the sub-list for method output_type
|
||||
53, // [53:53] is the sub-list for method input_type
|
||||
53, // [53:53] is the sub-list for extension type_name
|
||||
53, // [53:53] is the sub-list for extension extendee
|
||||
0, // [0:53] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_messages_proto_init() }
|
||||
|
||||
@@ -6,7 +6,16 @@ from base.client_v2_base import TestMilvusClientV2Base
|
||||
from common import common_func as cf
|
||||
from common import common_type as ct
|
||||
from common.common_type import CaseLabel, CheckTasks
|
||||
from pymilvus import AnnSearchRequest, DataType, FieldSchema, Function, FunctionType, RRFRanker, WeightedRanker
|
||||
from pymilvus import (
|
||||
AnnSearchRequest,
|
||||
DataType,
|
||||
FieldSchema,
|
||||
Function,
|
||||
FunctionType,
|
||||
MilvusException,
|
||||
RRFRanker,
|
||||
WeightedRanker,
|
||||
)
|
||||
from utils.util_log import test_log as log
|
||||
|
||||
|
||||
@@ -101,7 +110,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
# Step 1: Execute add_function_field.
|
||||
# Step 1: Execute add_function_field with the bound sparse index params.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -109,17 +118,16 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
# Step 2: Verify the new output field and function are visible in collection schema.
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "sparse" in [field["name"] for field in desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
# Step 3: Explicitly create sparse index for the added BM25 output field.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params)
|
||||
# Step 3: The bound sparse index is created atomically by add_function_field; wait for build.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
|
||||
|
||||
# Step 4: Load collection after sparse index is ready.
|
||||
@@ -192,13 +200,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
self.drop_collection(client, collection_name)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
def test_add_bm25_function_field_no_auto_index(self):
|
||||
def test_add_bm25_function_field_creates_bound_index(self):
|
||||
"""
|
||||
TC-02: add_function_field does not auto-create index.
|
||||
TC-02: add_function_field atomically creates the bound index meta.
|
||||
|
||||
target: verify schema evolution and index management boundary
|
||||
method: add BM25 function field, search before index, then create index and search again
|
||||
expected: add_function_field does not create index implicitly; BM25 search works after explicit index/load
|
||||
target: verify the bound index meta is created together with the schema change
|
||||
method: add BM25 function field with bound index params, then describe the index right away
|
||||
expected: describe_index shows the bound SPARSE_INVERTED_INDEX immediately; BM25 search works after load
|
||||
"""
|
||||
client = self._client()
|
||||
collection_name = cf.gen_collection_name_by_testcase_name()
|
||||
@@ -228,7 +236,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
# Step 1: Execute add_function_field.
|
||||
# Step 1: Execute add_function_field with the bound sparse index params.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -236,41 +244,26 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "sparse" in [field["name"] for field in desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
# Expected: add_function_field should not create sparse index implicitly.
|
||||
assert client.describe_index(collection_name, index_name="sparse") is None
|
||||
# Step 2: The bound index meta is created atomically with the schema change,
|
||||
# so describe_index shows it right after add_function_field returns.
|
||||
bound_index_info = client.describe_index(collection_name, index_name="sparse")
|
||||
assert bound_index_info is not None
|
||||
assert bound_index_info["index_type"] == "SPARSE_INVERTED_INDEX"
|
||||
|
||||
# Step 2: Search on anns_field="sparse" before creating sparse index.
|
||||
# Expected: no crash; field-not-loaded, no-index, empty result, or equivalent behavior is acceptable.
|
||||
search_res, is_succ = self.search(
|
||||
client,
|
||||
collection_name,
|
||||
data=["alpha"],
|
||||
anns_field="sparse",
|
||||
limit=5,
|
||||
output_fields=["id"],
|
||||
check_task=CheckTasks.check_nothing,
|
||||
)
|
||||
if is_succ:
|
||||
assert len(search_res[0]) == 0
|
||||
else:
|
||||
err = str(search_res).lower()
|
||||
assert any(token in err for token in ["sparse", "index", "load", "loaded", "not exist", "not found"]), err
|
||||
|
||||
# Step 3: Create sparse index and load collection.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params)
|
||||
# Step 3: Wait for the bound index build and load collection.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
# Step 4: Search again and wait until BM25 search becomes ready.
|
||||
# Expected: BM25 search succeeds after explicit index creation and load.
|
||||
# Step 4: Search and wait until BM25 search becomes ready.
|
||||
# Expected: BM25 search succeeds through the bound index after load.
|
||||
search_timeout = 30
|
||||
poll_interval = 1
|
||||
poll_start = time.time()
|
||||
@@ -352,7 +345,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
]
|
||||
client.insert(collection_name=collection_name, data=growing_rows)
|
||||
|
||||
# Step 4: Execute add_function_field while pre-add growing rows exist.
|
||||
# Step 4: Execute add_function_field with bound sparse index params while pre-add growing rows exist.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -360,16 +353,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "sparse" in [field["name"] for field in desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
# Step 5: Create sparse index and load collection.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params)
|
||||
# Step 5: Wait for the bound sparse index build and load collection.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=30)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
@@ -454,7 +446,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert "bm25_fn" not in [func["name"] for func in before_desc.get("functions", [])]
|
||||
assert int(before_desc["properties"]["max_field_id"]) == before_max_field_id
|
||||
|
||||
# Step 2: Execute add_function_field.
|
||||
# Step 2: Execute add_function_field with the bound sparse index params.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -462,7 +454,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
# Step 3: Describe collection again and verify metadata has been updated.
|
||||
after_desc = client.describe_collection(collection_name)
|
||||
@@ -714,6 +708,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
),
|
||||
]
|
||||
|
||||
# Valid bound index params so the field/function validation error stays the trigger.
|
||||
invalid_case_index_params = client.prepare_index_params()
|
||||
invalid_case_index_params.add_index(field_name="", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
|
||||
for case_name, field_schema, function, error in invalid_cases:
|
||||
# Step 2: Each invalid request must fail clearly.
|
||||
self.add_function_field(
|
||||
@@ -721,6 +719,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
collection_name,
|
||||
field_schema,
|
||||
function,
|
||||
index_params=invalid_case_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items=error,
|
||||
)
|
||||
@@ -732,6 +731,61 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert [func["name"] for func in after_desc.get("functions", [])] == before_function_names, case_name
|
||||
assert after_desc["properties"]["max_field_id"] == before_max_field_id, case_name
|
||||
|
||||
# Step 3.5: Bound index params negative matrix.
|
||||
bound_params_field = FieldSchema(name="sparse_bound_neg", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bound_params_function = Function(
|
||||
name="bm25_bound_neg",
|
||||
function_type=FunctionType.BM25,
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse_bound_neg"],
|
||||
)
|
||||
|
||||
# Case a: empty IndexParams is rejected client-side with ParamError.
|
||||
self.add_function_field(
|
||||
client,
|
||||
collection_name,
|
||||
bound_params_field,
|
||||
bound_params_function,
|
||||
index_params=client.prepare_index_params(),
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={
|
||||
ct.err_code: 1,
|
||||
ct.err_msg: "index_params must contain exactly one index for the function output field",
|
||||
},
|
||||
)
|
||||
|
||||
# Case b: AUTOINDEX index_type is rejected; the bound index requires an explicit index_type.
|
||||
autoindex_params = client.prepare_index_params()
|
||||
autoindex_params.add_index(field_name="sparse_bound_neg", index_type="AUTOINDEX", metric_type="BM25")
|
||||
self.add_function_field(
|
||||
client,
|
||||
collection_name,
|
||||
bound_params_field,
|
||||
bound_params_function,
|
||||
index_params=autoindex_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={
|
||||
ct.err_code: 1100,
|
||||
ct.err_msg: "an explicit index_type is required for the bound index of function output field",
|
||||
},
|
||||
)
|
||||
|
||||
# Case c: the server mandates bound index params for vector function output fields;
|
||||
# a raw alter_collection_schema without index params must be rejected.
|
||||
with pytest.raises(MilvusException, match="index params are required"):
|
||||
client._get_connection().alter_collection_schema(
|
||||
collection_name=collection_name,
|
||||
field_schema=bound_params_field,
|
||||
func=bound_params_function,
|
||||
)
|
||||
|
||||
# Bound index params failures must not partially mutate schema either.
|
||||
after_desc = client.describe_collection(collection_name)
|
||||
assert after_desc["schema_version"] == before_schema_version
|
||||
assert [field["name"] for field in after_desc["fields"]] == before_field_names
|
||||
assert [func["name"] for func in after_desc.get("functions", [])] == before_function_names
|
||||
assert after_desc["properties"]["max_field_id"] == before_max_field_id
|
||||
|
||||
# Step 4: Non-existent collection should fail and not affect the original collection.
|
||||
sparse_field = FieldSchema(name="sparse_missing_collection", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
@@ -745,6 +799,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
f"{collection_name}_missing",
|
||||
sparse_field,
|
||||
bm25_function,
|
||||
index_params=invalid_case_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={ct.err_code: 100, ct.err_msg: "can't find collection"},
|
||||
)
|
||||
@@ -791,6 +846,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse_duplicate_function"],
|
||||
),
|
||||
index_params=invalid_case_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={ct.err_code: 1100, ct.err_msg: "duplicate function name: bm25_fn"},
|
||||
)
|
||||
@@ -812,7 +868,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "sparse" in [field["name"] for field in desc["fields"]]
|
||||
@@ -856,11 +914,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
|
||||
self.release_collection(client, collection_name)
|
||||
client.load_collection(collection_name)
|
||||
@@ -1006,7 +1063,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
# Step 3: Execute add_function_field.
|
||||
# Step 3: Execute add_function_field with the bound sparse index params.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -1014,12 +1071,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
# Step 4: Create sparse BM25 index and reload collection.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=30)
|
||||
# Step 4: Wait for the bound sparse index build and reload collection.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=30)
|
||||
self.release_collection(client, collection_name)
|
||||
client.load_collection(collection_name)
|
||||
@@ -1166,11 +1222,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=180)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
@@ -1307,7 +1362,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="vec", timeout=120)
|
||||
client.load_collection(collection_name)
|
||||
|
||||
# Step 1: add_function_field adds bm25_fn and sparse.
|
||||
# Step 1: add_function_field adds bm25_fn and sparse with the bound sparse index.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -1315,7 +1370,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
fields = {field["name"]: field for field in desc["fields"]}
|
||||
@@ -1324,9 +1381,6 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert "bm25_fn" in functions
|
||||
old_sparse_field_id = fields["sparse"]["field_id"]
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
|
||||
self.release_collection(client, collection_name)
|
||||
client.load_collection(collection_name)
|
||||
@@ -1412,7 +1466,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
fields = {field["name"]: field for field in desc["fields"]}
|
||||
@@ -1421,9 +1477,6 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert "bm25_fn" in functions
|
||||
assert fields["sparse"]["field_id"] > old_sparse_field_id
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
|
||||
self.release_collection(client, collection_name)
|
||||
client.load_collection(collection_name)
|
||||
@@ -1487,7 +1540,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert "sparse" not in [field["name"] for field in alias_desc["fields"]]
|
||||
assert "bm25_fn" not in [func["name"] for func in alias_desc.get("functions", [])]
|
||||
|
||||
# Step 2: Execute add_function_field through collection name.
|
||||
# Step 2: Execute add_function_field through collection name with the bound sparse index.
|
||||
sparse_field = FieldSchema(name="sparse", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
bm25_function = Function(
|
||||
name="bm25_fn",
|
||||
@@ -1495,16 +1548,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
# Step 3: Verify alias describe/create_index/load/search observes the new schema.
|
||||
# Step 3: Verify alias describe/load/search observes the new schema and bound index.
|
||||
alias_desc = client.describe_collection(alias_name)
|
||||
assert "sparse" in [field["name"] for field in alias_desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in alias_desc.get("functions", [])]
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(alias_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, alias_name, index_name="sparse", timeout=60)
|
||||
client.load_collection(alias_name)
|
||||
|
||||
@@ -1562,15 +1614,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(reverse_alias_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(reverse_alias_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
reverse_desc = client.describe_collection(reverse_collection_name)
|
||||
assert "sparse" in [field["name"] for field in reverse_desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in reverse_desc.get("functions", [])]
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(reverse_collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, reverse_collection_name, index_name="sparse", timeout=60)
|
||||
client.load_collection(reverse_collection_name)
|
||||
|
||||
@@ -1655,12 +1706,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
# Step 5: Create sparse BM25 index and load the added function output field.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
# Step 5: Wait for the bound sparse index build and load the added function output field.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
|
||||
self.release_collection(client, collection_name)
|
||||
client.load_collection(collection_name)
|
||||
@@ -1764,16 +1814,15 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "sparse" in [field["name"] for field in desc["fields"]]
|
||||
assert "bm25_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
# Step 4: Create sparse BM25 index for the added output field.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
# Step 4: Wait for the bound sparse index build of the added output field.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=60)
|
||||
|
||||
# Step 5: Do not call load_collection/release_collection here.
|
||||
@@ -1872,12 +1921,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse_added"],
|
||||
)
|
||||
client.add_function_field(bm25_collection_name, sparse_added, bm25_added_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse_added", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(
|
||||
bm25_collection_name, sparse_added, bm25_added_function, index_params=bound_index_params
|
||||
)
|
||||
|
||||
# Step 6 and Step 7: Create index for added output and wait for backfill/search convergence.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse_added", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(bm25_collection_name, index_params, timeout=60)
|
||||
# Step 6 and Step 7: Wait for the bound index build and backfill/search convergence.
|
||||
assert self.wait_for_index_ready(client, bm25_collection_name, index_name="sparse_added", timeout=60)
|
||||
|
||||
# Step 8 and Step 9: Search added output and compare topK id order and distances when present.
|
||||
@@ -2013,17 +2063,18 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
output_field_names=["mh_added"],
|
||||
params=minhash_params,
|
||||
)
|
||||
client.add_function_field(minhash_collection_name, mh_added, minhash_added_function)
|
||||
|
||||
# Step 6 and Step 7: Create index for added MinHash output and wait for backfill/search convergence.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(
|
||||
field_name="mh_added",
|
||||
index_type="MINHASH_LSH",
|
||||
metric_type="MHJACCARD",
|
||||
params={"mh_lsh_band": 8},
|
||||
)
|
||||
client.create_index(minhash_collection_name, index_params, timeout=60)
|
||||
client.add_function_field(
|
||||
minhash_collection_name, mh_added, minhash_added_function, index_params=bound_index_params
|
||||
)
|
||||
|
||||
# Step 6 and Step 7: Wait for the bound MinHash index build and backfill/search convergence.
|
||||
assert self.wait_for_index_ready(client, minhash_collection_name, index_name="mh_added", timeout=60)
|
||||
|
||||
# Step 8 and Step 9: Search added output and compare topK id order and distances when present.
|
||||
@@ -2143,7 +2194,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["title"],
|
||||
output_field_names=["sparse_title"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_title, title_function)
|
||||
title_bound_index_params = client.prepare_index_params()
|
||||
title_bound_index_params.add_index(
|
||||
field_name="sparse_title", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_title, title_function, index_params=title_bound_index_params)
|
||||
|
||||
# Step 3: Add body -> sparse_body.
|
||||
sparse_body = FieldSchema(name="sparse_body", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
@@ -2153,7 +2208,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["body"],
|
||||
output_field_names=["sparse_body"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_body, body_function)
|
||||
body_bound_index_params = client.prepare_index_params()
|
||||
body_bound_index_params.add_index(
|
||||
field_name="sparse_body", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_body, body_function, index_params=body_bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
field_names = [field["name"] for field in desc["fields"]]
|
||||
@@ -2163,11 +2222,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert "bm25_title_fn" in function_names
|
||||
assert "bm25_body_fn" in function_names
|
||||
|
||||
# Step 4: Create indexes for both added BM25 output fields.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse_title", index_type="AUTOINDEX", metric_type="BM25")
|
||||
index_params.add_index(field_name="sparse_body", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
# Step 4: Wait for the bound index builds of both added BM25 output fields.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_title", timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_body", timeout=60)
|
||||
|
||||
@@ -2309,7 +2364,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["standard_text"],
|
||||
output_field_names=["sparse_standard"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_standard, standard_function)
|
||||
standard_bound_index_params = client.prepare_index_params()
|
||||
standard_bound_index_params.add_index(
|
||||
field_name="sparse_standard", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
|
||||
)
|
||||
client.add_function_field(
|
||||
collection_name, sparse_standard, standard_function, index_params=standard_bound_index_params
|
||||
)
|
||||
|
||||
sparse_filtered = FieldSchema(name="sparse_filtered", dtype=DataType.SPARSE_FLOAT_VECTOR)
|
||||
filtered_function = Function(
|
||||
@@ -2318,7 +2379,13 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["filtered_text"],
|
||||
output_field_names=["sparse_filtered"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_filtered, filtered_function)
|
||||
filtered_bound_index_params = client.prepare_index_params()
|
||||
filtered_bound_index_params.add_index(
|
||||
field_name="sparse_filtered", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25"
|
||||
)
|
||||
client.add_function_field(
|
||||
collection_name, sparse_filtered, filtered_function, index_params=filtered_bound_index_params
|
||||
)
|
||||
|
||||
after_add_desc = client.describe_collection(collection_name)
|
||||
field_names = [field["name"] for field in after_add_desc["fields"]]
|
||||
@@ -2330,11 +2397,7 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
assert analyzer_params_from_desc(after_add_desc, "standard_text") == standard_analyzer_params
|
||||
assert analyzer_params_from_desc(after_add_desc, "filtered_text") == filtered_analyzer_params
|
||||
|
||||
# Step 3: Create indexes and verify analyzer-specific BM25 behavior.
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse_standard", index_type="AUTOINDEX", metric_type="BM25")
|
||||
index_params.add_index(field_name="sparse_filtered", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
# Step 3: Wait for the bound index builds and verify analyzer-specific BM25 behavior.
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_standard", timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse_filtered", timeout=60)
|
||||
|
||||
@@ -2485,11 +2548,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
|
||||
|
||||
self.release_collection(client, collection_name)
|
||||
@@ -2593,11 +2655,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
|
||||
|
||||
self.release_collection(client, collection_name)
|
||||
@@ -2744,15 +2805,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
after_add_desc = client.describe_collection(collection_name)
|
||||
text_field_after = next(field for field in after_add_desc["fields"] if field["name"] == "text")
|
||||
assert json.loads(text_field_after["params"]["multi_analyzer_params"]) == multi_analyzer_params
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=120)
|
||||
|
||||
self.release_collection(client, collection_name)
|
||||
@@ -2839,7 +2899,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(bm25_collection_name, bm25_field, bm25_function)
|
||||
bm25_bound_index_params = client.prepare_index_params()
|
||||
bm25_bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(bm25_collection_name, bm25_field, bm25_function, index_params=bm25_bound_index_params)
|
||||
|
||||
bm25_desc = client.describe_collection(bm25_collection_name)
|
||||
assert "sparse" in [field["name"] for field in bm25_desc["fields"]]
|
||||
@@ -2872,7 +2934,11 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
output_field_names=["mh"],
|
||||
params={"num_hashes": minhash_num_hashes, "shingle_size": 3},
|
||||
)
|
||||
client.add_function_field(minhash_collection_name, minhash_field, minhash_function)
|
||||
minhash_bound_index_params = client.prepare_index_params()
|
||||
minhash_bound_index_params.add_index(field_name="mh", index_type="MINHASH_LSH", metric_type="MHJACCARD")
|
||||
client.add_function_field(
|
||||
minhash_collection_name, minhash_field, minhash_function, index_params=minhash_bound_index_params
|
||||
)
|
||||
|
||||
minhash_desc = client.describe_collection(minhash_collection_name)
|
||||
assert "mh" in [field["name"] for field in minhash_desc["fields"]]
|
||||
@@ -2938,12 +3004,17 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
),
|
||||
]
|
||||
|
||||
# Valid bound index params so the unsupported-function-type error stays the trigger.
|
||||
unsupported_case_index_params = client.prepare_index_params()
|
||||
unsupported_case_index_params.add_index(field_name="", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
|
||||
for case_name, field_schema, function, function_type_value in unsupported_cases:
|
||||
self.add_function_field(
|
||||
client,
|
||||
unsupported_collection_name,
|
||||
field_schema,
|
||||
function,
|
||||
index_params=unsupported_case_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={
|
||||
ct.err_code: 1,
|
||||
@@ -3046,20 +3117,19 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
output_field_names=["mh"],
|
||||
params={"num_hashes": num_hashes, "shingle_size": 5},
|
||||
)
|
||||
client.add_function_field(collection_name, mh_field, mh_function)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "mh" in [field["name"] for field in desc["fields"]]
|
||||
assert "minhash_params_backfill_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(
|
||||
field_name="mh",
|
||||
index_type="MINHASH_LSH",
|
||||
metric_type="MHJACCARD",
|
||||
params={"mh_lsh_band": 8},
|
||||
)
|
||||
client.create_index(collection_name, index_params, timeout=60)
|
||||
client.add_function_field(collection_name, mh_field, mh_function, index_params=bound_index_params)
|
||||
|
||||
desc = client.describe_collection(collection_name)
|
||||
assert "mh" in [field["name"] for field in desc["fields"]]
|
||||
assert "minhash_params_backfill_fn" in [func["name"] for func in desc.get("functions", [])]
|
||||
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="mh", timeout=120)
|
||||
|
||||
self.release_collection(client, collection_name)
|
||||
@@ -3133,11 +3203,14 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
output_field_names=["mh"],
|
||||
params={"num_hashes": 16, "shingle_size": 3},
|
||||
)
|
||||
mismatch_bound_index_params = client.prepare_index_params()
|
||||
mismatch_bound_index_params.add_index(field_name="mh", index_type="MINHASH_LSH", metric_type="MHJACCARD")
|
||||
self.add_function_field(
|
||||
client,
|
||||
mismatch_collection_name,
|
||||
mismatch_field,
|
||||
mismatch_function,
|
||||
index_params=mismatch_bound_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={
|
||||
ct.err_code: 1100,
|
||||
@@ -3160,11 +3233,16 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
output_field_names=["mh_bad_shingle"],
|
||||
params={"num_hashes": 16, "shingle_size": 0},
|
||||
)
|
||||
bad_shingle_bound_index_params = client.prepare_index_params()
|
||||
bad_shingle_bound_index_params.add_index(
|
||||
field_name="mh_bad_shingle", index_type="MINHASH_LSH", metric_type="MHJACCARD"
|
||||
)
|
||||
self.add_function_field(
|
||||
client,
|
||||
mismatch_collection_name,
|
||||
bad_shingle_field,
|
||||
bad_shingle_function,
|
||||
index_params=bad_shingle_bound_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={
|
||||
ct.err_code: 1100,
|
||||
@@ -3233,11 +3311,9 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
|
||||
sparse_index = client.prepare_index_params()
|
||||
sparse_index.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, sparse_index, timeout=180)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=240)
|
||||
sparse_index_info = client.describe_index(collection_name, index_name="sparse")
|
||||
@@ -3346,11 +3422,10 @@ class TestMilvusClientAddFunctionFieldFeature(TestMilvusClientV2Base):
|
||||
input_field_names=["text"],
|
||||
output_field_names=["sparse"],
|
||||
)
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function)
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(field_name="sparse", index_type="SPARSE_INVERTED_INDEX", metric_type="BM25")
|
||||
client.add_function_field(collection_name, sparse_field, bm25_function, index_params=bound_index_params)
|
||||
|
||||
sparse_index = client.prepare_index_params()
|
||||
sparse_index.add_index(field_name="sparse", index_type="AUTOINDEX", metric_type="BM25")
|
||||
client.create_index(collection_name, sparse_index, timeout=180)
|
||||
assert self.wait_for_index_ready(client, collection_name, index_name="sparse", timeout=240)
|
||||
|
||||
post_add_rows = [
|
||||
|
||||
@@ -2936,11 +2936,18 @@ class TestMilvusClientExternalTableAddField(ExternalTableTestBase):
|
||||
ct.err_code: 1100,
|
||||
ct.err_msg: "alter collection schema operation is not supported for external collection",
|
||||
}
|
||||
bound_index_params = client.prepare_index_params()
|
||||
bound_index_params.add_index(
|
||||
field_name="bm25_sparse",
|
||||
index_type="SPARSE_INVERTED_INDEX",
|
||||
metric_type="BM25",
|
||||
)
|
||||
self.add_function_field(
|
||||
client,
|
||||
collection_name=coll,
|
||||
field_schema=bm25_field,
|
||||
func=bm25_function,
|
||||
index_params=bound_index_params,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items=error,
|
||||
)
|
||||
|
||||
@@ -1,17 +1,15 @@
|
||||
import pytest
|
||||
|
||||
from base.client_v2_base import TestMilvusClientV2Base
|
||||
from common import common_func as cf
|
||||
from common import common_type as ct
|
||||
from common.common_type import CaseLabel, CheckTasks
|
||||
from utils.util_pymilvus import *
|
||||
|
||||
from utils.util_pymilvus import DataType, gen_vectors
|
||||
|
||||
default_dim = ct.default_dim
|
||||
|
||||
|
||||
class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
""" Test case of partition key params """
|
||||
"""Test case of partition key params"""
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.parametrize("par_key_field", [ct.default_int64_field_name, ct.default_string_field_name])
|
||||
@@ -25,10 +23,19 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
client = self._client()
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64,
|
||||
is_partition_key=(par_key_field == ct.default_int64_field_name))
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
|
||||
is_partition_key=(par_key_field == ct.default_string_field_name))
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_int64_field_name,
|
||||
DataType.INT64,
|
||||
is_partition_key=(par_key_field == ct.default_int64_field_name),
|
||||
)
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_string_field_name,
|
||||
DataType.VARCHAR,
|
||||
max_length=ct.default_length,
|
||||
is_partition_key=(par_key_field == ct.default_string_field_name),
|
||||
)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema)
|
||||
@@ -41,19 +48,26 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
entities_per_parkey = 10
|
||||
for _ in range(entities_per_parkey):
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
self.insert(client, c_name, data)
|
||||
|
||||
# flush
|
||||
self.flush(client, c_name)
|
||||
# build index
|
||||
index_params = self.prepare_index_params(client)[0]
|
||||
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="IVF_SQ8",
|
||||
metric_type="COSINE", params={"nlist": 128})
|
||||
index_params.add_index(
|
||||
field_name=ct.default_float_vec_field_name,
|
||||
index_type="IVF_SQ8",
|
||||
metric_type="COSINE",
|
||||
params={"nlist": 128},
|
||||
)
|
||||
self.create_index(client, c_name, index_params)
|
||||
# load
|
||||
self.load_collection(client, c_name)
|
||||
@@ -61,46 +75,81 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
nq = 10
|
||||
search_vectors = gen_vectors(nq, ct.default_dim)
|
||||
# search with mixed filtered
|
||||
res1 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey})[0]
|
||||
res1 = self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey},
|
||||
)[0]
|
||||
# search with partition key filter only or with non partition key
|
||||
res2 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey})[0]
|
||||
res2 = self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f"{ct.default_int64_field_name} in [1,3,5]",
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey},
|
||||
)[0]
|
||||
# search with partition key filter only or with non partition key
|
||||
res3 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey})[0]
|
||||
res3 = self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f'{ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey},
|
||||
)[0]
|
||||
# assert the results persist
|
||||
for i in range(nq):
|
||||
assert res1[i].ids == res2[i].ids == res3[i].ids
|
||||
|
||||
# search with 'or' to verify no partition key optimization local with or binary expr
|
||||
query_res1 = self.query(client, c_name,
|
||||
filter=f'{ct.default_string_field_name} == "{string_prefix}5" || {ct.default_int64_field_name} in [2,4,6]',
|
||||
output_fields=['count(*)'])[0]
|
||||
query_res2 = self.query(client, c_name,
|
||||
filter=f'{ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"] || {ct.default_int64_field_name}==5',
|
||||
output_fields=['count(*)'])[0]
|
||||
query_res3 = self.query(client, c_name,
|
||||
filter=f'{ct.default_int64_field_name}==5 or {ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"]',
|
||||
output_fields=['count(*)'])[0]
|
||||
query_res4 = self.query(client, c_name,
|
||||
filter=f'{ct.default_int64_field_name} in [2,4,6] || {ct.default_string_field_name} == "{string_prefix}5"',
|
||||
output_fields=['count(*)'])[0]
|
||||
query_res1 = self.query(
|
||||
client,
|
||||
c_name,
|
||||
filter=f'{ct.default_string_field_name} == "{string_prefix}5" || {ct.default_int64_field_name} in [2,4,6]',
|
||||
output_fields=["count(*)"],
|
||||
)[0]
|
||||
query_res2 = self.query(
|
||||
client,
|
||||
c_name,
|
||||
filter=f'{ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"] || {ct.default_int64_field_name}==5',
|
||||
output_fields=["count(*)"],
|
||||
)[0]
|
||||
query_res3 = self.query(
|
||||
client,
|
||||
c_name,
|
||||
filter=f'{ct.default_int64_field_name}==5 or {ct.default_string_field_name} in ["{string_prefix}2","{string_prefix}4", "{string_prefix}6"]',
|
||||
output_fields=["count(*)"],
|
||||
)[0]
|
||||
query_res4 = self.query(
|
||||
client,
|
||||
c_name,
|
||||
filter=f'{ct.default_int64_field_name} in [2,4,6] || {ct.default_string_field_name} == "{string_prefix}5"',
|
||||
output_fields=["count(*)"],
|
||||
)[0]
|
||||
# assert the results persist
|
||||
assert query_res1[0].get('count(*)') == query_res2[0].get('count(*)') \
|
||||
== query_res3[0].get('count(*)') == query_res4[0].get('count(*)') == 40
|
||||
assert (
|
||||
query_res1[0].get("count(*)")
|
||||
== query_res2[0].get("count(*)")
|
||||
== query_res3[0].get("count(*)")
|
||||
== query_res4[0].get("count(*)")
|
||||
== 40
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.parametrize("par_key_field", [ct.default_int64_field_name, ct.default_string_field_name])
|
||||
@@ -130,20 +179,24 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
entities_per_parkey = 20
|
||||
for n in range(entities_per_parkey):
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
"pk": str(n * nb + i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
"pk": str(n * nb + i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
self.insert(client, c_name, data)
|
||||
|
||||
# flush
|
||||
self.flush(client, c_name)
|
||||
# build index
|
||||
index_params = self.prepare_index_params(client)[0]
|
||||
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="FLAT",
|
||||
metric_type="COSINE", params={})
|
||||
index_params.add_index(
|
||||
field_name=ct.default_float_vec_field_name, index_type="FLAT", metric_type="COSINE", params={}
|
||||
)
|
||||
if index_on_par_key_field:
|
||||
index_params.add_index(field_name=par_key_field)
|
||||
self.create_index(client, c_name, index_params)
|
||||
@@ -153,12 +206,18 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
nq = 10
|
||||
search_vectors = gen_vectors(nq, ct.default_dim)
|
||||
# search with mixed filtered
|
||||
self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey})
|
||||
self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": entities_per_parkey},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
def test_partition_key_off_in_field_but_enable_in_schema(self):
|
||||
@@ -168,8 +227,7 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
2. verify the collection created successfully and partition key is enabled
|
||||
"""
|
||||
client = self._client()
|
||||
schema = self.create_schema(client, auto_id=True,
|
||||
partition_key_field=ct.default_int64_field_name)[0]
|
||||
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_int64_field_name)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=False)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
@@ -187,7 +245,7 @@ class TestPartitionKeyParams(TestMilvusClientV2Base):
|
||||
|
||||
|
||||
class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
""" Test case of partition key invalid params """
|
||||
"""Test case of partition key invalid params"""
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
def test_max_partitions(self):
|
||||
@@ -204,8 +262,9 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
|
||||
is_partition_key=True)
|
||||
self.add_field(
|
||||
schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True
|
||||
)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=max_partition)
|
||||
@@ -217,11 +276,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
string_prefix = cf.gen_str_by_length(length=6)
|
||||
for _ in range(5):
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
self.insert(client, c_name, data)
|
||||
|
||||
# drop collection
|
||||
@@ -231,9 +293,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
num_partitions = max_partition + 1
|
||||
err_msg = f"partition number ({num_partitions}) exceeds max configuration ({max_partition})"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=num_partitions,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1100, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
num_partitions=num_partitions,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1100, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
def test_min_partitions(self):
|
||||
@@ -262,12 +329,15 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
string_prefix = cf.gen_str_by_length(length=6)
|
||||
for n in range(5):
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
"pk": str(n * nb + i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
"pk": str(n * nb + i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
self.insert(client, c_name, data)
|
||||
self.flush(client, c_name)
|
||||
|
||||
@@ -277,12 +347,22 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
# create a collection with min partitions - 1
|
||||
err_msg = "The specified num_partitions should be greater than or equal to 1"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=min_partition - 1,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=min_partition - 3,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
num_partitions=min_partition - 1,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
num_partitions=min_partition - 3,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.parametrize("is_par_key", [None, "", "invalid", 0.1, [], {}, ()])
|
||||
@@ -295,10 +375,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
client = self._client()
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
err_msg = "Param is_partition_key must be bool type"
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64,
|
||||
is_partition_key=is_par_key,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_int64_field_name,
|
||||
DataType.INT64,
|
||||
is_partition_key=is_par_key,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
@pytest.mark.parametrize("num_partitions", [True, False, "", "invalid", 0.1, [], {}, ()])
|
||||
@@ -316,9 +400,14 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "invalid num_partitions type"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=num_partitions,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
num_partitions=num_partitions,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
def test_partition_key_on_multi_fields(self):
|
||||
@@ -330,37 +419,45 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
client = self._client()
|
||||
|
||||
# sub-case 1: both defined in field schema via add_field
|
||||
# pymilvus(>=3.1.0rc61) validates partition key uniqueness client-side at add_field
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
|
||||
is_partition_key=True)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "Expected only one partition key field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_string_field_name,
|
||||
DataType.VARCHAR,
|
||||
max_length=ct.default_length,
|
||||
is_partition_key=True,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# sub-case 2: partition_key_field passed as list in create_schema
|
||||
err_msg = "Param partition_key_field must be str type"
|
||||
self.create_schema(client, auto_id=True,
|
||||
partition_key_field=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_schema(
|
||||
client,
|
||||
auto_id=True,
|
||||
partition_key_field=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# sub-case 3: one defined in field schema, one defined in create_schema
|
||||
schema = self.create_schema(client, auto_id=True,
|
||||
partition_key_field=ct.default_string_field_name)[0]
|
||||
# pymilvus raises client-side when adding the field named by partition_key_field
|
||||
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_string_field_name)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "Expected only one partition key field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_string_field_name,
|
||||
DataType.VARCHAR,
|
||||
max_length=ct.default_length,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L0)
|
||||
@pytest.mark.parametrize("is_int64_primary", [True, False])
|
||||
@@ -377,16 +474,21 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
if is_int64_primary:
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True, is_partition_key=True)
|
||||
else:
|
||||
self.add_field(schema, "pk", DataType.VARCHAR, max_length=ct.default_length,
|
||||
is_primary=True, is_partition_key=True)
|
||||
self.add_field(
|
||||
schema, "pk", DataType.VARCHAR, max_length=ct.default_length, is_primary=True, is_partition_key=True
|
||||
)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "the partition key field must not be primary field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# sub-case 2: partition key set on primary field via create_schema partition_key_field
|
||||
schema = self.create_schema(client, auto_id=False, partition_key_field="pk")[0]
|
||||
@@ -399,9 +501,13 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "the partition key field must not be primary field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
def test_partition_key_on_and_off(self):
|
||||
@@ -413,35 +519,51 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
client = self._client()
|
||||
|
||||
# sub-case 1: int64 field is_partition_key=True, schema partition_key_field=vector field
|
||||
schema = self.create_schema(client, auto_id=True,
|
||||
partition_key_field=ct.default_float_vec_field_name)[0]
|
||||
# pymilvus raises client-side when adding the field named by partition_key_field
|
||||
schema = self.create_schema(client, auto_id=True, partition_key_field=ct.default_float_vec_field_name)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64, is_partition_key=True)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "Expected only one partition key field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_float_vec_field_name,
|
||||
DataType.FLOAT_VECTOR,
|
||||
dim=default_dim,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# sub-case 2: string1 is_partition_key=True, schema partition_key_field=string2
|
||||
schema = self.create_schema(client, auto_id=True,
|
||||
partition_key_field="string2")[0]
|
||||
# pymilvus raises client-side when adding the field named by partition_key_field
|
||||
schema = self.create_schema(client, auto_id=True, partition_key_field="string2")[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, "string1", DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True)
|
||||
self.add_field(schema, "string2", DataType.VARCHAR, max_length=ct.default_length)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "Expected only one partition key field"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.add_field(
|
||||
schema,
|
||||
"string2",
|
||||
DataType.VARCHAR,
|
||||
max_length=ct.default_length,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 1, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
@pytest.mark.parametrize("field_type", [DataType.FLOAT_VECTOR, DataType.BINARY_VECTOR, DataType.FLOAT,
|
||||
DataType.DOUBLE, DataType.BOOL, DataType.INT8,
|
||||
DataType.INT16, DataType.INT32, DataType.JSON])
|
||||
@pytest.mark.parametrize(
|
||||
"field_type",
|
||||
[
|
||||
DataType.FLOAT_VECTOR,
|
||||
DataType.BINARY_VECTOR,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
DataType.BOOL,
|
||||
DataType.INT8,
|
||||
DataType.INT16,
|
||||
DataType.INT32,
|
||||
DataType.JSON,
|
||||
],
|
||||
)
|
||||
def test_partition_key_on_invalid_type_fields(self, field_type):
|
||||
"""
|
||||
Method:
|
||||
@@ -461,16 +583,26 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
if field_type == DataType.BINARY_VECTOR:
|
||||
self.add_field(schema, ct.default_binary_vec_field_name, DataType.BINARY_VECTOR,
|
||||
dim=default_dim, is_partition_key=True)
|
||||
self.add_field(
|
||||
schema, ct.default_binary_vec_field_name, DataType.BINARY_VECTOR, dim=default_dim, is_partition_key=True
|
||||
)
|
||||
else:
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim,
|
||||
is_partition_key=(field_type == DataType.FLOAT_VECTOR))
|
||||
self.add_field(
|
||||
schema,
|
||||
ct.default_float_vec_field_name,
|
||||
DataType.FLOAT_VECTOR,
|
||||
dim=default_dim,
|
||||
is_partition_key=(field_type == DataType.FLOAT_VECTOR),
|
||||
)
|
||||
err_msg = "Partition key field type must be DataType.INT64 or DataType.VARCHAR"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
def test_partition_key_on_not_existed_fields(self):
|
||||
@@ -480,17 +612,20 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
2. verify the error raised
|
||||
"""
|
||||
client = self._client()
|
||||
schema = self.create_schema(client, auto_id=True,
|
||||
partition_key_field="non_existing_field")[0]
|
||||
schema = self.create_schema(client, auto_id=True, partition_key_field="non_existing_field")[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "the specified partition key field {non_existing_field} not exist"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
def test_partition_key_on_empty_and_num_partitions_set(self):
|
||||
@@ -509,9 +644,13 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "the specified partition key field {} not exist"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# sub-case 2: no partition key but num_partitions set → server error
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
@@ -521,13 +660,18 @@ class TestPartitionKeyInvalidParams(TestMilvusClientV2Base):
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
err_msg = "num_partitions should only be specified with partition key field enabled"
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema, num_partitions=200,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_collection(
|
||||
client,
|
||||
c_name,
|
||||
schema=schema,
|
||||
num_partitions=200,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
|
||||
class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
|
||||
""" Test case of partition key insert invalid data """
|
||||
"""Test case of partition key insert invalid data"""
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L2)
|
||||
@pytest.mark.parametrize("invalid_data", [99, True, None, [], {}, ()])
|
||||
@@ -539,8 +683,7 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
|
||||
3. verify the error raised
|
||||
"""
|
||||
client = self._client()
|
||||
schema = self.create_schema(client, auto_id=False,
|
||||
partition_key_field=ct.default_string_field_name)[0]
|
||||
schema = self.create_schema(client, auto_id=False, partition_key_field=ct.default_string_field_name)[0]
|
||||
self.add_field(schema, "pk", DataType.VARCHAR, max_length=ct.default_length, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length)
|
||||
@@ -552,12 +695,15 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
|
||||
nb = 10
|
||||
string_prefix = cf.gen_str_by_length(length=6)
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
"pk": str(i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
"pk": str(i),
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
data[1][ct.default_string_field_name] = invalid_data # inject invalid data
|
||||
|
||||
if invalid_data is None:
|
||||
@@ -568,13 +714,13 @@ class TestPartitionKeyInsertInvalid(TestMilvusClientV2Base):
|
||||
# non-string types trigger DataNotMatchException in row-based insert
|
||||
err_msg = "The Input data type is inconsistent with defined schema"
|
||||
err_code = 1
|
||||
self.insert(client, c_name, data,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": err_code, "err_msg": err_msg})
|
||||
self.insert(
|
||||
client, c_name, data, check_task=CheckTasks.err_res, check_items={"err_code": err_code, "err_msg": err_msg}
|
||||
)
|
||||
|
||||
|
||||
class TestPartitionApiForbidden(TestMilvusClientV2Base):
|
||||
""" Test case of partition api forbidden when partition key is on """
|
||||
"""Test case of partition api forbidden when partition key is on"""
|
||||
|
||||
@pytest.mark.tags(CaseLabel.L1)
|
||||
def test_create_partition(self):
|
||||
@@ -593,8 +739,9 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
|
||||
schema = self.create_schema(client, auto_id=True)[0]
|
||||
self.add_field(schema, "pk", DataType.INT64, is_primary=True)
|
||||
self.add_field(schema, ct.default_int64_field_name, DataType.INT64)
|
||||
self.add_field(schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length,
|
||||
is_partition_key=True)
|
||||
self.add_field(
|
||||
schema, ct.default_string_field_name, DataType.VARCHAR, max_length=ct.default_length, is_partition_key=True
|
||||
)
|
||||
self.add_field(schema, ct.default_float_vec_field_name, DataType.FLOAT_VECTOR, dim=default_dim)
|
||||
c_name = cf.gen_collection_name_by_testcase_name()
|
||||
self.create_collection(client, c_name, schema=schema)
|
||||
@@ -602,9 +749,13 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
|
||||
# create partition → error
|
||||
err_msg = "disable create partition if partition key mode is used"
|
||||
partition_name = cf.gen_unique_str("partition")
|
||||
self.create_partition(client, c_name, partition_name,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.create_partition(
|
||||
client,
|
||||
c_name,
|
||||
partition_name,
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# list/has partition → allowed
|
||||
partitions = self.list_partitions(client, c_name)[0]
|
||||
@@ -617,53 +768,81 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
|
||||
entities_per_parkey = 10
|
||||
for _ in range(entities_per_parkey):
|
||||
float_vec_values = gen_vectors(nb, ct.default_dim)
|
||||
data = [{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i]
|
||||
} for i in range(nb)]
|
||||
data = [
|
||||
{
|
||||
ct.default_int64_field_name: i,
|
||||
ct.default_string_field_name: string_prefix + str(i),
|
||||
ct.default_float_vec_field_name: float_vec_values[i],
|
||||
}
|
||||
for i in range(nb)
|
||||
]
|
||||
self.insert(client, c_name, data)
|
||||
|
||||
# insert with partition_name → error
|
||||
err_msg = "not support manually specifying the partition names if partition key mode is used"
|
||||
self.insert(client, c_name, data, partition_name=partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.insert(
|
||||
client,
|
||||
c_name,
|
||||
data,
|
||||
partition_name=partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# load partitions → error
|
||||
err_msg = "disable load partitions if partition key mode is used"
|
||||
self.load_partitions(client, c_name, [partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.load_partitions(
|
||||
client,
|
||||
c_name,
|
||||
[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# flush + index + load collection → allowed
|
||||
self.flush(client, c_name)
|
||||
index_params = self.prepare_index_params(client)[0]
|
||||
index_params.add_index(field_name=ct.default_float_vec_field_name, index_type="IVF_SQ8",
|
||||
metric_type="COSINE", params={"nlist": 128})
|
||||
index_params.add_index(
|
||||
field_name=ct.default_float_vec_field_name,
|
||||
index_type="IVF_SQ8",
|
||||
metric_type="COSINE",
|
||||
params={"nlist": 128},
|
||||
)
|
||||
self.create_index(client, c_name, index_params)
|
||||
self.load_collection(client, c_name)
|
||||
|
||||
# search without partition_names → allowed
|
||||
nq = 10
|
||||
search_vectors = gen_vectors(nq, ct.default_dim)
|
||||
res1 = self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": ct.default_limit})[0]
|
||||
res1 = self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5] && {ct.default_string_field_name} in ["{string_prefix}1","{string_prefix}3","{string_prefix}5"]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
check_task=CheckTasks.check_search_results,
|
||||
check_items={"nq": nq, "limit": ct.default_limit},
|
||||
)[0]
|
||||
pks = res1[0].ids[:3]
|
||||
|
||||
# search with partition_names → error
|
||||
err_msg = "not support manually specifying the partition names if partition key mode is used"
|
||||
self.search(client, c_name, data=search_vectors, anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params, limit=entities_per_parkey,
|
||||
filter=f'{ct.default_int64_field_name} in [1,3,5]',
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
partition_names=[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.search(
|
||||
client,
|
||||
c_name,
|
||||
data=search_vectors,
|
||||
anns_field=ct.default_float_vec_field_name,
|
||||
search_params=ct.default_search_params,
|
||||
limit=entities_per_parkey,
|
||||
filter=f"{ct.default_int64_field_name} in [1,3,5]",
|
||||
output_fields=[ct.default_int64_field_name, ct.default_string_field_name],
|
||||
partition_names=[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# get_load_state with partition → allowed (v1: loading_progress/wait_for_loading_complete)
|
||||
load_state = self.get_load_state(client, c_name, partition_name=partitions[0])[0]
|
||||
@@ -678,23 +857,41 @@ class TestPartitionApiForbidden(TestMilvusClientV2Base):
|
||||
|
||||
# delete with partition_name → error
|
||||
err_msg = "not support manually specifying the partition names if partition key mode is used"
|
||||
self.delete(client, c_name, filter=f'pk in {pks}', partition_name=partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.delete(
|
||||
client,
|
||||
c_name,
|
||||
filter=f"pk in {pks}",
|
||||
partition_name=partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# query with partition_names → error
|
||||
self.query(client, c_name, filter=f'pk in {pks}', partition_names=[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.query(
|
||||
client,
|
||||
c_name,
|
||||
filter=f"pk in {pks}",
|
||||
partition_names=[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# release partitions → error
|
||||
err_msg = "disable release partitions if partition key mode is used"
|
||||
self.release_partitions(client, c_name, [partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.release_partitions(
|
||||
client,
|
||||
c_name,
|
||||
[partitions[0]],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
# drop partition → error
|
||||
err_msg = "disable drop partition if partition key mode is used"
|
||||
self.drop_partition(client, c_name, partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg})
|
||||
self.drop_partition(
|
||||
client,
|
||||
c_name,
|
||||
partitions[0],
|
||||
check_task=CheckTasks.err_res,
|
||||
check_items={"err_code": 2, "err_msg": err_msg},
|
||||
)
|
||||
|
||||
@@ -23,8 +23,8 @@ pytest-sugar==0.9.5
|
||||
pytest-random-order
|
||||
|
||||
# pymilvus
|
||||
pymilvus==3.1.0rc54
|
||||
pymilvus[bulk_writer]==3.1.0rc54
|
||||
pymilvus==3.1.0rc61
|
||||
pymilvus[bulk_writer]==3.1.0rc61
|
||||
# for protobuf
|
||||
protobuf>=5.29.5
|
||||
|
||||
|
||||
Reference in New Issue
Block a user