enhance: enforce function-field binding principle for function DDL (#51360)

## What

Enforce a function–field binding principle for function DDL (add / drop
/ alter function on an existing collection), so a function and its
output field stay coupled and already-stored vectors can never be
silently invalidated by DDL.

BM25 and MinHash follow the strict binding: add/drop only via the
unified `add_function_field` / `drop_function_field`, output field
always coupled. **TextEmbedding is a temporary carve-out** — its legacy
`AddCollectionFunction` / `DropCollectionFunction` paths are retained,
so attach-over-existing-field and detach are still possible for it,
because `add_function_field` embedding backfill is not yet implemented.
It will be folded into the unified path in a follow-up. So the "always
coupled" invariant currently holds for BM25/MinHash, not TextEmbedding.

## Changes

- **AlterCollectionSchema invariant**: reject standalone add-function (a
function must be added together with its new output field), reject
detaching a function without dropping its output field, and reject
function cascade (a new function's input being another function's
output).
- **Legacy RPCs**: `AddCollectionFunction` / `DropCollectionFunction`
are rejected for BM25/MinHash (must use `add_function_field` /
`drop_function_field`) and retained only for TextEmbedding (see
carve-out). `AlterCollectionFunction` is kept and gains the whitelist
below. In the alter and legacy-add paths, function field IDs are always
re-derived from field names (never trusted from the request), so a
request cannot inject an unrelated field ID that a later
`drop_function_field` would delete.
- **alter_function whitelist**: only connection/runtime params may
change (TextEmbedding: `url`, `credential`, `timeout_ms`,
`max_client_batch_size`, `region`, `location`, `projectid`, `user`);
BM25/MinHash have no alterable params. Function identity (type, name,
input/output fields) and output-shaping params (`dim`, `model_name`,
`endpoint` — the TEI model identity, `normalize`, `truncate*`, prompts,
...) are immutable. Params are normalized (keys lowercased, duplicate
keys rejected) so a crafted duplicate key cannot bypass the diff.
- **drop_function_field**: allow any output-producing function (dropping
needs no backfill), removing the previous BM25/MinHash-only restriction.

## Not in scope / follow-up

- Extending `add_function_field` to TextEmbedding (post-creation add
would require backfilling every existing row through the external
embedding model) — deferred; folding the TextEmbedding legacy paths into
the unified binding follows this.
- MinHash input-field analyzer immutability lives on a different DDL
path (`AlterCollectionField`); tracked as a separate follow-up.

## Breaking changes

- `DropCollectionFunction` on a **MinHash** function (detach — remove
the function but keep its output field) is no longer supported; use
`drop_function_field` instead (drops the function together with its
output field). If a released version supported MinHash detach, migrate
any such call to `drop_function_field`.

issue: #51348

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Signed-off-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: MrPresent-Han <chun.han@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Chun Han
2026-07-18 05:04:39 +08:00
committed by GitHub
co-authored by MrPresent-Han Claude Opus 4.8
parent 59e3ce9f9f
commit 5def5ced4a
25 changed files with 1136 additions and 2193 deletions
@@ -76,6 +76,8 @@ const (
AddFunctionAction = "add_function"
AlterFunctionAction = "alter_function"
DropFunctionAction = "drop_function"
AddFunctionFieldAction = "add_function_field"
DropFunctionFieldAction = "drop_function_field"
AddAction = `add`
DropPropertiesAction = "drop_properties"
CompactAction = "compact"
@@ -90,6 +90,8 @@ var routeToMethod = map[string]string{ //nolint:gosec // not credentials, just a
"/v2/vectordb/collections/add_function": "AddCollectionFunction",
"/v2/vectordb/collections/alter_function": "AlterCollectionFunction",
"/v2/vectordb/collections/drop_function": "DropCollectionFunction",
"/v2/vectordb/collections/add_function_field": "AlterCollectionSchema",
"/v2/vectordb/collections/drop_function_field": "AlterCollectionSchema",
"/v2/vectordb/collections/drop_properties": "AlterCollection",
"/v2/vectordb/collections/compact": "ManualCompaction",
"/v2/vectordb/collections/get_compaction_state": "GetCompactionState",
@@ -206,6 +208,8 @@ func (h *HandlersV2) RegisterRoutesToV2(router gin.IRouter) {
router.POST(CollectionCategory+AddFunctionAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionAddFunction{} }, wrapperTraceLog(h.addCollectionFunction))))
router.POST(CollectionCategory+AlterFunctionAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionAlterFunction{} }, wrapperTraceLog(h.alterCollectionFunction))))
router.POST(CollectionCategory+DropFunctionAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionDropFunction{} }, wrapperTraceLog(h.dropCollectionFunction))))
router.POST(CollectionCategory+AddFunctionFieldAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionAddFunctionField{} }, wrapperTraceLog(h.addCollectionFunctionField))))
router.POST(CollectionCategory+DropFunctionFieldAction, timeoutMiddleware(wrapperPost(func() any { return &CollectionDropFunctionField{} }, wrapperTraceLog(h.dropCollectionFunctionField))))
router.POST(CollectionCategory+DropPropertiesAction, timeoutMiddleware(wrapperPost(func() any { return &DropCollectionPropertiesReq{} }, wrapperTraceLog(h.dropCollectionProperties))))
router.POST(CollectionCategory+CompactAction, timeoutMiddleware(wrapperPost(func() any { return &CompactReq{} }, wrapperTraceLog(h.compact))))
router.POST(CollectionCategory+CompactionStateAction, timeoutMiddleware(wrapperPost(func() any { return &GetCompactionStateReq{} }, wrapperTraceLog(h.getcompactionState))))
@@ -996,6 +1000,9 @@ func (h *HandlersV2) alterCollectionFunction(ctx context.Context, c *gin.Context
return resp, err
}
// dropCollectionFunction backs the deprecated /collections/drop_function endpoint,
// which mapped to the legacy detach RPC. A function is coupled to its output field,
// so the RPC is rejected; use /collections/drop_function_field instead.
func (h *HandlersV2) dropCollectionFunction(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionDropFunction)
req := &milvuspb.DropCollectionFunctionRequest{
@@ -1013,6 +1020,111 @@ func (h *HandlersV2) dropCollectionFunction(ctx context.Context, c *gin.Context,
return resp, err
}
// addCollectionFunctionField backs /collections/add_function_field: it attaches a
// BM25/MinHash function together with its output field and index via AlterCollectionSchema.
// A function is coupled to its output field, so both are added in one request.
func (h *HandlersV2) addCollectionFunctionField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionAddFunctionField)
// The index always targets the newly-added output field; reject a mismatched
// indexParams.fieldName rather than silently building the index on outputField.
if httpReq.IndexParam.FieldName != httpReq.OutputField.FieldName {
err := merr.WrapErrParameterInvalidMsg(
"indexParams.fieldName %q must match outputField.fieldName %q",
httpReq.IndexParam.FieldName, httpReq.OutputField.FieldName)
HTTPAbortReturn(c, http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: err.Error(),
})
return nil, err
}
fSchema, err := genFunctionSchema(ctx, &httpReq.Function)
if err != nil {
HTTPAbortReturn(c, http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: err.Error(),
})
return nil, err
}
fieldSchema, err := httpReq.OutputField.GetProto(ctx)
if err != nil {
HTTPAbortReturn(c, http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: err.Error(),
})
return nil, err
}
extraParams, err := convertToExtraParams(httpReq.IndexParam)
if err != nil {
HTTPAbortReturn(c, http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: err.Error(),
})
return nil, err
}
req := &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: httpReq.CollectionName,
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
Op: &milvuspb.AlterCollectionSchemaRequest_Action_AddRequest{
AddRequest: &milvuspb.AlterCollectionSchemaRequest_AddRequest{
FieldInfos: []*milvuspb.AlterCollectionSchemaRequest_FieldInfo{
{
FieldSchema: fieldSchema,
IndexName: httpReq.IndexParam.IndexName,
ExtraParams: extraParams,
},
},
FuncSchema: []*schemapb.FunctionSchema{fSchema},
},
},
},
}
c.Set(ContextRequest, req)
resp, err := wrapperProxyWithLimit(ctx, c, req, h.checkAuth, false, "/milvus.proto.milvus.MilvusService/AlterCollectionSchema", true, h.proxy, func(reqCtx context.Context, req any) (interface{}, error) {
r, callErr := h.proxy.AlterCollectionSchema(reqCtx, req.(*milvuspb.AlterCollectionSchemaRequest))
if callErr == nil {
callErr = merr.Error(r.GetAlterStatus())
}
return r, callErr
})
if err == nil {
HTTPReturn(c, http.StatusOK, wrapperReturnDefault())
}
return resp, err
}
// dropCollectionFunctionField backs /collections/drop_function_field: it detaches a
// function and drops its output field together via AlterCollectionSchema.
func (h *HandlersV2) dropCollectionFunctionField(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionDropFunctionField)
req := &milvuspb.AlterCollectionSchemaRequest{
DbName: dbName,
CollectionName: httpReq.CollectionName,
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
Op: &milvuspb.AlterCollectionSchemaRequest_Action_DropRequest{
DropRequest: &milvuspb.AlterCollectionSchemaRequest_DropRequest{
Identifier: &milvuspb.AlterCollectionSchemaRequest_DropRequest_FunctionName{
FunctionName: httpReq.FunctionName,
},
DropFunctionOutputFields: true,
},
},
},
}
c.Set(ContextRequest, req)
resp, err := wrapperProxyWithLimit(ctx, c, req, h.checkAuth, false, "/milvus.proto.milvus.MilvusService/AlterCollectionSchema", true, h.proxy, func(reqCtx context.Context, req any) (interface{}, error) {
r, callErr := h.proxy.AlterCollectionSchema(reqCtx, req.(*milvuspb.AlterCollectionSchemaRequest))
if callErr == nil {
callErr = merr.Error(r.GetAlterStatus())
}
return r, callErr
})
if err == nil {
HTTPReturn(c, http.StatusOK, wrapperReturnDefault())
}
return resp, err
}
func (h *HandlersV2) dropCollectionProperties(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*DropCollectionPropertiesReq)
req := &milvuspb.AlterCollectionRequest{
@@ -4914,6 +4914,80 @@ func (s *CollectionFunctionSuite) TestDropCollectionFunctionNormal() {
})
}
func (s *CollectionFunctionSuite) TestAddCollectionFunctionFieldNormal() {
s.Run("success", func() {
addFunctionFieldTestCases := []requestBodyTestCase{
{
path: versionalV2(CollectionCategory, AddFunctionFieldAction),
requestBody: []byte(`{"dbName": "db", "collectionName": "coll", "function": {"name": "bm25_fn", "type": "BM25", "inputFieldNames": ["text"], "outputFieldNames": ["sparse"]}, "outputField": {"fieldName": "sparse", "dataType": "SparseFloatVector"}, "indexParams": {"fieldName": "sparse", "indexName": "sparse_idx", "metricType": "BM25", "indexType": "SPARSE_INVERTED_INDEX"}}`),
errCode: 0,
errMsg: "",
},
}
s.mp.EXPECT().AlterCollectionSchema(mock.Anything, mock.Anything).Return(&milvuspb.AlterCollectionSchemaResponse{AlterStatus: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}}, nil).Maybe()
validateRequestBodyTestCases(s.T(), s.testEngine, addFunctionFieldTestCases, false)
})
s.Run("bad_request", func() {
addFunctionFieldTestCases := []requestBodyTestCase{
{
path: versionalV2(CollectionCategory, AddFunctionFieldAction),
requestBody: []byte(`{"dbName": "db", "collectionName": "", "function": {"name": "bm25_fn", "type": "BM25", "inputFieldNames": ["text"], "outputFieldNames": ["sparse"]}, "outputField": {"fieldName": "sparse", "dataType": "SparseFloatVector"}, "indexParams": {"fieldName": "sparse"}}`),
errCode: 1802,
errMsg: "missing required parameters, error: Key: 'CollectionAddFunctionField.CollectionName' Error:Field validation for 'CollectionName' failed on the 'required' tag",
},
{
path: versionalV2(CollectionCategory, AddFunctionFieldAction),
requestBody: []byte(`{"dbName": "db", "collectionName": "coll", "function": {"name": "bm25_fn", "type": "BM25", "inputFieldNames": ["text"], "outputFieldNames": ["sparse"]}, "outputField": {"fieldName": "sparse", "dataType": "SparseFloatVector"}, "indexParams": {"fieldName": "wrong_field", "indexName": "sparse_idx", "metricType": "BM25", "indexType": "SPARSE_INVERTED_INDEX"}}`),
errCode: 1100,
errMsg: `indexParams.fieldName "wrong_field" must match outputField.fieldName "sparse": invalid parameter`,
},
{
path: versionalV2(CollectionCategory, AddFunctionFieldAction),
requestBody: []byte(`invalid json`),
errCode: 1801,
errMsg: "can only accept json format request, error: invalid character 'i' looking for beginning of value",
},
}
validateRequestBodyTestCases(s.T(), s.testEngine, addFunctionFieldTestCases, false)
})
}
func (s *CollectionFunctionSuite) TestDropCollectionFunctionFieldNormal() {
s.Run("success", func() {
dropFunctionFieldTestCases := []requestBodyTestCase{
{
path: versionalV2(CollectionCategory, DropFunctionFieldAction),
requestBody: []byte(`{"dbName": "db", "collectionName": "coll", "functionName": "bm25_fn"}`),
errCode: 0,
errMsg: "",
},
}
s.mp.EXPECT().AlterCollectionSchema(mock.Anything, mock.Anything).Return(&milvuspb.AlterCollectionSchemaResponse{AlterStatus: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}}, nil).Maybe()
validateRequestBodyTestCases(s.T(), s.testEngine, dropFunctionFieldTestCases, false)
})
s.Run("bad_request", func() {
dropFunctionFieldTestCases := []requestBodyTestCase{
{
path: versionalV2(CollectionCategory, DropFunctionFieldAction),
requestBody: []byte(`{"dbName": "db", "collectionName": "coll", "functionName": ""}`),
errCode: 1802,
errMsg: "missing required parameters, error: Key: 'CollectionDropFunctionField.FunctionName' Error:Field validation for 'FunctionName' failed on the 'required' tag",
},
{
path: versionalV2(CollectionCategory, DropFunctionFieldAction),
requestBody: []byte(`invalid json`),
errCode: 1801,
errMsg: "can only accept json format request, error: invalid character 'i' looking for beginning of value",
},
}
validateRequestBodyTestCases(s.T(), s.testEngine, dropFunctionFieldTestCases, false)
})
}
func TestTruncateCollection(t *testing.T) {
paramtable.Init()
// disable rate limit
@@ -124,6 +124,32 @@ func (req *CollectionAddFunction) GetFunction() *FunctionSchema {
return &req.Function
}
type CollectionAddFunctionField struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Function FunctionSchema `json:"function" binding:"required"`
OutputField FieldSchema `json:"outputField" binding:"required"`
IndexParam IndexParam `json:"indexParams" binding:"required"`
}
func (req *CollectionAddFunctionField) GetDbName() string { return req.DbName }
func (req *CollectionAddFunctionField) GetCollectionName() string {
return req.CollectionName
}
type CollectionDropFunctionField struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
FunctionName string `json:"functionName" binding:"required"`
}
func (req *CollectionDropFunctionField) GetDbName() string { return req.DbName }
func (req *CollectionDropFunctionField) GetCollectionName() string {
return req.CollectionName
}
type CollectionAlterFunction struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
+3 -188
View File
@@ -42,99 +42,6 @@ func rejectExternalCollectionFunctionMutation(schema *schemapb.CollectionSchema)
return nil
}
type addCollectionFunctionTask struct {
baseTask
Condition
*milvuspb.AddCollectionFunctionRequest
ctx context.Context
mixCoord types.MixCoordClient
result *commonpb.Status
}
func (t *addCollectionFunctionTask) TraceCtx() context.Context {
return t.ctx
}
func (t *addCollectionFunctionTask) ID() UniqueID {
return t.Base.MsgID
}
func (t *addCollectionFunctionTask) SetID(uid UniqueID) {
t.Base.MsgID = uid
}
func (t *addCollectionFunctionTask) Type() commonpb.MsgType {
return t.Base.MsgType
}
func (t *addCollectionFunctionTask) BeginTs() Timestamp {
return t.Base.Timestamp
}
func (t *addCollectionFunctionTask) EndTs() Timestamp {
return t.Base.Timestamp
}
func (t *addCollectionFunctionTask) SetTs(ts Timestamp) {
t.Base.Timestamp = ts
}
func (t *addCollectionFunctionTask) OnEnqueue() error {
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
t.Base.MsgType = commonpb.MsgType_AddCollectionFunction
t.Base.SourceID = paramtable.GetNodeID()
return nil
}
func (t *addCollectionFunctionTask) Name() string {
return AddCollectionFunctionTask
}
func (t *addCollectionFunctionTask) PreExecute(ctx context.Context) error {
if t.FunctionSchema == nil {
return merr.WrapErrParameterInvalidMsg("function schema is empty")
}
if t.FunctionSchema.Type == schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("currently does not support adding BM25 function")
}
if err := validateAddFunctionRequiresStorageV3(); err != nil {
return err
}
coll, err := getCollectionInfo(ctx, t.GetDbName(), t.GetCollectionName())
if err != nil {
mlog.Error(t.ctx, "AddCollectionTask, get collection info failed",
mlog.String("dbName", t.GetDbName()),
mlog.String("collectionName", t.GetCollectionName()),
mlog.Err(err))
return err
}
if err := validateAddFunctionInputNotText(coll.schema.CollectionSchema, t.FunctionSchema); err != nil {
return err
}
newColl := proto.Clone(coll.schema.CollectionSchema).(*schemapb.CollectionSchema)
newColl.Functions = append(coll.schema.Functions, t.FunctionSchema)
if err := validator.ValidateFunction(newColl, t.FunctionSchema.Name, false); err != nil {
return err
}
return nil
}
func (t *addCollectionFunctionTask) Execute(ctx context.Context) error {
var err error
t.result, err = t.mixCoord.AddCollectionFunction(ctx, t.AddCollectionFunctionRequest)
if err = merr.CheckRPCCall(t.result, err); err != nil {
return err
}
return nil
}
func (t *addCollectionFunctionTask) PostExecute(ctx context.Context) error {
return nil
}
type alterCollectionFunctionTask struct {
baseTask
Condition
@@ -213,6 +120,9 @@ func (t *alterCollectionFunctionTask) PreExecute(ctx context.Context) error {
if fSchema.Type == schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("currently does not support alter BM25 function")
}
if err := validator.CheckFunctionAlterAllowed(fSchema, t.FunctionSchema); err != nil {
return err
}
newFunctions = append(newFunctions, t.FunctionSchema)
funcExist = true
} else {
@@ -244,101 +154,6 @@ func (t *alterCollectionFunctionTask) PostExecute(ctx context.Context) error {
return nil
}
type dropCollectionFunctionTask struct {
baseTask
Condition
*milvuspb.DropCollectionFunctionRequest
ctx context.Context
mixCoord types.MixCoordClient
result *commonpb.Status
fSchema *schemapb.FunctionSchema
}
func (t *dropCollectionFunctionTask) TraceCtx() context.Context {
return t.ctx
}
func (t *dropCollectionFunctionTask) ID() UniqueID {
return t.Base.MsgID
}
func (t *dropCollectionFunctionTask) SetID(uid UniqueID) {
t.Base.MsgID = uid
}
func (t *dropCollectionFunctionTask) Type() commonpb.MsgType {
return t.Base.MsgType
}
func (t *dropCollectionFunctionTask) BeginTs() Timestamp {
return t.Base.Timestamp
}
func (t *dropCollectionFunctionTask) EndTs() Timestamp {
return t.Base.Timestamp
}
func (t *dropCollectionFunctionTask) SetTs(ts Timestamp) {
t.Base.Timestamp = ts
}
func (t *dropCollectionFunctionTask) OnEnqueue() error {
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
t.Base.MsgType = commonpb.MsgType_DropCollectionFunction
t.Base.SourceID = paramtable.GetNodeID()
return nil
}
func (t *dropCollectionFunctionTask) Name() string {
return DropCollectionFunctionTask
}
func (t *dropCollectionFunctionTask) PreExecute(ctx context.Context) error {
coll, err := getCollectionInfo(ctx, t.GetDbName(), t.GetCollectionName())
if err != nil {
mlog.Error(t.ctx, "DropFunctionTask, get collection info failed",
mlog.String("dbName", t.GetDbName()),
mlog.String("collectionName", t.GetCollectionName()),
mlog.Err(err))
return err
}
if err := rejectExternalCollectionFunctionMutation(coll.schema.CollectionSchema); err != nil {
return err
}
for _, f := range coll.schema.Functions {
if f.Name == t.FunctionName {
t.fSchema = f
break
}
}
return nil
}
func (t *dropCollectionFunctionTask) Execute(ctx context.Context) error {
if t.fSchema == nil {
return nil
}
if t.fSchema.Type == schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("currently does not support droping BM25 function")
}
var err error
t.result, err = t.mixCoord.DropCollectionFunction(ctx, t.DropCollectionFunctionRequest)
if err = merr.CheckRPCCall(t.result, err); err != nil {
return err
}
return nil
}
func (t *dropCollectionFunctionTask) PostExecute(ctx context.Context) error {
return nil
}
func getCollectionInfo(ctx context.Context, dbName string, collectionName string) (*collectionInfo, error) {
collID, err := globalMetaCache.GetCollectionID(ctx, dbName, collectionName)
if err != nil {
+33 -419
View File
@@ -96,34 +96,6 @@ func (f *FunctionTaskSuite) TestValidateAddFunctionInputNotText() {
}
func (f *FunctionTaskSuite) TestFunctionOnType() {
{
task := &addCollectionFunctionTask{
AddCollectionFunctionRequest: &milvuspb.AddCollectionFunctionRequest{},
}
err := task.OnEnqueue()
f.NoError(err)
f.Equal(commonpb.MsgType_AddCollectionFunction, task.Type())
f.Equal(task.TraceCtx(), task.ctx)
task.SetID(1)
f.Equal(task.ID(), int64(1))
task.SetTs(2)
f.Equal(task.EndTs(), uint64(2))
f.Equal(task.Name(), AddCollectionFunctionTask)
}
{
task := &dropCollectionFunctionTask{
DropCollectionFunctionRequest: &milvuspb.DropCollectionFunctionRequest{},
}
err := task.OnEnqueue()
f.NoError(err)
f.Equal(commonpb.MsgType_DropCollectionFunction, task.Type())
f.Equal(task.TraceCtx(), task.ctx)
task.SetID(1)
f.Equal(task.ID(), int64(1))
task.SetTs(2)
f.Equal(task.EndTs(), uint64(2))
f.Equal(task.Name(), DropCollectionFunctionTask)
}
{
task := &alterCollectionFunctionTask{
AlterCollectionFunctionRequest: &milvuspb.AlterCollectionFunctionRequest{},
@@ -140,136 +112,6 @@ func (f *FunctionTaskSuite) TestFunctionOnType() {
}
}
func (f *FunctionTaskSuite) TestAddCollectionFunctionTaskPreExecute() {
ctx := context.Background()
// Adding a function requires StorageV3 + schema-bump/storage-version compaction enabled
// (see validateAddFunctionRequiresStorageV3). storageVersion.enabled defaults true;
// bumpSchemaVersion.enabled defaults false, so it must be set explicitly.
paramtable.Get().Save(paramtable.Get().CommonCfg.UseLoonFFI.Key, "true")
paramtable.Get().Save(paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key, "true")
defer paramtable.Get().Reset(paramtable.Get().CommonCfg.UseLoonFFI.Key)
defer paramtable.Get().Reset(paramtable.Get().DataCoordCfg.BumpSchemaVersionCompactionEnabled.Key)
{
mixc := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
DbName: "db",
CollectionName: "NotExist",
CollectionID: 1,
FunctionSchema: &schemapb.FunctionSchema{},
}
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
mixCoord: mixc,
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(0, fmt.Errorf("Mock Error")).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.ErrorContains(err, "Mock Error")
}
{
// Test with invalid function schema
mixc := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionSchema: &schemapb.FunctionSchema{},
}
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
mixCoord: mixc,
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(nil, fmt.Errorf("Mock info error")).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.ErrorContains(err, "Mock info error")
}
{
// Test with valid request
functionSchema := &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"sparse_field"},
Params: []*commonpb.KeyValuePair{},
}
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionSchema: functionSchema,
}
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(nil, nil).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.ErrorContains(err, "not support adding BM25")
}
{
functionSchema := &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text"},
OutputFieldNames: []string{"vec"},
Params: []*commonpb.KeyValuePair{},
}
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionSchema: functionSchema,
}
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
}
coll := &collectionInfo{
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Functions: []*schemapb.FunctionSchema{},
},
},
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(coll, nil).Maybe()
m := mockey.Mock(validator.ValidateFunction).Return(nil).Build()
defer m.UnPatch()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.NoError(err)
}
}
func (f *FunctionTaskSuite) TestAlterCollectionFunctionTaskPreExecute() {
ctx := context.Background()
@@ -496,7 +338,8 @@ func (f *FunctionTaskSuite) TestAlterCollectionFunctionTaskPreExecute() {
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Functions: []*schemapb.FunctionSchema{
{Name: "test_function", Type: schemapb.FunctionType_TextEmbedding},
// identity matches the request; a valid alter changes only params
{Name: "test_function", Type: schemapb.FunctionType_TextEmbedding, InputFieldNames: []string{"text"}, OutputFieldNames: []string{"vec"}},
{Name: "f2", Type: schemapb.FunctionType_TextEmbedding},
},
},
@@ -511,130 +354,39 @@ func (f *FunctionTaskSuite) TestAlterCollectionFunctionTaskPreExecute() {
err := task.PreExecute(ctx)
f.NoError(err)
}
}
func (f *FunctionTaskSuite) TestDropCollectionFunctionTaskPreExecute() {
ctx := context.Background()
{
// Test with valid request
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
// Altering the output field is rejected: function identity is immutable.
req := &milvuspb.AlterCollectionFunctionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_AlterCollectionFunction},
CollectionName: "test_collection",
CollectionID: 1,
FunctionName: "test_function",
}
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(nil, fmt.Errorf("mock error")).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.ErrorContains(err, "mock error")
}
{
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
FunctionName: "test_function",
}
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
}
coll := &collectionInfo{
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Functions: []*schemapb.FunctionSchema{},
},
FunctionSchema: &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text"},
OutputFieldNames: []string{"other_vec"},
},
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(coll, nil).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.NoError(err)
}
{
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
FunctionName: "test_function",
task := &alterCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AlterCollectionFunctionRequest: req,
}
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
}
coll := &collectionInfo{
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Functions: []*schemapb.FunctionSchema{
{Name: req.FunctionName},
{Name: "test_function", Type: schemapb.FunctionType_TextEmbedding, InputFieldNames: []string{"text"}, OutputFieldNames: []string{"vec"}},
},
},
},
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(coll, nil).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.NoError(err)
}
{
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
FunctionName: "test_function",
}
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
}
coll := &collectionInfo{
schema: &schemaInfo{
CollectionSchema: &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "text", DataType: schemapb.DataType_VarChar, ExternalField: "text_col"},
{Name: "vec", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{Name: req.FunctionName},
},
},
},
}
cache := NewMockCache(f.T())
cache.EXPECT().GetCollectionID(ctx, req.DbName, req.CollectionName).Return(int64(1), nil).Maybe()
cache.EXPECT().GetCollectionInfo(ctx, req.DbName, req.CollectionName, int64(1)).Return(coll, nil).Maybe()
globalMetaCache = cache
err := task.PreExecute(ctx)
f.ErrorContains(err, externalCollectionFunctionMutationUnsupportedMsg)
f.ErrorContains(err, "output fields cannot be altered")
}
}
@@ -732,162 +484,24 @@ func (f *FunctionTaskSuite) TestAlterCollectionFunctionTaskExecute() {
}
}
func (f *FunctionTaskSuite) TestAddCollectionFunctionTaskExecute() {
func (f *FunctionTaskSuite) TestAlterCollectionFunctionTaskExecuteRPCError() {
ctx := context.Background()
{
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
functionSchema := &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"sparse_field"},
Params: []*commonpb.KeyValuePair{},
}
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionSchema: functionSchema,
}
mockRootCoord.EXPECT().AddCollectionFunction(mock.Anything, req).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil)
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
}
err := task.Execute(ctx)
f.NoError(err)
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.AlterCollectionFunctionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_AlterCollectionFunction},
CollectionName: "test_collection",
FunctionName: "test_function",
FunctionSchema: &schemapb.FunctionSchema{Name: "test_function", Type: schemapb.FunctionType_TextEmbedding},
}
{
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
functionSchema := &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"sparse_field"},
Params: []*commonpb.KeyValuePair{},
}
req := &milvuspb.AddCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_AddCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionSchema: functionSchema,
}
mockRootCoord.EXPECT().AddCollectionFunction(mock.Anything, req).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: "test error",
}, nil)
task := &addCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
}
err := task.Execute(ctx)
f.Error(err)
}
}
func (f *FunctionTaskSuite) TestDropCollectionFunctionTaskExecute() {
ctx := context.Background()
{
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionName: "test_function",
}
mockRootCoord.EXPECT().DropCollectionFunction(mock.Anything, req).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil)
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
fSchema: &schemapb.FunctionSchema{
Type: schemapb.FunctionType_TextEmbedding,
},
}
err := task.Execute(ctx)
f.NoError(err)
}
{
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionName: "test_function",
}
mockRootCoord.EXPECT().DropCollectionFunction(mock.Anything, req).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: "test error",
}, nil)
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
fSchema: &schemapb.FunctionSchema{
Type: schemapb.FunctionType_TextEmbedding,
},
}
err := task.Execute(ctx)
f.Error(err)
}
{
mockRootCoord := mocks.NewMockMixCoordClient(f.T())
req := &milvuspb.DropCollectionFunctionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollectionFunction,
},
CollectionName: "test_collection",
CollectionID: 1,
FunctionName: "test_function",
}
task := &dropCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
fSchema: &schemapb.FunctionSchema{
Type: schemapb.FunctionType_BM25,
},
}
err := task.Execute(ctx)
f.ErrorContains(err, "currently does not support droping BM25 function")
mockRootCoord.EXPECT().AlterCollectionFunction(mock.Anything, req).Return(&commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: "test error",
}, nil)
task := &alterCollectionFunctionTask{
Condition: NewTaskCondition(ctx),
AlterCollectionFunctionRequest: req,
mixCoord: mockRootCoord,
}
err := task.Execute(ctx)
f.Error(err)
}
+13 -86
View File
@@ -1325,53 +1325,17 @@ func (node *Proxy) AlterCollection(ctx context.Context, request *milvuspb.AlterC
return act.result, nil
}
// AddCollectionFunction is the deprecated legacy attach RPC. A function is coupled
// to its output field: BM25/MinHash add a brand-new output field via
// add_function_field, and TextEmbedding is defined at collection creation (runtime
// add awaits embedding backfill). This RPC only allowed the unsafe attach-over-
// existing-field path, so it is rejected.
func (node *Proxy) AddCollectionFunction(ctx context.Context, request *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return merr.Status(err), nil
}
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-AddCollectionFunction")
defer sp.End()
method := "AddCollectionFunction"
tr := timerecord.NewTimeRecorder(method)
task := &addCollectionFunctionTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
AddCollectionFunctionRequest: request,
mixCoord: node.mixCoord,
}
mlog.Info(ctx, rpcReceived(method))
if err := node.sched.ddQueue.Enqueue(task); err != nil {
mlog.Warn(ctx,
rpcFailedToEnqueue(method),
mlog.Err(err))
return merr.Status(err), nil
}
mlog.Debug(ctx,
rpcEnqueued(method),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()),
mlog.Uint64("timestamp", request.Base.Timestamp))
if err := task.WaitToFinish(); err != nil {
mlog.Warn(ctx,
rpcFailedToWaitToFinish(method),
mlog.Err(err),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()))
return merr.Status(err), nil
}
mlog.Info(ctx,
rpcDone(method),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()))
metrics.ProxyReqLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method).Observe(float64(tr.ElapseSpan().Milliseconds()))
return task.result, nil
return merr.Status(merr.WrapErrParameterInvalidMsg(
"AddCollectionFunction RPC is no longer supported; add BM25/MinHash via add_function_field, and define a TextEmbedding function at collection creation")), nil
}
func (node *Proxy) AlterCollectionFunction(ctx context.Context, request *milvuspb.AlterCollectionFunctionRequest) (*commonpb.Status, error) {
@@ -1423,53 +1387,16 @@ func (node *Proxy) AlterCollectionFunction(ctx context.Context, request *milvusp
return task.result, nil
}
// DropCollectionFunction is the deprecated legacy detach RPC. pymilvus
// drop_collection_function routes through AlterCollectionSchema (drop_function_field
// / detach) instead, so this RPC is unused; reject to avoid a second, divergent DDL
// path.
func (node *Proxy) DropCollectionFunction(ctx context.Context, request *milvuspb.DropCollectionFunctionRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return merr.Status(err), nil
}
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-DropCollectionFunction")
defer sp.End()
method := "DropCollectionFunction"
tr := timerecord.NewTimeRecorder(method)
task := &dropCollectionFunctionTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
DropCollectionFunctionRequest: request,
mixCoord: node.mixCoord,
}
mlog.Info(ctx, rpcReceived(method))
if err := node.sched.ddQueue.Enqueue(task); err != nil {
mlog.Warn(ctx,
rpcFailedToEnqueue(method),
mlog.Err(err))
return merr.Status(err), nil
}
mlog.Debug(ctx,
rpcEnqueued(method),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()),
mlog.Uint64("timestamp", request.Base.Timestamp))
if err := task.WaitToFinish(); err != nil {
mlog.Warn(ctx,
rpcFailedToWaitToFinish(method),
mlog.Err(err),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()))
return merr.Status(err), nil
}
mlog.Info(ctx,
rpcDone(method),
mlog.Uint64("BeginTs", task.BeginTs()),
mlog.Uint64("EndTs", task.EndTs()))
metrics.ProxyReqLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), method).Observe(float64(tr.ElapseSpan().Milliseconds()))
return task.result, nil
return merr.Status(merr.WrapErrParameterInvalidMsg(
"DropCollectionFunction RPC is no longer supported; drop a function via drop_function_field")), nil
}
func (node *Proxy) AlterCollectionField(ctx context.Context, request *milvuspb.AlterCollectionFieldRequest) (*commonpb.Status, error) {
+14 -14
View File
@@ -114,9 +114,7 @@ const (
ListAliasesTaskName = "ListAliasesTask"
AlterCollectionTaskName = "AlterCollectionTask"
AlterCollectionFieldTaskName = "AlterCollectionFieldTask"
AddCollectionFunctionTask = "AddCollectionFunctionTask"
AlterCollectionFunctionTask = "AlterCollectionFunctionTask"
DropCollectionFunctionTask = "DropCollectionFunctionTask"
UpsertTaskName = "UpsertTask"
CreateResourceGroupTaskName = "CreateResourceGroupTask"
UpdateResourceGroupsTaskName = "UpdateResourceGroupsTask"
@@ -1182,6 +1180,9 @@ func (t *alterCollectionSchemaTask) preExecuteAdd(ctx context.Context) error {
if err := schemautil.ValidateAlterSchemaAddFunctionPlan(plan); err != nil {
return err
}
if err := schemautil.CheckNoFunctionCascade(t.oldSchema.GetFunctions(), plan.Function); err != nil {
return err
}
}
// Validate the bound index params against the new field, mirroring the
@@ -1418,18 +1419,15 @@ func validateDropFunction(schema *schemapb.CollectionSchema, functionName string
}
if !dropOutputFields {
if targetFunc.GetType() == schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("BM25 function must be dropped with its output field in drop_function_field interface: %s", functionName)
}
return nil
}
switch targetFunc.GetType() {
case schemapb.FunctionType_BM25, schemapb.FunctionType_MinHash:
default:
return merr.WrapErrParameterInvalidMsg("only BM25 and MinHash functions support dropping output fields: %s", functionName)
// A function is coupled to its output field for all types: detaching (removing
// the function but keeping the field) is never allowed. drop_function always
// removes the function together with its output field.
return merr.WrapErrParameterInvalidMsg(
"detaching a function without dropping its output field is not supported; drop_function always removes the function together with its output field: %s", functionName)
}
// Drop is uniform across function types (no backfill); unlike add_function_field
// it is not type-restricted.
removedVectors := 0
for _, name := range targetFunc.OutputFieldNames {
if f := typeutil.GetFieldByName(schema, name); f != nil && typeutil.IsVectorType(f.DataType) {
@@ -2580,9 +2578,11 @@ func validateAlterAnalyzerFieldParam(collSchema *schemapb.CollectionSchema, fiel
return merr.WrapErrParameterInvalidMsg("can not alter analyzer params for non-string field %s", fieldName)
}
if typeutil.CreateFieldSchemaHelper(field).EnableMatch() || typeutil.IsBm25FunctionInputField(collSchema, field) {
if typeutil.CreateFieldSchemaHelper(field).EnableMatch() ||
typeutil.IsBm25FunctionInputField(collSchema, field) ||
typeutil.IsMinHashFunctionInputField(collSchema, field) {
return merr.WrapErrParameterInvalidMsg(
"can not alter analyzer params for field %s after text match is enabled or BM25 function depends on it",
"can not alter analyzer params for field %s after text match is enabled or a BM25/MinHash function depends on it",
fieldName,
)
}
+36 -17
View File
@@ -8012,7 +8012,9 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
t.Run("PreExecute add function without fieldInfos", func(t *testing.T) {
task := buildTask(buildAddFunctionRequest(minHashFunctionOnlySchema), functionOnlyOldSchema)
err := task.PreExecute(ctx)
assert.NoError(t, err)
assert.Error(t, err)
assert.ErrorIs(t, err, merr.ErrParameterInvalid)
assert.ErrorContains(t, err, "adding a function over existing fields is not supported")
})
t.Run("PreExecute rejects function-only BM25 request", func(t *testing.T) {
@@ -8191,7 +8193,7 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
task := buildTask(req, oldSchema)
err := task.PreExecute(ctx)
assert.Error(t, err)
assert.ErrorContains(t, err, "function output field not found")
assert.ErrorContains(t, err, "adding a function over existing fields is not supported")
})
t.Run("PreExecute happy path", func(t *testing.T) {
@@ -8257,16 +8259,13 @@ func TestAlterCollectionSchemaTask(t *testing.T) {
}
})
t.Run("Execute supports function-only add request", func(t *testing.T) {
t.Run("PreExecute rejects function-only add request", func(t *testing.T) {
req := buildAddFunctionRequest(minHashFunctionOnlySchema)
task := buildTask(req, functionOnlyOldSchema)
err := task.PreExecute(ctx)
assert.NoError(t, err)
err = task.Execute(ctx)
assert.NoError(t, err)
assert.Empty(t, req.GetAction().GetAddRequest().GetFieldInfos())
assert.Error(t, err)
assert.ErrorContains(t, err, "adding a function over existing fields is not supported")
})
t.Run("Execute skips nil field infos", func(t *testing.T) {
@@ -8754,7 +8753,7 @@ func TestAlterCollectionSchemaTask_PreExecute(t *testing.T) {
assert.NoError(t, err)
})
t.Run("drop by function_name - detach minhash success", func(t *testing.T) {
t.Run("drop by function_name - detach minhash rejected", func(t *testing.T) {
schemaWithFunc := &schemapb.CollectionSchema{
Name: "test_collection",
Fields: []*schemapb.FieldSchema{
@@ -8783,7 +8782,8 @@ func TestAlterCollectionSchemaTask_PreExecute(t *testing.T) {
},
}
err := task.PreExecute(ctx)
assert.NoError(t, err)
assert.Error(t, err)
assert.ErrorContains(t, err, "detaching a function without dropping its output field is not supported")
})
t.Run("drop by function_name - bm25 function field success", func(t *testing.T) {
@@ -8976,7 +8976,7 @@ func TestValidateDropFunction(t *testing.T) {
assert.Contains(t, err.Error(), "function not found")
})
t.Run("detach minhash function succeeds", func(t *testing.T) {
t.Run("detach minhash function rejected", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
@@ -8991,10 +8991,11 @@ func TestValidateDropFunction(t *testing.T) {
},
}
err := validateDropFunction(schema, "minhash_func", false)
assert.NoError(t, err)
assert.Error(t, err)
assert.Contains(t, err.Error(), "detaching a function without dropping its output field is not supported")
})
t.Run("detach bm25 function fails", func(t *testing.T) {
t.Run("detach bm25 function rejected", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
@@ -9010,7 +9011,26 @@ func TestValidateDropFunction(t *testing.T) {
}
err := validateDropFunction(schema, "bm25_func", false)
assert.Error(t, err)
assert.Contains(t, err.Error(), "BM25 function must be dropped with its output field")
assert.Contains(t, err.Error(), "detaching a function without dropping its output field is not supported")
})
t.Run("detach text embedding function rejected (coupled like all types)", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 102, Name: "vec_func_out", DataType: schemapb.DataType_FloatVector},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "embed_func", Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text"}, OutputFieldNames: []string{"vec_func_out"},
},
},
}
err := validateDropFunction(schema, "embed_func", false)
assert.Error(t, err)
assert.Contains(t, err.Error(), "detaching a function without dropping its output field is not supported")
})
t.Run("drop function field leaves another vector field", func(t *testing.T) {
@@ -9051,7 +9071,7 @@ func TestValidateDropFunction(t *testing.T) {
assert.NoError(t, err)
})
t.Run("drop unsupported function field fails", func(t *testing.T) {
t.Run("drop text embedding function field succeeds", func(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "id", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
@@ -9067,8 +9087,7 @@ func TestValidateDropFunction(t *testing.T) {
},
}
err := validateDropFunction(schema, "embed_func", true)
assert.Error(t, err)
assert.Contains(t, err.Error(), "only BM25 and MinHash functions support dropping output fields")
assert.NoError(t, err)
})
t.Run("drop function field would leave no vector field", func(t *testing.T) {
@@ -61,9 +61,11 @@ func validateAlterCollectionFieldAnalyzerMutation(schema *schemapb.CollectionSch
return merr.WrapErrParameterInvalidMsg("can not alter analyzer params for non-string field %s", fieldName)
}
fieldHelper := typeutil.CreateFieldSchemaHelper(field)
if fieldHelper.EnableMatch() || typeutil.IsBm25FunctionInputField(schema, field) {
if fieldHelper.EnableMatch() ||
typeutil.IsBm25FunctionInputField(schema, field) ||
typeutil.IsMinHashFunctionInputField(schema, field) {
return merr.WrapErrParameterInvalidMsg(
"can not alter analyzer params for field %s after text match is enabled or BM25 function depends on it",
"can not alter analyzer params for field %s after text match is enabled or a BM25/MinHash function depends on it",
fieldName,
)
}
@@ -108,6 +108,9 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
if err := schemautil.ValidateAlterSchemaAddFunctionPlan(plan); err != nil {
return err
}
if err := schemautil.CheckNoFunctionCascade(coll.ToCollectionSchemaPB().GetFunctions(), plan.Function); err != nil {
return err
}
for _, function := range coll.Functions {
if function.Name == plan.Function.GetName() {
return merr.WrapErrParameterInvalidMsg("function already exists, name: %s", plan.Function.GetName())
@@ -371,11 +374,11 @@ func (c *Core) broadcastAlterCollectionSchemaDrop(ctx context.Context, broadcast
switch id := dropReq.GetIdentifier().(type) {
case *milvuspb.AlterCollectionSchemaRequest_DropRequest_FunctionName:
if dropReq.GetDropFunctionOutputFields() {
schema, properties, droppedFieldIds, err = buildSchemaForDropFunctionField(coll, id.FunctionName)
} else {
schema, properties, droppedFieldIds, err = buildSchemaForDetachFunction(coll, id.FunctionName)
if !dropReq.GetDropFunctionOutputFields() {
return merr.WrapErrParameterInvalidMsg(
"detaching a function without dropping its output field is not supported; drop_function always removes the function together with its output field: %s", id.FunctionName)
}
schema, properties, droppedFieldIds, err = buildSchemaForDropFunctionField(coll, id.FunctionName)
case *milvuspb.AlterCollectionSchemaRequest_DropRequest_FieldName:
schema, properties, droppedFieldIds, err = buildSchemaForDropField(coll, id.FieldName, 0)
case *milvuspb.AlterCollectionSchemaRequest_DropRequest_FieldId:
@@ -461,6 +464,12 @@ func buildSchemaForDropField(coll *model.Collection, fieldName string, fieldID i
newFields = append(newFields, model.MarshalFieldModel(f))
}
if droppedField != nil {
// Mirror the proxy guard for direct-coord callers: a field a function
// depends on must be dropped via the function DDL, not directly, else the
// function is orphaned and its stored output invalidated.
if fn, kind := functionReferencing(coll.Functions, droppedField.Name); fn != "" {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("field is referenced by function %s as %s, drop function first", fn, kind)
}
schema = coll.ToCollectionSchemaPB()
maxFieldID := maxAssignedFieldIDFromSchema(schema)
properties = updateMaxFieldIDProperty(coll.Properties, maxFieldID)
@@ -486,6 +495,11 @@ func buildSchemaForDropField(coll *model.Collection, fieldName string, fieldID i
newStructs = append(newStructs, model.MarshalStructArrayFieldModel(s))
}
if droppedStruct != nil {
for _, sub := range droppedStruct.Fields {
if fn, kind := functionReferencing(coll.Functions, sub.Name); fn != "" {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("cannot drop struct array field %s: sub-field %s is referenced by function %s as %s", droppedStruct.Name, sub.Name, fn, kind)
}
}
schema = coll.ToCollectionSchemaPB()
maxFieldID := maxAssignedFieldIDFromSchema(schema)
properties = updateMaxFieldIDProperty(coll.Properties, maxFieldID)
@@ -505,55 +519,44 @@ func buildSchemaForDropField(coll *model.Collection, fieldName string, fieldID i
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("field not found with id: %d", fieldID)
}
func buildSchemaForDetachFunction(coll *model.Collection, functionName string) (
schema *schemapb.CollectionSchema,
properties []*commonpb.KeyValuePair,
droppedFieldIds []int64,
err error,
) {
var targetFunc *model.Function
for _, fn := range coll.Functions {
if fn.Name == functionName {
targetFunc = fn
// resolveOutputFieldIDsFromNames re-derives a function's output field IDs from its
// OutputFieldNames against the current schema (the authoritative name->id mapping),
// and rejects if the persisted OutputFieldIDs disagree as a set. A drop deletes by
// field id, but the proxy guard authorizes by name; a stale/injected persisted id
// that no name resolves to (e.g. a primary key) would then be deleted past that
// guard. Re-resolving keeps authorization (name) and action (id) on one carrier.
// Read-only, unlike resolveFunctionFieldIDs which mutates the schema on add/alter.
func resolveOutputFieldIDsFromNames(fn *model.Function, fields []*model.Field) ([]int64, error) {
nameToID := make(map[string]int64, len(fields))
for _, f := range fields {
nameToID[f.Name] = f.FieldID
}
resolved := make([]int64, 0, len(fn.OutputFieldNames))
resolvedSet := make(map[int64]struct{}, len(fn.OutputFieldNames))
for _, name := range fn.OutputFieldNames {
id, ok := nameToID[name]
if !ok {
return nil, merr.WrapErrParameterInvalidMsg("function %s output field %s not found in schema", fn.Name, name)
}
resolved = append(resolved, id)
resolvedSet[id] = struct{}{}
}
persistedSet := make(map[int64]struct{}, len(fn.OutputFieldIDs))
for _, id := range fn.OutputFieldIDs {
persistedSet[id] = struct{}{}
}
mismatch := len(resolvedSet) != len(persistedSet)
for id := range persistedSet {
if _, ok := resolvedSet[id]; !ok {
mismatch = true
break
}
}
if targetFunc == nil {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("function not found: %s", functionName)
if mismatch {
return nil, merr.WrapErrParameterInvalidMsg(
"function %s persisted output field ids %v do not align with output field names %v; metadata may be corrupt", fn.Name, fn.OutputFieldIDs, fn.OutputFieldNames)
}
if targetFunc.Type == schemapb.FunctionType_BM25 {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("BM25 function must be dropped with its output field in drop_function_field interface: %s", functionName)
}
outputFieldIDSet := make(map[int64]struct{}, len(targetFunc.OutputFieldIDs))
for _, fid := range targetFunc.OutputFieldIDs {
outputFieldIDSet[fid] = struct{}{}
}
newFields := make([]*schemapb.FieldSchema, 0, len(coll.Fields))
for _, field := range coll.Fields {
fieldSchema := model.MarshalFieldModel(field)
if _, ok := outputFieldIDSet[field.FieldID]; ok {
fieldSchema.IsFunctionOutput = false
}
newFields = append(newFields, fieldSchema)
}
newFunctions := make([]*schemapb.FunctionSchema, 0, len(coll.Functions)-1)
for _, fn := range coll.Functions {
if fn.Name != functionName {
newFunctions = append(newFunctions, model.MarshalFunctionModel(fn))
}
}
schema = coll.ToCollectionSchemaPB()
properties = coll.Properties
schema.Fields = newFields
schema.Functions = newFunctions
schema.Properties = properties
schema.Version = coll.SchemaVersion + 1
return schema, properties, nil, nil
return resolved, nil
}
func buildSchemaForDropFunctionField(coll *model.Collection, functionName string) (
@@ -572,16 +575,29 @@ func buildSchemaForDropFunctionField(coll *model.Collection, functionName string
if targetFunc == nil {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("function not found: %s", functionName)
}
switch targetFunc.Type {
case schemapb.FunctionType_BM25, schemapb.FunctionType_MinHash:
default:
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("only BM25 and MinHash functions support dropping output fields: %s", functionName)
// Drop is uniform across function types (no backfill); unlike add_function_field
// it is not type-restricted. Re-resolve the delete set from names so an injected
// persisted id cannot be deleted past the name-based proxy guard.
outputFieldIDs, err := resolveOutputFieldIDsFromNames(targetFunc, coll.Fields)
if err != nil {
return nil, nil, nil, err
}
droppedFieldIds = append(droppedFieldIds, outputFieldIDs...)
outputFieldIDSet := make(map[int64]struct{}, len(outputFieldIDs))
for _, fid := range outputFieldIDs {
outputFieldIDSet[fid] = struct{}{}
}
droppedFieldIds = append(droppedFieldIds, targetFunc.OutputFieldIDs...)
outputFieldIDSet := make(map[int64]struct{}, len(targetFunc.OutputFieldIDs))
for _, fid := range targetFunc.OutputFieldIDs {
outputFieldIDSet[fid] = struct{}{}
// Mirror the proxy guard for direct-coord callers: dropping the output vector
// field(s) must not leave the collection with no vector field.
removedVectors := 0
for _, field := range coll.Fields {
if _, ok := outputFieldIDSet[field.FieldID]; ok && typeutil.IsVectorType(field.DataType) {
removedVectors++
}
}
if removedVectors > 0 && removedVectors >= len(typeutil.GetVectorFieldSchemas(coll.ToCollectionSchemaPB())) {
return nil, nil, nil, merr.WrapErrParameterInvalidMsg("cannot drop function %s: it would leave no vector field in the collection", functionName)
}
newFields := make([]*schemapb.FieldSchema, 0, len(coll.Fields))
@@ -608,3 +624,22 @@ func buildSchemaForDropFunctionField(coll *model.Collection, functionName string
return schema, properties, droppedFieldIds, nil
}
// functionReferencing returns the name of the first function referencing fieldName
// and its role ("input"/"output"), or "" if none. A field a function depends on
// must not be dropped directly (mirrors the proxy validateDropField guard).
func functionReferencing(functions []*model.Function, fieldName string) (string, string) {
for _, fn := range functions {
for _, in := range fn.InputFieldNames {
if in == fieldName {
return fn.Name, "input"
}
}
for _, out := range fn.OutputFieldNames {
if out == fieldName {
return fn.Name, "output"
}
}
}
return "", ""
}
@@ -346,7 +346,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
},
})
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
require.Contains(t, resp.GetAlterStatus().GetReason(), "output field missing_minhash_output")
require.Contains(t, resp.GetAlterStatus().GetReason(), "adding a function over existing fields is not supported")
// case 4.3: BM25 function with multiple output fields is rejected.
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
@@ -608,9 +608,9 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
},
}))
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
require.Contains(t, resp.GetAlterStatus().GetReason(), "output field missing_minhash_output_late")
require.Contains(t, resp.GetAlterStatus().GetReason(), "adding a function over existing fields is not supported")
// Function-only add cannot relabel an existing field as a function output.
// function-only add (marking an existing field as output) is no longer supported.
functionOnlyReq := buildAlterSchemaAddFunctionReq(dbName, collectionName, &schemapb.FunctionSchema{
Name: "minhash_existing_fn",
Type: schemapb.FunctionType_MinHash,
@@ -624,36 +624,9 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
},
})
resp, err = core.AlterCollectionSchema(ctx, functionOnlyReq)
alterErr = merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
require.ErrorContains(t, alterErr, "cannot repurpose existing field")
assertSchemaVersion(t, ctx, core, dbName, collectionName, 6)
coll, err = core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
require.NoError(t, err)
require.Len(t, coll.Functions, 1)
existingOutputFound := false
for _, field := range coll.Fields {
if field.Name == "existing_minhash_output" {
existingOutputFound = true
require.False(t, field.IsFunctionOutput)
}
}
require.True(t, existingOutputFound)
// function-only rejects output fields already owned by another function.
resp, err = core.AlterCollectionSchema(ctx, buildAlterSchemaAddFunctionReq(dbName, collectionName, &schemapb.FunctionSchema{
Name: "minhash_existing_fn2",
Type: schemapb.FunctionType_MinHash,
InputFieldNames: []string{"text_input"},
OutputFieldNames: []string{"existing_minhash_output"},
Params: []*commonpb.KeyValuePair{
{Key: "num_hashes", Value: "128"},
{Key: "shingle_size", Value: "3"},
{Key: "hash_function", Value: "xxhash64"},
{Key: "seed", Value: "42"},
},
}))
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
functionOnlyErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, functionOnlyErr, merr.ErrParameterInvalid)
require.ErrorContains(t, functionOnlyErr, "adding a function over existing fields is not supported")
// happy path: add sparse vector output field + BM25 function.
firstAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output", "bm25_fn")
@@ -797,7 +770,7 @@ func TestDDLCallbacksAlterCollectionSchemaAnalyzerFileResourceRefs(t *testing.T)
assertFieldNotExists(t, ctx, core, dbName, collectionName, fieldName)
}
func TestDDLCallbacksAlterCollectionSchemaRejectsFunctionOnlyMaterializedOutput(t *testing.T) {
func TestDDLCallbacksAlterCollectionSchemaRejectsFunctionOnlyAdd(t *testing.T) {
core := initStreamingSystemAndCore(t)
ctx := context.Background()
@@ -853,7 +826,7 @@ func TestDDLCallbacksAlterCollectionSchemaRejectsFunctionOnlyMaterializedOutput(
}))
alterErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
require.ErrorContains(t, alterErr, "cannot repurpose existing field")
require.ErrorContains(t, alterErr, "adding a function over existing fields is not supported")
assertSchemaVersion(t, ctx, core, dbName, collectionName, 2)
}
@@ -1084,83 +1057,6 @@ func mustParseInt64(s string) int64 {
return v
}
func TestBuildSchemaForDetachFunction(t *testing.T) {
t.Run("function not found", func(t *testing.T) {
coll := &model.Collection{
Functions: []*model.Function{
{Name: "func1", OutputFieldIDs: []int64{103}},
},
}
_, _, _, err := buildSchemaForDetachFunction(coll, "nonexistent")
require.Error(t, err)
require.Contains(t, err.Error(), "function not found")
})
t.Run("detach function keeps output fields", func(t *testing.T) {
coll := &model.Collection{
Name: "test_coll",
Fields: []*model.Field{
{FieldID: 100, Name: "pk"},
{FieldID: 101, Name: "text"},
{FieldID: 102, Name: "minhash_vec", IsFunctionOutput: true},
{FieldID: 103, Name: "dense_vec"},
},
Functions: []*model.Function{
{
Name: "minhash_func",
Type: schemapb.FunctionType_MinHash,
InputFieldIDs: []int64{101},
InputFieldNames: []string{"text"},
OutputFieldIDs: []int64{102},
OutputFieldNames: []string{"minhash_vec"},
},
{
Name: "embed_func",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldIDs: []int64{101},
InputFieldNames: []string{"text"},
OutputFieldIDs: []int64{103},
OutputFieldNames: []string{"dense_vec"},
},
},
Properties: []*commonpb.KeyValuePair{
{Key: common.MaxFieldIDKey, Value: "103"},
},
SchemaVersion: 3,
}
schema, properties, droppedFieldIDs, err := buildSchemaForDetachFunction(coll, "minhash_func")
require.NoError(t, err)
require.Empty(t, droppedFieldIDs)
require.Equal(t, coll.Properties, properties)
require.Equal(t, int32(4), schema.Version)
require.Len(t, schema.Fields, 4)
var minhashField *schemapb.FieldSchema
for _, field := range schema.Fields {
if field.GetName() == "minhash_vec" {
minhashField = field
break
}
}
require.NotNil(t, minhashField)
require.False(t, minhashField.GetIsFunctionOutput())
require.Len(t, schema.Functions, 1)
require.Equal(t, "embed_func", schema.Functions[0].GetName())
})
t.Run("detach bm25 function fails", func(t *testing.T) {
coll := &model.Collection{
Functions: []*model.Function{
{Name: "bm25_func", Type: schemapb.FunctionType_BM25, OutputFieldIDs: []int64{102}},
},
}
_, _, _, err := buildSchemaForDetachFunction(coll, "bm25_func")
require.Error(t, err)
require.Contains(t, err.Error(), "BM25 function must be dropped with its output field")
})
}
func TestBuildSchemaForDropFunctionField(t *testing.T) {
t.Run("function not found", func(t *testing.T) {
coll := &model.Collection{
@@ -1173,15 +1069,43 @@ func TestBuildSchemaForDropFunctionField(t *testing.T) {
require.Contains(t, err.Error(), "function not found")
})
t.Run("unsupported function type", func(t *testing.T) {
t.Run("dropping the last vector field rejected", func(t *testing.T) {
coll := &model.Collection{
Functions: []*model.Function{
{Name: "embed_func", Type: schemapb.FunctionType_TextEmbedding, OutputFieldIDs: []int64{103}},
Fields: []*model.Field{
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 102, Name: "only_vec", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
},
Functions: []*model.Function{
{Name: "embed_func", Type: schemapb.FunctionType_TextEmbedding, InputFieldIDs: []int64{101}, OutputFieldIDs: []int64{102}, OutputFieldNames: []string{"only_vec"}},
},
SchemaVersion: 1,
}
_, _, _, err := buildSchemaForDropFunctionField(coll, "embed_func")
require.Error(t, err)
require.Contains(t, err.Error(), "only BM25 and MinHash functions support dropping output fields")
require.ErrorContains(t, err, "leave no vector field")
})
t.Run("text embedding function can be dropped", func(t *testing.T) {
coll := &model.Collection{
Name: "test_coll",
Fields: []*model.Field{
{FieldID: 100, Name: "pk"},
{FieldID: 101, Name: "text"},
{FieldID: 102, Name: "keep_vec"},
{FieldID: 103, Name: "dense_vec", IsFunctionOutput: true},
},
Functions: []*model.Function{
{Name: "embed_func", Type: schemapb.FunctionType_TextEmbedding, InputFieldIDs: []int64{101}, OutputFieldIDs: []int64{103}, OutputFieldNames: []string{"dense_vec"}},
},
SchemaVersion: 2,
}
schema, _, droppedFieldIDs, err := buildSchemaForDropFunctionField(coll, "embed_func")
require.NoError(t, err)
require.Equal(t, []int64{103}, droppedFieldIDs)
require.Equal(t, 0, len(schema.Functions))
for _, f := range schema.Fields {
require.NotEqual(t, "dense_vec", f.Name)
}
})
t.Run("drop function removes function and output fields", func(t *testing.T) {
@@ -1272,6 +1196,32 @@ func TestBuildSchemaForDropFunctionField(t *testing.T) {
require.Equal(t, 1, len(schema.Functions))
require.Equal(t, "embed_func", schema.Functions[0].Name)
})
t.Run("corrupt persisted output ids rejected (primary key protection)", func(t *testing.T) {
// Stored with output name "sparse_vec" (id 102) but a stale/injected pk id
// (100) in OutputFieldIDs; drop must re-resolve from the name and reject the
// mismatch instead of deleting the primary key past the name-based guard.
coll := &model.Collection{
Name: "test_coll",
Fields: []*model.Field{
{FieldID: 100, Name: "pk", DataType: schemapb.DataType_Int64, IsPrimaryKey: true},
{FieldID: 101, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 102, Name: "sparse_vec", DataType: schemapb.DataType_SparseFloatVector},
{FieldID: 103, Name: "keep_vec", DataType: schemapb.DataType_FloatVector},
},
Functions: []*model.Function{
{
Name: "bm25_func", Type: schemapb.FunctionType_BM25,
InputFieldIDs: []int64{101},
OutputFieldIDs: []int64{100, 102}, // 100 = pk injected
OutputFieldNames: []string{"sparse_vec"},
},
},
SchemaVersion: 3,
}
_, _, _, err := buildSchemaForDropFunctionField(coll, "bm25_func")
require.ErrorContains(t, err, "do not align with output field names")
})
}
func TestBuildSchemaForDropField(t *testing.T) {
@@ -1296,6 +1246,18 @@ func TestBuildSchemaForDropField(t *testing.T) {
require.Equal(t, int32(4), schema.Version)
})
t.Run("field referenced by function rejected", func(t *testing.T) {
coll := baseColl()
coll.Functions = []*model.Function{
{Name: "bm25", InputFieldNames: []string{"extra"}, OutputFieldNames: []string{"vec"}},
}
_, _, _, err := buildSchemaForDropField(coll, "extra", 0)
require.ErrorContains(t, err, "referenced by function bm25 as input, drop function first")
_, _, _, err = buildSchemaForDropField(coll, "vec", 0)
require.ErrorContains(t, err, "referenced by function bm25 as output, drop function first")
})
t.Run("drop by field id", func(t *testing.T) {
schema, _, droppedFieldIDs, err := buildSchemaForDropField(baseColl(), "", 101)
require.NoError(t, err)
@@ -26,6 +26,7 @@ import (
"github.com/milvus-io/milvus/internal/distributed/streaming"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/streamingcoord/server/broadcaster"
"github.com/milvus-io/milvus/internal/util/function/validator"
"github.com/milvus-io/milvus/pkg/v3/mlog"
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
@@ -89,6 +90,41 @@ func callAlterCollection(ctx context.Context, c *Core, broadcaster broadcaster.B
return nil
}
// resolveFunctionFieldIDs re-derives the function's input/output field IDs from
// its field names (the source of truth), discarding any client-supplied IDs — else
// a request could inject an unrelated field ID that a later drop_function_field
// would delete. It rejects unknown fields and output fields already owned by
// another function, and marks the resolved output fields as function outputs.
func resolveFunctionFieldIDs(ctx context.Context, fSchema *schemapb.FunctionSchema, fieldMapping map[string]*model.Field) error {
fSchema.InputFieldIds = make([]int64, 0, len(fSchema.InputFieldNames))
for _, name := range fSchema.InputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's input field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
fSchema.InputFieldIds = append(fSchema.InputFieldIds, field.FieldID)
}
fSchema.OutputFieldIds = make([]int64, 0, len(fSchema.OutputFieldNames))
for _, name := range fSchema.OutputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's output field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
if field.IsFunctionOutput {
err := merr.WrapErrParameterInvalidMsg("function's output field %s is already of other functions", name)
mlog.Error(ctx, "Incorrect function configuration: ", mlog.Err(err))
return err
}
fSchema.OutputFieldIds = append(fSchema.OutputFieldIds, field.FieldID)
field.IsFunctionOutput = true
}
return nil
}
func alterFunctionGenNewCollection(ctx context.Context, fSchema *schemapb.FunctionSchema, collection *model.Collection) error {
if fSchema == nil {
return merr.WrapErrParameterInvalidMsg("function schema is empty")
@@ -124,29 +160,8 @@ func alterFunctionGenNewCollection(ctx context.Context, fSchema *schemapb.Functi
field.IsFunctionOutput = false
}
for _, name := range fSchema.InputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's input field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
fSchema.InputFieldIds = append(fSchema.InputFieldIds, field.FieldID)
}
for _, name := range fSchema.OutputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's output field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
if field.IsFunctionOutput {
err := merr.WrapErrParameterInvalidMsg("function's output field %s is already of other functions", name)
mlog.Error(ctx, "Incorrect function configuration: ", mlog.Err(err))
return err
}
fSchema.OutputFieldIds = append(fSchema.OutputFieldIds, field.FieldID)
field.IsFunctionOutput = true
if err := resolveFunctionFieldIDs(ctx, fSchema, fieldMapping); err != nil {
return err
}
newFunc := model.UnmarshalFunctionModel(fSchema)
newFuncs = append(newFuncs, newFunc)
@@ -169,6 +184,18 @@ func (c *Core) broadcastAlterCollectionForAlterFunction(ctx context.Context, req
return err
}
// Only whitelisted params may be altered; function identity (type, name,
// input/output fields) is immutable. Skip when the function is absent so the
// mutation helper below emits the canonical "not exists" error.
for _, fn := range oldColl.Functions {
if fn.Name == req.GetFunctionSchema().GetName() {
if err := validator.CheckFunctionAlterAllowed(model.MarshalFunctionModel(fn), req.GetFunctionSchema()); err != nil {
return err
}
break
}
}
newColl := oldColl.Clone()
if err := alterFunctionGenNewCollection(ctx, req.FunctionSchema, newColl); err != nil {
return err
@@ -176,113 +203,3 @@ func (c *Core) broadcastAlterCollectionForAlterFunction(ctx context.Context, req
return callAlterCollection(ctx, c, broadcaster, oldColl, newColl, req.GetDbName(), req.GetCollectionName())
}
func (c *Core) broadcastAlterCollectionForDropFunction(ctx context.Context, req *milvuspb.DropCollectionFunctionRequest) error {
broadcaster, err := c.startBroadcastWithAliasOrCollectionLock(ctx, req.GetDbName(), req.GetCollectionName())
if err != nil {
return err
}
defer broadcaster.Close()
oldColl, err := c.meta.GetCollectionByName(ctx, req.GetDbName(), req.GetCollectionName(), typeutil.MaxTimestamp, false)
if err != nil {
return err
}
if err := rejectExternalCollectionFunctionMutation(oldColl.ToCollectionSchemaPB()); err != nil {
return err
}
var needDelFunc *model.Function
for _, f := range oldColl.Functions {
if f.Name == req.FunctionName {
needDelFunc = f
break
}
}
if needDelFunc == nil {
return nil
}
newColl := oldColl.Clone()
newFuncs := []*model.Function{}
for _, f := range newColl.Functions {
if f.Name != needDelFunc.Name {
newFuncs = append(newFuncs, f)
}
}
newColl.Functions = newFuncs
fieldMapping := map[int64]*model.Field{}
for _, field := range newColl.Fields {
fieldMapping[field.FieldID] = field
}
for _, id := range needDelFunc.OutputFieldIDs {
field, exists := fieldMapping[id]
if !exists {
return merr.WrapErrServiceInternalMsg("function's output field %d not exists", id)
}
field.IsFunctionOutput = false
}
return callAlterCollection(ctx, c, broadcaster, oldColl, newColl, req.GetDbName(), req.GetCollectionName())
}
func (c *Core) broadcastAlterCollectionForAddFunction(ctx context.Context, req *milvuspb.AddCollectionFunctionRequest) error {
broadcaster, err := c.startBroadcastWithAliasOrCollectionLock(ctx, req.GetDbName(), req.GetCollectionName())
if err != nil {
return err
}
defer broadcaster.Close()
oldColl, err := c.meta.GetCollectionByName(ctx, req.GetDbName(), req.GetCollectionName(), typeutil.MaxTimestamp, false)
if err != nil {
return err
}
newColl := oldColl.Clone()
fSchema := req.FunctionSchema
if fSchema == nil {
return merr.WrapErrParameterInvalidMsg("function schema is empty")
}
nextFunctionID := int64(StartOfUserFunctionID)
for _, f := range newColl.Functions {
if f.Name == fSchema.Name {
return merr.WrapErrParameterInvalidMsg("function name already exists: %s", f.Name)
}
nextFunctionID = max(nextFunctionID, f.ID+1)
}
fSchema.Id = nextFunctionID
fieldMapping := map[string]*model.Field{}
for _, field := range newColl.Fields {
fieldMapping[field.Name] = field
}
for _, name := range fSchema.InputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's input field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
fSchema.InputFieldIds = append(fSchema.InputFieldIds, field.FieldID)
}
for _, name := range fSchema.OutputFieldNames {
field, exists := fieldMapping[name]
if !exists {
err := merr.WrapErrParameterInvalidMsg("function's output field %s not exists", name)
mlog.Error(ctx, "Incorrect function configuration:", mlog.Err(err))
return err
}
if field.IsFunctionOutput {
err := merr.WrapErrParameterInvalidMsg("function's output field %s is already of other functions", name)
mlog.Error(ctx, "Incorrect function configuration: ", mlog.Err(err))
return err
}
fSchema.OutputFieldIds = append(fSchema.OutputFieldIds, field.FieldID)
field.IsFunctionOutput = true
}
newFunc := model.UnmarshalFunctionModel(fSchema)
newColl.Functions = append(newColl.Functions, newFunc)
return callAlterCollection(ctx, c, broadcaster, oldColl, newColl, req.GetDbName(), req.GetCollectionName())
}
@@ -235,6 +235,28 @@ func (suite *DDLCallbacksCollectionFunctionTestSuite) TestAlterFunctionGenNewCol
suite.Len(coll.Functions, 1) // Original + new
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestAlterFunctionGenNewCollection_DiscardsClientFieldIds() {
ctx := context.Background()
coll := suite.createTestCollection()
// Client injects bogus field IDs (output_field_ids points at the unrelated
// "vector" field 102). Only the name-resolved IDs must persist, else a later
// drop_function_field would delete field 102.
fSchema := &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text_field"},
InputFieldIds: []int64{999},
OutputFieldNames: []string{"output_field"},
OutputFieldIds: []int64{102},
}
err := alterFunctionGenNewCollection(ctx, fSchema, coll)
suite.NoError(err)
suite.Equal([]int64{103}, fSchema.InputFieldIds) // text_field only, 999 discarded
suite.Equal([]int64{104}, fSchema.OutputFieldIds) // output_field only, 102 discarded
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestAlterFunctionGenNewCollection_FunctionNotExists() {
ctx := context.Background()
coll := suite.createTestCollection()
@@ -277,126 +299,6 @@ func (suite *DDLCallbacksCollectionFunctionTestSuite) TestAlterFunctionGenNewCol
suite.Contains(err.Error(), "function's output field non_existent_field not exists")
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForAddFunction_FunctionNameExists() {
coll := suite.createTestCollection()
// Create request with existing function name
req := &milvuspb.AddCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "test_function", // Same as existing function
},
}
newColl := coll.Clone()
fSchema := req.FunctionSchema
// Check for function name conflict
for _, f := range newColl.Functions {
if f.Name == fSchema.Name {
err := errors.New("function name already exists")
suite.Error(err)
return
}
}
}
// Test broadcastAlterCollectionForDropFunction
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForDropFunction_Success() {
coll := suite.createTestCollection()
req := &milvuspb.DropCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionName: "test_function",
}
// Find function to delete
var needDelFunc *model.Function
for _, f := range coll.Functions {
if f.Name == req.FunctionName {
needDelFunc = f
break
}
}
suite.NotNil(needDelFunc, "Function should exist")
newColl := coll.Clone()
// Remove function from collection
newFuncs := []*model.Function{}
for _, f := range newColl.Functions {
if f.Name != needDelFunc.Name {
newFuncs = append(newFuncs, f)
}
}
newColl.Functions = newFuncs
// Reset output field flags
fieldMapping := map[int64]*model.Field{}
for _, field := range newColl.Fields {
fieldMapping[field.FieldID] = field
}
for _, id := range needDelFunc.OutputFieldIDs {
field, exists := fieldMapping[id]
suite.True(exists, "Output field should exist")
field.IsFunctionOutput = false
}
// Verify function was removed
suite.Len(newColl.Functions, 0)
// Verify output field is no longer marked as function output
outputField := fieldMapping[104] // output_field
suite.False(outputField.IsFunctionOutput)
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForDropFunction_FunctionNotExists() {
coll := suite.createTestCollection()
req := &milvuspb.DropCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionName: "non_existent_function",
}
// Find function to delete
var needDelFunc *model.Function
for _, f := range coll.Functions {
if f.Name == req.FunctionName {
needDelFunc = f
break
}
}
suite.Nil(needDelFunc, "Function should not exist")
// This should return nil (no error) as per the original implementation
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForDropFunction_OutputFieldNotExists() {
coll := suite.createTestCollection()
// Create a function with non-existent output field ID
needDelFunc := &model.Function{
ID: 1001,
Name: "test_function",
OutputFieldIDs: []int64{999}, // Non-existent field ID
}
newColl := coll.Clone()
fieldMapping := map[int64]*model.Field{}
for _, field := range newColl.Fields {
fieldMapping[field.FieldID] = field
}
for _, id := range needDelFunc.OutputFieldIDs {
_, exists := fieldMapping[id]
if !exists {
err := errors.New("function's output field not exists")
suite.Error(err)
return
}
}
}
// Test edge cases and error conditions
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestAlterFunctionGenNewCollection_OldOutputFieldNotExists() {
coll := suite.createTestCollection()
@@ -468,9 +370,14 @@ func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollecti
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_BM25,
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"vector"},
OutputFieldNames: []string{"output_field"},
// identity unchanged; only a whitelisted connection param ("url") is altered
Params: []*commonpb.KeyValuePair{
{Key: "param1", Value: "value1"},
{Key: "url", Value: "http://new-endpoint"},
},
},
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
@@ -480,6 +387,30 @@ func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollecti
suite.NoError(err)
})
suite.Run("altering non-whitelisted param rejected", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.AlterCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"output_field"},
Params: []*commonpb.KeyValuePair{
{Key: "param1", Value: "changed"},
},
},
}
err := suite.core.broadcastAlterCollectionForAlterFunction(context.Background(), req)
suite.ErrorContains(err, "cannot be altered")
})
suite.Run("external collection rejected", func() {
coll := suite.createTestCollection()
coll.Fields[2].ExternalField = "text_col"
@@ -502,141 +433,3 @@ func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollecti
suite.ErrorContains(err, externalCollectionFunctionMutationUnsupportedMsg)
})
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForDropFunction() {
suite.Run("success", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.DropCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionName: "test_function",
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
defer mocker.UnPatch()
err := suite.core.broadcastAlterCollectionForDropFunction(context.Background(), req)
suite.NoError(err)
})
suite.Run("external collection rejected", func() {
coll := suite.createTestCollection()
coll.Fields[2].ExternalField = "text_col"
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.DropCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionName: "test_function",
}
err := suite.core.broadcastAlterCollectionForDropFunction(context.Background(), req)
suite.ErrorContains(err, externalCollectionFunctionMutationUnsupportedMsg)
})
suite.Run("function not exists", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.DropCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionName: "non_existent_function",
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
defer mocker.UnPatch()
err := suite.core.broadcastAlterCollectionForDropFunction(context.Background(), req)
suite.NoError(err)
})
}
func (suite *DDLCallbacksCollectionFunctionTestSuite) TestBroadcastAlterCollectionForAddFunction() {
suite.Run("function name already exists", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.AddCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "test_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"vector"},
},
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
defer mocker.UnPatch()
err := suite.core.broadcastAlterCollectionForAddFunction(context.Background(), req)
suite.Error(err)
})
suite.Run("function input field not exists", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.AddCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "new_function",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"non_existent_field"},
OutputFieldNames: []string{"vector"},
},
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
defer mocker.UnPatch()
err := suite.core.broadcastAlterCollectionForAddFunction(context.Background(), req)
suite.Error(err)
suite.Contains(err.Error(), "function's input field non_existent_field not exists")
})
suite.Run("function output field not exists", func() {
coll := suite.createTestCollection()
mockMeta := mockrootcoord.NewIMetaTable(suite.T())
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil)
suite.core.meta = mockMeta
req := &milvuspb.AddCollectionFunctionRequest{
DbName: "test_db",
CollectionName: "test_collection",
FunctionSchema: &schemapb.FunctionSchema{
Name: "new_function2",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text_field"},
OutputFieldNames: []string{"non_existent_field"},
},
}
mocker := mockey.Mock(callAlterCollection).Return(nil).Build()
defer mocker.UnPatch()
err := suite.core.broadcastAlterCollectionForAddFunction(context.Background(), req)
suite.Error(err)
suite.Contains(err.Error(), "function's output field non_existent_field not exists")
})
}
+11 -42
View File
@@ -1423,31 +1423,16 @@ func (c *Core) AlterCollection(ctx context.Context, in *milvuspb.AlterCollection
return merr.Success(), nil
}
// AddCollectionFunction is the deprecated legacy attach RPC; it only allowed the
// unsafe attach-over-existing-field path. A function is coupled to its output field
// (BM25/MinHash via add_function_field; TextEmbedding at collection creation), so
// this path is rejected.
func (c *Core) AddCollectionFunction(ctx context.Context, in *milvuspb.AddCollectionFunctionRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionFunction", metrics.TotalLabel).Inc()
tr := timerecord.NewTimeRecorder("AddCollectionFunction")
mlog.Info(context.TODO(), "received request to Add collection function")
if err := c.broadcastAlterCollectionForAddFunction(ctx, in); err != nil {
if errors.Is(err, errIgnoredAlterCollection) {
mlog.Info(context.TODO(), "add collection function make no changes, ignore it")
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionFunction", metrics.SuccessLabel).Inc()
return merr.Success(), nil
}
mlog.Warn(context.TODO(), "failed to alter collection function", mlog.Err(err))
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionFunction", metrics.FailLabel).Inc()
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("AddCollectionFunction", metrics.SuccessLabel).Inc()
metrics.RootCoordDDLReqLatency.WithLabelValues("AddCollectionFunction").Observe(float64(tr.ElapseSpan().Milliseconds()))
mlog.Info(context.TODO(), "done to add collection function")
return merr.Success(), nil
return merr.Status(merr.WrapErrParameterInvalidMsg(
"AddCollectionFunction RPC is no longer supported; add BM25/MinHash via add_function_field, and define a TextEmbedding function at collection creation")), nil
}
func (c *Core) AlterCollectionFunction(ctx context.Context, in *milvuspb.AlterCollectionFunctionRequest) (*commonpb.Status, error) {
@@ -1477,31 +1462,15 @@ func (c *Core) AlterCollectionFunction(ctx context.Context, in *milvuspb.AlterCo
return merr.Success(), nil
}
// DropCollectionFunction is the deprecated legacy detach RPC; pymilvus routes
// drop through AlterCollectionSchema (drop_function_field / detach), so this path
// is unused. Reject to avoid a second, divergent DDL path.
func (c *Core) DropCollectionFunction(ctx context.Context, in *milvuspb.DropCollectionFunctionRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(c.GetStateCode()); err != nil {
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("DropCollectionFunction", metrics.TotalLabel).Inc()
tr := timerecord.NewTimeRecorder("DropCollectionFunction")
mlog.Info(context.TODO(), "received request to drop collection function")
if err := c.broadcastAlterCollectionForDropFunction(ctx, in); err != nil {
if errors.Is(err, errIgnoredAlterCollection) {
mlog.Info(context.TODO(), "Drop collection function make no changes, ignore it")
metrics.RootCoordDDLReqCounter.WithLabelValues("DropCollectionFunction", metrics.SuccessLabel).Inc()
return merr.Success(), nil
}
mlog.Warn(context.TODO(), "failed to drop collection function", mlog.Err(err))
metrics.RootCoordDDLReqCounter.WithLabelValues("DropCollectionFunction", metrics.FailLabel).Inc()
return merr.Status(err), nil
}
metrics.RootCoordDDLReqCounter.WithLabelValues("DropCollectionFunction", metrics.SuccessLabel).Inc()
metrics.RootCoordDDLReqLatency.WithLabelValues("DropCollectionFunction").Observe(float64(tr.ElapseSpan().Milliseconds()))
mlog.Info(context.TODO(), "done to drop collection function")
return merr.Success(), nil
return merr.Status(merr.WrapErrParameterInvalidMsg(
"DropCollectionFunction RPC is no longer supported; drop a function via drop_function_field")), nil
}
func (c *Core) AlterCollectionField(ctx context.Context, in *milvuspb.AlterCollectionFieldRequest) (*commonpb.Status, error) {
+9 -34
View File
@@ -1397,26 +1397,14 @@ func TestRootCoord_AddCollectionFunction(t *testing.T) {
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
t.Run("run ok", func(t *testing.T) {
// Deprecated legacy RPC: always rejects (add BM25/MinHash via add_function_field,
// define TextEmbedding at collection creation).
t.Run("rejected", func(t *testing.T) {
ctx := context.Background()
c := initStreamingSystemAndCore(t)
defer c.Stop()
mocker := mockey.Mock((*Core).broadcastAlterCollectionForAddFunction).Return(nil).Build()
defer mocker.UnPatch()
c := newTestCore(withHealthyCode())
resp, err := c.AddCollectionFunction(ctx, &milvuspb.AddCollectionFunctionRequest{})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
t.Run("run failed", func(t *testing.T) {
ctx := context.Background()
c := initStreamingSystemAndCore(t)
defer c.Stop()
mocker := mockey.Mock((*Core).broadcastAlterCollectionForAddFunction).Return(fmt.Errorf("")).Build()
defer mocker.UnPatch()
resp, err := c.AddCollectionFunction(ctx, &milvuspb.AddCollectionFunctionRequest{})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetErrorCode())
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
}
@@ -1429,26 +1417,13 @@ func TestRootCoord_DropCollectionFunction(t *testing.T) {
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
t.Run("run ok", func(t *testing.T) {
// Deprecated legacy RPC: always rejects (drop routes through drop_function_field).
t.Run("rejected", func(t *testing.T) {
ctx := context.Background()
c := initStreamingSystemAndCore(t)
defer c.Stop()
mocker := mockey.Mock((*Core).broadcastAlterCollectionForDropFunction).Return(nil).Build()
defer mocker.UnPatch()
c := newTestCore(withHealthyCode())
resp, err := c.DropCollectionFunction(ctx, &milvuspb.DropCollectionFunctionRequest{})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
t.Run("run failed", func(t *testing.T) {
ctx := context.Background()
c := initStreamingSystemAndCore(t)
defer c.Stop()
mocker := mockey.Mock((*Core).broadcastAlterCollectionForDropFunction).Return(fmt.Errorf("")).Build()
defer mocker.UnPatch()
resp, err := c.DropCollectionFunction(ctx, &milvuspb.DropCollectionFunctionRequest{})
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetErrorCode())
assert.NotEqual(t, commonpb.ErrorCode_Success, resp.GetErrorCode())
})
}
@@ -0,0 +1,126 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validator
import (
"slices"
"strings"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/util/function/models"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)
// alterableFunctionParams lists, per function type, the param keys that
// alter_function may change. Only pure connection/auth/perf knobs are allowed —
// params that never change the produced vector for a given input. Any param that
// can change the output vector's semantics or shape is immutable, because altering
// it on an already-backfilled output field would silently mix incompatible vectors
// in the same field. A function type absent from this map (BM25, MinHash) has no
// alterable params.
//
// Immutable (semantic) params, by first principle "same input must keep producing a
// compatible vector":
// - dim / model_name / provider / normalization / prompts: change the model or its
// output shape.
// - endpoint: for TEI the endpoint IS the model's identity, so repointing it can
// silently swap to a different model.
// - truncate / truncation_direction: for an over-length input they change which
// tokens are embedded (truncate on/off, head vs tail), so the produced vector
// differs — altering them mixes old and new vector semantics in one field.
//
// url stays alterable because model_name providers keep their identity in model_name
// and use url only as a relocatable API host.
var alterableFunctionParams = map[schemapb.FunctionType]typeutil.Set[string]{
schemapb.FunctionType_TextEmbedding: typeutil.NewSet(
models.URLParamKey,
models.CredentialParamKey,
models.TimeoutMsParamKey,
models.MaxClientBatchSizeParamKey,
models.RegionParamKey,
models.LocationParamKey,
models.ProjectIDParamKey,
models.UserParamKey,
),
}
// CheckFunctionAlterAllowed rejects an alter_function request that changes
// anything other than whitelisted params. The function identity (type, name,
// input/output fields) is immutable; only connection/runtime params may change.
func CheckFunctionAlterAllowed(oldFn, newFn *schemapb.FunctionSchema) error {
if oldFn.GetType() != newFn.GetType() {
return merr.WrapErrParameterInvalidMsg("function type cannot be altered: function %s", oldFn.GetName())
}
if oldFn.GetName() != newFn.GetName() {
return merr.WrapErrParameterInvalidMsg("function name cannot be altered: %s -> %s", oldFn.GetName(), newFn.GetName())
}
if !slices.Equal(oldFn.GetInputFieldNames(), newFn.GetInputFieldNames()) {
return merr.WrapErrParameterInvalidMsg("function input fields cannot be altered: function %s", oldFn.GetName())
}
if !slices.Equal(oldFn.GetOutputFieldNames(), newFn.GetOutputFieldNames()) {
return merr.WrapErrParameterInvalidMsg("function output fields cannot be altered: function %s", oldFn.GetName())
}
whitelist := alterableFunctionParams[oldFn.GetType()]
if len(whitelist) == 0 {
return merr.WrapErrParameterInvalidMsg("function type %s has no alterable params", oldFn.GetType().String())
}
oldParams, err := normalizeParams(oldFn.GetParams())
if err != nil {
return err
}
newParams, err := normalizeParams(newFn.GetParams())
if err != nil {
return err
}
changed := typeutil.NewSet[string]()
for k, ov := range oldParams {
if nv, ok := newParams[k]; !ok || nv != ov {
changed.Insert(k)
}
}
for k := range newParams {
if _, ok := oldParams[k]; !ok {
changed.Insert(k)
}
}
for k := range changed {
if !whitelist.Contain(k) {
return merr.WrapErrParameterInvalidMsg(
"function param %q cannot be altered for function %s; only connection/runtime params may be changed", k, oldFn.GetName())
}
}
return nil
}
// normalizeParams lowercases param keys (matching how providers read them, e.g.
// getProvider) and rejects duplicate keys, so a crafted duplicate key cannot slip
// a non-whitelisted change past the diff while the runtime binds to a different
// occurrence.
func normalizeParams(params []*commonpb.KeyValuePair) (map[string]string, error) {
m := make(map[string]string, len(params))
for _, p := range params {
k := strings.ToLower(p.GetKey())
if _, dup := m[k]; dup {
return nil, merr.WrapErrParameterInvalidMsg("duplicate function param key %q", p.GetKey())
}
m[k] = p.GetValue()
}
return m, nil
}
@@ -0,0 +1,127 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validator
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
)
func kv(k, v string) *commonpb.KeyValuePair { return &commonpb.KeyValuePair{Key: k, Value: v} }
func embFn(params ...*commonpb.KeyValuePair) *schemapb.FunctionSchema {
return &schemapb.FunctionSchema{
Name: "emb",
Type: schemapb.FunctionType_TextEmbedding,
InputFieldNames: []string{"text"},
OutputFieldNames: []string{"vec"},
Params: params,
}
}
func TestCheckFunctionAlterAllowed(t *testing.T) {
old := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://a"))
t.Run("change whitelisted param -> ok", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://b"))
assert.NoError(t, CheckFunctionAlterAllowed(old, newFn))
})
t.Run("add whitelisted param -> ok", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://a"), kv("timeout_ms", "1000"))
assert.NoError(t, CheckFunctionAlterAllowed(old, newFn))
})
t.Run("remove whitelisted param -> ok", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"))
assert.NoError(t, CheckFunctionAlterAllowed(old, newFn))
})
t.Run("change truncate/truncation_direction -> rejected (semantic for long inputs)", func(t *testing.T) {
oldT := embFn(kv("provider", "TEI"), kv("endpoint", "http://tei"))
newT := embFn(kv("provider", "TEI"), kv("endpoint", "http://tei"), kv("truncate", "true"), kv("truncation_direction", "Left"))
assert.ErrorContains(t, CheckFunctionAlterAllowed(oldT, newT), "cannot be altered")
})
t.Run("change non-whitelisted param -> rejected", func(t *testing.T) {
newFn := embFn(kv("model_name", "m2"), kv("dim", "8"), kv("url", "http://a"))
assert.ErrorContains(t, CheckFunctionAlterAllowed(old, newFn), "cannot be altered")
})
t.Run("change dim -> rejected", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "16"), kv("url", "http://a"))
assert.ErrorContains(t, CheckFunctionAlterAllowed(old, newFn), "cannot be altered")
})
t.Run("change endpoint -> rejected (TEI model identity)", func(t *testing.T) {
oldE := embFn(kv("model_name", "m1"), kv("endpoint", "http://tei-a"))
newE := embFn(kv("model_name", "m1"), kv("endpoint", "http://tei-b"))
assert.ErrorContains(t, CheckFunctionAlterAllowed(oldE, newE), "cannot be altered")
})
t.Run("duplicate provider key -> rejected (no last-wins bypass)", func(t *testing.T) {
// last-wins map would see provider unchanged; runtime getProvider is first-wins.
oldD := embFn(kv("provider", "openai"), kv("url", "http://a"))
newD := embFn(kv("provider", "evil"), kv("provider", "openai"), kv("url", "http://a"))
assert.ErrorContains(t, CheckFunctionAlterAllowed(oldD, newD), "duplicate function param key")
})
t.Run("change type -> rejected", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://a"))
newFn.Type = schemapb.FunctionType_BM25
assert.ErrorContains(t, CheckFunctionAlterAllowed(old, newFn), "type cannot be altered")
})
t.Run("change input field -> rejected", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://a"))
newFn.InputFieldNames = []string{"other"}
assert.ErrorContains(t, CheckFunctionAlterAllowed(old, newFn), "input fields cannot be altered")
})
t.Run("change output field -> rejected", func(t *testing.T) {
newFn := embFn(kv("model_name", "m1"), kv("dim", "8"), kv("url", "http://a"))
newFn.OutputFieldNames = []string{"other"}
assert.ErrorContains(t, CheckFunctionAlterAllowed(old, newFn), "output fields cannot be altered")
})
t.Run("bm25 has no alterable params", func(t *testing.T) {
oldBM := &schemapb.FunctionSchema{
Name: "b", Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"t"}, OutputFieldNames: []string{"s"},
}
newBM := &schemapb.FunctionSchema{
Name: "b", Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"t"}, OutputFieldNames: []string{"s"},
Params: []*commonpb.KeyValuePair{kv("k1", "1.5")},
}
assert.ErrorContains(t, CheckFunctionAlterAllowed(oldBM, newBM), "has no alterable params")
})
t.Run("minhash has no alterable params (even with no change)", func(t *testing.T) {
mh := func() *schemapb.FunctionSchema {
return &schemapb.FunctionSchema{
Name: "m", Type: schemapb.FunctionType_MinHash,
InputFieldNames: []string{"t"}, OutputFieldNames: []string{"sig"},
}
}
assert.ErrorContains(t, CheckFunctionAlterAllowed(mh(), mh()), "has no alterable params")
})
}
@@ -87,6 +87,17 @@ func ValidateFunction(coll *schemapb.CollectionSchema, needValidateFunctionName
return err
}
}
// No cascade: a function's input must not be another function's output. A field
// cannot be a function's own input and output (rejected above), so any input in
// usedOutputField belongs to a different function. Enforced here so it holds on
// every path (create, legacy add, alter), not only add_function_field.
for _, function := range coll.GetFunctions() {
for _, name := range function.GetInputFieldNames() {
if usedOutputField.Contain(name) {
return merr.WrapErrParameterInvalidMsg("function %s input field %s is the output of another function; function cascade is not supported", function.GetName(), name)
}
}
}
if !disableRuntimeCheck {
if err := embedding.ValidateFunctions(coll, needValidateFunctionName, &models.ModelExtraInfo{ClusterID: paramtable.Get().CommonCfg.ClusterPrefix.GetValue(), DBName: coll.DbName}); err != nil {
return err
+24 -4
View File
@@ -113,10 +113,8 @@ func ValidateAlterSchemaAddFunctionPlan(plan *AlterSchemaAddPlan) error {
function := plan.Function
switch plan.Kind {
case AlterSchemaAddFunction:
if function.GetType() == schemapb.FunctionType_BM25 {
return merr.WrapErrParameterInvalidMsg("BM25 function must be added with its output field in add_function_field interface")
}
return nil
return merr.WrapErrParameterInvalidMsg(
"adding a function over existing fields is not supported; use add_function_field to add the function together with its new output field")
case AlterSchemaAddFunctionField:
if err := validateAddFunctionFieldAllowed(function); err != nil {
return err
@@ -173,3 +171,25 @@ func validateAddFunctionFieldInputOutput(function *schemapb.FunctionSchema) erro
return merr.WrapErrParameterInvalidMsg("unsupported function type in alter schema task: %s", function.GetType().String())
}
}
// CheckNoFunctionCascade rejects a new function whose input field is the output
// field of an existing function. Cascade (function-on-function) is unsupported:
// the executor has no topological execution and backfill has no dependency
// ordering, so a chained output can never be materialized in a defined order.
func CheckNoFunctionCascade(existingFunctions []*schemapb.FunctionSchema, newFunction *schemapb.FunctionSchema) error {
if newFunction == nil {
return nil
}
existingOutputs := typeutil.NewSet[string]()
for _, fn := range existingFunctions {
existingOutputs.Insert(fn.GetOutputFieldNames()...)
}
for _, input := range newFunction.GetInputFieldNames() {
if existingOutputs.Contain(input) {
return merr.WrapErrParameterInvalidMsg(
"function %s input field %s is the output of another function; function cascade is not supported",
newFunction.GetName(), input)
}
}
return nil
}
@@ -0,0 +1,57 @@
// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package schemautil
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
)
func TestValidateAlterSchemaAddFunctionPlan_StandaloneAddRejected(t *testing.T) {
plan := &AlterSchemaAddPlan{
Kind: AlterSchemaAddFunction,
Function: &schemapb.FunctionSchema{Name: "f", Type: schemapb.FunctionType_TextEmbedding},
}
err := ValidateAlterSchemaAddFunctionPlan(plan)
assert.Error(t, err)
assert.Contains(t, err.Error(), "adding a function over existing fields is not supported")
}
func TestCheckNoFunctionCascade(t *testing.T) {
existing := []*schemapb.FunctionSchema{
{Name: "bm25", OutputFieldNames: []string{"sparse"}},
}
t.Run("input is another function's output -> rejected", func(t *testing.T) {
newFn := &schemapb.FunctionSchema{Name: "f2", InputFieldNames: []string{"sparse"}, OutputFieldNames: []string{"vec"}}
err := CheckNoFunctionCascade(existing, newFn)
assert.Error(t, err)
assert.Contains(t, err.Error(), "cascade is not supported")
})
t.Run("input is a free field -> ok", func(t *testing.T) {
newFn := &schemapb.FunctionSchema{Name: "f2", InputFieldNames: []string{"text"}, OutputFieldNames: []string{"vec"}}
assert.NoError(t, CheckNoFunctionCascade(existing, newFn))
})
t.Run("nil function -> ok", func(t *testing.T) {
assert.NoError(t, CheckNoFunctionCascade(existing, nil))
})
}
+14 -2
View File
@@ -4073,12 +4073,12 @@ func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb
return false
}
func IsBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
func isFunctionInputFieldOfType(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema, fnType schemapb.FunctionType) bool {
if coll == nil || field == nil {
return false
}
for _, fn := range coll.GetFunctions() {
if fn.GetType() != schemapb.FunctionType_BM25 {
if fn.GetType() != fnType {
continue
}
if field.GetFieldID() != 0 && len(fn.GetInputFieldIds()) > 0 {
@@ -4098,6 +4098,18 @@ func IsBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.F
return false
}
func IsBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
return isFunctionInputFieldOfType(coll, field, schemapb.FunctionType_BM25)
}
// IsMinHashFunctionInputField reports whether field is the input of a MinHash
// function. Like BM25, MinHash tokenizes the input text through the field's
// analyzer, so its analyzer params must not change after backfill or stored
// signatures would silently disagree with newly written ones.
func IsMinHashFunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
return isFunctionInputFieldOfType(coll, field, schemapb.FunctionType_MinHash)
}
func IsMinHashFunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
if !IsFunctionOutputField(collSchema, field) || field.GetDataType() != schemapb.DataType_BinaryVector {
return false
+39
View File
@@ -5060,6 +5060,45 @@ func TestIsBm25FunctionInputField(t *testing.T) {
assert.False(t, IsBm25FunctionInputField(nilSchema, nilSchema.Fields[0]))
}
func TestIsMinHashFunctionInputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{FieldID: 100, Name: "text", DataType: schemapb.DataType_VarChar},
{FieldID: 101, Name: "sig", DataType: schemapb.DataType_BinaryVector, IsFunctionOutput: true},
{FieldID: 102, Name: "bm25_in", DataType: schemapb.DataType_VarChar},
{FieldID: 103, Name: "sparse", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "minhash_func", Type: schemapb.FunctionType_MinHash,
InputFieldIds: []int64{100}, InputFieldNames: []string{"text"},
OutputFieldIds: []int64{101}, OutputFieldNames: []string{"sig"},
},
{
Name: "bm25_func", Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{102}, InputFieldNames: []string{"bm25_in"},
OutputFieldIds: []int64{103}, OutputFieldNames: []string{"sparse"},
},
},
}
// each helper matches only its own function type's input field
assert.True(t, IsMinHashFunctionInputField(schema, schema.Fields[0])) // text -> minhash input
assert.False(t, IsBm25FunctionInputField(schema, schema.Fields[0]))
assert.True(t, IsBm25FunctionInputField(schema, schema.Fields[2])) // bm25_in -> bm25 input
assert.False(t, IsMinHashFunctionInputField(schema, schema.Fields[2]))
assert.False(t, IsMinHashFunctionInputField(schema, schema.Fields[1])) // output field
// name fallback (no field ids assigned yet) + nil safety
schemaNoIDs := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{{Name: "text", DataType: schemapb.DataType_VarChar}},
Functions: []*schemapb.FunctionSchema{
{Name: "mh", Type: schemapb.FunctionType_MinHash, InputFieldNames: []string{"text"}},
},
}
assert.True(t, IsMinHashFunctionInputField(schemaNoIDs, schemaNoIDs.Fields[0]))
assert.False(t, IsMinHashFunctionInputField(nil, schemaNoIDs.Fields[0]))
}
func TestIsMinHashFunctionOutputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
@@ -301,7 +301,8 @@ class TestMilvusClientDropFieldFeature(TestMilvusClientV2Base):
check_items={
ct.err_code: 1100,
ct.err_msg: (
"BM25 function must be dropped with its output field in drop_function_field interface: "
"detaching a function without dropping its output field is not supported; "
"drop_function always removes the function together with its output field: "
"bm25: invalid parameter"
),
},
@@ -1250,7 +1251,8 @@ class TestMilvusClientDropFieldFeature(TestMilvusClientV2Base):
check_items={
ct.err_code: 1100,
ct.err_msg: (
"BM25 function must be dropped with its output field in drop_function_field interface: "
"detaching a function without dropping its output field is not supported; "
"drop_function always removes the function together with its output field: "
"bm25: invalid parameter"
),
},
@@ -1327,9 +1329,9 @@ class TestMilvusClientDropFieldFeature(TestMilvusClientV2Base):
"""
TC-L1-07a: Drop Function detach vs cascade semantics for MinHash.
target: verify MinHash supports both detach-only and cascade-output-field drop paths
method: drop_collection_function keeps output field/index; drop_function_field removes output field/index
expected: detach removes only the function; cascade removes function, output field, and output index
target: verify MinHash detach-only drop is rejected and cascade-output-field drop works
method: drop_collection_function (detach) is rejected; drop_function_field removes function + output field/index
expected: detach rejected (function/field unchanged); cascade removes function, output field, and output index
"""
client = self._client()
detach_collection_name = f"{cf.gen_collection_name_by_testcase_name()}_detach"
@@ -1393,30 +1395,31 @@ class TestMilvusClientDropFieldFeature(TestMilvusClientV2Base):
assert len(search_res[0]) > 0
assert "doc" in search_res[0][0]["entity"]
# Step 1: Detach-only MinHash drop removes the function but keeps output field/index.
# Step 1: Detach-only MinHash drop is rejected; the function keeps its output field.
create_minhash_collection(detach_collection_name)
client.drop_collection_function(detach_collection_name, "text_to_minhash")
self.drop_collection_function(
client,
detach_collection_name,
"text_to_minhash",
check_task=ct.CheckTasks.err_res,
check_items={
ct.err_code: 1100,
ct.err_msg: (
"detaching a function without dropping its output field is not supported; "
"drop_function always removes the function together with its output field: "
"text_to_minhash: invalid parameter"
),
},
)
desc = client.describe_collection(detach_collection_name)
field_names = [field["name"] for field in desc["fields"]]
function_names = [func["name"] for func in desc.get("functions", [])]
assert "text_to_minhash" not in function_names
assert "text_to_minhash" in function_names
assert "mh" in field_names
assert "vec" in field_names
assert "mh" in client.list_indexes(detach_collection_name)
# Once detached, mh is a normal field and new writes must provide it or fail clearly.
self.insert(
client,
collection_name=detach_collection_name,
data=[{"id": 100, "doc": "after detach", "vec": [1.0, 0.0, 0.0, 0.0]}],
check_task=ct.CheckTasks.err_res,
check_items={
ct.err_code: 1,
ct.err_msg: "Insert missed an field `mh` to collection without set nullable==true or set default_value",
},
)
# Step 2: Cascade MinHash drop removes the function, output field, and output index.
create_minhash_collection(cascade_collection_name)
self.drop_function_field(client, cascade_collection_name, "text_to_minhash")
File diff suppressed because it is too large Load Diff