mirror of
https://github.com/milvus-io/milvus.git
synced 2026-07-21 02:05:41 +00:00
fix: validate collection schema mutations in rootcoord (#51368)
## Problem Milvus supports online collection-schema mutations — add/drop field, add/drop function, alter field, enable dynamic field — through several RootCoord DDL paths. Existing segments are written under the committed schema, so a mutation that reinterprets an existing field (its data type, nullability, or structural role) or relabels existing data as a function output makes already-written binlogs read incorrectly. Before this change there was no single authoritative check that a proposed schema is a **safe structural successor** of the committed one; validation was scattered and incomplete across DDL paths, so an unsafe mutation could reach the WAL and silently corrupt or misinterpret existing data. ## What this PR does Adds an authoritative, validation-only, fail-closed gate `ValidateSchemaEvolution(oldSchema, newSchema)`, invoked in every RootCoord schema-mutating callback **before** any side effect (analyzer reservation, bound-index allocation, WAL broadcast). It rejects: - in-place field reinterpretation — name, data type, element type, nullability, or structural role (primary / partition / clustering key, autoID, dynamic); - relabeling an existing field as a function output — function outputs must use brand-new fields, never existing data; - unsafe field additions — a newly added field may not arrive already marked primary / partition / clustering key, function output, or dynamic; - unsafe drops of protected fields; - more than one clustering key; - a dynamic field that is also a function output; - corruption of the reserved `max_field_id` watermark. Property-only / TTL updates, analyzer rollback, BM25 / MinHash `add_function_field`, and legacy function Drop / detach paths are preserved unchanged. ## Scope Non-function structural invariants only. Function-graph validation (single producer, output binding, alter-time immutability, cascade) is intentionally **out of scope here and owned by #51360**. This PR does not include schema propagation, Proxy barriers, DML draining, backfill / readiness, index promotion, TextEmbedding enablement, protobuf, configuration, or C++ changes. ## Validation - `go test -tags dynamic,test -gcflags='all=-N -l' ./internal/util/schemautil -count=1` — PASS (the gate's own unit tests) - `git diff --check origin/master...HEAD` — PASS issue: #51340 --------- Signed-off-by: xiaofanluan <xf@hjjaq.com> Co-authored-by: xiaofanluan <xf@hjjaq.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
xiaofanluan
Claude Opus 4.8
parent
85c3aeb8c0
commit
66069bfc13
@@ -83,6 +83,9 @@ func (c *Core) broadcastAlterCollectionForAddField(ctx context.Context, req *mil
|
||||
if err := typeutil.ValidateTextRequiresStorageV3(schema, Params.CommonCfg.UseLoonFFI.GetAsBool()); err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("%s", err.Error())
|
||||
}
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
|
||||
@@ -221,6 +221,7 @@ func getFieldSchema(fieldName string) []byte {
|
||||
fieldSchema := &schemapb.FieldSchema{
|
||||
Name: fieldName,
|
||||
DataType: schemapb.DataType_Int64,
|
||||
Nullable: true,
|
||||
}
|
||||
schemaBytes, _ := proto.Marshal(fieldSchema)
|
||||
return schemaBytes
|
||||
|
||||
@@ -78,6 +78,9 @@ func (c *Core) broadcastAlterCollectionForAddStructField(ctx context.Context, re
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, structArrayField)
|
||||
properties := updateMaxFieldIDProperty(coll.Properties, maxAssignedFieldIDFromSchema(schema))
|
||||
schema.Properties = properties
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
|
||||
@@ -119,6 +119,9 @@ func (c *Core) broadcastAlterCollectionV2ForAlterCollectionField(ctx context.Con
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -44,6 +44,10 @@ func (c *Core) broadcastAlterCollectionForAlterCollection(ctx context.Context, r
|
||||
return merr.WrapErrParameterInvalidMsg("can not provide properties and deletekeys at the same time")
|
||||
}
|
||||
|
||||
if err := validateReservedCollectionProperties(req.GetProperties(), req.GetDeleteKeys()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if hookutil.ContainsCipherProperties(req.GetProperties(), req.GetDeleteKeys()) {
|
||||
return merr.WrapErrParameterInvalidMsg("can not alter cipher related properties")
|
||||
}
|
||||
@@ -212,6 +216,18 @@ func (c *Core) broadcastAlterCollectionForAlterCollection(ctx context.Context, r
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateReservedCollectionProperties(properties []*commonpb.KeyValuePair, deleteKeys []string) error {
|
||||
for _, property := range properties {
|
||||
if property.GetKey() == common.MaxFieldIDKey {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot alter reserved collection property %s", common.MaxFieldIDKey)
|
||||
}
|
||||
}
|
||||
if funcutil.SliceContain(deleteKeys, common.MaxFieldIDKey) {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot delete reserved collection property %s", common.MaxFieldIDKey)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateNamespaceModeImmutable(properties []*commonpb.KeyValuePair, deleteKeys []string) error {
|
||||
for _, prop := range properties {
|
||||
if prop.GetKey() == common.NamespaceModeKey {
|
||||
@@ -293,6 +309,9 @@ func (c *Core) broadcastAlterCollectionForAlterDynamicField(ctx context.Context,
|
||||
schema.Fields = append(schema.Fields, fieldSchema)
|
||||
properties := updateMaxFieldIDProperty(coll.Properties, fieldSchema.GetFieldID())
|
||||
schema.Properties = properties
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
channels := make([]string, 0, len(coll.VirtualChannelNames)+1)
|
||||
channels = append(channels, streaming.WAL().ControlChannel())
|
||||
@@ -349,6 +368,9 @@ func (c *Core) broadcastDisableDynamicField(ctx context.Context, req *milvuspb.A
|
||||
schema.EnableDynamicField = false
|
||||
schema.Properties = properties
|
||||
schema.Version = coll.SchemaVersion + 1
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
channels := make([]string, 0, len(coll.VirtualChannelNames)+1)
|
||||
channels = append(channels, streaming.WAL().ControlChannel())
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/bytedance/mockey"
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -27,13 +28,16 @@ import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/distributed/streaming"
|
||||
imocks "github.com/milvus-io/milvus/internal/mocks"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_balancer"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_broadcaster"
|
||||
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
|
||||
"github.com/milvus-io/milvus/internal/streamingcoord/server/balancer/balance"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/messagespb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/types"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
@@ -215,6 +219,50 @@ func TestDDLCallbacksAlterCollectionProperties(t *testing.T) {
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
}
|
||||
|
||||
func TestAlterCollectionRejectsReservedMaxFieldIDProperty(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
properties []*commonpb.KeyValuePair
|
||||
deleteKeys []string
|
||||
}{
|
||||
{
|
||||
name: "alter",
|
||||
properties: []*commonpb.KeyValuePair{{Key: common.MaxFieldIDKey, Value: "999"}},
|
||||
},
|
||||
{
|
||||
name: "delete",
|
||||
deleteKeys: []string{common.MaxFieldIDKey},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
streaming.SetupNoopWALForTest()
|
||||
defer streaming.SetWALForTest(nil)
|
||||
|
||||
coll := new(DDLCallbacksCollectionFunctionTestSuite).createTestCollection()
|
||||
mockMeta := mockrootcoord.NewIMetaTable(t)
|
||||
mockMeta.EXPECT().GetCollectionByName(mock.Anything, "test_db", "test_collection", typeutil.MaxTimestamp, mock.Anything).Return(coll, nil).Maybe()
|
||||
core := newTestCore(withHealthyCode(), withMeta(mockMeta), withBroker(newValidMockBroker()))
|
||||
|
||||
mockBroadcaster := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
mockBroadcaster.EXPECT().Close().Maybe()
|
||||
mockBroadcaster.EXPECT().Broadcast(mock.Anything, mock.Anything).Return(&types.BroadcastAppendResult{}, nil).Maybe()
|
||||
lockMocker := mockey.Mock((*Core).startBroadcastWithAliasOrCollectionLock).Return(mockBroadcaster, nil).Build()
|
||||
defer lockMocker.UnPatch()
|
||||
cacheMocker := mockey.Mock((*Core).getCacheExpireForCollection).Return(nil, nil).Build()
|
||||
defer cacheMocker.UnPatch()
|
||||
|
||||
err := core.broadcastAlterCollectionForAlterCollection(context.Background(), &milvuspb.AlterCollectionRequest{
|
||||
DbName: "test_db",
|
||||
CollectionName: "test_collection",
|
||||
Properties: tc.properties,
|
||||
DeleteKeys: tc.deleteKeys,
|
||||
})
|
||||
require.ErrorIs(t, err, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, err, common.MaxFieldIDKey)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNamespaceModeImmutable(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
|
||||
@@ -119,6 +119,9 @@ func (c *Core) broadcastAlterCollectionSchemaAdd(ctx context.Context, broadcaste
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
if plan.HasFunction() {
|
||||
if err := validator.ValidateFunction(schema, plan.Function.GetName(), true); err != nil {
|
||||
return merr.Wrap(err, "invalid function schema")
|
||||
@@ -383,6 +386,9 @@ func (c *Core) broadcastAlterCollectionSchemaDrop(ctx context.Context, broadcast
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateSchemaEvolution(coll, schema); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheExpirations, err := c.getCacheExpireForCollection(ctx, req.GetDbName(), req.GetCollectionName())
|
||||
if err != nil {
|
||||
|
||||
@@ -581,6 +581,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
existingOutputFieldSchema := &schemapb.FieldSchema{
|
||||
Name: "existing_minhash_output",
|
||||
DataType: schemapb.DataType_BinaryVector,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "4096"},
|
||||
},
|
||||
@@ -609,7 +610,7 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
|
||||
require.Contains(t, resp.GetAlterStatus().GetReason(), "output field missing_minhash_output_late")
|
||||
|
||||
// happy path: add only a function and mark an existing output field.
|
||||
// Function-only add cannot relabel an existing field as a function output.
|
||||
functionOnlyReq := buildAlterSchemaAddFunctionReq(dbName, collectionName, &schemapb.FunctionSchema{
|
||||
Name: "minhash_existing_fn",
|
||||
Type: schemapb.FunctionType_MinHash,
|
||||
@@ -623,16 +624,18 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
},
|
||||
})
|
||||
resp, err = core.AlterCollectionSchema(ctx, functionOnlyReq)
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 7)
|
||||
alterErr = merr.CheckRPCCall(resp.GetAlterStatus(), err)
|
||||
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, alterErr, "cannot repurpose existing field")
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 6)
|
||||
coll, err = core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, coll.Functions, 2)
|
||||
require.Len(t, coll.Functions, 1)
|
||||
existingOutputFound := false
|
||||
for _, field := range coll.Fields {
|
||||
if field.Name == "existing_minhash_output" {
|
||||
existingOutputFound = true
|
||||
require.True(t, field.IsFunctionOutput)
|
||||
require.False(t, field.IsFunctionOutput)
|
||||
}
|
||||
}
|
||||
require.True(t, existingOutputFound)
|
||||
@@ -656,19 +659,19 @@ func TestDDLCallbacksBroadcastAlterCollectionSchema(t *testing.T) {
|
||||
firstAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output", "bm25_fn")
|
||||
resp, err = core.AlterCollectionSchema(ctx, firstAlterReq)
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 8)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 7)
|
||||
|
||||
// second happy path with DoPhysicalBackfill=true: the flag is ignored by alter schema.
|
||||
secondAlterReq := buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output2", "bm25_fn2")
|
||||
secondAlterReq.GetAction().GetAddRequest().DoPhysicalBackfill = true
|
||||
resp, err = core.AlterCollectionSchema(ctx, secondAlterReq)
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 9)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 8)
|
||||
updated, err := core.meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
schema := updated.ToCollectionSchemaPB()
|
||||
require.False(t, schema.GetDoPhysicalBackfill())
|
||||
require.EqualValues(t, 9, schema.GetVersion())
|
||||
require.EqualValues(t, 8, schema.GetVersion())
|
||||
|
||||
// case 9: function already exists (same name "bm25_fn")
|
||||
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
|
||||
@@ -794,7 +797,7 @@ func TestDDLCallbacksAlterCollectionSchemaAnalyzerFileResourceRefs(t *testing.T)
|
||||
assertFieldNotExists(t, ctx, core, dbName, collectionName, fieldName)
|
||||
}
|
||||
|
||||
func TestDDLCallbacksAlterCollectionSchemaValidatesFunctionOnlyFinalSchema(t *testing.T) {
|
||||
func TestDDLCallbacksAlterCollectionSchemaRejectsFunctionOnlyMaterializedOutput(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -822,6 +825,7 @@ func TestDDLCallbacksAlterCollectionSchemaValidatesFunctionOnlyFinalSchema(t *te
|
||||
invalidOutputBytes, err := proto.Marshal(&schemapb.FieldSchema{
|
||||
Name: "invalid_minhash_output",
|
||||
DataType: schemapb.DataType_FloatVector,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "128"},
|
||||
},
|
||||
@@ -849,7 +853,7 @@ func TestDDLCallbacksAlterCollectionSchemaValidatesFunctionOnlyFinalSchema(t *te
|
||||
}))
|
||||
alterErr := merr.CheckRPCCall(resp.GetAlterStatus(), err)
|
||||
require.ErrorIs(t, alterErr, merr.ErrParameterInvalid)
|
||||
require.ErrorContains(t, alterErr, "MinHash function output field must be a BinaryVector field")
|
||||
require.ErrorContains(t, alterErr, "cannot repurpose existing field")
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 2)
|
||||
}
|
||||
|
||||
@@ -1023,6 +1027,7 @@ func TestDDLCallbacksAlterCollectionSchemaAddSkipsSchemaDropReady(t *testing.T)
|
||||
varcharFieldSchema := &schemapb.FieldSchema{
|
||||
Name: "text_input",
|
||||
DataType: schemapb.DataType_VarChar,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "256"},
|
||||
{Key: common.EnableAnalyzerKey, Value: "true"},
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rootcoord
|
||||
|
||||
import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/internal/util/schemautil"
|
||||
)
|
||||
|
||||
func validateSchemaEvolution(oldColl *model.Collection, newSchema *schemapb.CollectionSchema) error {
|
||||
var oldSchema *schemapb.CollectionSchema
|
||||
if oldColl != nil {
|
||||
oldSchema = oldColl.ToCollectionSchemaPB()
|
||||
}
|
||||
return schemautil.ValidateSchemaEvolution(oldSchema, newSchema)
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package rootcoord
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/bytedance/mockey"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/internal/mocks"
|
||||
"github.com/milvus-io/milvus/internal/mocks/streamingcoord/server/mock_broadcaster"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/proto/querypb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v3/streaming/util/types"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
func TestDDLCallbacksSchemaEvolutionRejectsUnsafeAddCollectionFieldBeforeSideEffects(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
field *schemapb.FieldSchema
|
||||
expectsAnalyzer bool
|
||||
}{
|
||||
{
|
||||
name: "non-nullable field without default",
|
||||
field: &schemapb.FieldSchema{
|
||||
Name: "required",
|
||||
DataType: schemapb.DataType_Int64,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
createCollectionForTest(t, ctx, core, dbName, collectionName)
|
||||
|
||||
analyzerCalls := 0
|
||||
if test.expectsAnalyzer {
|
||||
mixCoord := core.mixCoord.(*mocks.MixCoord)
|
||||
mixCoord.EXPECT().ValidateAnalyzer(mock.Anything, mock.Anything).Run(func(context.Context, *querypb.ValidateAnalyzerRequest) {
|
||||
analyzerCalls++
|
||||
}).Return(&querypb.ValidateAnalyzerResponse{Status: merr.Success()}, nil).Maybe()
|
||||
}
|
||||
broadcasts := 0
|
||||
mockBroadcastAPI := mock_broadcaster.NewMockBroadcastAPI(t)
|
||||
mockBroadcastAPI.EXPECT().Close().Return().Maybe()
|
||||
mockBroadcastAPI.EXPECT().Broadcast(mock.Anything, mock.Anything).Run(func(context.Context, message.BroadcastMutableMessage) {
|
||||
broadcasts++
|
||||
}).Return(&types.BroadcastAppendResult{}, nil).Maybe()
|
||||
lockMocker := mockey.Mock((*Core).startBroadcastWithAliasOrCollectionLock).Return(mockBroadcastAPI, nil).Build()
|
||||
t.Cleanup(func() { lockMocker.UnPatch() })
|
||||
|
||||
schemaBytes, err := proto.Marshal(test.field)
|
||||
require.NoError(t, err)
|
||||
resp, err := core.AddCollectionField(ctx, &milvuspb.AddCollectionFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
Schema: schemaBytes,
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 0)
|
||||
assertFieldNotExists(t, ctx, core, dbName, collectionName, test.field.GetName())
|
||||
|
||||
meta := core.meta.(*MetaTable)
|
||||
require.Empty(t, meta.fileResourceRefCnt, "validation must reject before analyzer resource reservation")
|
||||
require.Zero(t, analyzerCalls, "validation must reject before ValidateAnalyzer")
|
||||
require.Zero(t, broadcasts, "validation must reject before broadcast")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDDLCallbacksSchemaEvolutionRejectsInPlaceFieldMutation(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
|
||||
resp, err := core.CreateDatabase(ctx, &milvuspb.CreateDatabaseRequest{DbName: dbName})
|
||||
require.NoError(t, merr.CheckRPCCall(resp, err))
|
||||
schemaBytes, err := proto.Marshal(&schemapb.CollectionSchema{
|
||||
Name: collectionName,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "text", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: common.MaxLengthKey, Value: "128"}}},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
resp, err = core.CreateCollection(ctx, &milvuspb.CreateCollectionRequest{DbName: dbName, CollectionName: collectionName, Schema: schemaBytes})
|
||||
require.NoError(t, merr.CheckRPCCall(resp, err))
|
||||
|
||||
// Resizing max_length (grow or shrink) is allowed; removing the bound
|
||||
// entirely is still rejected, since it would drop the write-time bound.
|
||||
resp, err = core.AlterCollectionField(ctx, &milvuspb.AlterCollectionFieldRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
FieldName: "text",
|
||||
DeleteKeys: []string{common.MaxLengthKey},
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 0)
|
||||
assertFieldProperties(t, ctx, core, dbName, collectionName, "text", common.MaxLengthKey, "128")
|
||||
}
|
||||
|
||||
func TestDDLCallbacksSchemaEvolutionRejectsGraphBreakingAlterCollectionSchemaDrop(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
createCollectionForTest(t, ctx, core, dbName, collectionName)
|
||||
|
||||
resp, err := core.AlterCollectionSchema(ctx, buildAlterSchemaAddFieldSchemaReq(dbName, collectionName, &schemapb.FieldSchema{
|
||||
Name: "text_input",
|
||||
DataType: schemapb.DataType_VarChar,
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "128"},
|
||||
{Key: common.EnableAnalyzerKey, Value: "true"},
|
||||
},
|
||||
}, false))
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
resp, err = core.AlterCollectionSchema(ctx, buildAlterSchemaReq(dbName, collectionName, "text_input", "sparse_output", "bm25"))
|
||||
require.NoError(t, merr.CheckRPCCall(resp.GetAlterStatus(), err))
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 2)
|
||||
|
||||
resp, err = core.AlterCollectionSchema(ctx, &milvuspb.AlterCollectionSchemaRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
Action: &milvuspb.AlterCollectionSchemaRequest_Action{
|
||||
Op: &milvuspb.AlterCollectionSchemaRequest_Action_DropRequest{
|
||||
DropRequest: &milvuspb.AlterCollectionSchemaRequest_DropRequest{
|
||||
Identifier: &milvuspb.AlterCollectionSchemaRequest_DropRequest_FieldName{FieldName: "text_input"},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp.GetAlterStatus(), err), merr.ErrParameterInvalid)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 2)
|
||||
assertFieldExists(t, ctx, core, dbName, collectionName, "text_input", 101)
|
||||
}
|
||||
|
||||
func TestDDLCallbacksSchemaEvolutionRejectsUnsafeDynamicEnable(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
createCollectionForTest(t, ctx, core, dbName, collectionName)
|
||||
|
||||
meta := core.meta.(*MetaTable)
|
||||
coll, err := meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
meta.ddLock.Lock()
|
||||
meta.collID2Meta[coll.CollectionID].Fields = append(meta.collID2Meta[coll.CollectionID].Fields, &model.Field{
|
||||
FieldID: 101,
|
||||
Name: common.MetaFieldName,
|
||||
DataType: schemapb.DataType_JSON,
|
||||
IsDynamic: true,
|
||||
Nullable: true,
|
||||
})
|
||||
meta.ddLock.Unlock()
|
||||
|
||||
resp, err := core.AlterCollection(ctx, &milvuspb.AlterCollectionRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
Properties: []*commonpb.KeyValuePair{{Key: common.EnableDynamicSchemaKey, Value: "true"}},
|
||||
})
|
||||
require.ErrorIs(t, merr.CheckRPCCall(resp, err), merr.ErrParameterInvalid)
|
||||
assertSchemaVersion(t, ctx, core, dbName, collectionName, 0)
|
||||
}
|
||||
|
||||
func TestDDLCallbacksPropertyOnlyAlterSkipsSchemaEvolutionValidation(t *testing.T) {
|
||||
core := initStreamingSystemAndCore(t)
|
||||
ctx := context.Background()
|
||||
dbName := "testDB" + funcutil.RandomString(10)
|
||||
collectionName := "testCollection" + funcutil.RandomString(10)
|
||||
createCollectionForTest(t, ctx, core, dbName, collectionName)
|
||||
|
||||
meta := core.meta.(*MetaTable)
|
||||
coll, err := meta.GetCollectionByName(ctx, dbName, collectionName, typeutil.MaxTimestamp, false)
|
||||
require.NoError(t, err)
|
||||
meta.ddLock.Lock()
|
||||
meta.collID2Meta[coll.CollectionID].Fields[0].IsFunctionOutput = true
|
||||
meta.ddLock.Unlock()
|
||||
|
||||
resp, err := core.AlterCollection(ctx, &milvuspb.AlterCollectionRequest{
|
||||
DbName: dbName,
|
||||
CollectionName: collectionName,
|
||||
Properties: []*commonpb.KeyValuePair{{Key: "property_only", Value: "updated"}},
|
||||
})
|
||||
require.NoError(t, merr.CheckRPCCall(resp, err))
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package schemautil
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/merr"
|
||||
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
||||
)
|
||||
|
||||
// ValidateSchemaEvolution validates that a proposed schema can safely replace
|
||||
// the committed schema without reinterpreting existing rows or leaving an
|
||||
// inconsistent field graph.
|
||||
func ValidateSchemaEvolution(oldSchema, newSchema *schemapb.CollectionSchema) error {
|
||||
if oldSchema == nil || newSchema == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("old and new collection schemas must not be nil")
|
||||
}
|
||||
|
||||
oldFields, err := buildEvolutionFieldMaps(oldSchema)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newFields, err := buildEvolutionFieldMaps(newSchema)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateKeptEvolutionFields(oldFields, newFields); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAddedEvolutionFields(oldFields, newFields, newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateDroppedEvolutionFields(oldFields, newFields, newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
// Function-graph validation (single producer, output binding, alter-immutability,
|
||||
// cascade) is owned by internal/util/function/validator (see #51360); this gate
|
||||
// intentionally only enforces the non-function structural invariants below.
|
||||
if err := validateEvolutionClusteringKey(newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateEvolutionDynamicGraph(newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateMaxFieldIDEvolution(oldSchema, newSchema, oldFields, newFields)
|
||||
}
|
||||
|
||||
type evolutionFieldMaps struct {
|
||||
fields map[int64]*schemapb.FieldSchema
|
||||
structFields map[int64]*schemapb.StructArrayFieldSchema
|
||||
structSubFields map[int64]struct{}
|
||||
structParents map[int64]int64
|
||||
allIDs map[int64]string
|
||||
}
|
||||
|
||||
func buildEvolutionFieldMaps(schema *schemapb.CollectionSchema) (*evolutionFieldMaps, error) {
|
||||
result := &evolutionFieldMaps{
|
||||
fields: make(map[int64]*schemapb.FieldSchema),
|
||||
structFields: make(map[int64]*schemapb.StructArrayFieldSchema),
|
||||
structSubFields: make(map[int64]struct{}),
|
||||
structParents: make(map[int64]int64),
|
||||
allIDs: make(map[int64]string),
|
||||
}
|
||||
addID := func(id int64, kind string) error {
|
||||
if previous, ok := result.allIDs[id]; ok {
|
||||
return merr.WrapErrParameterInvalidMsg("duplicate field id %d is used by both %s and %s", id, previous, kind)
|
||||
}
|
||||
result.allIDs[id] = kind
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, field := range schema.GetFields() {
|
||||
if field == nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("collection schema contains a nil field")
|
||||
}
|
||||
if err := addID(field.GetFieldID(), "field"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.fields[field.GetFieldID()] = field
|
||||
}
|
||||
for _, structField := range schema.GetStructArrayFields() {
|
||||
if structField == nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("collection schema contains a nil struct field")
|
||||
}
|
||||
if err := addID(structField.GetFieldID(), "struct field"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.structFields[structField.GetFieldID()] = structField
|
||||
for _, field := range structField.GetFields() {
|
||||
if field == nil {
|
||||
return nil, merr.WrapErrParameterInvalidMsg("struct field %q contains a nil sub-field", structField.GetName())
|
||||
}
|
||||
if err := addID(field.GetFieldID(), "struct sub-field"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result.fields[field.GetFieldID()] = field
|
||||
result.structSubFields[field.GetFieldID()] = struct{}{}
|
||||
result.structParents[field.GetFieldID()] = structField.GetFieldID()
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func validateKeptEvolutionFields(oldFields, newFields *evolutionFieldMaps) error {
|
||||
for id, oldKind := range oldFields.allIDs {
|
||||
if newKind, kept := newFields.allIDs[id]; kept && oldKind != newKind {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change field id %d from %s to %s", id, oldKind, newKind)
|
||||
}
|
||||
}
|
||||
for id, oldParent := range oldFields.structParents {
|
||||
if newParent, kept := newFields.structParents[id]; kept && oldParent != newParent {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot move struct sub-field id %d from parent %d to parent %d", id, oldParent, newParent)
|
||||
}
|
||||
}
|
||||
for id, oldField := range oldFields.fields {
|
||||
newField, kept := newFields.fields[id]
|
||||
if !kept {
|
||||
continue
|
||||
}
|
||||
if oldFields.allIDs[id] != newFields.allIDs[id] {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the kind of field id %d in place", id)
|
||||
}
|
||||
if err := validateKeptEvolutionField(oldField, newField); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for id, oldField := range oldFields.structFields {
|
||||
newField, kept := newFields.structFields[id]
|
||||
if !kept {
|
||||
continue
|
||||
}
|
||||
if oldField.GetName() != newField.GetName() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot rename struct field id %d from %q to %q", id, oldField.GetName(), newField.GetName())
|
||||
}
|
||||
if oldField.GetNullable() != newField.GetNullable() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the nullability of struct field %q in place", oldField.GetName())
|
||||
}
|
||||
oldChildren := make(map[int64]struct{}, len(oldField.GetFields()))
|
||||
for _, child := range oldField.GetFields() {
|
||||
oldChildren[child.GetFieldID()] = struct{}{}
|
||||
}
|
||||
newChildren := make(map[int64]struct{}, len(newField.GetFields()))
|
||||
for _, child := range newField.GetFields() {
|
||||
newChildren[child.GetFieldID()] = struct{}{}
|
||||
}
|
||||
for childID := range oldChildren {
|
||||
if _, kept := newChildren[childID]; !kept {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop sub-field id %d from kept struct field %q", childID, oldField.GetName())
|
||||
}
|
||||
}
|
||||
for childID := range newChildren {
|
||||
if _, existed := oldChildren[childID]; !existed {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add sub-field id %d to kept struct field %q", childID, oldField.GetName())
|
||||
}
|
||||
}
|
||||
if err := validateNumericBoundsEvolution(oldField.GetName(), oldField.GetTypeParams(), newField.GetTypeParams()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateKeptEvolutionField(oldField, newField *schemapb.FieldSchema) error {
|
||||
name := oldField.GetName()
|
||||
switch {
|
||||
case name != newField.GetName():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot rename field id %d from %q to %q", oldField.GetFieldID(), name, newField.GetName())
|
||||
case oldField.GetDataType() != newField.GetDataType():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the data type of field %q in place", name)
|
||||
case oldField.GetElementType() != newField.GetElementType():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the element type of field %q in place", name)
|
||||
case oldField.GetNullable() != newField.GetNullable():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the nullability of field %q in place", name)
|
||||
case !oldField.GetIsFunctionOutput() && newField.GetIsFunctionOutput():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot repurpose existing field %q as a function output", name)
|
||||
case oldField.GetIsPrimaryKey() != newField.GetIsPrimaryKey(),
|
||||
oldField.GetIsPartitionKey() != newField.GetIsPartitionKey(),
|
||||
oldField.GetIsClusteringKey() != newField.GetIsClusteringKey(),
|
||||
oldField.GetAutoID() != newField.GetAutoID(),
|
||||
oldField.GetIsDynamic() != newField.GetIsDynamic():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change the structural role of field %q in place", name)
|
||||
}
|
||||
return validateNumericBoundsEvolution(name, oldField.GetTypeParams(), newField.GetTypeParams())
|
||||
}
|
||||
|
||||
func validateAddedEvolutionFields(oldFields, newFields *evolutionFieldMaps, newSchema *schemapb.CollectionSchema) error {
|
||||
for id, field := range newFields.fields {
|
||||
if _, existed := oldFields.allIDs[id]; existed {
|
||||
continue
|
||||
}
|
||||
if _, isSubField := newFields.structSubFields[id]; isSubField {
|
||||
parentID := newFields.structParents[id]
|
||||
if _, parentExisted := oldFields.structFields[parentID]; parentExisted {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add sub-field %q to kept struct field id %d", field.GetName(), parentID)
|
||||
}
|
||||
if field.GetIsPrimaryKey() || field.GetAutoID() || field.GetIsPartitionKey() || field.GetIsClusteringKey() || field.GetIsFunctionOutput() || field.GetIsDynamic() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add struct sub-field %q with a protected role", field.GetName())
|
||||
}
|
||||
if !field.GetNullable() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add non-nullable sub-field %q in a new struct field", field.GetName())
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := validateAddedEvolutionField(field, newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for id, structField := range newFields.structFields {
|
||||
if _, existed := oldFields.allIDs[id]; existed {
|
||||
continue
|
||||
}
|
||||
if structField.GetFieldID() < common.StartOfUserFieldID || isReservedEvolutionFieldName(structField.GetName()) {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add system struct field %q online", structField.GetName())
|
||||
}
|
||||
if !structField.GetNullable() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add non-nullable struct field %q online", structField.GetName())
|
||||
}
|
||||
if len(structField.GetFields()) == 0 {
|
||||
return merr.WrapErrParameterInvalidMsg("new struct field %q must contain at least one sub-field", structField.GetName())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAddedEvolutionField(field *schemapb.FieldSchema, newSchema *schemapb.CollectionSchema) error {
|
||||
name := field.GetName()
|
||||
dynamicEnabled := newSchema.GetEnableDynamicField()
|
||||
if field.GetIsPrimaryKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add primary key field %q online", name)
|
||||
}
|
||||
if field.GetAutoID() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add auto-ID field %q online", name)
|
||||
}
|
||||
if field.GetIsPartitionKey() {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add partition key field %q online", name)
|
||||
}
|
||||
if field.GetFieldID() < common.StartOfUserFieldID {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add system field %q online", name)
|
||||
}
|
||||
if isReservedEvolutionFieldName(name) && (!dynamicEnabled || !field.GetIsDynamic() || name != common.MetaFieldName) {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add system field %q online", name)
|
||||
}
|
||||
if field.GetIsFunctionOutput() {
|
||||
// A function-output field is only legitimate when a function in the new
|
||||
// schema actually produces it. Ordinary AddField does not run the function
|
||||
// graph validator, so without this check a request could persist an orphan
|
||||
// output field with no owning function (later index creation then fails).
|
||||
if !evolutionFieldHasFunctionProducer(newSchema, field) {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add field %q marked as a function output with no producing function", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if field.GetIsDynamic() {
|
||||
return nil
|
||||
}
|
||||
if !field.GetNullable() && field.GetDefaultValue() == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot add non-nullable field %q without a default value", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDroppedEvolutionFields(oldFields, newFields *evolutionFieldMaps, newSchema *schemapb.CollectionSchema) error {
|
||||
for id, field := range oldFields.fields {
|
||||
if _, kept := newFields.allIDs[id]; kept {
|
||||
continue
|
||||
}
|
||||
if parentID, isSubField := oldFields.structParents[id]; isSubField {
|
||||
if _, parentKept := newFields.structFields[parentID]; parentKept {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop sub-field %q from kept struct field id %d", field.GetName(), parentID)
|
||||
}
|
||||
}
|
||||
if err := validateDroppedEvolutionField(field, newSchema); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDroppedEvolutionField(field *schemapb.FieldSchema, newSchema *schemapb.CollectionSchema) error {
|
||||
name := field.GetName()
|
||||
switch {
|
||||
case field.GetIsPrimaryKey():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop primary key field %q", name)
|
||||
case field.GetIsPartitionKey():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop partition key field %q", name)
|
||||
case field.GetIsClusteringKey():
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop clustering key field %q", name)
|
||||
case field.GetFieldID() < common.StartOfUserFieldID:
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop system field %q", name)
|
||||
case isReservedEvolutionFieldName(name) && (!field.GetIsDynamic() || newSchema.GetEnableDynamicField()):
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop system field %q", name)
|
||||
}
|
||||
if function := evolutionFunctionReferencing(newSchema, field.GetFieldID()); function != "" {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop field %q while function %q still references it", name, function)
|
||||
}
|
||||
if typeutil.IsVectorType(field.GetDataType()) && !evolutionHasVectorField(newSchema) {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot drop the last vector field %q", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateEvolutionClusteringKey allows a clustering-key field to be added
|
||||
// online, but enforces that a collection is clustered on at most one field.
|
||||
func validateEvolutionClusteringKey(schema *schemapb.CollectionSchema) error {
|
||||
clusteringKey := ""
|
||||
for _, field := range schema.GetFields() {
|
||||
if !field.GetIsClusteringKey() {
|
||||
continue
|
||||
}
|
||||
if clusteringKey != "" {
|
||||
return merr.WrapErrParameterInvalidMsg("collection can only have one clustering key, but got both %q and %q", clusteringKey, field.GetName())
|
||||
}
|
||||
clusteringKey = field.GetName()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateEvolutionDynamicGraph(schema *schemapb.CollectionSchema) error {
|
||||
var dynamicField *schemapb.FieldSchema
|
||||
for _, field := range schema.GetFields() {
|
||||
if !field.GetIsDynamic() {
|
||||
continue
|
||||
}
|
||||
if dynamicField != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("collection schema contains more than one dynamic field")
|
||||
}
|
||||
dynamicField = field
|
||||
}
|
||||
if !schema.GetEnableDynamicField() {
|
||||
if dynamicField != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("dynamic field %q exists while dynamic fields are disabled", dynamicField.GetName())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if dynamicField == nil {
|
||||
return merr.WrapErrParameterInvalidMsg("dynamic fields are enabled but the dynamic field is missing")
|
||||
}
|
||||
if dynamicField.GetName() != common.MetaFieldName || dynamicField.GetDataType() != schemapb.DataType_JSON {
|
||||
return merr.WrapErrParameterInvalidMsg("dynamic field must be the JSON field %q", common.MetaFieldName)
|
||||
}
|
||||
if dynamicField.GetIsFunctionOutput() {
|
||||
return merr.WrapErrParameterInvalidMsg("dynamic field %q cannot be a function output", dynamicField.GetName())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateNumericBoundsEvolution rejects field-param changes that would
|
||||
// reinterpret already-written data. A vector dimension is immutable: reading
|
||||
// existing vectors under a new dim yields garbage. max_length / max_capacity
|
||||
// are write-time bounds only — existing rows stay readable and new inserts are
|
||||
// validated against the new bound — so they may grow or shrink freely; they may
|
||||
// not be removed or invalidated (that would drop the bound entirely).
|
||||
func validateNumericBoundsEvolution(fieldName string, oldParams, newParams []*commonpb.KeyValuePair) error {
|
||||
for _, key := range []string{common.MaxLengthKey, common.MaxCapacityKey} {
|
||||
_, existed, err := evolutionNumericValue(oldParams, key)
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("field %q has an invalid old %s bound", fieldName, key)
|
||||
}
|
||||
if !existed {
|
||||
continue
|
||||
}
|
||||
newValue, kept, err := evolutionNumericValue(newParams, key)
|
||||
if err != nil || !kept {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot remove or invalidate %s of field %q", key, fieldName)
|
||||
}
|
||||
if newValue <= 0 {
|
||||
return merr.WrapErrParameterInvalidMsg("%s of field %q must be a positive integer, got %d", key, fieldName, newValue)
|
||||
}
|
||||
}
|
||||
|
||||
oldDim, existed, err := evolutionNumericValue(oldParams, common.DimKey)
|
||||
if err != nil {
|
||||
return merr.WrapErrParameterInvalidMsg("field %q has an invalid old dimension", fieldName)
|
||||
}
|
||||
if !existed {
|
||||
return nil
|
||||
}
|
||||
newDim, kept, err := evolutionNumericValue(newParams, common.DimKey)
|
||||
if err != nil || !kept || oldDim != newDim {
|
||||
return merr.WrapErrParameterInvalidMsg("cannot change or remove the dimension of field %q", fieldName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func evolutionNumericValue(params []*commonpb.KeyValuePair, key string) (int64, bool, error) {
|
||||
for _, param := range params {
|
||||
if param.GetKey() != key {
|
||||
continue
|
||||
}
|
||||
value, err := strconv.ParseInt(param.GetValue(), 10, 64)
|
||||
return value, true, err
|
||||
}
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
func validateMaxFieldIDEvolution(oldSchema, newSchema *schemapb.CollectionSchema, oldFields, newFields *evolutionFieldMaps) error {
|
||||
oldProperty, err := evolutionSchemaIntProperty(oldSchema, common.MaxFieldIDKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newProperty, err := evolutionSchemaIntProperty(newSchema, common.MaxFieldIDKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if proto.Equal(oldSchema, newSchema) {
|
||||
return nil
|
||||
}
|
||||
|
||||
oldFloor := maxEvolutionFieldID(oldFields)
|
||||
if oldProperty.valid && oldProperty.value > oldFloor {
|
||||
oldFloor = oldProperty.value
|
||||
}
|
||||
expectedMax := oldFloor
|
||||
if newLiveMax := maxEvolutionFieldID(newFields); newLiveMax > expectedMax {
|
||||
expectedMax = newLiveMax
|
||||
}
|
||||
if !newProperty.present || !newProperty.valid {
|
||||
return merr.WrapErrParameterInvalidMsg("schema evolution must contain a valid %s property", common.MaxFieldIDKey)
|
||||
}
|
||||
if newProperty.value != expectedMax {
|
||||
return merr.WrapErrParameterInvalidMsg("%s must be %d after this evolution, got %d", common.MaxFieldIDKey, expectedMax, newProperty.value)
|
||||
}
|
||||
|
||||
for id := range newFields.allIDs {
|
||||
if _, existed := oldFields.allIDs[id]; !existed && id <= oldFloor {
|
||||
return merr.WrapErrParameterInvalidMsg("new field id %d reuses an already allocated field id", id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type evolutionIntProperty struct {
|
||||
value int64
|
||||
present bool
|
||||
valid bool
|
||||
}
|
||||
|
||||
func evolutionSchemaIntProperty(schema *schemapb.CollectionSchema, key string) (evolutionIntProperty, error) {
|
||||
result := evolutionIntProperty{}
|
||||
for _, property := range schema.GetProperties() {
|
||||
if property.GetKey() != key {
|
||||
continue
|
||||
}
|
||||
if result.present {
|
||||
return evolutionIntProperty{}, merr.WrapErrParameterInvalidMsg("schema contains duplicate property %q", key)
|
||||
}
|
||||
result.present = true
|
||||
parsed, err := strconv.ParseInt(property.GetValue(), 10, 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
result.value = parsed
|
||||
result.valid = true
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func maxEvolutionFieldID(fields *evolutionFieldMaps) int64 {
|
||||
maxID := int64(common.StartOfUserFieldID)
|
||||
for id := range fields.allIDs {
|
||||
if id > maxID {
|
||||
maxID = id
|
||||
}
|
||||
}
|
||||
return maxID
|
||||
}
|
||||
|
||||
func isReservedEvolutionFieldName(name string) bool {
|
||||
switch name {
|
||||
case common.RowIDFieldName, common.TimeStampFieldName, common.MetaFieldName, common.NamespaceFieldName, common.VirtualPKFieldName:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func evolutionFunctionReferencing(schema *schemapb.CollectionSchema, fieldID int64) string {
|
||||
for _, function := range schema.GetFunctions() {
|
||||
for _, inputID := range function.GetInputFieldIds() {
|
||||
if inputID == fieldID {
|
||||
return function.GetName()
|
||||
}
|
||||
}
|
||||
for _, outputID := range function.GetOutputFieldIds() {
|
||||
if outputID == fieldID {
|
||||
return function.GetName()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// evolutionFieldHasFunctionProducer reports whether some function in the schema
|
||||
// produces the given field as an output. Field id and name are matched because
|
||||
// they are populated at different stages of the add-function-field flow.
|
||||
func evolutionFieldHasFunctionProducer(schema *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
|
||||
for _, function := range schema.GetFunctions() {
|
||||
for _, outputID := range function.GetOutputFieldIds() {
|
||||
if outputID == field.GetFieldID() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, outputName := range function.GetOutputFieldNames() {
|
||||
if outputName == field.GetName() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func evolutionHasVectorField(schema *schemapb.CollectionSchema) bool {
|
||||
for _, field := range typeutil.GetAllFieldSchemas(schema) {
|
||||
if typeutil.IsVectorType(field.GetDataType()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,680 @@
|
||||
// Licensed to the LF AI & Data foundation under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package schemautil
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/v3/common"
|
||||
)
|
||||
|
||||
func TestValidateSchemaEvolutionAllowed(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*schemapb.CollectionSchema)
|
||||
}{
|
||||
{name: "unchanged", mutate: func(*schemapb.CollectionSchema) {}},
|
||||
{name: "add nullable field", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.Fields = append(schema.Fields, evolutionField(106, "nullable", schemapb.DataType_Int64, true))
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
{name: "add field with default", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
field := evolutionField(106, "with_default", schemapb.DataType_Int64, false)
|
||||
field.DefaultValue = &schemapb.ValueField{Data: &schemapb.ValueField_LongData{LongData: 7}}
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
{name: "add nullable struct container", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
child := evolutionField(107, "profile[values]", schemapb.DataType_Array, true)
|
||||
child.ElementType = schemapb.DataType_Int64
|
||||
child.TypeParams = []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "32"}}
|
||||
schema.StructArrayFields = append(schema.StructArrayFields, &schemapb.StructArrayFieldSchema{
|
||||
FieldID: 106,
|
||||
Name: "profile",
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxCapacityKey, Value: "32"},
|
||||
},
|
||||
Fields: []*schemapb.FieldSchema{child},
|
||||
})
|
||||
setMaxFieldID(schema, 107)
|
||||
}},
|
||||
{name: "drop ordinary field", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.Fields = removeEvolutionField(schema.Fields, 105)
|
||||
}},
|
||||
{name: "grow max length", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
setEvolutionTypeParam(evolutionFieldByID(schema, 104), common.MaxLengthKey, "512")
|
||||
}},
|
||||
{name: "grow max capacity", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
setEvolutionTypeParam(evolutionFieldByID(schema, 105), common.MaxCapacityKey, "128")
|
||||
}},
|
||||
{name: "add function output field with producer", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
output := evolutionField(106, "func_output", schemapb.DataType_SparseFloatVector, false)
|
||||
output.IsFunctionOutput = true
|
||||
schema.Fields = append(schema.Fields, output)
|
||||
schema.Functions = append(schema.Functions, &schemapb.FunctionSchema{
|
||||
Name: "bm25",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
OutputFieldNames: []string{"func_output"},
|
||||
OutputFieldIds: []int64{106},
|
||||
})
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
{name: "enable dynamic field", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.EnableDynamicField = true
|
||||
field := evolutionField(106, common.MetaFieldName, schemapb.DataType_JSON, true)
|
||||
field.IsDynamic = true
|
||||
field.DefaultValue = &schemapb.ValueField{Data: &schemapb.ValueField_BytesData{BytesData: []byte("{}")}}
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
test.mutate(newSchema)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("disable dynamic field", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
oldSchema.EnableDynamicField = true
|
||||
field := evolutionField(106, common.MetaFieldName, schemapb.DataType_JSON, true)
|
||||
field.IsDynamic = true
|
||||
oldSchema.Fields = append(oldSchema.Fields, field)
|
||||
setMaxFieldID(oldSchema, 106)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.EnableDynamicField = false
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 106)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("drop function and clear former output role", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithFunction()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Functions = nil
|
||||
evolutionFieldByID(newSchema, 106).IsFunctionOutput = false
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("drop whole struct container", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithStruct()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.StructArrayFields = nil
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
oldValue string
|
||||
}{
|
||||
{name: "repair missing max field id", oldValue: ""},
|
||||
{name: "repair malformed max field id", oldValue: "not-a-number"},
|
||||
{name: "repair stale low max field id", oldValue: "102"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
if test.oldValue == "" {
|
||||
removeEvolutionProperty(oldSchema, common.MaxFieldIDKey)
|
||||
} else {
|
||||
setEvolutionProperty(oldSchema, common.MaxFieldIDKey, test.oldValue)
|
||||
}
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
setMaxFieldID(newSchema, 105)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("repair stale low max field id using struct child floor", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithStruct()
|
||||
setMaxFieldID(oldSchema, 102)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
setMaxFieldID(newSchema, 107)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("preserve stale high max field id", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
setMaxFieldID(oldSchema, 120)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 105)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{name: "unchanged malformed max field id", value: "not-a-number"},
|
||||
{name: "unchanged stale low max field id", value: "102"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
setEvolutionProperty(oldSchema, common.MaxFieldIDKey, test.value)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsKeptFieldReinterpretation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*schemapb.FieldSchema)
|
||||
}{
|
||||
{name: "data type", mutate: func(field *schemapb.FieldSchema) { field.DataType = schemapb.DataType_Int32 }},
|
||||
{name: "element type", mutate: func(field *schemapb.FieldSchema) { field.ElementType = schemapb.DataType_Int32 }},
|
||||
{name: "nullability", mutate: func(field *schemapb.FieldSchema) { field.Nullable = false }},
|
||||
{name: "dimension", mutate: func(field *schemapb.FieldSchema) { setEvolutionTypeParam(field, common.DimKey, "256") }},
|
||||
{name: "function output role", mutate: func(field *schemapb.FieldSchema) { field.IsFunctionOutput = true }},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
var field *schemapb.FieldSchema
|
||||
switch test.name {
|
||||
case "element type", "nullability":
|
||||
field = evolutionFieldByID(newSchema, 105)
|
||||
case "dimension":
|
||||
field = evolutionFieldByID(newSchema, 103)
|
||||
default:
|
||||
field = evolutionFieldByID(newSchema, 102)
|
||||
}
|
||||
test.mutate(field)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsNonMonotonicBounds(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
field int64
|
||||
key string
|
||||
value string
|
||||
remove bool
|
||||
}{
|
||||
{name: "remove max length", field: 104, key: common.MaxLengthKey, remove: true},
|
||||
{name: "remove max capacity", field: 105, key: common.MaxCapacityKey, remove: true},
|
||||
{name: "zero max length", field: 104, key: common.MaxLengthKey, value: "0"},
|
||||
{name: "negative max length", field: 104, key: common.MaxLengthKey, value: "-8"},
|
||||
{name: "zero max capacity", field: 105, key: common.MaxCapacityKey, value: "0"},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
field := evolutionFieldByID(newSchema, test.field)
|
||||
if test.remove {
|
||||
removeEvolutionTypeParam(field, test.key)
|
||||
} else {
|
||||
setEvolutionTypeParam(field, test.key, test.value)
|
||||
}
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("shrink max field id", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
setMaxFieldID(newSchema, 104)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("remove max field id", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
removeEvolutionProperty(newSchema, common.MaxFieldIDKey)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("alter max field id without allocating a field", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
setMaxFieldID(newSchema, 110)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionAllowsResizingFieldBounds(t *testing.T) {
|
||||
// max_length / max_capacity are write-time bounds, not a reinterpretation of
|
||||
// already-written data, so they may grow or shrink freely.
|
||||
tests := []struct {
|
||||
name string
|
||||
field int64
|
||||
key string
|
||||
value string
|
||||
}{
|
||||
{name: "shrink max length", field: 104, key: common.MaxLengthKey, value: "64"},
|
||||
{name: "grow max length", field: 104, key: common.MaxLengthKey, value: "1024"},
|
||||
{name: "shrink max capacity", field: 105, key: common.MaxCapacityKey, value: "16"},
|
||||
{name: "grow max capacity", field: 105, key: common.MaxCapacityKey, value: "256"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
setEvolutionTypeParam(evolutionFieldByID(newSchema, test.field), test.key, test.value)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsUnsafeAddedFields(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
field *schemapb.FieldSchema
|
||||
}{
|
||||
{name: "non-nullable field without default", field: evolutionField(106, "required", schemapb.DataType_Int64, false)},
|
||||
{name: "orphan function output", field: func() *schemapb.FieldSchema {
|
||||
field := evolutionField(106, "orphan_output", schemapb.DataType_SparseFloatVector, false)
|
||||
field.IsFunctionOutput = true
|
||||
return field
|
||||
}()},
|
||||
{name: "primary key", field: func() *schemapb.FieldSchema {
|
||||
field := evolutionField(106, "new_pk", schemapb.DataType_Int64, true)
|
||||
field.IsPrimaryKey = true
|
||||
return field
|
||||
}()},
|
||||
{name: "auto id", field: func() *schemapb.FieldSchema {
|
||||
field := evolutionField(106, "auto_id", schemapb.DataType_Int64, true)
|
||||
field.AutoID = true
|
||||
return field
|
||||
}()},
|
||||
{name: "partition key", field: func() *schemapb.FieldSchema {
|
||||
field := evolutionField(106, "partition_key", schemapb.DataType_Int64, true)
|
||||
field.IsPartitionKey = true
|
||||
return field
|
||||
}()},
|
||||
{name: "system field id", field: evolutionField(int64(common.RowIDField), "fake_system", schemapb.DataType_Int64, true)},
|
||||
{name: "system field name", field: evolutionField(106, common.TimeStampFieldName, schemapb.DataType_Int64, true)},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = append(newSchema.Fields, test.field)
|
||||
setMaxFieldID(newSchema, 106)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("reused dropped field id", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 105)
|
||||
newSchema.Fields = append(newSchema.Fields, evolutionField(105, "replacement", schemapb.DataType_Int64, true))
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("legacy schema must establish max field id before adding", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
removeEvolutionProperty(oldSchema, common.MaxFieldIDKey)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = append(newSchema.Fields, evolutionField(106, "nullable", schemapb.DataType_Int64, true))
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsStructSubFieldReparenting(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
oldSchema.StructArrayFields = []*schemapb.StructArrayFieldSchema{
|
||||
{
|
||||
FieldID: 106,
|
||||
Name: "left",
|
||||
Nullable: true,
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
evolutionField(107, "left[value]", schemapb.DataType_Int64, true),
|
||||
},
|
||||
},
|
||||
{FieldID: 108, Name: "right", Nullable: true},
|
||||
}
|
||||
setMaxFieldID(oldSchema, 108)
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
moved := newSchema.StructArrayFields[0].Fields[0]
|
||||
newSchema.StructArrayFields[0].Fields = nil
|
||||
newSchema.StructArrayFields[1].Fields = []*schemapb.FieldSchema{moved}
|
||||
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsIndependentStructSubFieldMutation(t *testing.T) {
|
||||
t.Run("add child to kept container", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithStruct()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
child := evolutionField(108, "profile[extra]", schemapb.DataType_Array, true)
|
||||
child.ElementType = schemapb.DataType_Int64
|
||||
child.TypeParams = []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "16"}}
|
||||
newSchema.StructArrayFields[0].Fields = append(newSchema.StructArrayFields[0].Fields, child)
|
||||
setMaxFieldID(newSchema, 108)
|
||||
require.ErrorContains(t, ValidateSchemaEvolution(oldSchema, newSchema), "cannot add sub-field")
|
||||
})
|
||||
|
||||
t.Run("drop child from kept container", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithStruct()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.StructArrayFields[0].Fields = nil
|
||||
require.ErrorContains(t, ValidateSchemaEvolution(oldSchema, newSchema), "cannot drop sub-field")
|
||||
})
|
||||
|
||||
t.Run("add whole container with non-nullable child", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
child := evolutionField(107, "profile[values]", schemapb.DataType_Array, false)
|
||||
child.ElementType = schemapb.DataType_Int64
|
||||
child.TypeParams = []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "16"}}
|
||||
newSchema.StructArrayFields = []*schemapb.StructArrayFieldSchema{
|
||||
{FieldID: 106, Name: "profile", Nullable: true, Fields: []*schemapb.FieldSchema{child}},
|
||||
}
|
||||
setMaxFieldID(newSchema, 107)
|
||||
require.ErrorContains(t, ValidateSchemaEvolution(oldSchema, newSchema), "non-nullable sub-field")
|
||||
})
|
||||
|
||||
t.Run("add empty whole container", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.StructArrayFields = []*schemapb.StructArrayFieldSchema{
|
||||
{FieldID: 106, Name: "profile", Nullable: true},
|
||||
}
|
||||
setMaxFieldID(newSchema, 106)
|
||||
require.ErrorContains(t, ValidateSchemaEvolution(oldSchema, newSchema), "must contain at least one sub-field")
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsProtectedDrops(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*schemapb.CollectionSchema)
|
||||
}{
|
||||
{name: "primary key", mutate: func(schema *schemapb.CollectionSchema) { schema.Fields = removeEvolutionField(schema.Fields, 100) }},
|
||||
{name: "system field", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.Fields = removeEvolutionField(schema.Fields, int64(common.RowIDField))
|
||||
}},
|
||||
{name: "last vector field", mutate: func(schema *schemapb.CollectionSchema) { schema.Fields = removeEvolutionField(schema.Fields, 103) }},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
test.mutate(newSchema)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
mark func(*schemapb.FieldSchema)
|
||||
}{
|
||||
{name: "partition key", mark: func(field *schemapb.FieldSchema) { field.IsPartitionKey = true }},
|
||||
{name: "clustering key", mark: func(field *schemapb.FieldSchema) { field.IsClusteringKey = true }},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
test.mark(evolutionFieldByID(oldSchema, 104))
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 104)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("surviving function input", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithFunction()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 102)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("surviving function output", func(t *testing.T) {
|
||||
oldSchema := evolutionSchemaWithFunction()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
newSchema.Fields = removeEvolutionField(newSchema.Fields, 106)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionRejectsMalformedDynamicGraphs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
mutate func(*schemapb.CollectionSchema)
|
||||
}{
|
||||
{name: "enabled without dynamic field", mutate: func(schema *schemapb.CollectionSchema) { schema.EnableDynamicField = true }},
|
||||
{name: "disabled with dynamic field", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
field := evolutionField(106, common.MetaFieldName, schemapb.DataType_JSON, true)
|
||||
field.IsDynamic = true
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
{name: "multiple dynamic fields", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.EnableDynamicField = true
|
||||
for id := int64(106); id <= 107; id++ {
|
||||
field := evolutionField(id, common.MetaFieldName+strconv.FormatInt(id, 10), schemapb.DataType_JSON, true)
|
||||
field.IsDynamic = true
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
}
|
||||
setMaxFieldID(schema, 107)
|
||||
}},
|
||||
{name: "dynamic field has wrong type", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.EnableDynamicField = true
|
||||
field := evolutionField(106, common.MetaFieldName, schemapb.DataType_Int64, true)
|
||||
field.IsDynamic = true
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
{name: "dynamic field has wrong name", mutate: func(schema *schemapb.CollectionSchema) {
|
||||
schema.EnableDynamicField = true
|
||||
field := evolutionField(106, "dynamic", schemapb.DataType_JSON, true)
|
||||
field.IsDynamic = true
|
||||
schema.Fields = append(schema.Fields, field)
|
||||
setMaxFieldID(schema, 106)
|
||||
}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
test.mutate(newSchema)
|
||||
require.Error(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSchemaEvolutionClusteringKey(t *testing.T) {
|
||||
longDefault := func() *schemapb.ValueField {
|
||||
return &schemapb.ValueField{Data: &schemapb.ValueField_LongData{LongData: 0}}
|
||||
}
|
||||
|
||||
t.Run("add clustering key allowed", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
ck := evolutionField(106, "ck", schemapb.DataType_Int64, false)
|
||||
ck.IsClusteringKey = true
|
||||
ck.DefaultValue = longDefault()
|
||||
newSchema.Fields = append(newSchema.Fields, ck)
|
||||
setMaxFieldID(newSchema, 106)
|
||||
require.NoError(t, ValidateSchemaEvolution(oldSchema, newSchema))
|
||||
})
|
||||
|
||||
t.Run("second clustering key rejected", func(t *testing.T) {
|
||||
oldSchema := evolutionBaseSchema()
|
||||
evolutionFieldByID(oldSchema, 104).IsClusteringKey = true
|
||||
newSchema := proto.Clone(oldSchema).(*schemapb.CollectionSchema)
|
||||
ck := evolutionField(106, "ck2", schemapb.DataType_Int64, false)
|
||||
ck.IsClusteringKey = true
|
||||
ck.DefaultValue = longDefault()
|
||||
newSchema.Fields = append(newSchema.Fields, ck)
|
||||
setMaxFieldID(newSchema, 106)
|
||||
require.ErrorContains(t, ValidateSchemaEvolution(oldSchema, newSchema), "one clustering key")
|
||||
})
|
||||
}
|
||||
|
||||
func evolutionBaseSchema() *schemapb.CollectionSchema {
|
||||
rowID := evolutionField(int64(common.RowIDField), common.RowIDFieldName, schemapb.DataType_Int64, false)
|
||||
rowID.AutoID = true
|
||||
timestamp := evolutionField(int64(common.TimeStampField), common.TimeStampFieldName, schemapb.DataType_Int64, false)
|
||||
pk := evolutionField(100, "pk", schemapb.DataType_Int64, false)
|
||||
pk.IsPrimaryKey = true
|
||||
body := evolutionField(102, "body", schemapb.DataType_Text, true)
|
||||
body.TypeParams = []*commonpb.KeyValuePair{{Key: common.EnableAnalyzerKey, Value: "true"}}
|
||||
vector := evolutionField(103, "embedding", schemapb.DataType_FloatVector, false)
|
||||
vector.TypeParams = []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "128"}}
|
||||
title := evolutionField(104, "title", schemapb.DataType_VarChar, true)
|
||||
title.TypeParams = []*commonpb.KeyValuePair{{Key: common.MaxLengthKey, Value: "128"}}
|
||||
tags := evolutionField(105, "tags", schemapb.DataType_Array, true)
|
||||
tags.ElementType = schemapb.DataType_VarChar
|
||||
tags.TypeParams = []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxCapacityKey, Value: "64"},
|
||||
{Key: common.MaxLengthKey, Value: "32"},
|
||||
}
|
||||
return &schemapb.CollectionSchema{
|
||||
Name: "evolution",
|
||||
Fields: []*schemapb.FieldSchema{rowID, timestamp, pk, body, vector, title, tags},
|
||||
Properties: []*commonpb.KeyValuePair{{Key: common.MaxFieldIDKey, Value: "105"}},
|
||||
}
|
||||
}
|
||||
|
||||
func evolutionSchemaWithFunction() *schemapb.CollectionSchema {
|
||||
schema := evolutionBaseSchema()
|
||||
output := evolutionField(106, "sparse", schemapb.DataType_SparseFloatVector, false)
|
||||
output.IsFunctionOutput = true
|
||||
schema.Fields = append(schema.Fields, output)
|
||||
schema.Functions = []*schemapb.FunctionSchema{evolutionFunction(200, 102, 106)}
|
||||
setMaxFieldID(schema, 106)
|
||||
return schema
|
||||
}
|
||||
|
||||
func evolutionSchemaWithStruct() *schemapb.CollectionSchema {
|
||||
schema := evolutionBaseSchema()
|
||||
child := evolutionField(107, "profile[values]", schemapb.DataType_Array, true)
|
||||
child.ElementType = schemapb.DataType_Int64
|
||||
child.TypeParams = []*commonpb.KeyValuePair{{Key: common.MaxCapacityKey, Value: "32"}}
|
||||
schema.StructArrayFields = []*schemapb.StructArrayFieldSchema{
|
||||
{
|
||||
FieldID: 106,
|
||||
Name: "profile",
|
||||
Nullable: true,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxCapacityKey, Value: "64"},
|
||||
},
|
||||
Fields: []*schemapb.FieldSchema{child},
|
||||
},
|
||||
}
|
||||
setMaxFieldID(schema, 107)
|
||||
return schema
|
||||
}
|
||||
|
||||
func evolutionFunction(id, inputID, outputID int64) *schemapb.FunctionSchema {
|
||||
return &schemapb.FunctionSchema{
|
||||
Id: id,
|
||||
Name: "bm25",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldIds: []int64{inputID},
|
||||
InputFieldNames: []string{"body"},
|
||||
OutputFieldIds: []int64{outputID},
|
||||
OutputFieldNames: []string{"sparse"},
|
||||
}
|
||||
}
|
||||
|
||||
func evolutionField(id int64, name string, dataType schemapb.DataType, nullable bool) *schemapb.FieldSchema {
|
||||
return &schemapb.FieldSchema{FieldID: id, Name: name, DataType: dataType, Nullable: nullable}
|
||||
}
|
||||
|
||||
func evolutionFieldByID(schema *schemapb.CollectionSchema, fieldID int64) *schemapb.FieldSchema {
|
||||
for _, field := range schema.GetFields() {
|
||||
if field.GetFieldID() == fieldID {
|
||||
return field
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeEvolutionField(fields []*schemapb.FieldSchema, fieldID int64) []*schemapb.FieldSchema {
|
||||
result := make([]*schemapb.FieldSchema, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
if field.GetFieldID() != fieldID {
|
||||
result = append(result, field)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func setEvolutionTypeParam(field *schemapb.FieldSchema, key, value string) {
|
||||
for _, param := range field.GetTypeParams() {
|
||||
if param.GetKey() == key {
|
||||
param.Value = value
|
||||
return
|
||||
}
|
||||
}
|
||||
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
|
||||
}
|
||||
|
||||
func removeEvolutionTypeParam(field *schemapb.FieldSchema, key string) {
|
||||
params := make([]*commonpb.KeyValuePair, 0, len(field.GetTypeParams()))
|
||||
for _, param := range field.GetTypeParams() {
|
||||
if param.GetKey() != key {
|
||||
params = append(params, param)
|
||||
}
|
||||
}
|
||||
field.TypeParams = params
|
||||
}
|
||||
|
||||
func setMaxFieldID(schema *schemapb.CollectionSchema, value int64) {
|
||||
for _, property := range schema.GetProperties() {
|
||||
if property.GetKey() == common.MaxFieldIDKey {
|
||||
property.Value = strconv.FormatInt(value, 10)
|
||||
return
|
||||
}
|
||||
}
|
||||
schema.Properties = append(schema.Properties, &commonpb.KeyValuePair{
|
||||
Key: common.MaxFieldIDKey,
|
||||
Value: strconv.FormatInt(value, 10),
|
||||
})
|
||||
}
|
||||
|
||||
func removeEvolutionProperty(schema *schemapb.CollectionSchema, key string) {
|
||||
properties := make([]*commonpb.KeyValuePair, 0, len(schema.GetProperties()))
|
||||
for _, property := range schema.GetProperties() {
|
||||
if property.GetKey() != key {
|
||||
properties = append(properties, property)
|
||||
}
|
||||
}
|
||||
schema.Properties = properties
|
||||
}
|
||||
|
||||
func setEvolutionProperty(schema *schemapb.CollectionSchema, key, value string) {
|
||||
for _, property := range schema.GetProperties() {
|
||||
if property.GetKey() == key {
|
||||
property.Value = value
|
||||
return
|
||||
}
|
||||
}
|
||||
schema.Properties = append(schema.Properties, &commonpb.KeyValuePair{Key: key, Value: value})
|
||||
}
|
||||
Reference in New Issue
Block a user